From 872cbc613245db7a1fc5e6656ed0135d2e115f50 Mon Sep 17 00:00:00 2001 From: Juhi Chandalia Date: Tue, 2 May 2023 19:00:17 -0700 Subject: [PATCH 001/204] GH-103963: Make dis display names of args for intrinsics opcodes (#104029) --- Include/internal/pycore_intrinsics.h | 26 ++++++------ Lib/dis.py | 8 ++++ Lib/opcode.py | 15 +++++++ Lib/test/test_dis.py | 41 ++++++++++++++++++- ...-05-01-17-58-28.gh-issue-103963.XWlHx7.rst | 1 + Tools/build/generate_opcode_h.py | 36 +++++++++++++++- 6 files changed, 110 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-01-17-58-28.gh-issue-103963.XWlHx7.rst diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index 46a52740eb8a0c..3902059a04b9da 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -1,26 +1,24 @@ +// Auto-generated by Tools/build/generate_opcode_h.py from Lib/opcode.py /* Unary Functions: */ +#define INTRINSIC_1_INVALID 0 +#define INTRINSIC_PRINT 1 +#define INTRINSIC_IMPORT_STAR 2 +#define INTRINSIC_STOPITERATION_ERROR 3 +#define INTRINSIC_ASYNC_GEN_WRAP 4 +#define INTRINSIC_UNARY_POSITIVE 5 +#define INTRINSIC_LIST_TO_TUPLE 6 -#define INTRINSIC_PRINT 1 -#define INTRINSIC_IMPORT_STAR 2 -#define INTRINSIC_STOPITERATION_ERROR 3 -#define INTRINSIC_ASYNC_GEN_WRAP 4 -#define INTRINSIC_UNARY_POSITIVE 5 -#define INTRINSIC_LIST_TO_TUPLE 6 - -#define MAX_INTRINSIC_1 6 +#define MAX_INTRINSIC_1 6 /* Binary Functions: */ +#define INTRINSIC_2_INVALID 0 +#define INTRINSIC_PREP_RERAISE_STAR 1 -#define INTRINSIC_PREP_RERAISE_STAR 1 - -#define MAX_INTRINSIC_2 1 - +#define MAX_INTRINSIC_2 1 typedef PyObject *(*instrinsic_func1)(PyThreadState* tstate, PyObject *value); typedef PyObject *(*instrinsic_func2)(PyThreadState* tstate, PyObject *value1, PyObject *value2); - extern const instrinsic_func1 _PyIntrinsics_UnaryFunctions[]; extern const instrinsic_func2 _PyIntrinsics_BinaryFunctions[]; - diff --git a/Lib/dis.py b/Lib/dis.py index 85c109584bf94f..3a8e6ac3bf5ace 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -11,6 +11,8 @@ _cache_format, _inline_cache_entries, _nb_ops, + _intrinsic_1_descs, + _intrinsic_2_descs, _specializations, _specialized_instructions, ) @@ -42,6 +44,8 @@ SEND = opmap['SEND'] LOAD_ATTR = opmap['LOAD_ATTR'] LOAD_SUPER_ATTR = opmap['LOAD_SUPER_ATTR'] +CALL_INTRINSIC_1 = opmap['CALL_INTRINSIC_1'] +CALL_INTRINSIC_2 = opmap['CALL_INTRINSIC_2'] CACHE = opmap["CACHE"] @@ -506,6 +510,10 @@ def _get_instructions_bytes(code, varname_from_oparg=None, if arg & (1<> COPY 3 POP_EXCEPT RERAISE 1 - >> CALL_INTRINSIC_1 3 + >> CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) RERAISE 1 ExceptionTable: 12 rows @@ -942,6 +971,16 @@ def test_kw_names(self): # Test that value is displayed for KW_NAMES self.do_disassembly_test(wrap_func_w_kwargs, dis_kw_names) + def test_intrinsic_1(self): + # Test that argrepr is displayed for CALL_INTRINSIC_1 + self.do_disassembly_test("from math import *", dis_intrinsic_1_2) + self.do_disassembly_test("+a", dis_intrinsic_1_5) + self.do_disassembly_test("(*a,)", dis_intrinsic_1_6) + + def test_intrinsic_2(self): + self.assertIn("CALL_INTRINSIC_2 1 (INTRINSIC_PREP_RERAISE_STAR)", + self.get_disassembly("try: pass\nexcept* Exception: x")) + def test_big_linenos(self): def func(count): namespace = {} diff --git a/Misc/NEWS.d/next/Library/2023-05-01-17-58-28.gh-issue-103963.XWlHx7.rst b/Misc/NEWS.d/next/Library/2023-05-01-17-58-28.gh-issue-103963.XWlHx7.rst new file mode 100644 index 00000000000000..cb06ad5d22e8a9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-01-17-58-28.gh-issue-103963.XWlHx7.rst @@ -0,0 +1 @@ +Make :mod:`dis` display the names of the args for :opcode:`CALL_INTRINSIC_*`. diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index 645b9f1de1170b..adcbaf2b8e08bf 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -52,6 +52,18 @@ #endif // !Py_INTERNAL_OPCODE_H """ +intrinsic_header = f""" +// Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE} + +""".lstrip() + +intrinsic_footer = """ +typedef PyObject *(*instrinsic_func1)(PyThreadState* tstate, PyObject *value); +typedef PyObject *(*instrinsic_func2)(PyThreadState* tstate, PyObject *value1, PyObject *value2); +extern const instrinsic_func1 _PyIntrinsics_UnaryFunctions[]; +extern const instrinsic_func2 _PyIntrinsics_BinaryFunctions[]; +""" + DEFINE = "#define {:<38} {:>3}\n" UINT32_MASK = (1<<32)-1 @@ -67,7 +79,9 @@ def write_int_array_from_ops(name, ops, out): assert bits == 0 out.write(f"}};\n") -def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/internal/pycore_opcode.h'): +def main(opcode_py, outfile='Include/opcode.h', + internaloutfile='Include/internal/pycore_opcode.h', + intrinsicoutfile='Include/internal/pycore_intrinsics.h'): opcode = {} if hasattr(tokenize, 'open'): fp = tokenize.open(opcode_py) # Python 3.2+ @@ -107,9 +121,11 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna opname_including_specialized[next_op] = name used[next_op] = True - with open(outfile, 'w') as fobj, open(internaloutfile, 'w') as iobj: + with open(outfile, 'w') as fobj, open(internaloutfile, 'w') as iobj, open( + intrinsicoutfile, "w") as nobj: fobj.write(header) iobj.write(internal_header) + nobj.write(intrinsic_header) for name in opname: if name in opmap: @@ -172,6 +188,22 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna for i, (op, _) in enumerate(opcode["_nb_ops"]): fobj.write(DEFINE.format(op, i)) + nobj.write("/* Unary Functions: */") + nobj.write("\n") + for i, op in enumerate(opcode["_intrinsic_1_descs"]): + nobj.write(DEFINE.format(op, i)) + nobj.write("\n") + nobj.write(DEFINE.format("MAX_INTRINSIC_1", i)) + + nobj.write("\n\n") + nobj.write("/* Binary Functions: */\n") + for i, op in enumerate(opcode["_intrinsic_2_descs"]): + nobj.write(DEFINE.format(op, i)) + nobj.write("\n") + nobj.write(DEFINE.format("MAX_INTRINSIC_2", i)) + + nobj.write(intrinsic_footer) + fobj.write("\n") fobj.write("/* Defined in Lib/opcode.py */\n") fobj.write(f"#define ENABLE_SPECIALIZATION {int(ENABLE_SPECIALIZATION)}") From de64e7561680fdc5358001e9488091e75d4174a3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 2 May 2023 21:30:03 -0600 Subject: [PATCH 002/204] gh-94673: More Per-Interpreter Fields for Builtin Static Types (gh-103912) his involves moving tp_dict, tp_bases, and tp_mro to PyInterpreterState, in the same way we did for tp_subclasses. Those three fields are effectively const for builtin static types (unlike tp_subclasses). In theory we only need to make their values immortal, along with their contents. However, that isn't such a simple proposition. (See gh-103823.) In the meantime the simplest solution is to move the fields into the interpreter. One alternative is to statically allocate the values, but that's its own can of worms. --- Include/internal/pycore_typeobject.h | 7 + Modules/_abc.c | 1 + Modules/gcmodule.c | 31 +--- Objects/structseq.c | 9 +- Objects/typeobject.c | 223 +++++++++++++++++++++------ 5 files changed, 185 insertions(+), 86 deletions(-) diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index f865e51aeba503..6a5ab7e63f85c3 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -44,6 +44,13 @@ struct type_cache { typedef struct { PyTypeObject *type; + int readying; + int ready; + // XXX tp_dict, tp_bases, and tp_mro can probably be statically + // allocated, instead of dynamically and stored on the interpreter. + PyObject *tp_dict; + PyObject *tp_bases; + PyObject *tp_mro; PyObject *tp_subclasses; /* We never clean up weakrefs for static builtin types since they will effectively never get triggered. However, there diff --git a/Modules/_abc.c b/Modules/_abc.c index 997b618d557ab2..9694331339aa78 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -7,6 +7,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_object.h" // _PyType_GetSubclasses() #include "pycore_runtime.h" // _Py_ID() +#include "pycore_typeobject.h" // _PyType_GetMRO() #include "clinic/_abc.c.h" /*[clinic input] diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 8a4d1a4398281e..f4d5186ff1552e 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -2174,23 +2174,6 @@ _PyGC_DumpShutdownStats(PyInterpreterState *interp) } -static void -gc_fini_untrack(PyGC_Head *list) -{ - PyGC_Head *gc; - for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(list)) { - PyObject *op = FROM_GC(gc); - _PyObject_GC_UNTRACK(op); - // gh-92036: If a deallocator function expect the object to be tracked - // by the GC (ex: func_dealloc()), it can crash if called on an object - // which is no longer tracked by the GC. Leak one strong reference on - // purpose so the object is never deleted and its deallocator is not - // called. - Py_INCREF(op); - } -} - - void _PyGC_Fini(PyInterpreterState *interp) { @@ -2198,17 +2181,9 @@ _PyGC_Fini(PyInterpreterState *interp) Py_CLEAR(gcstate->garbage); Py_CLEAR(gcstate->callbacks); - if (!_Py_IsMainInterpreter(interp)) { - // bpo-46070: Explicitly untrack all objects currently tracked by the - // GC. Otherwise, if an object is used later by another interpreter, - // calling PyObject_GC_UnTrack() on the object crashs if the previous - // or the next object of the PyGC_Head structure became a dangling - // pointer. - for (int i = 0; i < NUM_GENERATIONS; i++) { - PyGC_Head *gen = GEN_HEAD(gcstate, i); - gc_fini_untrack(gen); - } - } + /* We expect that none of this interpreters objects are shared + with other interpreters. + See https://github.com/python/cpython/issues/90228. */ } /* for debugging */ diff --git a/Objects/structseq.c b/Objects/structseq.c index f63660acb639c3..8b1895957101a4 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -511,7 +511,6 @@ _PyStructSequence_InitBuiltinWithFlags(PyInterpreterState *interp, Py_ssize_t n_members = count_members(desc, &n_unnamed_members); PyMemberDef *members = NULL; - int initialized = 1; if ((type->tp_flags & Py_TPFLAGS_READY) == 0) { assert(type->tp_name == NULL); assert(type->tp_members == NULL); @@ -524,7 +523,6 @@ _PyStructSequence_InitBuiltinWithFlags(PyInterpreterState *interp, initialize_static_fields(type, desc, members, tp_flags); _Py_SetImmortal(type); - initialized = 0; } #ifndef NDEBUG else { @@ -543,13 +541,10 @@ _PyStructSequence_InitBuiltinWithFlags(PyInterpreterState *interp, desc->name); goto error; } - // This should be dropped if tp_dict is made per-interpreter. - if (initialized) { - return 0; - } if (initialize_structseq_dict( - desc, _PyType_GetDict(type), n_members, n_unnamed_members) < 0) { + desc, _PyType_GetDict(type), n_members, n_unnamed_members) < 0) + { goto error; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index a9d3a69263fb40..cf0efe199b2828 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -167,29 +167,93 @@ static_builtin_state_clear(PyInterpreterState *interp, PyTypeObject *self) /* end static builtin helpers */ +static inline void +start_readying(PyTypeObject *type) +{ + if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = static_builtin_state_get(interp, type); + assert(state != NULL); + assert(!state->readying); + state->readying = 1; + return; + } + assert((type->tp_flags & Py_TPFLAGS_READYING) == 0); + type->tp_flags |= Py_TPFLAGS_READYING; +} + +static inline void +stop_readying(PyTypeObject *type) +{ + if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = static_builtin_state_get(interp, type); + assert(state != NULL); + assert(state->readying); + state->readying = 0; + return; + } + assert(type->tp_flags & Py_TPFLAGS_READYING); + type->tp_flags &= ~Py_TPFLAGS_READYING; +} + +static inline int +is_readying(PyTypeObject *type) +{ + if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = static_builtin_state_get(interp, type); + assert(state != NULL); + return state->readying; + } + return (type->tp_flags & Py_TPFLAGS_READYING) != 0; +} + + /* accessors for objects stored on PyTypeObject */ static inline PyObject * lookup_tp_dict(PyTypeObject *self) { + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + assert(state != NULL); + return state->tp_dict; + } return self->tp_dict; } PyObject * _PyType_GetDict(PyTypeObject *self) { + /* It returns a borrowed reference. */ return lookup_tp_dict(self); } static inline void set_tp_dict(PyTypeObject *self, PyObject *dict) { + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + assert(state != NULL); + state->tp_dict = dict; + return; + } self->tp_dict = dict; } static inline void clear_tp_dict(PyTypeObject *self) { + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + assert(state != NULL); + Py_CLEAR(state->tp_dict); + return; + } Py_CLEAR(self->tp_dict); } @@ -197,24 +261,45 @@ clear_tp_dict(PyTypeObject *self) static inline PyObject * lookup_tp_bases(PyTypeObject *self) { + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + assert(state != NULL); + return state->tp_bases; + } return self->tp_bases; } PyObject * _PyType_GetBases(PyTypeObject *self) { + /* It returns a borrowed reference. */ return lookup_tp_bases(self); } static inline void set_tp_bases(PyTypeObject *self, PyObject *bases) { + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + assert(state != NULL); + state->tp_bases = bases; + return; + } self->tp_bases = bases; } static inline void clear_tp_bases(PyTypeObject *self) { + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + assert(state != NULL); + Py_CLEAR(state->tp_bases); + return; + } Py_CLEAR(self->tp_bases); } @@ -222,24 +307,45 @@ clear_tp_bases(PyTypeObject *self) static inline PyObject * lookup_tp_mro(PyTypeObject *self) { + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + assert(state != NULL); + return state->tp_mro; + } return self->tp_mro; } PyObject * _PyType_GetMRO(PyTypeObject *self) { + /* It returns a borrowed reference. */ return lookup_tp_mro(self); } static inline void set_tp_mro(PyTypeObject *self, PyObject *mro) { + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + assert(state != NULL); + state->tp_mro = mro; + return; + } self->tp_mro = mro; } static inline void clear_tp_mro(PyTypeObject *self) { + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + assert(state != NULL); + Py_CLEAR(state->tp_mro); + return; + } Py_CLEAR(self->tp_mro); } @@ -408,7 +514,7 @@ _PyType_CheckConsistency(PyTypeObject *type) CHECK(Py_REFCNT(type) >= 1); CHECK(PyType_Check(type)); - CHECK(!(type->tp_flags & Py_TPFLAGS_READYING)); + CHECK(!is_readying(type)); CHECK(lookup_tp_dict(type) != NULL); if (type->tp_flags & Py_TPFLAGS_HAVE_GC) { @@ -809,7 +915,6 @@ static PyMemberDef type_members[] = { {"__base__", T_OBJECT, offsetof(PyTypeObject, tp_base), READONLY}, {"__dictoffset__", T_PYSSIZET, offsetof(PyTypeObject, tp_dictoffset), READONLY}, - {"__mro__", T_OBJECT, offsetof(PyTypeObject, tp_mro), READONLY}, {0} }; @@ -1023,7 +1128,21 @@ type_set_abstractmethods(PyTypeObject *type, PyObject *value, void *context) static PyObject * type_get_bases(PyTypeObject *type, void *context) { - return Py_NewRef(lookup_tp_bases(type)); + PyObject *bases = lookup_tp_bases(type); + if (bases == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(bases); +} + +static PyObject * +type_get_mro(PyTypeObject *type, void *context) +{ + PyObject *mro = lookup_tp_mro(type); + if (mro == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(mro); } static PyTypeObject *best_base(PyObject *); @@ -1402,6 +1521,7 @@ static PyGetSetDef type_getsets[] = { {"__name__", (getter)type_name, (setter)type_set_name, NULL}, {"__qualname__", (getter)type_qualname, (setter)type_set_qualname, NULL}, {"__bases__", (getter)type_get_bases, (setter)type_set_bases, NULL}, + {"__mro__", (getter)type_get_mro, NULL, NULL}, {"__module__", (getter)type_module, (setter)type_set_module, NULL}, {"__abstractmethods__", (getter)type_abstractmethods, (setter)type_set_abstractmethods, NULL}, @@ -4342,7 +4462,7 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) /* Look in tp_dict of types in MRO */ PyObject *mro = lookup_tp_mro(type); if (mro == NULL) { - if ((type->tp_flags & Py_TPFLAGS_READYING) == 0) { + if (!is_readying(type)) { if (PyType_Ready(type) < 0) { *error = -1; return NULL; @@ -4692,11 +4812,11 @@ static void clear_static_type_objects(PyInterpreterState *interp, PyTypeObject *type) { if (_Py_IsMainInterpreter(interp)) { - clear_tp_dict(type); - clear_tp_bases(type); - clear_tp_mro(type); Py_CLEAR(type->tp_cache); } + clear_tp_dict(type); + clear_tp_bases(type); + clear_tp_mro(type); clear_static_tp_subclasses(type); } @@ -6684,6 +6804,10 @@ type_ready_pre_checks(PyTypeObject *type) static int type_ready_set_bases(PyTypeObject *type) { + if (lookup_tp_bases(type) != NULL) { + return 0; + } + /* Initialize tp_base (defaults to BaseObject unless that's us) */ PyTypeObject *base = type->tp_base; if (base == NULL && type != &PyBaseObject_Type) { @@ -6997,7 +7121,7 @@ type_ready_add_subclasses(PyTypeObject *type) // Set tp_new and the "__new__" key in the type dictionary. // Use the Py_TPFLAGS_DISALLOW_INSTANTIATION flag. static int -type_ready_set_new(PyTypeObject *type) +type_ready_set_new(PyTypeObject *type, int rerunbuiltin) { PyTypeObject *base = type->tp_base; /* The condition below could use some explanation. @@ -7019,10 +7143,12 @@ type_ready_set_new(PyTypeObject *type) if (!(type->tp_flags & Py_TPFLAGS_DISALLOW_INSTANTIATION)) { if (type->tp_new != NULL) { - // If "__new__" key does not exists in the type dictionary, - // set it to tp_new_wrapper(). - if (add_tp_new_wrapper(type) < 0) { - return -1; + if (!rerunbuiltin || base == NULL || type->tp_new != base->tp_new) { + // If "__new__" key does not exists in the type dictionary, + // set it to tp_new_wrapper(). + if (add_tp_new_wrapper(type) < 0) { + return -1; + } } } else { @@ -7096,11 +7222,10 @@ type_ready_post_checks(PyTypeObject *type) static int -type_ready(PyTypeObject *type) +type_ready(PyTypeObject *type, int rerunbuiltin) { - _PyObject_ASSERT((PyObject *)type, - (type->tp_flags & Py_TPFLAGS_READYING) == 0); - type->tp_flags |= Py_TPFLAGS_READYING; + _PyObject_ASSERT((PyObject *)type, !is_readying(type)); + start_readying(type); if (type_ready_pre_checks(type) < 0) { goto error; @@ -7125,17 +7250,19 @@ type_ready(PyTypeObject *type) if (type_ready_mro(type) < 0) { goto error; } - if (type_ready_set_new(type) < 0) { + if (type_ready_set_new(type, rerunbuiltin) < 0) { goto error; } if (type_ready_fill_dict(type) < 0) { goto error; } - if (type_ready_inherit(type) < 0) { - goto error; - } - if (type_ready_preheader(type) < 0) { - goto error; + if (!rerunbuiltin) { + if (type_ready_inherit(type) < 0) { + goto error; + } + if (type_ready_preheader(type) < 0) { + goto error; + } } if (type_ready_set_hash(type) < 0) { goto error; @@ -7143,21 +7270,24 @@ type_ready(PyTypeObject *type) if (type_ready_add_subclasses(type) < 0) { goto error; } - if (type_ready_managed_dict(type) < 0) { - goto error; - } - if (type_ready_post_checks(type) < 0) { - goto error; + if (!rerunbuiltin) { + if (type_ready_managed_dict(type) < 0) { + goto error; + } + if (type_ready_post_checks(type) < 0) { + goto error; + } } /* All done -- set the ready flag */ - type->tp_flags = (type->tp_flags & ~Py_TPFLAGS_READYING) | Py_TPFLAGS_READY; + type->tp_flags = type->tp_flags | Py_TPFLAGS_READY; + stop_readying(type); assert(_PyType_CheckConsistency(type)); return 0; error: - type->tp_flags &= ~Py_TPFLAGS_READYING; + stop_readying(type); return -1; } @@ -7175,7 +7305,7 @@ PyType_Ready(PyTypeObject *type) type->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; } - return type_ready(type); + return type_ready(type, 0); } int @@ -7186,35 +7316,26 @@ _PyStaticType_InitBuiltin(PyInterpreterState *interp, PyTypeObject *self) assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_DICT)); assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_WEAKREF)); -#ifndef NDEBUG int ismain = _Py_IsMainInterpreter(interp); -#endif - if (self->tp_flags & Py_TPFLAGS_READY) { + if ((self->tp_flags & Py_TPFLAGS_READY) == 0) { + assert(ismain); + + self->tp_flags |= _Py_TPFLAGS_STATIC_BUILTIN; + self->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; + + assert(NEXT_GLOBAL_VERSION_TAG <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); + self->tp_version_tag = NEXT_GLOBAL_VERSION_TAG++; + self->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; + } + else { assert(!ismain); assert(self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); assert(self->tp_flags & Py_TPFLAGS_VALID_VERSION_TAG); - - static_builtin_state_init(interp, self); - - /* Per-interpreter tp_subclasses is done lazily. - Otherwise we would initialize it here. */ - - assert(_PyType_CheckConsistency(self)); - return 0; } - assert(ismain); - - self->tp_flags |= _Py_TPFLAGS_STATIC_BUILTIN; - self->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; - - assert(NEXT_GLOBAL_VERSION_TAG <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); - self->tp_version_tag = NEXT_GLOBAL_VERSION_TAG++; - self->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; - static_builtin_state_init(interp, self); - int res = type_ready(self); + int res = type_ready(self, !ismain); if (res < 0) { static_builtin_state_clear(interp, self); } From 292076a9aa29aba1023340a0d24252a7b27a454e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 2 May 2023 21:40:00 -0600 Subject: [PATCH 003/204] gh-104109: Expose Py_NewInterpreterFromConfig() in the Public C-API (gh-104110) We also expose PyInterpreterConfig. This is part of the PEP 684 (per-interpreter GIL) implementation. We will add docs as soon as we can. FYI, I'm adding the new config field for per-interpreter GIL in gh-99114. --- Include/cpython/initconfig.h | 2 +- Include/cpython/pylifecycle.h | 4 ++-- ...2023-05-02-21-05-54.gh-issue-104109.0tnDZV.rst | 5 +++++ Modules/_testcapimodule.c | 4 ++-- Modules/_xxsubinterpretersmodule.c | 8 ++++---- Python/pylifecycle.c | 15 ++++++++------- 6 files changed, 22 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-05-02-21-05-54.gh-issue-104109.0tnDZV.rst diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 79c1023baa9a0f..9c1783d272f1cd 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -252,7 +252,7 @@ typedef struct { int allow_threads; int allow_daemon_threads; int check_multi_interp_extensions; -} _PyInterpreterConfig; +} PyInterpreterConfig; #define _PyInterpreterConfig_INIT \ { \ diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index 79d55711319e55..08569ee683ce0d 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -62,9 +62,9 @@ PyAPI_FUNC(int) _Py_CoerceLegacyLocale(int warn); PyAPI_FUNC(int) _Py_LegacyLocaleDetected(int warn); PyAPI_FUNC(char *) _Py_SetLocaleFromEnv(int category); -PyAPI_FUNC(PyStatus) _Py_NewInterpreterFromConfig( +PyAPI_FUNC(PyStatus) Py_NewInterpreterFromConfig( PyThreadState **tstate_p, - const _PyInterpreterConfig *config); + const PyInterpreterConfig *config); typedef void (*atexit_datacallbackfunc)(void *); PyAPI_FUNC(int) _Py_AtExit( diff --git a/Misc/NEWS.d/next/C API/2023-05-02-21-05-54.gh-issue-104109.0tnDZV.rst b/Misc/NEWS.d/next/C API/2023-05-02-21-05-54.gh-issue-104109.0tnDZV.rst new file mode 100644 index 00000000000000..2ffc0fa81c014a --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-05-02-21-05-54.gh-issue-104109.0tnDZV.rst @@ -0,0 +1,5 @@ +We've added ``Py_NewInterpreterFromConfig()`` and ``PyInterpreterConfig`` to +the public C-API (but not the stable ABI; not yet at least). The new +function may be used to create a new interpreter with various features +configured. The function was added to support PEP 684 (per-interpreter +GIL). diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 30b8b6c6b3a87b..47e0ed9be8e709 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1538,7 +1538,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) PyThreadState_Swap(NULL); - const _PyInterpreterConfig config = { + const PyInterpreterConfig config = { .use_main_obmalloc = use_main_obmalloc, .allow_fork = allow_fork, .allow_exec = allow_exec, @@ -1546,7 +1546,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) .allow_daemon_threads = allow_daemon_threads, .check_multi_interp_extensions = check_multi_interp_extensions, }; - PyStatus status = _Py_NewInterpreterFromConfig(&substate, &config); + PyStatus status = Py_NewInterpreterFromConfig(&substate, &config); if (PyStatus_Exception(status)) { /* Since no new thread state was created, there is no exception to propagate; raise a fresh one after swapping in the old thread diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 884fb0d31f2b7f..95273ab278d996 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -513,12 +513,12 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) // Create and initialize the new interpreter. PyThreadState *save_tstate = _PyThreadState_GET(); - const _PyInterpreterConfig config = isolated - ? (_PyInterpreterConfig)_PyInterpreterConfig_INIT - : (_PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; + const PyInterpreterConfig config = isolated + ? (PyInterpreterConfig)_PyInterpreterConfig_INIT + : (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; // XXX Possible GILState issues? PyThreadState *tstate = NULL; - PyStatus status = _Py_NewInterpreterFromConfig(&tstate, &config); + PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); PyThreadState_Swap(save_tstate); if (PyStatus_Exception(status)) { /* Since no new thread state was created, there is no exception to diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index b8a115236900b9..b9add89b9c6c52 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -546,7 +546,8 @@ pycore_init_runtime(_PyRuntimeState *runtime, static PyStatus -init_interp_settings(PyInterpreterState *interp, const _PyInterpreterConfig *config) +init_interp_settings(PyInterpreterState *interp, + const PyInterpreterConfig *config) { assert(interp->feature_flags == 0); @@ -631,7 +632,7 @@ pycore_create_interpreter(_PyRuntimeState *runtime, return status; } - const _PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; + const PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; status = init_interp_settings(interp, &config); if (_PyStatus_EXCEPTION(status)) { return status; @@ -1991,7 +1992,7 @@ Py_Finalize(void) */ static PyStatus -new_interpreter(PyThreadState **tstate_p, const _PyInterpreterConfig *config) +new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) { PyStatus status; @@ -2079,8 +2080,8 @@ new_interpreter(PyThreadState **tstate_p, const _PyInterpreterConfig *config) } PyStatus -_Py_NewInterpreterFromConfig(PyThreadState **tstate_p, - const _PyInterpreterConfig *config) +Py_NewInterpreterFromConfig(PyThreadState **tstate_p, + const PyInterpreterConfig *config) { return new_interpreter(tstate_p, config); } @@ -2089,8 +2090,8 @@ PyThreadState * Py_NewInterpreter(void) { PyThreadState *tstate = NULL; - const _PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; - PyStatus status = _Py_NewInterpreterFromConfig(&tstate, &config); + const PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; + PyStatus status = new_interpreter(&tstate, &config); if (_PyStatus_EXCEPTION(status)) { Py_ExitStatusException(status); } From c7c3a60c88de61a79ded9fdaf6bc6a29da4efb9a Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Tue, 2 May 2023 20:42:00 -0700 Subject: [PATCH 004/204] gh-104049: do not expose on-disk location from SimpleHTTPRequestHandler (#104067) Do not expose the local server's on-disk location from `SimpleHTTPRequestHandler` when generating a directory index. (unnecessary information disclosure) --------- Co-authored-by: Gregory P. Smith Co-authored-by: Jelle Zijlstra --- Lib/http/server.py | 2 +- Lib/test/test_httpservers.py | 8 ++++++++ .../2023-05-01-15-03-25.gh-issue-104049.b01Y3g.rst | 2 ++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Security/2023-05-01-15-03-25.gh-issue-104049.b01Y3g.rst diff --git a/Lib/http/server.py b/Lib/http/server.py index 971f08046d50b5..a245ffb307860a 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -791,7 +791,7 @@ def list_directory(self, path): displaypath = urllib.parse.unquote(self.path, errors='surrogatepass') except UnicodeDecodeError: - displaypath = urllib.parse.unquote(path) + displaypath = urllib.parse.unquote(self.path) displaypath = html.escape(displaypath, quote=False) enc = sys.getfilesystemencoding() title = f'Directory listing for {displaypath}' diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index cbcf94136ac4eb..0382b5ec448d57 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -418,6 +418,14 @@ def test_undecodable_filename(self): self.check_status_and_reason(response, HTTPStatus.OK, data=os_helper.TESTFN_UNDECODABLE) + def test_undecodable_parameter(self): + # sanity check using a valid parameter + response = self.request(self.base_url + '/?x=123').read() + self.assertRegex(response, f'listing for {self.base_url}/\?x=123'.encode('latin1')) + # now the bogus encoding + response = self.request(self.base_url + '/?x=%bb').read() + self.assertRegex(response, f'listing for {self.base_url}/\?x=\xef\xbf\xbd'.encode('latin1')) + def test_get_dir_redirect_location_domain_injection_bug(self): """Ensure //evil.co/..%2f../../X does not put //evil.co/ in Location. diff --git a/Misc/NEWS.d/next/Security/2023-05-01-15-03-25.gh-issue-104049.b01Y3g.rst b/Misc/NEWS.d/next/Security/2023-05-01-15-03-25.gh-issue-104049.b01Y3g.rst new file mode 100644 index 00000000000000..969deb26bfeb95 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-05-01-15-03-25.gh-issue-104049.b01Y3g.rst @@ -0,0 +1,2 @@ +Do not expose the local on-disk location in directory indexes +produced by :class:`http.client.SimpleHTTPRequestHandler`. From af886ffa0612124598b5e8174c3b1be0f60eab38 Mon Sep 17 00:00:00 2001 From: andrei kulakov Date: Tue, 2 May 2023 23:50:10 -0400 Subject: [PATCH 005/204] GH-89769: `pathlib.Path.glob()`: do not follow symlinks when checking for precise match (GH-29655) Co-authored-by: Barney Gale --- Doc/library/pathlib.rst | 15 +++++++++------ Lib/pathlib.py | 10 +++++++--- Lib/test/test_pathlib.py | 4 ++++ .../2021-11-19-23-37-18.bpo-45606.UW5XE1.rst | 5 +++++ 4 files changed, 25 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2021-11-19-23-37-18.bpo-45606.UW5XE1.rst diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 8e91936680fab8..4847ac24c77513 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -819,9 +819,14 @@ call fails (for example because the path doesn't exist). .. versionchanged:: 3.10 The *follow_symlinks* parameter was added. -.. method:: Path.exists() +.. method:: Path.exists(*, follow_symlinks=True) - Whether the path points to an existing file or directory:: + Return ``True`` if the path points to an existing file or directory. + + This method normally follows symlinks; to check if a symlink exists, add + the argument ``follow_symlinks=False``. + + :: >>> Path('.').exists() True @@ -832,10 +837,8 @@ call fails (for example because the path doesn't exist). >>> Path('nonexistentfile').exists() False - .. note:: - If the path points to a symlink, :meth:`exists` returns whether the - symlink *points to* an existing file or directory. - + .. versionchanged:: 3.12 + The *follow_symlinks* parameter was added. .. method:: Path.expanduser() diff --git a/Lib/pathlib.py b/Lib/pathlib.py index c69089f4e1bc5d..dee19d1f89ad84 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -135,7 +135,8 @@ def __init__(self, name, child_parts, flavour): def _select_from(self, parent_path, is_dir, exists, scandir): try: path = parent_path._make_child_relpath(self.name) - if (is_dir if self.dironly else exists)(path): + follow = is_dir(path) if self.dironly else exists(path, follow_symlinks=False) + if follow: for p in self.successor._select_from(path, is_dir, exists, scandir): yield p except PermissionError: @@ -1122,12 +1123,15 @@ def hardlink_to(self, target): # Convenience functions for querying the stat results - def exists(self): + def exists(self, *, follow_symlinks=True): """ Whether this path exists. + + This method normally follows symlinks; to check whether a symlink exists, + add the argument follow_symlinks=False. """ try: - self.stat() + self.stat(follow_symlinks=follow_symlinks) except OSError as e: if not _ignore_error(e): raise diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 9902b7242205f3..620d480e37e2ba 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1700,6 +1700,8 @@ def test_exists(self): self.assertIs(True, (p / 'linkB').exists()) self.assertIs(True, (p / 'linkB' / 'fileB').exists()) self.assertIs(False, (p / 'linkA' / 'bah').exists()) + self.assertIs(False, (p / 'brokenLink').exists()) + self.assertIs(True, (p / 'brokenLink').exists(follow_symlinks=False)) self.assertIs(False, (p / 'foo').exists()) self.assertIs(False, P('/xyzzy').exists()) self.assertIs(False, P(BASE + '\udfff').exists()) @@ -1806,6 +1808,8 @@ def _check(glob, expected): _check(p.glob("*/fileB"), ['dirB/fileB']) else: _check(p.glob("*/fileB"), ['dirB/fileB', 'linkB/fileB']) + if os_helper.can_symlink(): + _check(p.glob("brokenLink"), ['brokenLink']) if not os_helper.can_symlink(): _check(p.glob("*/"), ["dirA", "dirB", "dirC", "dirE"]) diff --git a/Misc/NEWS.d/next/Library/2021-11-19-23-37-18.bpo-45606.UW5XE1.rst b/Misc/NEWS.d/next/Library/2021-11-19-23-37-18.bpo-45606.UW5XE1.rst new file mode 100644 index 00000000000000..531f4729220036 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-11-19-23-37-18.bpo-45606.UW5XE1.rst @@ -0,0 +1,5 @@ +Fixed the bug in :meth:`pathlib.Path.glob` -- previously a dangling symlink +would not be found by this method when the pattern is an exact match, but +would be found when the pattern contains a wildcard or the recursive +wildcard (``**``). With this change, a dangling symlink will be found in +both cases. From 5b05b013ff13032ffc4db07108a507c08b3a604d Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 3 May 2023 08:09:04 +0300 Subject: [PATCH 006/204] gh-101100: Fix Sphinx warnings in `curses` and `curses.ascii` modules (#103457) Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- Doc/howto/curses.rst | 44 +- Doc/library/curses.ascii.rst | 150 +++---- Doc/library/curses.rst | 774 ++++++++++++++++++----------------- 3 files changed, 500 insertions(+), 468 deletions(-) diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst index 83d80471ffc8ee..a3068d86d85bc4 100644 --- a/Doc/howto/curses.rst +++ b/Doc/howto/curses.rst @@ -4,6 +4,8 @@ Curses Programming with Python ********************************** +.. currentmodule:: curses + :Author: A.M. Kuchling, Eric S. Raymond :Release: 2.04 @@ -65,7 +67,7 @@ The Python module is a fairly simple wrapper over the C functions provided by curses; if you're already familiar with curses programming in C, it's really easy to transfer that knowledge to Python. The biggest difference is that the Python interface makes things simpler by merging different C functions such as -:c:func:`addstr`, :c:func:`mvaddstr`, and :c:func:`mvwaddstr` into a single +:c:func:`!addstr`, :c:func:`!mvaddstr`, and :c:func:`!mvwaddstr` into a single :meth:`~curses.window.addstr` method. You'll see this covered in more detail later. @@ -82,7 +84,7 @@ Before doing anything, curses must be initialized. This is done by calling the :func:`~curses.initscr` function, which will determine the terminal type, send any required setup codes to the terminal, and create various internal data structures. If successful, -:func:`initscr` returns a window object representing the entire +:func:`!initscr` returns a window object representing the entire screen; this is usually called ``stdscr`` after the name of the corresponding C variable. :: @@ -151,8 +153,8 @@ importing the :func:`curses.wrapper` function and using it like this:: The :func:`~curses.wrapper` function takes a callable object and does the initializations described above, also initializing colors if color -support is present. :func:`wrapper` then runs your provided callable. -Once the callable returns, :func:`wrapper` will restore the original +support is present. :func:`!wrapper` then runs your provided callable. +Once the callable returns, :func:`!wrapper` will restore the original state of the terminal. The callable is called inside a :keyword:`try`...\ :keyword:`except` that catches exceptions, restores the state of the terminal, and then re-raises the exception. Therefore @@ -200,7 +202,7 @@ This is because curses was originally written with slow 300-baud terminal connections in mind; with these terminals, minimizing the time required to redraw the screen was very important. Instead curses accumulates changes to the screen and displays them in the most -efficient manner when you call :meth:`refresh`. For example, if your +efficient manner when you call :meth:`!refresh`. For example, if your program displays some text in a window and then clears the window, there's no need to send the original text because they're never visible. @@ -210,7 +212,7 @@ really complicate programming with curses much. Most programs go into a flurry of activity, and then pause waiting for a keypress or some other action on the part of the user. All you have to do is to be sure that the screen has been redrawn before pausing to wait for user input, by first calling -``stdscr.refresh()`` or the :meth:`refresh` method of some other relevant +:meth:`!stdscr.refresh` or the :meth:`!refresh` method of some other relevant window. A pad is a special case of a window; it can be larger than the actual display @@ -234,7 +236,7 @@ displayed. :: # : filled with pad content. pad.refresh( 0,0, 5,5, 20,75) -The :meth:`refresh` call displays a section of the pad in the rectangle +The :meth:`!refresh` call displays a section of the pad in the rectangle extending from coordinate (5,5) to coordinate (20,75) on the screen; the upper left corner of the displayed section is coordinate (0,0) on the pad. Beyond that difference, pads are exactly like ordinary windows and support the same @@ -242,7 +244,7 @@ methods. If you have multiple windows and pads on screen there is a more efficient way to update the screen and prevent annoying screen flicker -as each part of the screen gets updated. :meth:`refresh` actually +as each part of the screen gets updated. :meth:`!refresh` actually does two things: 1) Calls the :meth:`~curses.window.noutrefresh` method of each window @@ -251,8 +253,8 @@ does two things: 2) Calls the function :func:`~curses.doupdate` function to change the physical screen to match the desired state recorded in the data structure. -Instead you can call :meth:`noutrefresh` on a number of windows to -update the data structure, and then call :func:`doupdate` to update +Instead you can call :meth:`!noutrefresh` on a number of windows to +update the data structure, and then call :func:`!doupdate` to update the screen. @@ -261,11 +263,11 @@ Displaying Text From a C programmer's point of view, curses may sometimes look like a twisty maze of functions, all subtly different. For example, -:c:func:`addstr` displays a string at the current cursor location in -the ``stdscr`` window, while :c:func:`mvaddstr` moves to a given y,x -coordinate first before displaying the string. :c:func:`waddstr` is just -like :c:func:`addstr`, but allows specifying a window to use instead of -using ``stdscr`` by default. :c:func:`mvwaddstr` allows specifying both +:c:func:`!addstr` displays a string at the current cursor location in +the ``stdscr`` window, while :c:func:`!mvaddstr` moves to a given y,x +coordinate first before displaying the string. :c:func:`!waddstr` is just +like :c:func:`!addstr`, but allows specifying a window to use instead of +using ``stdscr`` by default. :c:func:`!mvwaddstr` allows specifying both a window and a coordinate. Fortunately the Python interface hides all these details. ``stdscr`` @@ -298,7 +300,7 @@ the next subsection. The :meth:`~curses.window.addstr` method takes a Python string or bytestring as the value to be displayed. The contents of bytestrings are sent to the terminal as-is. Strings are encoded to bytes using -the value of the window's :attr:`encoding` attribute; this defaults to +the value of the window's :attr:`~window.encoding` attribute; this defaults to the default system encoding as returned by :func:`locale.getencoding`. The :meth:`~curses.window.addch` methods take a character, which can be @@ -444,15 +446,15 @@ There are two methods for getting input from a window: It's possible to not wait for the user using the :meth:`~curses.window.nodelay` window method. After ``nodelay(True)``, -:meth:`getch` and :meth:`getkey` for the window become -non-blocking. To signal that no input is ready, :meth:`getch` returns -``curses.ERR`` (a value of -1) and :meth:`getkey` raises an exception. +:meth:`!getch` and :meth:`!getkey` for the window become +non-blocking. To signal that no input is ready, :meth:`!getch` returns +``curses.ERR`` (a value of -1) and :meth:`!getkey` raises an exception. There's also a :func:`~curses.halfdelay` function, which can be used to (in -effect) set a timer on each :meth:`getch`; if no input becomes +effect) set a timer on each :meth:`!getch`; if no input becomes available within a specified delay (measured in tenths of a second), curses raises an exception. -The :meth:`getch` method returns an integer; if it's between 0 and 255, it +The :meth:`!getch` method returns an integer; if it's between 0 and 255, it represents the ASCII code of the key pressed. Values greater than 255 are special keys such as Page Up, Home, or the cursor keys. You can compare the value returned to constants such as :const:`curses.KEY_PPAGE`, diff --git a/Doc/library/curses.ascii.rst b/Doc/library/curses.ascii.rst index e1d1171927c9e2..410b76e77c025b 100644 --- a/Doc/library/curses.ascii.rst +++ b/Doc/library/curses.ascii.rst @@ -15,81 +15,81 @@ The :mod:`curses.ascii` module supplies name constants for ASCII characters and functions to test membership in various ASCII character classes. The constants supplied are names for control characters as follows: -+--------------+----------------------------------------------+ -| Name | Meaning | -+==============+==============================================+ -| :const:`NUL` | | -+--------------+----------------------------------------------+ -| :const:`SOH` | Start of heading, console interrupt | -+--------------+----------------------------------------------+ -| :const:`STX` | Start of text | -+--------------+----------------------------------------------+ -| :const:`ETX` | End of text | -+--------------+----------------------------------------------+ -| :const:`EOT` | End of transmission | -+--------------+----------------------------------------------+ -| :const:`ENQ` | Enquiry, goes with :const:`ACK` flow control | -+--------------+----------------------------------------------+ -| :const:`ACK` | Acknowledgement | -+--------------+----------------------------------------------+ -| :const:`BEL` | Bell | -+--------------+----------------------------------------------+ -| :const:`BS` | Backspace | -+--------------+----------------------------------------------+ -| :const:`TAB` | Tab | -+--------------+----------------------------------------------+ -| :const:`HT` | Alias for :const:`TAB`: "Horizontal tab" | -+--------------+----------------------------------------------+ -| :const:`LF` | Line feed | -+--------------+----------------------------------------------+ -| :const:`NL` | Alias for :const:`LF`: "New line" | -+--------------+----------------------------------------------+ -| :const:`VT` | Vertical tab | -+--------------+----------------------------------------------+ -| :const:`FF` | Form feed | -+--------------+----------------------------------------------+ -| :const:`CR` | Carriage return | -+--------------+----------------------------------------------+ -| :const:`SO` | Shift-out, begin alternate character set | -+--------------+----------------------------------------------+ -| :const:`SI` | Shift-in, resume default character set | -+--------------+----------------------------------------------+ -| :const:`DLE` | Data-link escape | -+--------------+----------------------------------------------+ -| :const:`DC1` | XON, for flow control | -+--------------+----------------------------------------------+ -| :const:`DC2` | Device control 2, block-mode flow control | -+--------------+----------------------------------------------+ -| :const:`DC3` | XOFF, for flow control | -+--------------+----------------------------------------------+ -| :const:`DC4` | Device control 4 | -+--------------+----------------------------------------------+ -| :const:`NAK` | Negative acknowledgement | -+--------------+----------------------------------------------+ -| :const:`SYN` | Synchronous idle | -+--------------+----------------------------------------------+ -| :const:`ETB` | End transmission block | -+--------------+----------------------------------------------+ -| :const:`CAN` | Cancel | -+--------------+----------------------------------------------+ -| :const:`EM` | End of medium | -+--------------+----------------------------------------------+ -| :const:`SUB` | Substitute | -+--------------+----------------------------------------------+ -| :const:`ESC` | Escape | -+--------------+----------------------------------------------+ -| :const:`FS` | File separator | -+--------------+----------------------------------------------+ -| :const:`GS` | Group separator | -+--------------+----------------------------------------------+ -| :const:`RS` | Record separator, block-mode terminator | -+--------------+----------------------------------------------+ -| :const:`US` | Unit separator | -+--------------+----------------------------------------------+ -| :const:`SP` | Space | -+--------------+----------------------------------------------+ -| :const:`DEL` | Delete | -+--------------+----------------------------------------------+ ++---------------+----------------------------------------------+ +| Name | Meaning | ++===============+==============================================+ +| .. data:: NUL | | ++---------------+----------------------------------------------+ +| .. data:: SOH | Start of heading, console interrupt | ++---------------+----------------------------------------------+ +| .. data:: STX | Start of text | ++---------------+----------------------------------------------+ +| .. data:: ETX | End of text | ++---------------+----------------------------------------------+ +| .. data:: EOT | End of transmission | ++---------------+----------------------------------------------+ +| .. data:: ENQ | Enquiry, goes with :const:`ACK` flow control | ++---------------+----------------------------------------------+ +| .. data:: ACK | Acknowledgement | ++---------------+----------------------------------------------+ +| .. data:: BEL | Bell | ++---------------+----------------------------------------------+ +| .. data:: BS | Backspace | ++---------------+----------------------------------------------+ +| .. data:: TAB | Tab | ++---------------+----------------------------------------------+ +| .. data:: HT | Alias for :const:`TAB`: "Horizontal tab" | ++---------------+----------------------------------------------+ +| .. data:: LF | Line feed | ++---------------+----------------------------------------------+ +| .. data:: NL | Alias for :const:`LF`: "New line" | ++---------------+----------------------------------------------+ +| .. data:: VT | Vertical tab | ++---------------+----------------------------------------------+ +| .. data:: FF | Form feed | ++---------------+----------------------------------------------+ +| .. data:: CR | Carriage return | ++---------------+----------------------------------------------+ +| .. data:: SO | Shift-out, begin alternate character set | ++---------------+----------------------------------------------+ +| .. data:: SI | Shift-in, resume default character set | ++---------------+----------------------------------------------+ +| .. data:: DLE | Data-link escape | ++---------------+----------------------------------------------+ +| .. data:: DC1 | XON, for flow control | ++---------------+----------------------------------------------+ +| .. data:: DC2 | Device control 2, block-mode flow control | ++---------------+----------------------------------------------+ +| .. data:: DC3 | XOFF, for flow control | ++---------------+----------------------------------------------+ +| .. data:: DC4 | Device control 4 | ++---------------+----------------------------------------------+ +| .. data:: NAK | Negative acknowledgement | ++---------------+----------------------------------------------+ +| .. data:: SYN | Synchronous idle | ++---------------+----------------------------------------------+ +| .. data:: ETB | End transmission block | ++---------------+----------------------------------------------+ +| .. data:: CAN | Cancel | ++---------------+----------------------------------------------+ +| .. data:: EM | End of medium | ++---------------+----------------------------------------------+ +| .. data:: SUB | Substitute | ++---------------+----------------------------------------------+ +| .. data:: ESC | Escape | ++---------------+----------------------------------------------+ +| .. data:: FS | File separator | ++---------------+----------------------------------------------+ +| .. data:: GS | Group separator | ++---------------+----------------------------------------------+ +| .. data:: RS | Record separator, block-mode terminator | ++---------------+----------------------------------------------+ +| .. data:: US | Unit separator | ++---------------+----------------------------------------------+ +| .. data:: SP | Space | ++---------------+----------------------------------------------+ +| .. data:: DEL | Delete | ++---------------+----------------------------------------------+ Note that many of these have little practical significance in modern usage. The mnemonics derive from teleprinter conventions that predate digital computers. diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index f50b51c3780ef0..cf208f3ba0db36 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -107,7 +107,7 @@ The module :mod:`curses` defines the following functions: Return the attribute value for displaying text in the specified color pair. Only the first 256 color pairs are supported. This attribute value can be combined with :const:`A_STANDOUT`, :const:`A_REVERSE`, - and the other :const:`A_\*` attributes. :func:`pair_number` is the counterpart + and the other :const:`!A_\*` attributes. :func:`pair_number` is the counterpart to this function. @@ -223,7 +223,7 @@ The module :mod:`curses` defines the following functions: .. function:: getwin(file) - Read window related data stored in the file by an earlier :func:`putwin` call. + Read window related data stored in the file by an earlier :func:`window.putwin` call. The routine then creates and initializes a new window using that data, returning the new window object. @@ -1323,9 +1323,9 @@ The :mod:`curses` module defines the following data members: .. data:: version +.. data:: __version__ - A bytes object representing the current version of the module. Also available as - :const:`__version__`. + A bytes object representing the current version of the module. .. data:: ncurses_version @@ -1339,51 +1339,55 @@ The :mod:`curses` module defines the following data members: .. versionadded:: 3.8 +.. data:: COLORS + + The maximum number of colors the terminal can support. + +.. data:: COLOR_PAIRS + + The maximum number of color pairs the terminal can support. Some constants are available to specify character cell attributes. The exact constants available are system dependent. -+------------------+-------------------------------+ -| Attribute | Meaning | -+==================+===============================+ -| ``A_ALTCHARSET`` | Alternate character set mode | -+------------------+-------------------------------+ -| ``A_BLINK`` | Blink mode | -+------------------+-------------------------------+ -| ``A_BOLD`` | Bold mode | -+------------------+-------------------------------+ -| ``A_DIM`` | Dim mode | -+------------------+-------------------------------+ -| ``A_INVIS`` | Invisible or blank mode | -+------------------+-------------------------------+ -| ``A_ITALIC`` | Italic mode | -+------------------+-------------------------------+ -| ``A_NORMAL`` | Normal attribute | -+------------------+-------------------------------+ -| ``A_PROTECT`` | Protected mode | -+------------------+-------------------------------+ -| ``A_REVERSE`` | Reverse background and | -| | foreground colors | -+------------------+-------------------------------+ -| ``A_STANDOUT`` | Standout mode | -+------------------+-------------------------------+ -| ``A_UNDERLINE`` | Underline mode | -+------------------+-------------------------------+ -| ``A_HORIZONTAL`` | Horizontal highlight | -+------------------+-------------------------------+ -| ``A_LEFT`` | Left highlight | -+------------------+-------------------------------+ -| ``A_LOW`` | Low highlight | -+------------------+-------------------------------+ -| ``A_RIGHT`` | Right highlight | -+------------------+-------------------------------+ -| ``A_TOP`` | Top highlight | -+------------------+-------------------------------+ -| ``A_VERTICAL`` | Vertical highlight | -+------------------+-------------------------------+ -| ``A_CHARTEXT`` | Bit-mask to extract a | -| | character | -+------------------+-------------------------------+ ++------------------------+-------------------------------+ +| Attribute | Meaning | ++========================+===============================+ +| .. data:: A_ALTCHARSET | Alternate character set mode | ++------------------------+-------------------------------+ +| .. data:: A_BLINK | Blink mode | ++------------------------+-------------------------------+ +| .. data:: A_BOLD | Bold mode | ++------------------------+-------------------------------+ +| .. data:: A_DIM | Dim mode | ++------------------------+-------------------------------+ +| .. data:: A_INVIS | Invisible or blank mode | ++------------------------+-------------------------------+ +| .. data:: A_ITALIC | Italic mode | ++------------------------+-------------------------------+ +| .. data:: A_NORMAL | Normal attribute | ++------------------------+-------------------------------+ +| .. data:: A_PROTECT | Protected mode | ++------------------------+-------------------------------+ +| .. data:: A_REVERSE | Reverse background and | +| | foreground colors | ++------------------------+-------------------------------+ +| .. data:: A_STANDOUT | Standout mode | ++------------------------+-------------------------------+ +| .. data:: A_UNDERLINE | Underline mode | ++------------------------+-------------------------------+ +| .. data:: A_HORIZONTAL | Horizontal highlight | ++------------------------+-------------------------------+ +| .. data:: A_LEFT | Left highlight | ++------------------------+-------------------------------+ +| .. data:: A_LOW | Low highlight | ++------------------------+-------------------------------+ +| .. data:: A_RIGHT | Right highlight | ++------------------------+-------------------------------+ +| .. data:: A_TOP | Top highlight | ++------------------------+-------------------------------+ +| .. data:: A_VERTICAL | Vertical highlight | ++------------------------+-------------------------------+ .. versionadded:: 3.7 ``A_ITALIC`` was added. @@ -1391,220 +1395,220 @@ The exact constants available are system dependent. Several constants are available to extract corresponding attributes returned by some methods. -+------------------+-------------------------------+ -| Bit-mask | Meaning | -+==================+===============================+ -| ``A_ATTRIBUTES`` | Bit-mask to extract | -| | attributes | -+------------------+-------------------------------+ -| ``A_CHARTEXT`` | Bit-mask to extract a | -| | character | -+------------------+-------------------------------+ -| ``A_COLOR`` | Bit-mask to extract | -| | color-pair field information | -+------------------+-------------------------------+ ++-------------------------+-------------------------------+ +| Bit-mask | Meaning | ++=========================+===============================+ +| .. data:: A_ATTRIBUTES | Bit-mask to extract | +| | attributes | ++-------------------------+-------------------------------+ +| .. data:: A_CHARTEXT | Bit-mask to extract a | +| | character | ++-------------------------+-------------------------------+ +| .. data:: A_COLOR | Bit-mask to extract | +| | color-pair field information | ++-------------------------+-------------------------------+ Keys are referred to by integer constants with names starting with ``KEY_``. The exact keycaps available are system dependent. .. XXX this table is far too large! should it be alphabetized? -+-------------------+--------------------------------------------+ -| Key constant | Key | -+===================+============================================+ -| ``KEY_MIN`` | Minimum key value | -+-------------------+--------------------------------------------+ -| ``KEY_BREAK`` | Break key (unreliable) | -+-------------------+--------------------------------------------+ -| ``KEY_DOWN`` | Down-arrow | -+-------------------+--------------------------------------------+ -| ``KEY_UP`` | Up-arrow | -+-------------------+--------------------------------------------+ -| ``KEY_LEFT`` | Left-arrow | -+-------------------+--------------------------------------------+ -| ``KEY_RIGHT`` | Right-arrow | -+-------------------+--------------------------------------------+ -| ``KEY_HOME`` | Home key (upward+left arrow) | -+-------------------+--------------------------------------------+ -| ``KEY_BACKSPACE`` | Backspace (unreliable) | -+-------------------+--------------------------------------------+ -| ``KEY_F0`` | Function keys. Up to 64 function keys are | -| | supported. | -+-------------------+--------------------------------------------+ -| ``KEY_Fn`` | Value of function key *n* | -+-------------------+--------------------------------------------+ -| ``KEY_DL`` | Delete line | -+-------------------+--------------------------------------------+ -| ``KEY_IL`` | Insert line | -+-------------------+--------------------------------------------+ -| ``KEY_DC`` | Delete character | -+-------------------+--------------------------------------------+ -| ``KEY_IC`` | Insert char or enter insert mode | -+-------------------+--------------------------------------------+ -| ``KEY_EIC`` | Exit insert char mode | -+-------------------+--------------------------------------------+ -| ``KEY_CLEAR`` | Clear screen | -+-------------------+--------------------------------------------+ -| ``KEY_EOS`` | Clear to end of screen | -+-------------------+--------------------------------------------+ -| ``KEY_EOL`` | Clear to end of line | -+-------------------+--------------------------------------------+ -| ``KEY_SF`` | Scroll 1 line forward | -+-------------------+--------------------------------------------+ -| ``KEY_SR`` | Scroll 1 line backward (reverse) | -+-------------------+--------------------------------------------+ -| ``KEY_NPAGE`` | Next page | -+-------------------+--------------------------------------------+ -| ``KEY_PPAGE`` | Previous page | -+-------------------+--------------------------------------------+ -| ``KEY_STAB`` | Set tab | -+-------------------+--------------------------------------------+ -| ``KEY_CTAB`` | Clear tab | -+-------------------+--------------------------------------------+ -| ``KEY_CATAB`` | Clear all tabs | -+-------------------+--------------------------------------------+ -| ``KEY_ENTER`` | Enter or send (unreliable) | -+-------------------+--------------------------------------------+ -| ``KEY_SRESET`` | Soft (partial) reset (unreliable) | -+-------------------+--------------------------------------------+ -| ``KEY_RESET`` | Reset or hard reset (unreliable) | -+-------------------+--------------------------------------------+ -| ``KEY_PRINT`` | Print | -+-------------------+--------------------------------------------+ -| ``KEY_LL`` | Home down or bottom (lower left) | -+-------------------+--------------------------------------------+ -| ``KEY_A1`` | Upper left of keypad | -+-------------------+--------------------------------------------+ -| ``KEY_A3`` | Upper right of keypad | -+-------------------+--------------------------------------------+ -| ``KEY_B2`` | Center of keypad | -+-------------------+--------------------------------------------+ -| ``KEY_C1`` | Lower left of keypad | -+-------------------+--------------------------------------------+ -| ``KEY_C3`` | Lower right of keypad | -+-------------------+--------------------------------------------+ -| ``KEY_BTAB`` | Back tab | -+-------------------+--------------------------------------------+ -| ``KEY_BEG`` | Beg (beginning) | -+-------------------+--------------------------------------------+ -| ``KEY_CANCEL`` | Cancel | -+-------------------+--------------------------------------------+ -| ``KEY_CLOSE`` | Close | -+-------------------+--------------------------------------------+ -| ``KEY_COMMAND`` | Cmd (command) | -+-------------------+--------------------------------------------+ -| ``KEY_COPY`` | Copy | -+-------------------+--------------------------------------------+ -| ``KEY_CREATE`` | Create | -+-------------------+--------------------------------------------+ -| ``KEY_END`` | End | -+-------------------+--------------------------------------------+ -| ``KEY_EXIT`` | Exit | -+-------------------+--------------------------------------------+ -| ``KEY_FIND`` | Find | -+-------------------+--------------------------------------------+ -| ``KEY_HELP`` | Help | -+-------------------+--------------------------------------------+ -| ``KEY_MARK`` | Mark | -+-------------------+--------------------------------------------+ -| ``KEY_MESSAGE`` | Message | -+-------------------+--------------------------------------------+ -| ``KEY_MOVE`` | Move | -+-------------------+--------------------------------------------+ -| ``KEY_NEXT`` | Next | -+-------------------+--------------------------------------------+ -| ``KEY_OPEN`` | Open | -+-------------------+--------------------------------------------+ -| ``KEY_OPTIONS`` | Options | -+-------------------+--------------------------------------------+ -| ``KEY_PREVIOUS`` | Prev (previous) | -+-------------------+--------------------------------------------+ -| ``KEY_REDO`` | Redo | -+-------------------+--------------------------------------------+ -| ``KEY_REFERENCE`` | Ref (reference) | -+-------------------+--------------------------------------------+ -| ``KEY_REFRESH`` | Refresh | -+-------------------+--------------------------------------------+ -| ``KEY_REPLACE`` | Replace | -+-------------------+--------------------------------------------+ -| ``KEY_RESTART`` | Restart | -+-------------------+--------------------------------------------+ -| ``KEY_RESUME`` | Resume | -+-------------------+--------------------------------------------+ -| ``KEY_SAVE`` | Save | -+-------------------+--------------------------------------------+ -| ``KEY_SBEG`` | Shifted Beg (beginning) | -+-------------------+--------------------------------------------+ -| ``KEY_SCANCEL`` | Shifted Cancel | -+-------------------+--------------------------------------------+ -| ``KEY_SCOMMAND`` | Shifted Command | -+-------------------+--------------------------------------------+ -| ``KEY_SCOPY`` | Shifted Copy | -+-------------------+--------------------------------------------+ -| ``KEY_SCREATE`` | Shifted Create | -+-------------------+--------------------------------------------+ -| ``KEY_SDC`` | Shifted Delete char | -+-------------------+--------------------------------------------+ -| ``KEY_SDL`` | Shifted Delete line | -+-------------------+--------------------------------------------+ -| ``KEY_SELECT`` | Select | -+-------------------+--------------------------------------------+ -| ``KEY_SEND`` | Shifted End | -+-------------------+--------------------------------------------+ -| ``KEY_SEOL`` | Shifted Clear line | -+-------------------+--------------------------------------------+ -| ``KEY_SEXIT`` | Shifted Exit | -+-------------------+--------------------------------------------+ -| ``KEY_SFIND`` | Shifted Find | -+-------------------+--------------------------------------------+ -| ``KEY_SHELP`` | Shifted Help | -+-------------------+--------------------------------------------+ -| ``KEY_SHOME`` | Shifted Home | -+-------------------+--------------------------------------------+ -| ``KEY_SIC`` | Shifted Input | -+-------------------+--------------------------------------------+ -| ``KEY_SLEFT`` | Shifted Left arrow | -+-------------------+--------------------------------------------+ -| ``KEY_SMESSAGE`` | Shifted Message | -+-------------------+--------------------------------------------+ -| ``KEY_SMOVE`` | Shifted Move | -+-------------------+--------------------------------------------+ -| ``KEY_SNEXT`` | Shifted Next | -+-------------------+--------------------------------------------+ -| ``KEY_SOPTIONS`` | Shifted Options | -+-------------------+--------------------------------------------+ -| ``KEY_SPREVIOUS`` | Shifted Prev | -+-------------------+--------------------------------------------+ -| ``KEY_SPRINT`` | Shifted Print | -+-------------------+--------------------------------------------+ -| ``KEY_SREDO`` | Shifted Redo | -+-------------------+--------------------------------------------+ -| ``KEY_SREPLACE`` | Shifted Replace | -+-------------------+--------------------------------------------+ -| ``KEY_SRIGHT`` | Shifted Right arrow | -+-------------------+--------------------------------------------+ -| ``KEY_SRSUME`` | Shifted Resume | -+-------------------+--------------------------------------------+ -| ``KEY_SSAVE`` | Shifted Save | -+-------------------+--------------------------------------------+ -| ``KEY_SSUSPEND`` | Shifted Suspend | -+-------------------+--------------------------------------------+ -| ``KEY_SUNDO`` | Shifted Undo | -+-------------------+--------------------------------------------+ -| ``KEY_SUSPEND`` | Suspend | -+-------------------+--------------------------------------------+ -| ``KEY_UNDO`` | Undo | -+-------------------+--------------------------------------------+ -| ``KEY_MOUSE`` | Mouse event has occurred | -+-------------------+--------------------------------------------+ -| ``KEY_RESIZE`` | Terminal resize event | -+-------------------+--------------------------------------------+ -| ``KEY_MAX`` | Maximum key value | -+-------------------+--------------------------------------------+ ++-------------------------+--------------------------------------------+ +| Key constant | Key | ++=========================+============================================+ +| .. data:: KEY_MIN | Minimum key value | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_BREAK | Break key (unreliable) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_DOWN | Down-arrow | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_UP | Up-arrow | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_LEFT | Left-arrow | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_RIGHT | Right-arrow | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_HOME | Home key (upward+left arrow) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_BACKSPACE | Backspace (unreliable) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_F0 | Function keys. Up to 64 function keys are | +| | supported. | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_Fn | Value of function key *n* | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_DL | Delete line | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_IL | Insert line | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_DC | Delete character | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_IC | Insert char or enter insert mode | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_EIC | Exit insert char mode | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_CLEAR | Clear screen | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_EOS | Clear to end of screen | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_EOL | Clear to end of line | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SF | Scroll 1 line forward | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SR | Scroll 1 line backward (reverse) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_NPAGE | Next page | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_PPAGE | Previous page | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_STAB | Set tab | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_CTAB | Clear tab | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_CATAB | Clear all tabs | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_ENTER | Enter or send (unreliable) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SRESET | Soft (partial) reset (unreliable) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_RESET | Reset or hard reset (unreliable) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_PRINT | Print | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_LL | Home down or bottom (lower left) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_A1 | Upper left of keypad | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_A3 | Upper right of keypad | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_B2 | Center of keypad | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_C1 | Lower left of keypad | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_C3 | Lower right of keypad | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_BTAB | Back tab | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_BEG | Beg (beginning) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_CANCEL | Cancel | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_CLOSE | Close | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_COMMAND | Cmd (command) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_COPY | Copy | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_CREATE | Create | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_END | End | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_EXIT | Exit | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_FIND | Find | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_HELP | Help | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_MARK | Mark | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_MESSAGE | Message | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_MOVE | Move | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_NEXT | Next | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_OPEN | Open | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_OPTIONS | Options | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_PREVIOUS | Prev (previous) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_REDO | Redo | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_REFERENCE | Ref (reference) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_REFRESH | Refresh | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_REPLACE | Replace | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_RESTART | Restart | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_RESUME | Resume | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SAVE | Save | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SBEG | Shifted Beg (beginning) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SCANCEL | Shifted Cancel | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SCOMMAND | Shifted Command | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SCOPY | Shifted Copy | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SCREATE | Shifted Create | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SDC | Shifted Delete char | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SDL | Shifted Delete line | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SELECT | Select | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SEND | Shifted End | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SEOL | Shifted Clear line | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SEXIT | Shifted Exit | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SFIND | Shifted Find | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SHELP | Shifted Help | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SHOME | Shifted Home | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SIC | Shifted Input | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SLEFT | Shifted Left arrow | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SMESSAGE | Shifted Message | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SMOVE | Shifted Move | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SNEXT | Shifted Next | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SOPTIONS | Shifted Options | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SPREVIOUS | Shifted Prev | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SPRINT | Shifted Print | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SREDO | Shifted Redo | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SREPLACE | Shifted Replace | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SRIGHT | Shifted Right arrow | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SRSUME | Shifted Resume | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SSAVE | Shifted Save | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SSUSPEND | Shifted Suspend | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SUNDO | Shifted Undo | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SUSPEND | Suspend | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_UNDO | Undo | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_MOUSE | Mouse event has occurred | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_RESIZE | Terminal resize event | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_MAX | Maximum key value | ++-------------------------+--------------------------------------------+ On VT100s and their software emulations, such as X terminal emulators, there are -normally at least four function keys (:const:`KEY_F1`, :const:`KEY_F2`, -:const:`KEY_F3`, :const:`KEY_F4`) available, and the arrow keys mapped to +normally at least four function keys (:const:`KEY_F1 `, :const:`KEY_F2 `, +:const:`KEY_F3 `, :const:`KEY_F4 `) available, and the arrow keys mapped to :const:`KEY_UP`, :const:`KEY_DOWN`, :const:`KEY_LEFT` and :const:`KEY_RIGHT` in the obvious way. If your machine has a PC keyboard, it is safe to expect arrow keys and twelve function keys (older PC keyboards may have only ten function @@ -1635,117 +1639,143 @@ falls back on a crude printable ASCII approximation. These are available only after :func:`initscr` has been called. -+------------------+------------------------------------------+ -| ACS code | Meaning | -+==================+==========================================+ -| ``ACS_BBSS`` | alternate name for upper right corner | -+------------------+------------------------------------------+ -| ``ACS_BLOCK`` | solid square block | -+------------------+------------------------------------------+ -| ``ACS_BOARD`` | board of squares | -+------------------+------------------------------------------+ -| ``ACS_BSBS`` | alternate name for horizontal line | -+------------------+------------------------------------------+ -| ``ACS_BSSB`` | alternate name for upper left corner | -+------------------+------------------------------------------+ -| ``ACS_BSSS`` | alternate name for top tee | -+------------------+------------------------------------------+ -| ``ACS_BTEE`` | bottom tee | -+------------------+------------------------------------------+ -| ``ACS_BULLET`` | bullet | -+------------------+------------------------------------------+ -| ``ACS_CKBOARD`` | checker board (stipple) | -+------------------+------------------------------------------+ -| ``ACS_DARROW`` | arrow pointing down | -+------------------+------------------------------------------+ -| ``ACS_DEGREE`` | degree symbol | -+------------------+------------------------------------------+ -| ``ACS_DIAMOND`` | diamond | -+------------------+------------------------------------------+ -| ``ACS_GEQUAL`` | greater-than-or-equal-to | -+------------------+------------------------------------------+ -| ``ACS_HLINE`` | horizontal line | -+------------------+------------------------------------------+ -| ``ACS_LANTERN`` | lantern symbol | -+------------------+------------------------------------------+ -| ``ACS_LARROW`` | left arrow | -+------------------+------------------------------------------+ -| ``ACS_LEQUAL`` | less-than-or-equal-to | -+------------------+------------------------------------------+ -| ``ACS_LLCORNER`` | lower left-hand corner | -+------------------+------------------------------------------+ -| ``ACS_LRCORNER`` | lower right-hand corner | -+------------------+------------------------------------------+ -| ``ACS_LTEE`` | left tee | -+------------------+------------------------------------------+ -| ``ACS_NEQUAL`` | not-equal sign | -+------------------+------------------------------------------+ -| ``ACS_PI`` | letter pi | -+------------------+------------------------------------------+ -| ``ACS_PLMINUS`` | plus-or-minus sign | -+------------------+------------------------------------------+ -| ``ACS_PLUS`` | big plus sign | -+------------------+------------------------------------------+ -| ``ACS_RARROW`` | right arrow | -+------------------+------------------------------------------+ -| ``ACS_RTEE`` | right tee | -+------------------+------------------------------------------+ -| ``ACS_S1`` | scan line 1 | -+------------------+------------------------------------------+ -| ``ACS_S3`` | scan line 3 | -+------------------+------------------------------------------+ -| ``ACS_S7`` | scan line 7 | -+------------------+------------------------------------------+ -| ``ACS_S9`` | scan line 9 | -+------------------+------------------------------------------+ -| ``ACS_SBBS`` | alternate name for lower right corner | -+------------------+------------------------------------------+ -| ``ACS_SBSB`` | alternate name for vertical line | -+------------------+------------------------------------------+ -| ``ACS_SBSS`` | alternate name for right tee | -+------------------+------------------------------------------+ -| ``ACS_SSBB`` | alternate name for lower left corner | -+------------------+------------------------------------------+ -| ``ACS_SSBS`` | alternate name for bottom tee | -+------------------+------------------------------------------+ -| ``ACS_SSSB`` | alternate name for left tee | -+------------------+------------------------------------------+ -| ``ACS_SSSS`` | alternate name for crossover or big plus | -+------------------+------------------------------------------+ -| ``ACS_STERLING`` | pound sterling | -+------------------+------------------------------------------+ -| ``ACS_TTEE`` | top tee | -+------------------+------------------------------------------+ -| ``ACS_UARROW`` | up arrow | -+------------------+------------------------------------------+ -| ``ACS_ULCORNER`` | upper left corner | -+------------------+------------------------------------------+ -| ``ACS_URCORNER`` | upper right corner | -+------------------+------------------------------------------+ -| ``ACS_VLINE`` | vertical line | -+------------------+------------------------------------------+ ++------------------------+------------------------------------------+ +| ACS code | Meaning | ++========================+==========================================+ +| .. data:: ACS_BBSS | alternate name for upper right corner | ++------------------------+------------------------------------------+ +| .. data:: ACS_BLOCK | solid square block | ++------------------------+------------------------------------------+ +| .. data:: ACS_BOARD | board of squares | ++------------------------+------------------------------------------+ +| .. data:: ACS_BSBS | alternate name for horizontal line | ++------------------------+------------------------------------------+ +| .. data:: ACS_BSSB | alternate name for upper left corner | ++------------------------+------------------------------------------+ +| .. data:: ACS_BSSS | alternate name for top tee | ++------------------------+------------------------------------------+ +| .. data:: ACS_BTEE | bottom tee | ++------------------------+------------------------------------------+ +| .. data:: ACS_BULLET | bullet | ++------------------------+------------------------------------------+ +| .. data:: ACS_CKBOARD | checker board (stipple) | ++------------------------+------------------------------------------+ +| .. data:: ACS_DARROW | arrow pointing down | ++------------------------+------------------------------------------+ +| .. data:: ACS_DEGREE | degree symbol | ++------------------------+------------------------------------------+ +| .. data:: ACS_DIAMOND | diamond | ++------------------------+------------------------------------------+ +| .. data:: ACS_GEQUAL | greater-than-or-equal-to | ++------------------------+------------------------------------------+ +| .. data:: ACS_HLINE | horizontal line | ++------------------------+------------------------------------------+ +| .. data:: ACS_LANTERN | lantern symbol | ++------------------------+------------------------------------------+ +| .. data:: ACS_LARROW | left arrow | ++------------------------+------------------------------------------+ +| .. data:: ACS_LEQUAL | less-than-or-equal-to | ++------------------------+------------------------------------------+ +| .. data:: ACS_LLCORNER | lower left-hand corner | ++------------------------+------------------------------------------+ +| .. data:: ACS_LRCORNER | lower right-hand corner | ++------------------------+------------------------------------------+ +| .. data:: ACS_LTEE | left tee | ++------------------------+------------------------------------------+ +| .. data:: ACS_NEQUAL | not-equal sign | ++------------------------+------------------------------------------+ +| .. data:: ACS_PI | letter pi | ++------------------------+------------------------------------------+ +| .. data:: ACS_PLMINUS | plus-or-minus sign | ++------------------------+------------------------------------------+ +| .. data:: ACS_PLUS | big plus sign | ++------------------------+------------------------------------------+ +| .. data:: ACS_RARROW | right arrow | ++------------------------+------------------------------------------+ +| .. data:: ACS_RTEE | right tee | ++------------------------+------------------------------------------+ +| .. data:: ACS_S1 | scan line 1 | ++------------------------+------------------------------------------+ +| .. data:: ACS_S3 | scan line 3 | ++------------------------+------------------------------------------+ +| .. data:: ACS_S7 | scan line 7 | ++------------------------+------------------------------------------+ +| .. data:: ACS_S9 | scan line 9 | ++------------------------+------------------------------------------+ +| .. data:: ACS_SBBS | alternate name for lower right corner | ++------------------------+------------------------------------------+ +| .. data:: ACS_SBSB | alternate name for vertical line | ++------------------------+------------------------------------------+ +| .. data:: ACS_SBSS | alternate name for right tee | ++------------------------+------------------------------------------+ +| .. data:: ACS_SSBB | alternate name for lower left corner | ++------------------------+------------------------------------------+ +| .. data:: ACS_SSBS | alternate name for bottom tee | ++------------------------+------------------------------------------+ +| .. data:: ACS_SSSB | alternate name for left tee | ++------------------------+------------------------------------------+ +| .. data:: ACS_SSSS | alternate name for crossover or big plus | ++------------------------+------------------------------------------+ +| .. data:: ACS_STERLING | pound sterling | ++------------------------+------------------------------------------+ +| .. data:: ACS_TTEE | top tee | ++------------------------+------------------------------------------+ +| .. data:: ACS_UARROW | up arrow | ++------------------------+------------------------------------------+ +| .. data:: ACS_ULCORNER | upper left corner | ++------------------------+------------------------------------------+ +| .. data:: ACS_URCORNER | upper right corner | ++------------------------+------------------------------------------+ +| .. data:: ACS_VLINE | vertical line | ++------------------------+------------------------------------------+ + +The following table lists mouse button constants used by :meth:`getmouse`: + ++----------------------------------+---------------------------------------------+ +| Mouse button constant | Meaning | ++==================================+=============================================+ +| .. data:: BUTTONn_PRESSED | Mouse button *n* pressed | ++----------------------------------+---------------------------------------------+ +| .. data:: BUTTONn_RELEASED | Mouse button *n* released | ++----------------------------------+---------------------------------------------+ +| .. data:: BUTTONn_CLICKED | Mouse button *n* clicked | ++----------------------------------+---------------------------------------------+ +| .. data:: BUTTONn_DOUBLE_CLICKED | Mouse button *n* double clicked | ++----------------------------------+---------------------------------------------+ +| .. data:: BUTTONn_TRIPLE_CLICKED | Mouse button *n* triple clicked | ++----------------------------------+---------------------------------------------+ +| .. data:: BUTTON_SHIFT | Shift was down during button state change | ++----------------------------------+---------------------------------------------+ +| .. data:: BUTTON_CTRL | Control was down during button state change | ++----------------------------------+---------------------------------------------+ +| .. data:: BUTTON_ALT | Control was down during button state change | ++----------------------------------+---------------------------------------------+ + + .. versionchanged:: 3.10 + The ``BUTTON5_*`` constants are now exposed if they are provided by the + underlying curses library. The following table lists the predefined colors: -+-------------------+----------------------------+ -| Constant | Color | -+===================+============================+ -| ``COLOR_BLACK`` | Black | -+-------------------+----------------------------+ -| ``COLOR_BLUE`` | Blue | -+-------------------+----------------------------+ -| ``COLOR_CYAN`` | Cyan (light greenish blue) | -+-------------------+----------------------------+ -| ``COLOR_GREEN`` | Green | -+-------------------+----------------------------+ -| ``COLOR_MAGENTA`` | Magenta (purplish red) | -+-------------------+----------------------------+ -| ``COLOR_RED`` | Red | -+-------------------+----------------------------+ -| ``COLOR_WHITE`` | White | -+-------------------+----------------------------+ -| ``COLOR_YELLOW`` | Yellow | -+-------------------+----------------------------+ ++-------------------------+----------------------------+ +| Constant | Color | ++=========================+============================+ +| .. data:: COLOR_BLACK | Black | ++-------------------------+----------------------------+ +| .. data:: COLOR_BLUE | Blue | ++-------------------------+----------------------------+ +| .. data:: COLOR_CYAN | Cyan (light greenish blue) | ++-------------------------+----------------------------+ +| .. data:: COLOR_GREEN | Green | ++-------------------------+----------------------------+ +| .. data:: COLOR_MAGENTA | Magenta (purplish red) | ++-------------------------+----------------------------+ +| .. data:: COLOR_RED | Red | ++-------------------------+----------------------------+ +| .. data:: COLOR_WHITE | White | ++-------------------------+----------------------------+ +| .. data:: COLOR_YELLOW | Yellow | ++-------------------------+----------------------------+ :mod:`curses.textpad` --- Text input widget for curses programs @@ -1851,19 +1881,19 @@ You can instantiate a :class:`Textbox` object as follows: Move operations do nothing if the cursor is at an edge where the movement is not possible. The following synonyms are supported where possible: - +------------------------+------------------+ - | Constant | Keystroke | - +========================+==================+ - | :const:`KEY_LEFT` | :kbd:`Control-B` | - +------------------------+------------------+ - | :const:`KEY_RIGHT` | :kbd:`Control-F` | - +------------------------+------------------+ - | :const:`KEY_UP` | :kbd:`Control-P` | - +------------------------+------------------+ - | :const:`KEY_DOWN` | :kbd:`Control-N` | - +------------------------+------------------+ - | :const:`KEY_BACKSPACE` | :kbd:`Control-h` | - +------------------------+------------------+ + +--------------------------------+------------------+ + | Constant | Keystroke | + +================================+==================+ + | :const:`~curses.KEY_LEFT` | :kbd:`Control-B` | + +--------------------------------+------------------+ + | :const:`~curses.KEY_RIGHT` | :kbd:`Control-F` | + +--------------------------------+------------------+ + | :const:`~curses.KEY_UP` | :kbd:`Control-P` | + +--------------------------------+------------------+ + | :const:`~curses.KEY_DOWN` | :kbd:`Control-N` | + +--------------------------------+------------------+ + | :const:`~curses.KEY_BACKSPACE` | :kbd:`Control-h` | + +--------------------------------+------------------+ All other keystrokes are treated as a command to insert the given character and move right (with line wrapping). From fdb3ef8c0f94c7e55870a585dc6499aca46f9f90 Mon Sep 17 00:00:00 2001 From: Tim Hoffmann <2836374+timhoffm@users.noreply.github.com> Date: Wed, 3 May 2023 09:00:42 +0200 Subject: [PATCH 007/204] gh-82012: Deprecate bitwise inversion (~) of bool (#103487) The bitwise inversion operator on bool returns the bitwise inversion of the underlying int value; i.e. `~True == -2` such that `bool(~True) == True`. It's a common pitfall that users mistake `~` as negation operator and actually want `not`. Supporting `~` is an artifact of bool inheriting from int. Since there is no real use-case for the current behavior, let's deprecate `~` on bool and later raise an error. This removes a potential source errors for users. Full reasoning: https://github.com/python/cpython/issues/82012#issuecomment-1258705971 Co-authored-by: Jelle Zijlstra Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- Doc/library/functions.rst | 2 +- Doc/library/stdtypes.rst | 54 +++++++++++-------- Doc/whatsnew/3.12.rst | 6 +++ Lib/test/test_bool.py | 18 ++++++- ...3-04-12-19-55-24.gh-issue-82012.FlcJAh.rst | 5 ++ Objects/boolobject.c | 18 ++++++- 6 files changed, 78 insertions(+), 25 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-12-19-55-24.gh-issue-82012.FlcJAh.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index a5e86ef0f9eb59..085a11c3caa708 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -147,7 +147,7 @@ are always available. They are listed here in alphabetical order. or omitted, this returns ``False``; otherwise, it returns ``True``. The :class:`bool` class is a subclass of :class:`int` (see :ref:`typesnumeric`). It cannot be subclassed further. Its only instances are ``False`` and - ``True`` (see :ref:`bltin-boolean-values`). + ``True`` (see :ref:`typebool`). .. index:: pair: Boolean; type diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 2360472b31f175..f6662b4336c276 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -802,6 +802,39 @@ number, :class:`float`, or :class:`complex`:: hash_value = -2 return hash_value +.. _typebool: + +Boolean Type - :class:`bool` +============================ + +Booleans represent truth values. The :class:`bool` type has exactly two +constant instances: ``True`` and ``False``. + +.. index:: + single: False + single: True + pair: Boolean; values + +The built-in function :func:`bool` converts any value to a boolean, if the +value can be interpreted as a truth value (see section :ref:`truth` above). + +For logical operations, use the :ref:`boolean operators ` ``and``, +``or`` and ``not``. +When applying the bitwise operators ``&``, ``|``, ``^`` to two booleans, they +return a bool equivalent to the logical operations "and", "or", "xor". However, +the logical operators ``and``, ``or`` and ``!=`` should be preferred +over ``&``, ``|`` and ``^``. + +.. deprecated:: 3.12 + + The use of the bitwise inversion operator ``~`` is deprecated and will + raise an error in Python 3.14. + +:class:`bool` is a subclass of :class:`int` (see :ref:`typesnumeric`). In +many numeric contexts, ``False`` and ``True`` behave like the integers 0 and 1, respectively. +However, relying on this is discouraged; explicitly convert using :func:`int` +instead. + .. _typeiter: Iterator Types @@ -5394,27 +5427,6 @@ information. There is exactly one ``NotImplemented`` object. It is written as ``NotImplemented``. -.. _bltin-boolean-values: - -Boolean Values --------------- - -Boolean values are the two constant objects ``False`` and ``True``. They are -used to represent truth values (although other values can also be considered -false or true). In numeric contexts (for example when used as the argument to -an arithmetic operator), they behave like the integers 0 and 1, respectively. -The built-in function :func:`bool` can be used to convert any value to a -Boolean, if the value can be interpreted as a truth value (see section -:ref:`truth` above). - -.. index:: - single: False - single: True - pair: Boolean; values - -They are written as ``False`` and ``True``, respectively. - - .. _typesinternal: Internal Objects diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index a3fce7ccacf7c1..5f8a1f08026d0b 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -710,6 +710,12 @@ Deprecated replaced by :data:`calendar.Month.JANUARY` and :data:`calendar.Month.FEBRUARY`. (Contributed by Prince Roshan in :gh:`103636`.) +* The bitwise inversion operator (``~``) on bool is deprecated. It will throw an + error in Python 3.14. Use ``not`` for logical negation of bools instead. + In the rare case that you really need the bitwise inversion of the underlying + ``int``, convert to int explicitly with ``~int(x)``. (Contributed by Tim Hoffmann + in :gh:`103487`.) + Pending Removal in Python 3.13 ------------------------------ diff --git a/Lib/test/test_bool.py b/Lib/test/test_bool.py index 916e22a527a8e0..34ecb45f161dfe 100644 --- a/Lib/test/test_bool.py +++ b/Lib/test/test_bool.py @@ -58,8 +58,22 @@ def test_math(self): self.assertEqual(-True, -1) self.assertEqual(abs(True), 1) self.assertIsNot(abs(True), True) - self.assertEqual(~False, -1) - self.assertEqual(~True, -2) + with self.assertWarns(DeprecationWarning): + # We need to put the bool in a variable, because the constant + # ~False is evaluated at compile time due to constant folding; + # consequently the DeprecationWarning would be issued during + # module loading and not during test execution. + false = False + self.assertEqual(~false, -1) + with self.assertWarns(DeprecationWarning): + # also check that the warning is issued in case of constant + # folding at compile time + self.assertEqual(eval("~False"), -1) + with self.assertWarns(DeprecationWarning): + true = True + self.assertEqual(~true, -2) + with self.assertWarns(DeprecationWarning): + self.assertEqual(eval("~True"), -2) self.assertEqual(False+2, 2) self.assertEqual(True+2, 3) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-12-19-55-24.gh-issue-82012.FlcJAh.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-12-19-55-24.gh-issue-82012.FlcJAh.rst new file mode 100644 index 00000000000000..819a2359bf6fae --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-04-12-19-55-24.gh-issue-82012.FlcJAh.rst @@ -0,0 +1,5 @@ +The bitwise inversion operator (``~``) on bool is deprecated. +It returns the bitwise inversion of the underlying ``int`` representation such that +``bool(~True) == True``, which can be confusing. Use ``not`` for logical negation +of bools. In the rare case that you really need the bitwise inversion of the underlying ``int``, +convert to int explicitly ``~int(x)``. diff --git a/Objects/boolobject.c b/Objects/boolobject.c index 597a76fa5cb162..0300f7bb4e3dc0 100644 --- a/Objects/boolobject.c +++ b/Objects/boolobject.c @@ -73,6 +73,22 @@ bool_vectorcall(PyObject *type, PyObject * const*args, /* Arithmetic operations redefined to return bool if both args are bool. */ +static PyObject * +bool_invert(PyObject *v) +{ + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Bitwise inversion '~' on bool is deprecated. This " + "returns the bitwise inversion of the underlying int " + "object and is usually not what you expect from negating " + "a bool. Use the 'not' operator for boolean negation or " + "~int(x) if you really want the bitwise inversion of the " + "underlying int.", + 1) < 0) { + return NULL; + } + return PyLong_Type.tp_as_number->nb_invert(v); +} + static PyObject * bool_and(PyObject *a, PyObject *b) { @@ -119,7 +135,7 @@ static PyNumberMethods bool_as_number = { 0, /* nb_positive */ 0, /* nb_absolute */ 0, /* nb_bool */ - 0, /* nb_invert */ + (unaryfunc)bool_invert, /* nb_invert */ 0, /* nb_lshift */ 0, /* nb_rshift */ bool_and, /* nb_and */ From 8d34031068ece75667260f6526d3165efe34e054 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Wed, 3 May 2023 00:20:00 -0700 Subject: [PATCH 008/204] gh-104078: Improve performance of PyObject_HasAttrString (#104079) --- ...-05-01-21-05-47.gh-issue-104078.vRaBsU.rst | 1 + Objects/object.c | 23 ++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-01-21-05-47.gh-issue-104078.vRaBsU.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-21-05-47.gh-issue-104078.vRaBsU.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-21-05-47.gh-issue-104078.vRaBsU.rst new file mode 100644 index 00000000000000..6f24529bac3e0c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-21-05-47.gh-issue-104078.vRaBsU.rst @@ -0,0 +1 @@ +Improve the performance of :c:func:`PyObject_HasAttrString` diff --git a/Objects/object.c b/Objects/object.c index ee8690101d3cc0..c6ef5928164824 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -918,13 +918,24 @@ PyObject_GetAttrString(PyObject *v, const char *name) int PyObject_HasAttrString(PyObject *v, const char *name) { - PyObject *res = PyObject_GetAttrString(v, name); - if (res != NULL) { - Py_DECREF(res); - return 1; + if (Py_TYPE(v)->tp_getattr != NULL) { + PyObject *res = (*Py_TYPE(v)->tp_getattr)(v, (char*)name); + if (res != NULL) { + Py_DECREF(res); + return 1; + } + PyErr_Clear(); + return 0; } - PyErr_Clear(); - return 0; + + PyObject *attr_name = PyUnicode_FromString(name); + if (attr_name == NULL) { + PyErr_Clear(); + return 0; + } + int ok = PyObject_HasAttr(v, attr_name); + Py_DECREF(attr_name); + return ok; } int From 65c4a2b326086875ecbedf032204d1ff24ba74d7 Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Thu, 23 Mar 2023 10:36:47 -0400 Subject: [PATCH 009/204] GH-84976: Move Lib/datetime.py to Lib/_pydatetime This breaks the tests, but we are keeping it as a separate commit so that the move operation and editing of the moved files are separate, for a cleaner history. --- Lib/{datetime.py => _pydatetime.py} | 0 Lib/test/datetimetester.py | 6 +++++- Lib/test/test_datetime.py | 12 ++++++++---- Python/stdlib_module_names.h | 1 + 4 files changed, 14 insertions(+), 5 deletions(-) rename Lib/{datetime.py => _pydatetime.py} (100%) diff --git a/Lib/datetime.py b/Lib/_pydatetime.py similarity index 100% rename from Lib/datetime.py rename to Lib/_pydatetime.py diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index c5eb6e7f1643ee..fb07d2a5ad9b06 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -39,6 +39,10 @@ # Needed by test_datetime import _strptime +try: + import _pydatetime +except ImportError: + pass # pickle_loads = {pickle.loads, pickle._loads} @@ -92,7 +96,7 @@ def test_divide_and_round(self): if '_Fast' in self.__class__.__name__: self.skipTest('Only run for Pure Python implementation') - dar = datetime_module._divide_and_round + dar = _pydatetime._divide_and_round self.assertEqual(dar(-10, -3), 3) self.assertEqual(dar(5, -2), -2) diff --git a/Lib/test/test_datetime.py b/Lib/test/test_datetime.py index 7f9094fa7bd4e6..3859733a4fe65b 100644 --- a/Lib/test/test_datetime.py +++ b/Lib/test/test_datetime.py @@ -8,10 +8,12 @@ def load_tests(loader, tests, pattern): try: - pure_tests = import_fresh_module(TESTS, fresh=['datetime', '_strptime'], - blocked=['_datetime']) - fast_tests = import_fresh_module(TESTS, fresh=['datetime', - '_datetime', '_strptime']) + pure_tests = import_fresh_module(TESTS, + fresh=['datetime', '_pydatetime', '_strptime'], + blocked=['_datetime']) + fast_tests = import_fresh_module(TESTS, + fresh=['datetime', '_strptime'], + blocked=['_pydatetime']) finally: # XXX: import_fresh_module() is supposed to leave sys.module cache untouched, # XXX: but it does not, so we have to cleanup ourselves. @@ -42,6 +44,8 @@ def setUpClass(cls_, module=module): cls_._save_sys_modules = sys.modules.copy() sys.modules[TESTS] = module sys.modules['datetime'] = module.datetime_module + if hasattr(module, '_pydatetime'): + sys.modules['_pydatetime'] = module._pydatetime sys.modules['_strptime'] = module._strptime @classmethod def tearDownClass(cls_): diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 27f42e5202e571..ed4a0ac2dd32de 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -56,6 +56,7 @@ static const char* _Py_stdlib_module_names[] = { "_posixshmem", "_posixsubprocess", "_py_abc", +"_pydatetime", "_pydecimal", "_pyio", "_pylong", From 0a5cd984b215f28d3c205eadf0daf201b3388c90 Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Thu, 23 Mar 2023 11:36:22 -0400 Subject: [PATCH 010/204] GH-84976: Re-introduce `datetime.py` and fix reprs Without the change to the reprs, pure-python classes would have a repr of `datetime._pydatetime.time`, etc. --- Lib/_pydatetime.py | 42 ++++++------------- Lib/datetime.py | 9 ++++ ...3-04-19-16-08-53.gh-issue-84976.HwbzlD.rst | 5 +++ 3 files changed, 27 insertions(+), 29 deletions(-) create mode 100644 Lib/datetime.py create mode 100644 Misc/NEWS.d/next/Library/2023-04-19-16-08-53.gh-issue-84976.HwbzlD.rst diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index b0eb1c216a689d..f4fc2c58e5e293 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -16,6 +16,13 @@ def _cmp(x, y): return 0 if x == y else 1 if x > y else -1 +def _get_class_module(self): + module_name = self.__class__.__module__ + if module_name == '_pydatetime': + return 'datetime' + else: + return module_name + MINYEAR = 1 MAXYEAR = 9999 _MAXORDINAL = 3652059 # date.max.toordinal() @@ -706,7 +713,7 @@ def __repr__(self): args.append("microseconds=%d" % self._microseconds) if not args: args.append('0') - return "%s.%s(%s)" % (self.__class__.__module__, + return "%s.%s(%s)" % (_get_class_module(self), self.__class__.__qualname__, ', '.join(args)) @@ -1016,7 +1023,7 @@ def __repr__(self): >>> repr(dt) 'datetime.datetime(2010, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)' """ - return "%s.%s(%d, %d, %d)" % (self.__class__.__module__, + return "%s.%s(%d, %d, %d)" % (_get_class_module(self), self.__class__.__qualname__, self._year, self._month, @@ -1510,7 +1517,7 @@ def __repr__(self): s = ", %d" % self._second else: s = "" - s= "%s.%s(%d, %d%s)" % (self.__class__.__module__, + s= "%s.%s(%d, %d%s)" % (_get_class_module(self), self.__class__.__qualname__, self._hour, self._minute, s) if self._tzinfo is not None: @@ -2065,7 +2072,7 @@ def __repr__(self): del L[-1] if L[-1] == 0: del L[-1] - s = "%s.%s(%s)" % (self.__class__.__module__, + s = "%s.%s(%s)" % (_get_class_module(self), self.__class__.__qualname__, ", ".join(map(str, L))) if self._tzinfo is not None: @@ -2372,10 +2379,10 @@ def __repr__(self): if self is self.utc: return 'datetime.timezone.utc' if self._name is None: - return "%s.%s(%r)" % (self.__class__.__module__, + return "%s.%s(%r)" % (_get_class_module(self), self.__class__.__qualname__, self._offset) - return "%s.%s(%r, %r)" % (self.__class__.__module__, + return "%s.%s(%r, %r)" % (_get_class_module(self), self.__class__.__qualname__, self._offset, self._name) @@ -2638,26 +2645,3 @@ def _name_from_offset(delta): # small dst() may get within its bounds; and it doesn't even matter if some # perverse time zone returns a negative dst()). So a breaking case must be # pretty bizarre, and a tzinfo subclass can override fromutc() if it is. - -try: - from _datetime import * -except ImportError: - pass -else: - # Clean up unused names - del (_DAYNAMES, _DAYS_BEFORE_MONTH, _DAYS_IN_MONTH, _DI100Y, _DI400Y, - _DI4Y, _EPOCH, _MAXORDINAL, _MONTHNAMES, _build_struct_time, - _check_date_fields, _check_time_fields, - _check_tzinfo_arg, _check_tzname, _check_utc_offset, _cmp, _cmperror, - _date_class, _days_before_month, _days_before_year, _days_in_month, - _format_time, _format_offset, _index, _is_leap, _isoweek1monday, _math, - _ord2ymd, _time, _time_class, _tzinfo_class, _wrap_strftime, _ymd2ord, - _divide_and_round, _parse_isoformat_date, _parse_isoformat_time, - _parse_hh_mm_ss_ff, _IsoCalendarDate, _isoweek_to_gregorian, - _find_isoformat_datetime_separator, _FRACTION_CORRECTION, - _is_ascii_digit) - # XXX Since import * above excludes names that start with _, - # docstring does not get overwritten. In the future, it may be - # appropriate to maintain a single module level docstring and - # remove the following line. - from _datetime import __doc__ diff --git a/Lib/datetime.py b/Lib/datetime.py new file mode 100644 index 00000000000000..bad8beb4f6b026 --- /dev/null +++ b/Lib/datetime.py @@ -0,0 +1,9 @@ +try: + from _datetime import * + from _datetime import __doc__ +except ImportError: + from _pydatetime import * + from _pydatetime import __doc__ + +__all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo", + "MINYEAR", "MAXYEAR") diff --git a/Misc/NEWS.d/next/Library/2023-04-19-16-08-53.gh-issue-84976.HwbzlD.rst b/Misc/NEWS.d/next/Library/2023-04-19-16-08-53.gh-issue-84976.HwbzlD.rst new file mode 100644 index 00000000000000..8658627aeba434 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-19-16-08-53.gh-issue-84976.HwbzlD.rst @@ -0,0 +1,5 @@ +Create a new ``Lib/_pydatetime.py`` file that defines the Python version of +the ``datetime`` module, and make ``datetime`` import the contents of the +new library only if the C implementation is missing. Currently, the full +Python implementation is defined and then deleted if the C implementation is +not available, slowing down ``import datetime`` unnecessarily. From bcea36f8db9ad4fd542b38997e065987e829cb9f Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Wed, 3 May 2023 02:51:47 -0700 Subject: [PATCH 011/204] gh-103845: Remove line & instruction instrumentations before adding them back (GH-103851) --- Lib/test/test_monitoring.py | 36 ++++++++++++++++ ...-04-25-20-56-01.gh-issue-103845.V7NYFn.rst | 1 + Python/instrumentation.c | 41 +++++++++++++++---- 3 files changed, 70 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-25-20-56-01.gh-issue-103845.V7NYFn.rst diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 738ace923cc523..a493bb54d70d38 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -876,6 +876,42 @@ def func3(): ('instruction', 'func3', 34), ('line', 'check_events', 11)]) + def test_with_restart(self): + def func1(): + line1 = 1 + line2 = 2 + line3 = 3 + + self.check_events(func1, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('line', 'func1', 1), + ('instruction', 'func1', 2), + ('instruction', 'func1', 4), + ('line', 'func1', 2), + ('instruction', 'func1', 6), + ('instruction', 'func1', 8), + ('line', 'func1', 3), + ('instruction', 'func1', 10), + ('instruction', 'func1', 12), + ('instruction', 'func1', 14), + ('line', 'check_events', 11)]) + + sys.monitoring.restart_events() + + self.check_events(func1, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('line', 'func1', 1), + ('instruction', 'func1', 2), + ('instruction', 'func1', 4), + ('line', 'func1', 2), + ('instruction', 'func1', 6), + ('instruction', 'func1', 8), + ('line', 'func1', 3), + ('instruction', 'func1', 10), + ('instruction', 'func1', 12), + ('instruction', 'func1', 14), + ('line', 'check_events', 11)]) + class TestInstallIncrementallly(MonitoringTestBase, unittest.TestCase): def check_events(self, func, must_include, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-25-20-56-01.gh-issue-103845.V7NYFn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-25-20-56-01.gh-issue-103845.V7NYFn.rst new file mode 100644 index 00000000000000..e8434854cde632 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-04-25-20-56-01.gh-issue-103845.V7NYFn.rst @@ -0,0 +1 @@ +Remove both line and instruction instrumentation before adding new ones for monitoring, to avoid newly added instrumentation being removed immediately. diff --git a/Python/instrumentation.c b/Python/instrumentation.c index c5bbbdacbb851e..a1423240609699 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1477,25 +1477,25 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) } } } - uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE]; + + // GH-103845: We need to remove both the line and instruction instrumentation before + // adding new ones, otherwise we may remove the newly added instrumentation. + uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE]; - if (new_line_tools | removed_line_tools) { + uint8_t removed_per_instruction_tools = removed_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; + + if (removed_line_tools) { _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; for (int i = code->_co_firsttraceable; i < code_len;) { if (line_data[i].original_opcode) { if (removed_line_tools) { remove_line_tools(code, i, removed_line_tools); } - if (new_line_tools) { - add_line_tools(code, i, new_line_tools); - } } i += instruction_length(code, i); } } - uint8_t new_per_instruction_tools = new_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; - uint8_t removed_per_instruction_tools = removed_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; - if (new_per_instruction_tools | removed_per_instruction_tools) { + if (removed_per_instruction_tools) { for (int i = code->_co_firsttraceable; i < code_len;) { int opcode = _Py_GetBaseOpcode(code, i); if (opcode == RESUME || opcode == END_FOR) { @@ -1505,6 +1505,31 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) if (removed_per_instruction_tools) { remove_per_instruction_tools(code, i, removed_per_instruction_tools); } + i += instruction_length(code, i); + } + } + + uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE]; + uint8_t new_per_instruction_tools = new_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; + + if (new_line_tools) { + _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; + for (int i = code->_co_firsttraceable; i < code_len;) { + if (line_data[i].original_opcode) { + if (new_line_tools) { + add_line_tools(code, i, new_line_tools); + } + } + i += instruction_length(code, i); + } + } + if (new_per_instruction_tools) { + for (int i = code->_co_firsttraceable; i < code_len;) { + int opcode = _Py_GetBaseOpcode(code, i); + if (opcode == RESUME || opcode == END_FOR) { + i += instruction_length(code, i); + continue; + } if (new_per_instruction_tools) { add_per_instruction_tools(code, i, new_per_instruction_tools); } From 326997829d02458246dfd5b6d03297e2418bde52 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Wed, 3 May 2023 04:55:22 -0700 Subject: [PATCH 012/204] gh-98040: Remove find_loader, find_module and other deprecated APIs (#98059) * Remove deprecated classes from pkgutil * Remove some other PEP 302 obsolescence * Use find_spec instead of load_module * Remove more tests of PEP 302 obsolete APIs * Remove another bunch of tests using obsolete load_modules() * Remove deleted names from __all__ * Remove obsolete footnote * imp is removed * Remove `imp` from generated stdlib names * What's new and blurb * Update zipimport documentation for the removed methods * Fix some Windows tests * Remove any test (or part of a test) that references `find_module()`. * Use assertIsNone() / assertIsNotNone() consistently. * Update Doc/reference/import.rst * We don't need pkgutil._get_spec() any more either * test.test_importlib.fixtures.NullFinder * ...BadLoaderFinder.find_module * ...test_api.InvalidatingNullFinder.find_module * ...test.test_zipimport test of z.find_module * Suppress cross-references to find_loader and find_module * Suppress cross-references to Finder * Suppress cross-references to pkgutil.ImpImporter and pkgutil.ImpLoader --------- Co-authored-by: Oleg Iarygin Co-authored-by: Adam Turner <9087854+aa-turner@users.noreply.github.com> --- Doc/library/importlib.rst | 112 +----------------- Doc/library/pkgutil.rst | 27 ----- Doc/library/sys.rst | 2 +- Doc/library/zipimport.rst | 27 +---- Doc/reference/import.rst | 34 +++--- Doc/whatsnew/2.3.rst | 2 +- Doc/whatsnew/3.10.rst | 38 +++--- Doc/whatsnew/3.11.rst | 14 +-- Doc/whatsnew/3.12.rst | 16 ++- Doc/whatsnew/3.3.rst | 4 +- Doc/whatsnew/3.4.rst | 18 +-- Doc/whatsnew/3.7.rst | 4 +- Lib/importlib/__init__.py | 34 ------ Lib/importlib/_bootstrap.py | 41 +------ Lib/importlib/_bootstrap_external.py | 95 +-------------- Lib/importlib/abc.py | 92 +------------- Lib/pkgutil.py | 14 --- Lib/pydoc.py | 2 +- .../test_importlib/builtin/test_finder.py | 33 ------ .../extension/test_path_hook.py | 2 +- Lib/test/test_importlib/fixtures.py | 5 - Lib/test/test_importlib/frozen/test_finder.py | 40 ------- Lib/test/test_importlib/frozen/test_loader.py | 82 ------------- .../test_importlib/import_/test___loader__.py | 43 ------- .../import_/test___package__.py | 36 ------ Lib/test/test_importlib/import_/test_api.py | 5 - .../test_importlib/import_/test_caching.py | 25 ++-- .../test_importlib/import_/test_meta_path.py | 10 -- Lib/test/test_importlib/import_/test_path.py | 73 +----------- .../source/test_case_sensitivity.py | 13 -- Lib/test/test_importlib/source/test_finder.py | 29 ++--- .../test_importlib/source/test_path_hook.py | 9 -- Lib/test/test_importlib/test_abc.py | 78 +----------- Lib/test/test_importlib/test_api.py | 67 ++++------- Lib/test/test_importlib/test_windows.py | 18 +-- Lib/test/test_importlib/util.py | 27 +---- Lib/test/test_pkgutil.py | 10 +- Lib/test/test_zipimport.py | 1 - ...2-10-09-14-47-42.gh-issue-98040.IN3qab.rst | 2 + 39 files changed, 137 insertions(+), 1047 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-10-09-14-47-42.gh-issue-98040.IN3qab.rst diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 89efa64c6b5203..65aaad0df9ee66 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -127,28 +127,6 @@ Functions .. versionchanged:: 3.3 Parent packages are automatically imported. -.. function:: find_loader(name, path=None) - - Find the loader for a module, optionally within the specified *path*. If the - module is in :attr:`sys.modules`, then ``sys.modules[name].__loader__`` is - returned (unless the loader would be ``None`` or is not set, in which case - :exc:`ValueError` is raised). Otherwise a search using :attr:`sys.meta_path` - is done. ``None`` is returned if no loader is found. - - A dotted name does not have its parents implicitly imported as that requires - loading them and that may not be desired. To properly import a submodule you - will need to import all parent packages of the submodule and use the correct - argument to *path*. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.4 - If ``__loader__`` is not set, raise :exc:`ValueError`, just like when the - attribute is set to ``None``. - - .. deprecated:: 3.4 - Use :func:`importlib.util.find_spec` instead. - .. function:: invalidate_caches() Invalidate the internal caches of finders stored at @@ -247,7 +225,6 @@ are also provided to help in implementing the core ABCs. ABC hierarchy:: object - +-- Finder (deprecated) +-- MetaPathFinder +-- PathEntryFinder +-- Loader @@ -258,28 +235,6 @@ ABC hierarchy:: +-- SourceLoader -.. class:: Finder - - An abstract base class representing a :term:`finder`. - - .. deprecated:: 3.3 - Use :class:`MetaPathFinder` or :class:`PathEntryFinder` instead. - - .. abstractmethod:: find_module(fullname, path=None) - - An abstract method for finding a :term:`loader` for the specified - module. Originally specified in :pep:`302`, this method was meant - for use in :data:`sys.meta_path` and in the path-based import subsystem. - - .. versionchanged:: 3.4 - Returns ``None`` when called instead of raising - :exc:`NotImplementedError`. - - .. deprecated:: 3.10 - Implement :meth:`MetaPathFinder.find_spec` or - :meth:`PathEntryFinder.find_spec` instead. - - .. class:: MetaPathFinder An abstract base class representing a :term:`meta path finder`. @@ -287,7 +242,7 @@ ABC hierarchy:: .. versionadded:: 3.3 .. versionchanged:: 3.10 - No longer a subclass of :class:`Finder`. + No longer a subclass of :class:`!Finder`. .. method:: find_spec(fullname, path, target=None) @@ -303,25 +258,6 @@ ABC hierarchy:: .. versionadded:: 3.4 - .. method:: find_module(fullname, path) - - A legacy method for finding a :term:`loader` for the specified - module. If this is a top-level import, *path* will be ``None``. - Otherwise, this is a search for a subpackage or module and *path* - will be the value of :attr:`__path__` from the parent - package. If a loader cannot be found, ``None`` is returned. - - If :meth:`find_spec` is defined, backwards-compatible functionality is - provided. - - .. versionchanged:: 3.4 - Returns ``None`` when called instead of raising - :exc:`NotImplementedError`. Can use :meth:`find_spec` to provide - functionality. - - .. deprecated:: 3.4 - Use :meth:`find_spec` instead. - .. method:: invalidate_caches() An optional method which, when called, should invalidate any internal @@ -342,7 +278,7 @@ ABC hierarchy:: .. versionadded:: 3.3 .. versionchanged:: 3.10 - No longer a subclass of :class:`Finder`. + No longer a subclass of :class:`!Finder`. .. method:: find_spec(fullname, target=None) @@ -356,36 +292,6 @@ ABC hierarchy:: .. versionadded:: 3.4 - .. method:: find_loader(fullname) - - A legacy method for finding a :term:`loader` for the specified - module. Returns a 2-tuple of ``(loader, portion)`` where ``portion`` - is a sequence of file system locations contributing to part of a namespace - package. The loader may be ``None`` while specifying ``portion`` to - signify the contribution of the file system locations to a namespace - package. An empty list can be used for ``portion`` to signify the loader - is not part of a namespace package. If ``loader`` is ``None`` and - ``portion`` is the empty list then no loader or location for a namespace - package were found (i.e. failure to find anything for the module). - - If :meth:`find_spec` is defined then backwards-compatible functionality is - provided. - - .. versionchanged:: 3.4 - Returns ``(None, [])`` instead of raising :exc:`NotImplementedError`. - Uses :meth:`find_spec` when available to provide functionality. - - .. deprecated:: 3.4 - Use :meth:`find_spec` instead. - - .. method:: find_module(fullname) - - A concrete implementation of :meth:`Finder.find_module` which is - equivalent to ``self.find_loader(fullname)[0]``. - - .. deprecated:: 3.4 - Use :meth:`find_spec` instead. - .. method:: invalidate_caches() An optional method which, when called, should invalidate any internal @@ -881,13 +787,6 @@ find and load modules. is no longer valid then ``None`` is returned but no value is cached in :data:`sys.path_importer_cache`. - .. classmethod:: find_module(fullname, path=None) - - A legacy wrapper around :meth:`find_spec`. - - .. deprecated:: 3.4 - Use :meth:`find_spec` instead. - .. classmethod:: invalidate_caches() Calls :meth:`importlib.abc.PathEntryFinder.invalidate_caches` on all @@ -938,13 +837,6 @@ find and load modules. .. versionadded:: 3.4 - .. method:: find_loader(fullname) - - Attempt to find the loader to handle *fullname* within :attr:`path`. - - .. deprecated:: 3.10 - Use :meth:`find_spec` instead. - .. method:: invalidate_caches() Clear out the internal cache. diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index 64e617b82b48bc..98e6e294af0c5c 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -48,33 +48,6 @@ support. this function to raise an exception (in line with :func:`os.path.isdir` behavior). - -.. class:: ImpImporter(dirname=None) - - :pep:`302` Finder that wraps Python's "classic" import algorithm. - - If *dirname* is a string, a :pep:`302` finder is created that searches that - directory. If *dirname* is ``None``, a :pep:`302` finder is created that - searches the current :data:`sys.path`, plus any modules that are frozen or - built-in. - - Note that :class:`ImpImporter` does not currently support being used by - placement on :data:`sys.meta_path`. - - .. deprecated:: 3.3 - This emulation is no longer needed, as the standard import mechanism - is now fully :pep:`302` compliant and available in :mod:`importlib`. - - -.. class:: ImpLoader(fullname, file, filename, etc) - - :term:`Loader ` that wraps Python's "classic" import algorithm. - - .. deprecated:: 3.3 - This emulation is no longer needed, as the standard import mechanism - is now fully :pep:`302` compliant and available in :mod:`importlib`. - - .. function:: find_loader(fullname) Retrieve a module :term:`loader` for the given *fullname*. diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 95ad243bdde398..57a0d0a1258c2a 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1177,7 +1177,7 @@ always available. :term:`Module specs ` were introduced in Python 3.4, by :pep:`451`. Earlier versions of Python looked for a method called - :meth:`~importlib.abc.MetaPathFinder.find_module`. + :meth:`!find_module`. This is still called as a fallback if a :data:`meta_path` entry doesn't have a :meth:`~importlib.abc.MetaPathFinder.find_spec` method. diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst index fe1adcae163c23..11d19e8c863e9f 100644 --- a/Doc/library/zipimport.rst +++ b/Doc/library/zipimport.rst @@ -74,6 +74,11 @@ zipimporter Objects :exc:`ZipImportError` is raised if *archivepath* doesn't point to a valid ZIP archive. + .. versionchanged:: 3.12 + + Methods ``find_loader()`` and ``find_module()``, deprecated in 3.10 are + now removed. Use :meth:`find_spec` instead. + .. method:: create_module(spec) Implementation of :meth:`importlib.abc.Loader.create_module` that returns @@ -89,28 +94,6 @@ zipimporter Objects .. versionadded:: 3.10 - .. method:: find_loader(fullname, path=None) - - An implementation of :meth:`importlib.abc.PathEntryFinder.find_loader`. - - .. deprecated:: 3.10 - - Use :meth:`find_spec` instead. - - - .. method:: find_module(fullname, path=None) - - Search for a module specified by *fullname*. *fullname* must be the fully - qualified (dotted) module name. It returns the zipimporter instance itself - if the module was found, or :const:`None` if it wasn't. The optional - *path* argument is ignored---it's there for compatibility with the - importer protocol. - - .. deprecated:: 3.10 - - Use :meth:`find_spec` instead. - - .. method:: find_spec(fullname, target=None) An implementation of :meth:`importlib.abc.PathEntryFinder.find_spec`. diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 57eb5403243eef..8f30181ae0a10d 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -324,15 +324,18 @@ modules, and one that knows how to import modules from an :term:`import path` .. versionchanged:: 3.4 The :meth:`~importlib.abc.MetaPathFinder.find_spec` method of meta path - finders replaced :meth:`~importlib.abc.MetaPathFinder.find_module`, which + finders replaced :meth:`!find_module`, which is now deprecated. While it will continue to work without change, the import machinery will try it only if the finder does not implement ``find_spec()``. .. versionchanged:: 3.10 - Use of :meth:`~importlib.abc.MetaPathFinder.find_module` by the import system + Use of :meth:`!find_module` by the import system now raises :exc:`ImportWarning`. +.. versionchanged:: 3.12 + ``find_module()`` has been removed. Use :meth:`find_spec` instead. + Loading ======= @@ -837,7 +840,7 @@ stores finder objects rather than being limited to :term:`importer` objects). In this way, the expensive search for a particular :term:`path entry` location's :term:`path entry finder` need only be done once. User code is free to remove cache entries from :data:`sys.path_importer_cache` forcing -the path based finder to perform the path entry search again [#fnpic]_. +the path based finder to perform the path entry search again. If the path entry is not present in the cache, the path based finder iterates over every callable in :data:`sys.path_hooks`. Each of the :term:`path entry @@ -887,13 +890,13 @@ module. ``find_spec()`` returns a fully populated spec for the module. This spec will always have "loader" set (with one exception). To indicate to the import machinery that the spec represents a namespace -:term:`portion`, the path entry finder sets "submodule_search_locations" to +:term:`portion`, the path entry finder sets ``submodule_search_locations`` to a list containing the portion. .. versionchanged:: 3.4 :meth:`~importlib.abc.PathEntryFinder.find_spec` replaced - :meth:`~importlib.abc.PathEntryFinder.find_loader` and - :meth:`~importlib.abc.PathEntryFinder.find_module`, both of which + :meth:`!find_loader` and + :meth:`!find_module`, both of which are now deprecated, but will be used if ``find_spec()`` is not defined. Older path entry finders may implement one of these two deprecated methods @@ -901,7 +904,7 @@ a list containing the portion. sake of backward compatibility. However, if ``find_spec()`` is implemented on the path entry finder, the legacy methods are ignored. - :meth:`~importlib.abc.PathEntryFinder.find_loader` takes one argument, the + :meth:`!find_loader` takes one argument, the fully qualified name of the module being imported. ``find_loader()`` returns a 2-tuple where the first item is the loader and the second item is a namespace :term:`portion`. @@ -920,10 +923,13 @@ a list containing the portion. ``find_loader()`` in preference to ``find_module()``. .. versionchanged:: 3.10 - Calls to :meth:`~importlib.abc.PathEntryFinder.find_module` and - :meth:`~importlib.abc.PathEntryFinder.find_loader` by the import + Calls to :meth:`!find_module` and + :meth:`!find_loader` by the import system will raise :exc:`ImportWarning`. +.. versionchanged:: 3.12 + ``find_module()`` and ``find_loader()`` have been removed. + Replacing the standard import system ==================================== @@ -1045,8 +1051,8 @@ The original specification for :data:`sys.meta_path` was :pep:`302`, with subsequent extension in :pep:`420`. :pep:`420` introduced :term:`namespace packages ` for -Python 3.3. :pep:`420` also introduced the :meth:`find_loader` protocol as an -alternative to :meth:`find_module`. +Python 3.3. :pep:`420` also introduced the :meth:`!find_loader` protocol as an +alternative to :meth:`!find_module`. :pep:`366` describes the addition of the ``__package__`` attribute for explicit relative imports in main modules. @@ -1073,9 +1079,3 @@ methods to finders and loaders. module may replace itself in :data:`sys.modules`. This is implementation-specific behavior that is not guaranteed to work in other Python implementations. - -.. [#fnpic] In legacy code, it is possible to find instances of - :class:`imp.NullImporter` in the :data:`sys.path_importer_cache`. It - is recommended that code be changed to use ``None`` instead. See - :ref:`portingpythoncode` for more details. Note that the ``imp`` module - was removed in Python 3.12. diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index af489d7cb45c2a..4eb864f5092d30 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -728,7 +728,7 @@ module: Importer objects must have a single method, ``find_module(fullname, path=None)``. *fullname* will be a module or package name, e.g. ``string`` or -``distutils.core``. :meth:`find_module` must return a loader object that has a +``distutils.core``. :meth:`!find_module` must return a loader object that has a single method, ``load_module(fullname)``, that creates and returns the corresponding module object. diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index f6a48ed2680c14..44c8fa1e9eb2c4 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -1608,7 +1608,7 @@ Deprecated * Starting in this release, there will be a concerted effort to begin cleaning up old import semantics that were kept for Python 2.7 compatibility. Specifically, - :meth:`~importlib.abc.PathEntryFinder.find_loader`/:meth:`~importlib.abc.Finder.find_module` + :meth:`!find_loader`/:meth:`!find_module` (superseded by :meth:`~importlib.abc.Finder.find_spec`), :meth:`~importlib.abc.Loader.load_module` (superseded by :meth:`~importlib.abc.Loader.exec_module`), @@ -1645,8 +1645,8 @@ Deprecated :meth:`~importlib.abc.Loader.exec_module` is preferred. (Contributed by Brett Cannon in :issue:`26131`.) -* The use of :meth:`importlib.abc.MetaPathFinder.find_module` and - :meth:`importlib.abc.PathEntryFinder.find_module` by the import system now +* The use of :meth:`!importlib.abc.MetaPathFinder.find_module` and + :meth:`!importlib.abc.PathEntryFinder.find_module` by the import system now trigger an :exc:`ImportWarning` as :meth:`importlib.abc.MetaPathFinder.find_spec` and :meth:`importlib.abc.PathEntryFinder.find_spec` @@ -1654,40 +1654,40 @@ Deprecated :func:`importlib.util.spec_from_loader` to help in porting. (Contributed by Brett Cannon in :issue:`42134`.) -* The use of :meth:`importlib.abc.PathEntryFinder.find_loader` by the import +* The use of :meth:`!importlib.abc.PathEntryFinder.find_loader` by the import system now triggers an :exc:`ImportWarning` as :meth:`importlib.abc.PathEntryFinder.find_spec` is preferred. You can use :func:`importlib.util.spec_from_loader` to help in porting. (Contributed by Brett Cannon in :issue:`43672`.) * The various implementations of - :meth:`importlib.abc.MetaPathFinder.find_module` ( - :meth:`importlib.machinery.BuiltinImporter.find_module`, - :meth:`importlib.machinery.FrozenImporter.find_module`, - :meth:`importlib.machinery.WindowsRegistryFinder.find_module`, - :meth:`importlib.machinery.PathFinder.find_module`, - :meth:`importlib.abc.MetaPathFinder.find_module` ), - :meth:`importlib.abc.PathEntryFinder.find_module` ( - :meth:`importlib.machinery.FileFinder.find_module` ), and - :meth:`importlib.abc.PathEntryFinder.find_loader` ( - :meth:`importlib.machinery.FileFinder.find_loader` ) + :meth:`!importlib.abc.MetaPathFinder.find_module` ( + :meth:`!importlib.machinery.BuiltinImporter.find_module`, + :meth:`!importlib.machinery.FrozenImporter.find_module`, + :meth:`!importlib.machinery.WindowsRegistryFinder.find_module`, + :meth:`!importlib.machinery.PathFinder.find_module`, + :meth:`!importlib.abc.MetaPathFinder.find_module` ), + :meth:`!importlib.abc.PathEntryFinder.find_module` ( + :meth:`!importlib.machinery.FileFinder.find_module` ), and + :meth:`!importlib.abc.PathEntryFinder.find_loader` ( + :meth:`!importlib.machinery.FileFinder.find_loader` ) now raise :exc:`DeprecationWarning` and are slated for removal in Python 3.12 (previously they were documented as deprecated in Python 3.4). (Contributed by Brett Cannon in :issue:`42135`.) -* :class:`importlib.abc.Finder` is deprecated (including its sole method, - :meth:`~importlib.abc.Finder.find_module`). Both +* :class:`!importlib.abc.Finder` is deprecated (including its sole method, + :meth:`!find_module`). Both :class:`importlib.abc.MetaPathFinder` and :class:`importlib.abc.PathEntryFinder` no longer inherit from the class. Users should inherit from one of these two classes as appropriate instead. (Contributed by Brett Cannon in :issue:`42135`.) -* The deprecations of :mod:`imp`, :func:`importlib.find_loader`, +* The deprecations of :mod:`imp`, :func:`!importlib.find_loader`, :func:`importlib.util.set_package_wrapper`, :func:`importlib.util.set_loader_wrapper`, :func:`importlib.util.module_for_loader`, - :class:`pkgutil.ImpImporter`, and - :class:`pkgutil.ImpLoader` have all been updated to list Python 3.12 as the + :class:`!pkgutil.ImpImporter`, and + :class:`!pkgutil.ImpLoader` have all been updated to list Python 3.12 as the slated version of removal (they began raising :exc:`DeprecationWarning` in previous versions of Python). (Contributed by Brett Cannon in :issue:`43720`.) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 6b591d5e184ea7..918a6824618c1e 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1876,24 +1876,24 @@ C APIs pending removal are * The :class:`typing.io ` namespace * The :class:`typing.re ` namespace * :func:`!cgi.log` -* :func:`importlib.find_loader` +* :func:`!importlib.find_loader` * :meth:`importlib.abc.Loader.module_repr` -* :meth:`importlib.abc.MetaPathFinder.find_module` -* :meth:`importlib.abc.PathEntryFinder.find_loader` -* :meth:`importlib.abc.PathEntryFinder.find_module` +* :meth:`!importlib.abc.MetaPathFinder.find_module` +* :meth:`!importlib.abc.PathEntryFinder.find_loader` +* :meth:`!importlib.abc.PathEntryFinder.find_module` * :meth:`!importlib.machinery.BuiltinImporter.find_module` * :meth:`!importlib.machinery.BuiltinLoader.module_repr` * :meth:`!importlib.machinery.FileFinder.find_loader` * :meth:`!importlib.machinery.FileFinder.find_module` * :meth:`!importlib.machinery.FrozenImporter.find_module` * :meth:`!importlib.machinery.FrozenLoader.module_repr` -* :meth:`importlib.machinery.PathFinder.find_module` +* :meth:`!importlib.machinery.PathFinder.find_module` * :meth:`!importlib.machinery.WindowsRegistryFinder.find_module` * :func:`importlib.util.module_for_loader` * :func:`!importlib.util.set_loader_wrapper` * :func:`!importlib.util.set_package_wrapper` -* :class:`pkgutil.ImpImporter` -* :class:`pkgutil.ImpLoader` +* :class:`!pkgutil.ImpImporter` +* :class:`!pkgutil.ImpLoader` * :meth:`pathlib.Path.link_to` * :func:`!sqlite3.enable_shared_cache` * :func:`!sqlite3.OptimizedUnicode` diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 5f8a1f08026d0b..1139bb89f5cf0c 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -983,14 +983,20 @@ Removed * Many previously deprecated cleanups in :mod:`importlib` have now been completed: - * References to, and support for ``module_repr()`` has been eradicated. + * References to, and support for ``module_repr()`` has been removed. (Contributed by Barry Warsaw in :gh:`97850`.) -* ``importlib.util.set_package`` has been removed. - (Contributed by Brett Cannon in :gh:`65961`.) + * ``importlib.util.set_package`` has been removed. (Contributed by Brett + Cannon in :gh:`65961`.) + + * Support for ``find_loader()`` and ``find_module()`` APIs have been + removed. (Contributed by Barry Warsaw in :gh:`98040`.) + + * ``importlib.abc.Finder``, ``pkg.ImpImporter``, and ``pkg.ImpLoader`` have + been removed. (Contributed by Barry Warsaw in :gh:`98040`.) -* The ``imp`` module has been removed. (Contributed by Barry Warsaw in - :gh:`98040`.) + * The ``imp`` module has been removed. (Contributed by Barry Warsaw in + :gh:`98040`.) * Removed the ``suspicious`` rule from the documentation Makefile, and removed ``Doc/tools/rstlint.py``, both in favor of `sphinx-lint diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 5b6c3dcd45c6f5..df1f2ab775b047 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -714,7 +714,7 @@ to properly delineate between :term:`meta path finders ` and :term:`path entry finders ` by introducing :class:`importlib.abc.MetaPathFinder` and :class:`importlib.abc.PathEntryFinder`, respectively. The old ABC of -:class:`importlib.abc.Finder` is now only provided for backwards-compatibility +:class:`!importlib.abc.Finder` is now only provided for backwards-compatibility and does not enforce any method requirements. In terms of finders, :class:`importlib.machinery.FileFinder` exposes the @@ -2390,7 +2390,7 @@ Porting Python code :attr:`sys.path_importer_cache` where it represents the use of implicit finders, but semantically it should not change anything. -* :class:`importlib.abc.Finder` no longer specifies a ``find_module()`` abstract +* :class:`!importlib.abc.Finder` no longer specifies a ``find_module()`` abstract method that must be implemented. If you were relying on subclasses to implement that method, make sure to check for the method's existence first. You will probably want to check for ``find_loader()`` first, though, in the diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index b7bb505a818482..53f78e3621cfbd 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -2077,19 +2077,19 @@ Deprecations in the Python API ------------------------------ * As mentioned in :ref:`whatsnew-pep-451`, a number of :mod:`importlib` - methods and functions are deprecated: :meth:`importlib.find_loader` is + methods and functions are deprecated: :meth:`!importlib.find_loader` is replaced by :func:`importlib.util.find_spec`; - :meth:`importlib.machinery.PathFinder.find_module` is replaced by + :meth:`!importlib.machinery.PathFinder.find_module` is replaced by :meth:`importlib.machinery.PathFinder.find_spec`; - :meth:`importlib.abc.MetaPathFinder.find_module` is replaced by + :meth:`!importlib.abc.MetaPathFinder.find_module` is replaced by :meth:`importlib.abc.MetaPathFinder.find_spec`; - :meth:`importlib.abc.PathEntryFinder.find_loader` and - :meth:`~importlib.abc.PathEntryFinder.find_module` are replaced by + :meth:`!importlib.abc.PathEntryFinder.find_loader` and + :meth:`!find_module` are replaced by :meth:`importlib.abc.PathEntryFinder.find_spec`; all of the ``xxxLoader`` ABC - ``load_module`` methods (:meth:`importlib.abc.Loader.load_module`, - :meth:`importlib.abc.InspectLoader.load_module`, - :meth:`importlib.abc.FileLoader.load_module`, - :meth:`importlib.abc.SourceLoader.load_module`) should no longer be + ``load_module`` methods (:meth:`!importlib.abc.Loader.load_module`, + :meth:`!importlib.abc.InspectLoader.load_module`, + :meth:`!importlib.abc.FileLoader.load_module`, + :meth:`!importlib.abc.SourceLoader.load_module`) should no longer be implemented, instead loaders should implement an ``exec_module`` method (:meth:`importlib.abc.Loader.exec_module`, diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index df3b636cb9ec46..28f22836d8d09e 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -2004,11 +2004,11 @@ importlib --------- Methods -:meth:`MetaPathFinder.find_module() ` +:meth:`MetaPathFinder.find_module() ` (replaced by :meth:`MetaPathFinder.find_spec() `) and -:meth:`PathEntryFinder.find_loader() ` +:meth:`PathEntryFinder.find_loader() ` (replaced by :meth:`PathEntryFinder.find_spec() `) both deprecated in Python 3.4 now emit :exc:`DeprecationWarning`. diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py index 21d9dee652b3df..707c081cb2c5b6 100644 --- a/Lib/importlib/__init__.py +++ b/Lib/importlib/__init__.py @@ -70,40 +70,6 @@ def invalidate_caches(): finder.invalidate_caches() -def find_loader(name, path=None): - """Return the loader for the specified module. - - This is a backward-compatible wrapper around find_spec(). - - This function is deprecated in favor of importlib.util.find_spec(). - - """ - warnings.warn('Deprecated since Python 3.4 and slated for removal in ' - 'Python 3.12; use importlib.util.find_spec() instead', - DeprecationWarning, stacklevel=2) - try: - loader = sys.modules[name].__loader__ - if loader is None: - raise ValueError(f'{name}.__loader__ is None') - else: - return loader - except KeyError: - pass - except AttributeError: - raise ValueError(f'{name}.__loader__ is not set') from None - - spec = _bootstrap._find_spec(name, path) - # We won't worry about malformed specs (missing attributes). - if spec is None: - return None - if spec.loader is None: - if spec.submodule_search_locations is None: - raise ImportError(f'spec for {name} missing loader', name=name) - raise ImportError('namespace packages do not have loaders', - name=name) - return spec.loader - - def import_module(name, package=None): """Import a module. diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index e4fcaa61e6de29..c48fd506a0e4eb 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -892,21 +892,6 @@ def find_spec(cls, fullname, path=None, target=None): else: return None - @classmethod - def find_module(cls, fullname, path=None): - """Find the built-in module. - - If 'path' is ever specified then the search is considered a failure. - - This method is deprecated. Use find_spec() instead. - - """ - _warnings.warn("BuiltinImporter.find_module() is deprecated and " - "slated for removal in Python 3.12; use find_spec() instead", - DeprecationWarning) - spec = cls.find_spec(fullname, path) - return spec.loader if spec is not None else None - @staticmethod def create_module(spec): """Create a built-in module""" @@ -1076,18 +1061,6 @@ def find_spec(cls, fullname, path=None, target=None): spec.submodule_search_locations.insert(0, pkgdir) return spec - @classmethod - def find_module(cls, fullname, path=None): - """Find a frozen module. - - This method is deprecated. Use find_spec() instead. - - """ - _warnings.warn("FrozenImporter.find_module() is deprecated and " - "slated for removal in Python 3.12; use find_spec() instead", - DeprecationWarning) - return cls if _imp.is_frozen(fullname) else None - @staticmethod def create_module(spec): """Set __file__, if able.""" @@ -1170,16 +1143,6 @@ def _resolve_name(name, package, level): return f'{base}.{name}' if name else base -def _find_spec_legacy(finder, name, path): - msg = (f"{_object_name(finder)}.find_spec() not found; " - "falling back to find_module()") - _warnings.warn(msg, ImportWarning) - loader = finder.find_module(name, path) - if loader is None: - return None - return spec_from_loader(name, loader) - - def _find_spec(name, path, target=None): """Find a module's spec.""" meta_path = sys.meta_path @@ -1200,9 +1163,7 @@ def _find_spec(name, path, target=None): try: find_spec = finder.find_spec except AttributeError: - spec = _find_spec_legacy(finder, name, path) - if spec is None: - continue + continue else: spec = find_spec(name, path, target) if spec is not None: diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index cb227373ca2fd4..7a3fdbaebdf24e 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -659,26 +659,6 @@ def _wrap(new, old): return _check_name_wrapper -def _find_module_shim(self, fullname): - """Try to find a loader for the specified module by delegating to - self.find_loader(). - - This method is deprecated in favor of finder.find_spec(). - - """ - _warnings.warn("find_module() is deprecated and " - "slated for removal in Python 3.12; use find_spec() instead", - DeprecationWarning) - # Call find_loader(). If it returns a string (indicating this - # is a namespace package portion), generate a warning and - # return None. - loader, portions = self.find_loader(fullname) - if loader is None and len(portions): - msg = f'Not importing directory {portions[0]}: missing __init__' - _warnings.warn(msg, ImportWarning) - return loader - - def _classify_pyc(data, name, exc_details): """Perform basic validity checking of a pyc header and return the flags field, which determines how the pyc should be further validated against the source. @@ -985,22 +965,6 @@ def find_spec(cls, fullname, path=None, target=None): origin=filepath) return spec - @classmethod - def find_module(cls, fullname, path=None): - """Find module named in the registry. - - This method is deprecated. Use find_spec() instead. - - """ - _warnings.warn("WindowsRegistryFinder.find_module() is deprecated and " - "slated for removal in Python 3.12; use find_spec() instead", - DeprecationWarning) - spec = cls.find_spec(fullname, path) - if spec is not None: - return spec.loader - else: - return None - class _LoaderBasics: @@ -1517,27 +1481,6 @@ def _path_importer_cache(cls, path): sys.path_importer_cache[path] = finder return finder - @classmethod - def _legacy_get_spec(cls, fullname, finder): - # This would be a good place for a DeprecationWarning if - # we ended up going that route. - if hasattr(finder, 'find_loader'): - msg = (f"{_bootstrap._object_name(finder)}.find_spec() not found; " - "falling back to find_loader()") - _warnings.warn(msg, ImportWarning) - loader, portions = finder.find_loader(fullname) - else: - msg = (f"{_bootstrap._object_name(finder)}.find_spec() not found; " - "falling back to find_module()") - _warnings.warn(msg, ImportWarning) - loader = finder.find_module(fullname) - portions = [] - if loader is not None: - return _bootstrap.spec_from_loader(fullname, loader) - spec = _bootstrap.ModuleSpec(fullname, None) - spec.submodule_search_locations = portions - return spec - @classmethod def _get_spec(cls, fullname, path, target=None): """Find the loader or namespace_path for this module/package name.""" @@ -1549,10 +1492,7 @@ def _get_spec(cls, fullname, path, target=None): continue finder = cls._path_importer_cache(entry) if finder is not None: - if hasattr(finder, 'find_spec'): - spec = finder.find_spec(fullname, target) - else: - spec = cls._legacy_get_spec(fullname, finder) + spec = finder.find_spec(fullname, target) if spec is None: continue if spec.loader is not None: @@ -1594,22 +1534,6 @@ def find_spec(cls, fullname, path=None, target=None): else: return spec - @classmethod - def find_module(cls, fullname, path=None): - """find the module on sys.path or 'path' based on sys.path_hooks and - sys.path_importer_cache. - - This method is deprecated. Use find_spec() instead. - - """ - _warnings.warn("PathFinder.find_module() is deprecated and " - "slated for removal in Python 3.12; use find_spec() instead", - DeprecationWarning) - spec = cls.find_spec(fullname, path) - if spec is None: - return None - return spec.loader - @staticmethod def find_distributions(*args, **kwargs): """ @@ -1654,23 +1578,6 @@ def invalidate_caches(self): """Invalidate the directory mtime.""" self._path_mtime = -1 - find_module = _find_module_shim - - def find_loader(self, fullname): - """Try to find a loader for the specified module, or the namespace - package portions. Returns (loader, list-of-portions). - - This method is deprecated. Use find_spec() instead. - - """ - _warnings.warn("FileFinder.find_loader() is deprecated and " - "slated for removal in Python 3.12; use find_spec() instead", - DeprecationWarning) - spec = self.find_spec(fullname) - if spec is None: - return None, [] - return spec.loader, spec.submodule_search_locations or [] - def _get_spec(self, loader_class, fullname, path, smsl, target): loader = loader_class(fullname, path) return spec_from_file_location(fullname, path, loader=loader, diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 8fa9a0f3bc1e4b..b56fa94eb9c135 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -19,7 +19,7 @@ __all__ = [ - 'Loader', 'Finder', 'MetaPathFinder', 'PathEntryFinder', + 'Loader', 'MetaPathFinder', 'PathEntryFinder', 'ResourceLoader', 'InspectLoader', 'ExecutionLoader', 'FileLoader', 'SourceLoader', ] @@ -49,38 +49,6 @@ def _register(abstract_cls, *classes): abstract_cls.register(frozen_cls) -class Finder(metaclass=abc.ABCMeta): - - """Legacy abstract base class for import finders. - - It may be subclassed for compatibility with legacy third party - reimplementations of the import system. Otherwise, finder - implementations should derive from the more specific MetaPathFinder - or PathEntryFinder ABCs. - - Deprecated since Python 3.3 - """ - - def __init__(self): - warnings.warn("the Finder ABC is deprecated and " - "slated for removal in Python 3.12; use MetaPathFinder " - "or PathEntryFinder instead", - DeprecationWarning) - - @abc.abstractmethod - def find_module(self, fullname, path=None): - """An abstract method that should find a module. - The fullname is a str and the optional path is a str or None. - Returns a Loader object or None. - """ - warnings.warn("importlib.abc.Finder along with its find_module() " - "method are deprecated and " - "slated for removal in Python 3.12; use " - "MetaPathFinder.find_spec() or " - "PathEntryFinder.find_spec() instead", - DeprecationWarning) - - class MetaPathFinder(metaclass=abc.ABCMeta): """Abstract base class for import finders on sys.meta_path.""" @@ -88,27 +56,6 @@ class MetaPathFinder(metaclass=abc.ABCMeta): # We don't define find_spec() here since that would break # hasattr checks we do to support backward compatibility. - def find_module(self, fullname, path): - """Return a loader for the module. - - If no module is found, return None. The fullname is a str and - the path is a list of strings or None. - - This method is deprecated since Python 3.4 in favor of - finder.find_spec(). If find_spec() exists then backwards-compatible - functionality is provided for this method. - - """ - warnings.warn("MetaPathFinder.find_module() is deprecated since Python " - "3.4 in favor of MetaPathFinder.find_spec() and is " - "slated for removal in Python 3.12", - DeprecationWarning, - stacklevel=2) - if not hasattr(self, 'find_spec'): - return None - found = self.find_spec(fullname, path) - return found.loader if found is not None else None - def invalidate_caches(self): """An optional method for clearing the finder's cache, if any. This method is used by importlib.invalidate_caches(). @@ -122,43 +69,6 @@ class PathEntryFinder(metaclass=abc.ABCMeta): """Abstract base class for path entry finders used by PathFinder.""" - # We don't define find_spec() here since that would break - # hasattr checks we do to support backward compatibility. - - def find_loader(self, fullname): - """Return (loader, namespace portion) for the path entry. - - The fullname is a str. The namespace portion is a sequence of - path entries contributing to part of a namespace package. The - sequence may be empty. If loader is not None, the portion will - be ignored. - - The portion will be discarded if another path entry finder - locates the module as a normal module or package. - - This method is deprecated since Python 3.4 in favor of - finder.find_spec(). If find_spec() is provided than backwards-compatible - functionality is provided. - """ - warnings.warn("PathEntryFinder.find_loader() is deprecated since Python " - "3.4 in favor of PathEntryFinder.find_spec() " - "(available since 3.4)", - DeprecationWarning, - stacklevel=2) - if not hasattr(self, 'find_spec'): - return None, [] - found = self.find_spec(fullname) - if found is not None: - if not found.submodule_search_locations: - portions = [] - else: - portions = found.submodule_search_locations - return found.loader, portions - else: - return None, [] - - find_module = _bootstrap_external._find_module_shim - def invalidate_caches(self): """An optional method for clearing the finder's cache, if any. This method is used by PathFinder.invalidate_caches(). diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index fb977eaaa05767..f62eccb974d6f5 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -23,20 +23,6 @@ ModuleInfo.__doc__ = 'A namedtuple with minimal info about a module.' -def _get_spec(finder, name): - """Return the finder-specific module spec.""" - # Works with legacy finders. - try: - find_spec = finder.find_spec - except AttributeError: - loader = finder.find_module(name) - if loader is None: - return None - return importlib.util.spec_from_loader(name, loader) - else: - return find_spec(name) - - def read_code(stream): # This helper is needed in order for the PEP 302 emulation to # correctly handle compiled files diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 1c3443fa8469f7..b10a5da99402b5 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -2242,7 +2242,7 @@ def run(self, callback, key=None, completer=None, onerror=None): callback(None, modname, '') else: try: - spec = pkgutil._get_spec(importer, modname) + spec = importer.find_spec(modname) except SyntaxError: # raised by tests for bad coding cookies or BOM continue diff --git a/Lib/test/test_importlib/builtin/test_finder.py b/Lib/test/test_importlib/builtin/test_finder.py index 81dc5a3699d952..111c4af1ea7cfe 100644 --- a/Lib/test/test_importlib/builtin/test_finder.py +++ b/Lib/test/test_importlib/builtin/test_finder.py @@ -43,38 +43,5 @@ def test_failure(self): ) = util.test_both(FindSpecTests, machinery=machinery) -@unittest.skipIf(util.BUILTINS.good_name is None, 'no reasonable builtin module') -class FinderTests(abc.FinderTests): - - """Test find_module() for built-in modules.""" - - def test_module(self): - # Common case. - with util.uncache(util.BUILTINS.good_name): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - found = self.machinery.BuiltinImporter.find_module(util.BUILTINS.good_name) - self.assertTrue(found) - self.assertTrue(hasattr(found, 'load_module')) - - # Built-in modules cannot be a package. - test_package = test_package_in_package = test_package_over_module = None - - # Built-in modules cannot be in a package. - test_module_in_package = None - - def test_failure(self): - assert 'importlib' not in sys.builtin_module_names - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - loader = self.machinery.BuiltinImporter.find_module('importlib') - self.assertIsNone(loader) - - -(Frozen_FinderTests, - Source_FinderTests - ) = util.test_both(FinderTests, machinery=machinery) - - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/extension/test_path_hook.py b/Lib/test/test_importlib/extension/test_path_hook.py index a0adc70ad1ec4d..ec9644dc520534 100644 --- a/Lib/test/test_importlib/extension/test_path_hook.py +++ b/Lib/test/test_importlib/extension/test_path_hook.py @@ -19,7 +19,7 @@ def hook(self, entry): def test_success(self): # Path hook should handle a directory where a known extension module # exists. - self.assertTrue(hasattr(self.hook(util.EXTENSIONS.path), 'find_module')) + self.assertTrue(hasattr(self.hook(util.EXTENSIONS.path), 'find_spec')) (Frozen_PathHooksTests, diff --git a/Lib/test/test_importlib/fixtures.py b/Lib/test/test_importlib/fixtures.py index a364a977bce781..73e5da2ba92279 100644 --- a/Lib/test/test_importlib/fixtures.py +++ b/Lib/test/test_importlib/fixtures.py @@ -350,11 +350,6 @@ def DALS(str): return textwrap.dedent(str).lstrip() -class NullFinder: - def find_module(self, name): - pass - - @requires_zlib() class ZipFixtures: root = 'test.test_importlib.data' diff --git a/Lib/test/test_importlib/frozen/test_finder.py b/Lib/test/test_importlib/frozen/test_finder.py index 069755606b40af..469dcdbd09eaf7 100644 --- a/Lib/test/test_importlib/frozen/test_finder.py +++ b/Lib/test/test_importlib/frozen/test_finder.py @@ -182,45 +182,5 @@ def test_not_using_frozen(self): ) = util.test_both(FindSpecTests, machinery=machinery) -class FinderTests(abc.FinderTests): - - """Test finding frozen modules.""" - - def find(self, name, path=None): - finder = self.machinery.FrozenImporter - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - with import_helper.frozen_modules(): - return finder.find_module(name, path) - - def test_module(self): - name = '__hello__' - loader = self.find(name) - self.assertTrue(hasattr(loader, 'load_module')) - - def test_package(self): - loader = self.find('__phello__') - self.assertTrue(hasattr(loader, 'load_module')) - - def test_module_in_package(self): - loader = self.find('__phello__.spam', ['__phello__']) - self.assertTrue(hasattr(loader, 'load_module')) - - # No frozen package within another package to test with. - test_package_in_package = None - - # No easy way to test. - test_package_over_module = None - - def test_failure(self): - loader = self.find('') - self.assertIsNone(loader) - - -(Frozen_FinderTests, - Source_FinderTests - ) = util.test_both(FinderTests, machinery=machinery) - - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/frozen/test_loader.py b/Lib/test/test_importlib/frozen/test_loader.py index da1569e3d0681e..4f1af454b52c71 100644 --- a/Lib/test/test_importlib/frozen/test_loader.py +++ b/Lib/test/test_importlib/frozen/test_loader.py @@ -125,88 +125,6 @@ def test_unloadable(self): ) = util.test_both(ExecModuleTests, machinery=machinery) -class LoaderTests(abc.LoaderTests): - - def load_module(self, name): - with fresh(name, oldapi=True): - module = self.machinery.FrozenImporter.load_module(name) - with captured_stdout() as stdout: - module.main() - return module, stdout - - def test_module(self): - module, stdout = self.load_module('__hello__') - filename = resolve_stdlib_file('__hello__') - check = {'__name__': '__hello__', - '__package__': '', - '__loader__': self.machinery.FrozenImporter, - '__file__': filename, - } - for attr, value in check.items(): - self.assertEqual(getattr(module, attr, None), value) - self.assertEqual(stdout.getvalue(), 'Hello world!\n') - - def test_package(self): - module, stdout = self.load_module('__phello__') - filename = resolve_stdlib_file('__phello__', ispkg=True) - pkgdir = os.path.dirname(filename) - check = {'__name__': '__phello__', - '__package__': '__phello__', - '__path__': [pkgdir], - '__loader__': self.machinery.FrozenImporter, - '__file__': filename, - } - for attr, value in check.items(): - attr_value = getattr(module, attr, None) - self.assertEqual(attr_value, value, - "for __phello__.%s, %r != %r" % - (attr, attr_value, value)) - self.assertEqual(stdout.getvalue(), 'Hello world!\n') - - def test_lacking_parent(self): - with util.uncache('__phello__'): - module, stdout = self.load_module('__phello__.spam') - filename = resolve_stdlib_file('__phello__.spam') - check = {'__name__': '__phello__.spam', - '__package__': '__phello__', - '__loader__': self.machinery.FrozenImporter, - '__file__': filename, - } - for attr, value in check.items(): - attr_value = getattr(module, attr) - self.assertEqual(attr_value, value, - "for __phello__.spam.%s, %r != %r" % - (attr, attr_value, value)) - self.assertEqual(stdout.getvalue(), 'Hello world!\n') - - def test_module_reuse(self): - with fresh('__hello__', oldapi=True): - module1 = self.machinery.FrozenImporter.load_module('__hello__') - module2 = self.machinery.FrozenImporter.load_module('__hello__') - with captured_stdout() as stdout: - module1.main() - module2.main() - self.assertIs(module1, module2) - self.assertEqual(stdout.getvalue(), - 'Hello world!\nHello world!\n') - - # No way to trigger an error in a frozen module. - test_state_after_failure = None - - def test_unloadable(self): - with import_helper.frozen_modules(): - with deprecated(): - assert self.machinery.FrozenImporter.find_module('_not_real') is None - with self.assertRaises(ImportError) as cm: - self.load_module('_not_real') - self.assertEqual(cm.exception.name, '_not_real') - - -(Frozen_LoaderTests, - Source_LoaderTests - ) = util.test_both(LoaderTests, machinery=machinery) - - class InspectLoaderTests: """Tests for the InspectLoader methods for FrozenImporter.""" diff --git a/Lib/test/test_importlib/import_/test___loader__.py b/Lib/test/test_importlib/import_/test___loader__.py index eaf665a6f5b5af..a14163919af677 100644 --- a/Lib/test/test_importlib/import_/test___loader__.py +++ b/Lib/test/test_importlib/import_/test___loader__.py @@ -33,48 +33,5 @@ def test___loader__(self): ) = util.test_both(SpecLoaderAttributeTests, __import__=util.__import__) -class LoaderMock: - - def find_module(self, fullname, path=None): - return self - - def load_module(self, fullname): - sys.modules[fullname] = self.module - return self.module - - -class LoaderAttributeTests: - - def test___loader___missing(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - module = types.ModuleType('blah') - try: - del module.__loader__ - except AttributeError: - pass - loader = LoaderMock() - loader.module = module - with util.uncache('blah'), util.import_state(meta_path=[loader]): - module = self.__import__('blah') - self.assertEqual(loader, module.__loader__) - - def test___loader___is_None(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - module = types.ModuleType('blah') - module.__loader__ = None - loader = LoaderMock() - loader.module = module - with util.uncache('blah'), util.import_state(meta_path=[loader]): - returned_module = self.__import__('blah') - self.assertEqual(loader, module.__loader__) - - -(Frozen_Tests, - Source_Tests - ) = util.test_both(LoaderAttributeTests, __import__=util.__import__) - - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py index ab1b35ee3c1a4e..7130c99a6fc171 100644 --- a/Lib/test/test_importlib/import_/test___package__.py +++ b/Lib/test/test_importlib/import_/test___package__.py @@ -95,25 +95,6 @@ def __init__(self, parent): self.parent = parent -class Using__package__PEP302(Using__package__): - mock_modules = util.mock_modules - - def test_using___package__(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - super().test_using___package__() - - def test_spec_fallback(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - super().test_spec_fallback() - - -(Frozen_UsingPackagePEP302, - Source_UsingPackagePEP302 - ) = util.test_both(Using__package__PEP302, __import__=util.__import__) - - class Using__package__PEP451(Using__package__): mock_modules = util.mock_spec @@ -162,23 +143,6 @@ def test_submodule(self): module = getattr(pkg, 'mod') self.assertEqual(module.__package__, 'pkg') -class Setting__package__PEP302(Setting__package__, unittest.TestCase): - mock_modules = util.mock_modules - - def test_top_level(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - super().test_top_level() - - def test_package(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - super().test_package() - - def test_submodule(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - super().test_submodule() class Setting__package__PEP451(Setting__package__, unittest.TestCase): mock_modules = util.mock_spec diff --git a/Lib/test/test_importlib/import_/test_api.py b/Lib/test/test_importlib/import_/test_api.py index 0ee032b0206df9..d6ad590b3d46a0 100644 --- a/Lib/test/test_importlib/import_/test_api.py +++ b/Lib/test/test_importlib/import_/test_api.py @@ -28,11 +28,6 @@ def exec_module(module): class BadLoaderFinder: - @classmethod - def find_module(cls, fullname, path): - if fullname == SUBMOD_NAME: - return cls - @classmethod def load_module(cls, fullname): if fullname == SUBMOD_NAME: diff --git a/Lib/test/test_importlib/import_/test_caching.py b/Lib/test/test_importlib/import_/test_caching.py index 3ca765fb4ada97..aedf0fd4f9db02 100644 --- a/Lib/test/test_importlib/import_/test_caching.py +++ b/Lib/test/test_importlib/import_/test_caching.py @@ -52,12 +52,11 @@ class ImportlibUseCache(UseCache, unittest.TestCase): __import__ = util.__import__['Source'] def create_mock(self, *names, return_=None): - mock = util.mock_modules(*names) - original_load = mock.load_module - def load_module(self, fullname): - original_load(fullname) - return return_ - mock.load_module = MethodType(load_module, mock) + mock = util.mock_spec(*names) + original_spec = mock.find_spec + def find_spec(self, fullname, path, target=None): + return original_spec(fullname) + mock.find_spec = MethodType(find_spec, mock) return mock # __import__ inconsistent between loaders and built-in import when it comes @@ -86,14 +85,12 @@ def test_using_cache_for_assigning_to_attribute(self): # See test_using_cache_after_loader() for reasoning. def test_using_cache_for_fromlist(self): # [from cache for fromlist] - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - with self.create_mock('pkg.__init__', 'pkg.module') as importer: - with util.import_state(meta_path=[importer]): - module = self.__import__('pkg', fromlist=['module']) - self.assertTrue(hasattr(module, 'module')) - self.assertEqual(id(module.module), - id(sys.modules['pkg.module'])) + with self.create_mock('pkg.__init__', 'pkg.module') as importer: + with util.import_state(meta_path=[importer]): + module = self.__import__('pkg', fromlist=['module']) + self.assertTrue(hasattr(module, 'module')) + self.assertEqual(id(module.module), + id(sys.modules['pkg.module'])) if __name__ == '__main__': diff --git a/Lib/test/test_importlib/import_/test_meta_path.py b/Lib/test/test_importlib/import_/test_meta_path.py index c8b898ec237850..8689017ba43112 100644 --- a/Lib/test/test_importlib/import_/test_meta_path.py +++ b/Lib/test/test_importlib/import_/test_meta_path.py @@ -113,16 +113,6 @@ def test_with_path(self): super().test_no_path() -class CallSignaturePEP302(CallSignoreSuppressImportWarning): - mock_modules = util.mock_modules - finder_name = 'find_module' - - -(Frozen_CallSignaturePEP302, - Source_CallSignaturePEP302 - ) = util.test_both(CallSignaturePEP302, __import__=util.__import__) - - class CallSignaturePEP451(CallSignature): mock_modules = util.mock_spec finder_name = 'find_spec' diff --git a/Lib/test/test_importlib/import_/test_path.py b/Lib/test/test_importlib/import_/test_path.py index de620842bbc52b..89b52fbd1e1aff 100644 --- a/Lib/test/test_importlib/import_/test_path.py +++ b/Lib/test/test_importlib/import_/test_path.py @@ -116,46 +116,6 @@ def test_None_on_sys_path(self): if email is not missing: sys.modules['email'] = email - def test_finder_with_find_module(self): - class TestFinder: - def find_module(self, fullname): - return self.to_return - failing_finder = TestFinder() - failing_finder.to_return = None - path = 'testing path' - with util.import_state(path_importer_cache={path: failing_finder}): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - self.assertIsNone( - self.machinery.PathFinder.find_spec('whatever', [path])) - success_finder = TestFinder() - success_finder.to_return = __loader__ - with util.import_state(path_importer_cache={path: success_finder}): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - spec = self.machinery.PathFinder.find_spec('whatever', [path]) - self.assertEqual(spec.loader, __loader__) - - def test_finder_with_find_loader(self): - class TestFinder: - loader = None - portions = [] - def find_loader(self, fullname): - return self.loader, self.portions - path = 'testing path' - with util.import_state(path_importer_cache={path: TestFinder()}): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - self.assertIsNone( - self.machinery.PathFinder.find_spec('whatever', [path])) - success_finder = TestFinder() - success_finder.loader = __loader__ - with util.import_state(path_importer_cache={path: success_finder}): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - spec = self.machinery.PathFinder.find_spec('whatever', [path]) - self.assertEqual(spec.loader, __loader__) - def test_finder_with_find_spec(self): class TestFinder: spec = None @@ -228,9 +188,9 @@ def invalidate_caches(self): class FindModuleTests(FinderTests): def find(self, *args, **kwargs): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return self.machinery.PathFinder.find_module(*args, **kwargs) + spec = self.machinery.PathFinder.find_spec(*args, **kwargs) + return None if spec is None else spec.loader + def check_found(self, found, importer): self.assertIs(found, importer) @@ -255,16 +215,14 @@ def check_found(self, found, importer): class PathEntryFinderTests: def test_finder_with_failing_find_spec(self): - # PathEntryFinder with find_module() defined should work. - # Issue #20763. class Finder: - path_location = 'test_finder_with_find_module' + path_location = 'test_finder_with_find_spec' def __init__(self, path): if path != self.path_location: raise ImportError @staticmethod - def find_module(fullname): + def find_spec(fullname, target=None): return None @@ -274,27 +232,6 @@ def find_module(fullname): warnings.simplefilter("ignore", ImportWarning) self.machinery.PathFinder.find_spec('importlib') - def test_finder_with_failing_find_module(self): - # PathEntryFinder with find_module() defined should work. - # Issue #20763. - class Finder: - path_location = 'test_finder_with_find_module' - def __init__(self, path): - if path != self.path_location: - raise ImportError - - @staticmethod - def find_module(fullname): - return None - - - with util.import_state(path=[Finder.path_location]+sys.path[:], - path_hooks=[Finder]): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - warnings.simplefilter("ignore", DeprecationWarning) - self.machinery.PathFinder.find_module('importlib') - (Frozen_PEFTests, Source_PEFTests diff --git a/Lib/test/test_importlib/source/test_case_sensitivity.py b/Lib/test/test_importlib/source/test_case_sensitivity.py index 9d472707abe840..6a06313319dbcd 100644 --- a/Lib/test/test_importlib/source/test_case_sensitivity.py +++ b/Lib/test/test_importlib/source/test_case_sensitivity.py @@ -63,19 +63,6 @@ def test_insensitive(self): self.assertIn(self.name, insensitive.get_filename(self.name)) -class CaseSensitivityTestPEP302(CaseSensitivityTest): - def find(self, finder): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return finder.find_module(self.name) - - -(Frozen_CaseSensitivityTestPEP302, - Source_CaseSensitivityTestPEP302 - ) = util.test_both(CaseSensitivityTestPEP302, importlib=importlib, - machinery=machinery) - - class CaseSensitivityTestPEP451(CaseSensitivityTest): def find(self, finder): found = finder.find_spec(self.name) diff --git a/Lib/test/test_importlib/source/test_finder.py b/Lib/test/test_importlib/source/test_finder.py index bed9d56dca84ee..12db7c7d352a2f 100644 --- a/Lib/test/test_importlib/source/test_finder.py +++ b/Lib/test/test_importlib/source/test_finder.py @@ -120,7 +120,7 @@ def test_package_over_module(self): def test_failure(self): with util.create_modules('blah') as mapping: nothing = self.import_(mapping['.root'], 'sdfsadsadf') - self.assertIsNone(nothing) + self.assertEqual(nothing, self.NOT_FOUND) def test_empty_string_for_dir(self): # The empty string from sys.path means to search in the cwd. @@ -150,7 +150,7 @@ def test_dir_removal_handling(self): found = self._find(finder, 'mod', loader_only=True) self.assertIsNotNone(found) found = self._find(finder, 'mod', loader_only=True) - self.assertIsNone(found) + self.assertEqual(found, self.NOT_FOUND) @unittest.skipUnless(sys.platform != 'win32', 'os.chmod() does not support the needed arguments under Windows') @@ -196,10 +196,12 @@ class FinderTestsPEP420(FinderTests): NOT_FOUND = (None, []) def _find(self, finder, name, loader_only=False): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - loader_portions = finder.find_loader(name) - return loader_portions[0] if loader_only else loader_portions + spec = finder.find_spec(name) + if spec is None: + return self.NOT_FOUND + if loader_only: + return spec.loader + return spec.loader, spec.submodule_search_locations (Frozen_FinderTestsPEP420, @@ -207,20 +209,5 @@ def _find(self, finder, name, loader_only=False): ) = util.test_both(FinderTestsPEP420, machinery=machinery) -class FinderTestsPEP302(FinderTests): - - NOT_FOUND = None - - def _find(self, finder, name, loader_only=False): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return finder.find_module(name) - - -(Frozen_FinderTestsPEP302, - Source_FinderTestsPEP302 - ) = util.test_both(FinderTestsPEP302, machinery=machinery) - - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/source/test_path_hook.py b/Lib/test/test_importlib/source/test_path_hook.py index ead62f5e945e2a..f274330e0b333b 100644 --- a/Lib/test/test_importlib/source/test_path_hook.py +++ b/Lib/test/test_importlib/source/test_path_hook.py @@ -18,19 +18,10 @@ def test_success(self): self.assertTrue(hasattr(self.path_hook()(mapping['.root']), 'find_spec')) - def test_success_legacy(self): - with util.create_modules('dummy') as mapping: - self.assertTrue(hasattr(self.path_hook()(mapping['.root']), - 'find_module')) - def test_empty_string(self): # The empty string represents the cwd. self.assertTrue(hasattr(self.path_hook()(''), 'find_spec')) - def test_empty_string_legacy(self): - # The empty string represents the cwd. - self.assertTrue(hasattr(self.path_hook()(''), 'find_module')) - (Frozen_PathHookTest, Source_PathHooktest diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index 3c9149c4e45a92..603125f6d926f6 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -147,20 +147,13 @@ def ins(self): class MetaPathFinder: - def find_module(self, fullname, path): - return super().find_module(fullname, path) + pass class MetaPathFinderDefaultsTests(ABCTestHarness): SPLIT = make_abc_subclasses(MetaPathFinder) - def test_find_module(self): - # Default should return None. - with self.assertWarns(DeprecationWarning): - found = self.ins.find_module('something', None) - self.assertIsNone(found) - def test_invalidate_caches(self): # Calling the method is a no-op. self.ins.invalidate_caches() @@ -173,22 +166,13 @@ def test_invalidate_caches(self): class PathEntryFinder: - def find_loader(self, fullname): - return super().find_loader(fullname) + pass class PathEntryFinderDefaultsTests(ABCTestHarness): SPLIT = make_abc_subclasses(PathEntryFinder) - def test_find_loader(self): - with self.assertWarns(DeprecationWarning): - found = self.ins.find_loader('something') - self.assertEqual(found, (None, [])) - - def find_module(self): - self.assertEqual(None, self.ins.find_module('something')) - def test_invalidate_caches(self): # Should be a no-op. self.ins.invalidate_caches() @@ -201,8 +185,7 @@ def test_invalidate_caches(self): class Loader: - def load_module(self, fullname): - return super().load_module(fullname) + pass class LoaderDefaultsTests(ABCTestHarness): @@ -333,14 +316,6 @@ def find_spec(self, fullname, path, target=None): return MetaPathSpecFinder() - def test_find_module(self): - finder = self.finder(None) - path = ['a', 'b', 'c'] - name = 'blah' - with self.assertWarns(DeprecationWarning): - found = finder.find_module(name, path) - self.assertIsNone(found) - def test_find_spec_with_explicit_target(self): loader = object() spec = self.util.spec_from_loader('blah', loader) @@ -370,53 +345,6 @@ def test_spec(self): ) = test_util.test_both(MetaPathFinderFindModuleTests, abc=abc, util=util) -##### PathEntryFinder concrete methods ######################################### -class PathEntryFinderFindLoaderTests: - - @classmethod - def finder(cls, spec): - class PathEntrySpecFinder(cls.abc.PathEntryFinder): - - def find_spec(self, fullname, target=None): - self.called_for = fullname - return spec - - return PathEntrySpecFinder() - - def test_no_spec(self): - finder = self.finder(None) - name = 'blah' - with self.assertWarns(DeprecationWarning): - found = finder.find_loader(name) - self.assertIsNone(found[0]) - self.assertEqual([], found[1]) - self.assertEqual(name, finder.called_for) - - def test_spec_with_loader(self): - loader = object() - spec = self.util.spec_from_loader('blah', loader) - finder = self.finder(spec) - with self.assertWarns(DeprecationWarning): - found = finder.find_loader('blah') - self.assertIs(found[0], spec.loader) - - def test_spec_with_portions(self): - spec = self.machinery.ModuleSpec('blah', None) - paths = ['a', 'b', 'c'] - spec.submodule_search_locations = paths - finder = self.finder(spec) - with self.assertWarns(DeprecationWarning): - found = finder.find_loader('blah') - self.assertIsNone(found[0]) - self.assertEqual(paths, found[1]) - - -(Frozen_PEFFindLoaderTests, - Source_PEFFindLoaderTests - ) = test_util.test_both(PathEntryFinderFindLoaderTests, abc=abc, util=util, - machinery=machinery) - - ##### Loader concrete methods ################################################## class LoaderLoadModuleTests: diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py index b3a99dc2dd5731..ecf2c47c462e23 100644 --- a/Lib/test/test_importlib/test_api.py +++ b/Lib/test/test_importlib/test_api.py @@ -95,7 +95,8 @@ def load_b(): (Frozen_ImportModuleTests, Source_ImportModuleTests - ) = test_util.test_both(ImportModuleTests, init=init) + ) = test_util.test_both( + ImportModuleTests, init=init, util=util, machinery=machinery) class FindLoaderTests: @@ -103,29 +104,26 @@ class FindLoaderTests: FakeMetaFinder = None def test_sys_modules(self): - # If a module with __loader__ is in sys.modules, then return it. + # If a module with __spec__.loader is in sys.modules, then return it. name = 'some_mod' with test_util.uncache(name): module = types.ModuleType(name) loader = 'a loader!' - module.__loader__ = loader + module.__spec__ = self.machinery.ModuleSpec(name, loader) sys.modules[name] = module - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - found = self.init.find_loader(name) - self.assertEqual(loader, found) + spec = self.util.find_spec(name) + self.assertIsNotNone(spec) + self.assertEqual(spec.loader, loader) def test_sys_modules_loader_is_None(self): - # If sys.modules[name].__loader__ is None, raise ValueError. + # If sys.modules[name].__spec__.loader is None, raise ValueError. name = 'some_mod' with test_util.uncache(name): module = types.ModuleType(name) module.__loader__ = None sys.modules[name] = module with self.assertRaises(ValueError): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - self.init.find_loader(name) + self.util.find_spec(name) def test_sys_modules_loader_is_not_set(self): # Should raise ValueError @@ -134,24 +132,20 @@ def test_sys_modules_loader_is_not_set(self): with test_util.uncache(name): module = types.ModuleType(name) try: - del module.__loader__ + del module.__spec__.loader except AttributeError: pass sys.modules[name] = module with self.assertRaises(ValueError): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - self.init.find_loader(name) + self.util.find_spec(name) def test_success(self): # Return the loader found on sys.meta_path. name = 'some_mod' with test_util.uncache(name): with test_util.import_state(meta_path=[self.FakeMetaFinder]): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - warnings.simplefilter('ignore', ImportWarning) - self.assertEqual((name, None), self.init.find_loader(name)) + spec = self.util.find_spec(name) + self.assertEqual((name, (name, None)), (spec.name, spec.loader)) def test_success_path(self): # Searching on a path should work. @@ -159,17 +153,12 @@ def test_success_path(self): path = 'path to some place' with test_util.uncache(name): with test_util.import_state(meta_path=[self.FakeMetaFinder]): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - warnings.simplefilter('ignore', ImportWarning) - self.assertEqual((name, path), - self.init.find_loader(name, path)) + spec = self.util.find_spec(name, path) + self.assertEqual(name, spec.name) def test_nothing(self): # None is returned upon failure to find a loader. - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - self.assertIsNone(self.init.find_loader('nevergoingtofindthismodule')) + self.assertIsNone(self.util.find_spec('nevergoingtofindthismodule')) class FindLoaderPEP451Tests(FindLoaderTests): @@ -182,20 +171,8 @@ def find_spec(name, path=None, target=None): (Frozen_FindLoaderPEP451Tests, Source_FindLoaderPEP451Tests - ) = test_util.test_both(FindLoaderPEP451Tests, init=init) - - -class FindLoaderPEP302Tests(FindLoaderTests): - - class FakeMetaFinder: - @staticmethod - def find_module(name, path=None): - return name, path - - -(Frozen_FindLoaderPEP302Tests, - Source_FindLoaderPEP302Tests - ) = test_util.test_both(FindLoaderPEP302Tests, init=init) + ) = test_util.test_both( + FindLoaderPEP451Tests, init=init, util=util, machinery=machinery) class ReloadTests: @@ -380,7 +357,8 @@ def test_module_missing_spec(self): (Frozen_ReloadTests, Source_ReloadTests - ) = test_util.test_both(ReloadTests, init=init, util=util) + ) = test_util.test_both( + ReloadTests, init=init, util=util, machinery=machinery) class InvalidateCacheTests: @@ -390,8 +368,6 @@ def test_method_called(self): class InvalidatingNullFinder: def __init__(self, *ignored): self.called = False - def find_module(self, *args): - return None def invalidate_caches(self): self.called = True @@ -416,7 +392,8 @@ def test_method_lacking(self): (Frozen_InvalidateCacheTests, Source_InvalidateCacheTests - ) = test_util.test_both(InvalidateCacheTests, init=init) + ) = test_util.test_both( + InvalidateCacheTests, init=init, util=util, machinery=machinery) class FrozenImportlibTests(unittest.TestCase): diff --git a/Lib/test/test_importlib/test_windows.py b/Lib/test/test_importlib/test_windows.py index b7dfe865a03a97..40b8aa1787fe08 100644 --- a/Lib/test/test_importlib/test_windows.py +++ b/Lib/test/test_importlib/test_windows.py @@ -92,30 +92,16 @@ class WindowsRegistryFinderTests: def test_find_spec_missing(self): spec = self.machinery.WindowsRegistryFinder.find_spec('spam') - self.assertIs(spec, None) - - def test_find_module_missing(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - loader = self.machinery.WindowsRegistryFinder.find_module('spam') - self.assertIs(loader, None) + self.assertIsNone(spec) def test_module_found(self): with setup_module(self.machinery, self.test_module): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - loader = self.machinery.WindowsRegistryFinder.find_module(self.test_module) spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module) - self.assertIsNot(loader, None) - self.assertIsNot(spec, None) + self.assertIsNotNone(spec) def test_module_not_found(self): with setup_module(self.machinery, self.test_module, path="."): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - loader = self.machinery.WindowsRegistryFinder.find_module(self.test_module) spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module) - self.assertIsNone(loader) self.assertIsNone(spec) (Frozen_WindowsRegistryFinderTests, diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index e348733f6ce3c3..c25be096e52874 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -194,8 +194,7 @@ def import_state(**kwargs): new_value = default setattr(sys, attr, new_value) if len(kwargs): - raise ValueError( - 'unrecognized arguments: {0}'.format(kwargs.keys())) + raise ValueError('unrecognized arguments: {}'.format(kwargs)) yield finally: for attr, value in originals.items(): @@ -243,30 +242,6 @@ def __exit__(self, *exc_info): self._uncache.__exit__(None, None, None) -class mock_modules(_ImporterMock): - - """Importer mock using PEP 302 APIs.""" - - def find_module(self, fullname, path=None): - if fullname not in self.modules: - return None - else: - return self - - def load_module(self, fullname): - if fullname not in self.modules: - raise ImportError - else: - sys.modules[fullname] = self.modules[fullname] - if fullname in self.module_code: - try: - self.module_code[fullname]() - except Exception: - del sys.modules[fullname] - raise - return self.modules[fullname] - - class mock_spec(_ImporterMock): """Importer mock using PEP 451 APIs.""" diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index 4d9f5db3c6b3cf..902627088fc291 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -429,7 +429,7 @@ def test_iter_importers(self): importers = list(iter_importers(fullname)) expected_importer = get_importer(pathitem) for finder in importers: - spec = pkgutil._get_spec(finder, fullname) + spec = finder.find_spec(fullname) loader = spec.loader try: loader = loader.loader @@ -441,7 +441,7 @@ def test_iter_importers(self): self.assertEqual(finder, expected_importer) self.assertIsInstance(loader, importlib.machinery.SourceFileLoader) - self.assertIsNone(pkgutil._get_spec(finder, pkgname)) + self.assertIsNone(finder.find_spec(pkgname)) with self.assertRaises(ImportError): list(iter_importers('invalid.module')) @@ -535,12 +535,6 @@ class ImportlibMigrationTests(unittest.TestCase): # PEP 302 emulation in this module is in the process of being # deprecated in favour of importlib proper - def check_deprecated(self): - return check_warnings( - ("This emulation is deprecated and slated for removal in " - "Python 3.12; use 'importlib' instead", - DeprecationWarning)) - def test_get_loader_avoids_emulation(self): with check_warnings() as w: self.assertIsNotNone(pkgutil.get_loader("sys")) diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index 52d43bdead67f8..14c19719e260c4 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -836,7 +836,6 @@ def _testBogusZipFile(self): self.assertRaises(TypeError, z.get_source, None) error = zipimport.ZipImportError - self.assertIsNone(z.find_module('abc')) self.assertIsNone(z.find_spec('abc')) with warnings.catch_warnings(): diff --git a/Misc/NEWS.d/next/Library/2022-10-09-14-47-42.gh-issue-98040.IN3qab.rst b/Misc/NEWS.d/next/Library/2022-10-09-14-47-42.gh-issue-98040.IN3qab.rst new file mode 100644 index 00000000000000..ac185406844153 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-09-14-47-42.gh-issue-98040.IN3qab.rst @@ -0,0 +1,2 @@ +Remove more deprecated importlib APIs: ``find_loader()``, ``find_module()``, +``importlib.abc.Finder``, ``pkgutil.ImpImporter``, ``pkgutil.ImpLoader``. From 328435ed42d9d2d0aab7024540c745e730b9b9b1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 3 May 2023 13:29:42 +0100 Subject: [PATCH 013/204] GH-98040: Suppress cross-references to the removed ``imp`` module (#104131) Suppress cross-references to imp --- Doc/c-api/import.rst | 4 ++-- Doc/library/functions.rst | 1 + Doc/whatsnew/3.0.rst | 2 +- Doc/whatsnew/3.10.rst | 2 +- Doc/whatsnew/3.11.rst | 2 +- Doc/whatsnew/3.12.rst | 2 +- Doc/whatsnew/3.2.rst | 4 ++-- Doc/whatsnew/3.3.rst | 6 +++--- Doc/whatsnew/3.4.rst | 12 ++++++------ Doc/whatsnew/3.6.rst | 2 +- Misc/NEWS.d/3.8.0a1.rst | 2 +- 11 files changed, 20 insertions(+), 19 deletions(-) diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 474a64800044d0..8e5af32b65e02c 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -186,10 +186,10 @@ Importing Modules .. versionadded:: 3.2 .. versionchanged:: 3.3 - Uses :func:`imp.source_from_cache()` in calculating the source path if + Uses :func:`!imp.source_from_cache()` in calculating the source path if only the bytecode path is provided. .. versionchanged:: 3.12 - No longer uses the removed ``imp`` module. + No longer uses the removed :mod:`!imp` module. .. c:function:: long PyImport_GetMagicNumber() diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 085a11c3caa708..47e3880129595e 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1987,6 +1987,7 @@ are always available. They are listed here in alphabetical order. .. index:: statement: import + module: builtins .. note:: diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index 63b24748d8aab6..f9ac13036cbc8d 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -840,7 +840,7 @@ Builtins need it; however, 99 percent of the time an explicit :keyword:`for` loop is more readable. -* Removed :func:`reload`. Use :func:`imp.reload`. +* Removed :func:`reload`. Use :func:`!imp.reload`. * Removed. :meth:`dict.has_key` -- use the :keyword:`in` operator instead. diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 44c8fa1e9eb2c4..3d7bcf89694477 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -1682,7 +1682,7 @@ Deprecated classes as appropriate instead. (Contributed by Brett Cannon in :issue:`42135`.) -* The deprecations of :mod:`imp`, :func:`!importlib.find_loader`, +* The deprecations of :mod:`!imp`, :func:`!importlib.find_loader`, :func:`importlib.util.set_package_wrapper`, :func:`importlib.util.set_loader_wrapper`, :func:`importlib.util.module_for_loader`, diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 918a6824618c1e..6df30ad6c19c2c 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1872,7 +1872,7 @@ C APIs pending removal are * The :mod:`asynchat` module * The :mod:`asyncore` module * The :ref:`entire distutils package ` -* The :mod:`imp` module +* The :mod:`!imp` module * The :class:`typing.io ` namespace * The :class:`typing.re ` namespace * :func:`!cgi.log` diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 1139bb89f5cf0c..4ed08f3ca3b3d3 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -995,7 +995,7 @@ Removed * ``importlib.abc.Finder``, ``pkg.ImpImporter``, and ``pkg.ImpLoader`` have been removed. (Contributed by Barry Warsaw in :gh:`98040`.) - * The ``imp`` module has been removed. (Contributed by Barry Warsaw in + * The :mod:`!imp` module has been removed. (Contributed by Barry Warsaw in :gh:`98040`.) * Removed the ``suspicious`` rule from the documentation Makefile, and diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 8dbe2a1d828b4e..7af0c0288376fd 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -319,7 +319,7 @@ aspects that are visible to the programmer: >>> collections.__cached__ # doctest: +SKIP 'c:/py32/lib/__pycache__/collections.cpython-32.pyc' -* The tag that is unique to each interpreter is accessible from the :mod:`imp` +* The tag that is unique to each interpreter is accessible from the :mod:`!imp` module: >>> import imp # doctest: +SKIP @@ -328,7 +328,7 @@ aspects that are visible to the programmer: * Scripts that try to deduce source filename from the imported file now need to be smarter. It is no longer sufficient to simply strip the "c" from a ".pyc" - filename. Instead, use the new functions in the :mod:`imp` module: + filename. Instead, use the new functions in the :mod:`!imp` module: >>> imp.source_from_cache('c:/py32/lib/__pycache__/collections.cpython-32.pyc') # doctest: +SKIP 'c:/py32/lib/collections.py' diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index df1f2ab775b047..f121652ba51cbc 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -685,7 +685,7 @@ through normal attribute access. Using importlib as the Implementation of Import =============================================== :issue:`2377` - Replace __import__ w/ importlib.__import__ -:issue:`13959` - Re-implement parts of :mod:`imp` in pure Python +:issue:`13959` - Re-implement parts of :mod:`!imp` in pure Python :issue:`14605` - Make import machinery explicit :issue:`14646` - Require loaders set __loader__ and __package__ @@ -762,7 +762,7 @@ Loaders are also now expected to set the ``__package__`` attribute from from :mod:`importlib` and import itself is setting the attribute post-load. ``None`` is now inserted into :attr:`sys.path_importer_cache` when no finder -can be found on :attr:`sys.path_hooks`. Since :class:`imp.NullImporter` is not +can be found on :attr:`sys.path_hooks`. Since :class:`!imp.NullImporter` is not directly exposed on :attr:`sys.path_hooks` it could no longer be relied upon to always be available to use as a value representing no finder found. @@ -2385,7 +2385,7 @@ Porting Python code * Because ``None`` is now inserted into :attr:`sys.path_importer_cache`, if you are clearing out entries in the dictionary of paths that do not have a finder, you will need to remove keys paired with values of ``None`` **and** - :class:`imp.NullImporter` to be backwards-compatible. This will lead to extra + :class:`!imp.NullImporter` to be backwards-compatible. This will lead to extra overhead on older versions of Python that re-insert ``None`` into :attr:`sys.path_importer_cache` where it represents the use of implicit finders, but semantically it should not change anything. diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 53f78e3621cfbd..dabfdaa230ff08 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -991,18 +991,18 @@ for the :meth:`~importlib.abc.InspectLoader.get_code` method. However, it will normally be desirable to override the default implementation for performance reasons. (Contributed by Brett Cannon in :issue:`18072`.) -The :func:`~importlib.reload` function has been moved from :mod:`imp` to -:mod:`importlib` as part of the :mod:`imp` module deprecation. (Contributed by +The :func:`~importlib.reload` function has been moved from :mod:`!imp` to +:mod:`importlib` as part of the :mod:`!imp` module deprecation. (Contributed by Berker Peksag in :issue:`18193`.) :mod:`importlib.util` now has a :data:`~importlib.util.MAGIC_NUMBER` attribute providing access to the bytecode version number. This replaces the -:func:`~imp.get_magic` function in the deprecated :mod:`imp` module. +:func:`!get_magic` function in the deprecated :mod:`!imp` module. (Contributed by Brett Cannon in :issue:`18192`.) New :mod:`importlib.util` functions :func:`~importlib.util.cache_from_source` and :func:`~importlib.util.source_from_cache` replace the same-named functions -in the deprecated :mod:`imp` module. (Contributed by Brett Cannon in +in the deprecated :mod:`!imp` module. (Contributed by Brett Cannon in :issue:`18194`.) The :mod:`importlib` bootstrap :class:`.NamespaceLoader` now conforms to @@ -2101,7 +2101,7 @@ Deprecations in the Python API and :meth:`importlib.util.set_package` are no longer needed because their functions are now handled automatically by the import system. -* The :mod:`imp` module is pending deprecation. To keep compatibility with +* The :mod:`!imp` module is pending deprecation. To keep compatibility with Python 2/3 code bases, the module's removal is currently not scheduled. * The :mod:`formatter` module is pending deprecation and is slated for removal @@ -2300,7 +2300,7 @@ Changes in the Python API then you can see if the module's ``__spec__.location`` is set to ``'frozen'``, check if the loader is a subclass of :class:`importlib.machinery.FrozenImporter`, - or if Python 2 compatibility is necessary you can use :func:`imp.is_frozen`. + or if Python 2 compatibility is necessary you can use :func:`!imp.is_frozen`. * :func:`py_compile.compile` now raises :exc:`FileExistsError` if the file path it would write to is a symlink or a non-regular file. This is to act as a diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index c7faaebfed62b3..3a681754e25dd7 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -2180,7 +2180,7 @@ Changes in the Python API now raises :exc:`ValueError` for out-of-range values, rather than returning :const:`None`. See :issue:`20059`. -* The :mod:`imp` module now raises a :exc:`DeprecationWarning` instead of +* The :mod:`!imp` module now raises a :exc:`DeprecationWarning` instead of :exc:`PendingDeprecationWarning`. * The following modules have had missing APIs added to their :attr:`__all__` diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index db2eba32e6ea34..854458f2d1a994 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -3818,7 +3818,7 @@ user. .. section: Library The :2to3fixer:`reload` fixer now uses :func:`importlib.reload` instead of -deprecated :func:`imp.reload`. +deprecated :func:`!imp.reload`. .. From 8b03e5ff94d494109a84a106c1471175d42dd8fe Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 3 May 2023 13:57:23 +0100 Subject: [PATCH 014/204] GH-97850: Suppress cross-references to the removed ``module_repr`` method (#104133) Suppress cross-references to ``module_repr`` --- Doc/reference/import.rst | 2 +- Doc/whatsnew/3.10.rst | 10 +++++----- Doc/whatsnew/3.11.rst | 2 +- Doc/whatsnew/3.12.rst | 2 +- Doc/whatsnew/3.4.rst | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 8f30181ae0a10d..0f416a5c583f85 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -706,7 +706,7 @@ Here are the exact rules used: * Otherwise, just use the module's ``__name__`` in the repr. .. versionchanged:: 3.12 - Use of :meth:`module_repr`, having been deprecated since Python 3.4, was + Use of :meth:`!module_repr`, having been deprecated since Python 3.4, was removed in Python 3.12 and is no longer called during the resolution of a module's repr. diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 3d7bcf89694477..f689930ff0caca 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -1612,7 +1612,7 @@ Deprecated (superseded by :meth:`~importlib.abc.Finder.find_spec`), :meth:`~importlib.abc.Loader.load_module` (superseded by :meth:`~importlib.abc.Loader.exec_module`), - :meth:`~importlib.abc.Loader.module_repr` (which the import system + :meth:`!module_repr` (which the import system takes care of for you), the ``__package__`` attribute (superseded by ``__spec__.parent``), the ``__loader__`` attribute (superseded by ``__spec__.loader``), and the ``__cached__`` attribute @@ -1693,14 +1693,14 @@ Deprecated (Contributed by Brett Cannon in :issue:`43720`.) * The import system now uses the ``__spec__`` attribute on modules before - falling back on :meth:`~importlib.abc.Loader.module_repr` for a module's + falling back on :meth:`!module_repr` for a module's ``__repr__()`` method. Removal of the use of ``module_repr()`` is scheduled for Python 3.12. (Contributed by Brett Cannon in :issue:`42137`.) -* :meth:`importlib.abc.Loader.module_repr`, - :meth:`importlib.machinery.FrozenLoader.module_repr`, and - :meth:`importlib.machinery.BuiltinLoader.module_repr` are deprecated and +* :meth:`!importlib.abc.Loader.module_repr`, + :meth:`!importlib.machinery.FrozenLoader.module_repr`, and + :meth:`!importlib.machinery.BuiltinLoader.module_repr` are deprecated and slated for removal in Python 3.12. (Contributed by Brett Cannon in :issue:`42136`.) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 6df30ad6c19c2c..7d3639f67952f6 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1877,7 +1877,7 @@ C APIs pending removal are * The :class:`typing.re ` namespace * :func:`!cgi.log` * :func:`!importlib.find_loader` -* :meth:`importlib.abc.Loader.module_repr` +* :meth:`!importlib.abc.Loader.module_repr` * :meth:`!importlib.abc.MetaPathFinder.find_module` * :meth:`!importlib.abc.PathEntryFinder.find_loader` * :meth:`!importlib.abc.PathEntryFinder.find_module` diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 4ed08f3ca3b3d3..3381ce7b6b0d56 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -983,7 +983,7 @@ Removed * Many previously deprecated cleanups in :mod:`importlib` have now been completed: - * References to, and support for ``module_repr()`` has been removed. + * References to, and support for :meth:`!module_repr()` has been removed. (Contributed by Barry Warsaw in :gh:`97850`.) * ``importlib.util.set_package`` has been removed. (Contributed by Brett diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index dabfdaa230ff08..38f9449b9afe3a 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -2096,7 +2096,7 @@ Deprecations in the Python API :meth:`importlib.abc.InspectLoader.exec_module` :meth:`importlib.abc.SourceLoader.exec_module`) and let the import system take care of the rest; and - :meth:`importlib.abc.Loader.module_repr`, + :meth:`!importlib.abc.Loader.module_repr`, :meth:`importlib.util.module_for_loader`, :meth:`importlib.util.set_loader`, and :meth:`importlib.util.set_package` are no longer needed because their functions are now handled automatically by the import system. From 423d7faeb37b6c13b3ebbf9255165fefc651983e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 3 May 2023 14:13:03 +0100 Subject: [PATCH 015/204] GH-97850: Suppress cross-references to removed ``importlib.util`` functions (#104134) `importlib.utils` -> `importlib.util` in a few places --------- Co-authored-by: Alex Waygood --- Doc/whatsnew/3.10.rst | 6 +++--- Doc/whatsnew/3.11.rst | 2 +- Doc/whatsnew/3.4.rst | 6 +++--- Misc/NEWS.d/3.10.0a5.rst | 2 +- Misc/NEWS.d/3.12.0a1.rst | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index f689930ff0caca..661eeaedbfc0d0 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -1683,9 +1683,9 @@ Deprecated (Contributed by Brett Cannon in :issue:`42135`.) * The deprecations of :mod:`!imp`, :func:`!importlib.find_loader`, - :func:`importlib.util.set_package_wrapper`, - :func:`importlib.util.set_loader_wrapper`, - :func:`importlib.util.module_for_loader`, + :func:`!importlib.util.set_package_wrapper`, + :func:`!importlib.util.set_loader_wrapper`, + :func:`!importlib.util.module_for_loader`, :class:`!pkgutil.ImpImporter`, and :class:`!pkgutil.ImpLoader` have all been updated to list Python 3.12 as the slated version of removal (they began raising :exc:`DeprecationWarning` in diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 7d3639f67952f6..7a479c6e56a9f0 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1889,7 +1889,7 @@ C APIs pending removal are * :meth:`!importlib.machinery.FrozenLoader.module_repr` * :meth:`!importlib.machinery.PathFinder.find_module` * :meth:`!importlib.machinery.WindowsRegistryFinder.find_module` -* :func:`importlib.util.module_for_loader` +* :func:`!importlib.util.module_for_loader` * :func:`!importlib.util.set_loader_wrapper` * :func:`!importlib.util.set_package_wrapper` * :class:`!pkgutil.ImpImporter` diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 38f9449b9afe3a..45bb91833a352b 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -2097,8 +2097,8 @@ Deprecations in the Python API :meth:`importlib.abc.SourceLoader.exec_module`) and let the import system take care of the rest; and :meth:`!importlib.abc.Loader.module_repr`, - :meth:`importlib.util.module_for_loader`, :meth:`importlib.util.set_loader`, - and :meth:`importlib.util.set_package` are no longer needed because their + :meth:`!importlib.util.module_for_loader`, :meth:`!importlib.util.set_loader`, + and :meth:`!importlib.util.set_package` are no longer needed because their functions are now handled automatically by the import system. * The :mod:`!imp` module is pending deprecation. To keep compatibility with @@ -2277,7 +2277,7 @@ Changes in the Python API in a backwards-compatible fashion, use e.g. ``getattr(module, '__loader__', None) is not None``. (:issue:`17115`.) -* :meth:`importlib.util.module_for_loader` now sets ``__loader__`` and +* :meth:`!importlib.util.module_for_loader` now sets ``__loader__`` and ``__package__`` unconditionally to properly support reloading. If this is not desired then you will need to set these attributes manually. You can use :func:`importlib.util.module_to_load` for module management. diff --git a/Misc/NEWS.d/3.10.0a5.rst b/Misc/NEWS.d/3.10.0a5.rst index 1c7c7447cae065..497e3849171831 100644 --- a/Misc/NEWS.d/3.10.0a5.rst +++ b/Misc/NEWS.d/3.10.0a5.rst @@ -499,7 +499,7 @@ Araujo. .. nonce: HY2beA .. section: Documentation -Updated importlib.utils.resolve_name() doc to use __spec__.parent instead of +Updated importlib.util.resolve_name() doc to use __spec__.parent instead of __package__. (Thanks Yair Frid.) .. diff --git a/Misc/NEWS.d/3.12.0a1.rst b/Misc/NEWS.d/3.12.0a1.rst index 075e8da825a331..ff5064f89d8dd8 100644 --- a/Misc/NEWS.d/3.12.0a1.rst +++ b/Misc/NEWS.d/3.12.0a1.rst @@ -2028,8 +2028,8 @@ resources. .. nonce: NzdREm .. section: Library -Remove deprecated :func:`importlib.utils.set_loader` and -:func:`importlib.utils.module_for_loader` from :mod:`importlib.utils`. +Remove deprecated :func:`!importlib.util.set_loader` and +:func:`!importlib.util.module_for_loader` from :mod:`importlib.util`. .. From 524a7f77fd8244835e382f076dd4a76404580bb3 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 3 May 2023 15:17:14 +0200 Subject: [PATCH 016/204] gh-103968: Deprecate creating heap types whose metaclass has custom tp_new. (GH-103972) (That's a mouthful of an edge case!) Co-authored-by: Barney Gale --- Doc/c-api/type.rst | 26 ++++++++++++- Doc/whatsnew/3.12.rst | 20 ++++++++++ Lib/test/test_capi/test_misc.py | 14 +++++++ ...-04-28-18-04-38.gh-issue-103968.EnVvOx.rst | 4 ++ Modules/_testcapi/heaptype.c | 16 +++++++- Objects/typeobject.c | 38 ++++++++++++++----- 6 files changed, 106 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-04-28-18-04-38.gh-issue-103968.EnVvOx.rst diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 69b15296993301..9fd40e1008c4a8 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -256,8 +256,13 @@ The following functions and structs are used to create The metaclass *metaclass* is used to construct the resulting type object. When *metaclass* is ``NULL``, the metaclass is derived from *bases* (or *Py_tp_base[s]* slots if *bases* is ``NULL``, see below). - Note that metaclasses that override - :c:member:`~PyTypeObject.tp_new` are not supported. + + Metaclasses that override :c:member:`~PyTypeObject.tp_new` are not + supported. + (For backwards compatibility, other ``PyType_From*`` functions allow + such metaclasses. They ignore ``tp_new``, which may result in incomplete + initialization. This is deprecated and in Python 3.14+ such metaclasses will + not be supported.) The *bases* argument can be used to specify base classes; it can either be only one class or a tuple of classes. @@ -305,6 +310,11 @@ The following functions and structs are used to create The function now finds and uses a metaclass corresponding to the provided base classes. Previously, only :class:`type` instances were returned. + The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. + which may result in incomplete initialization. + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it + will be no longer allowed. .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) @@ -317,6 +327,12 @@ The following functions and structs are used to create The function now finds and uses a metaclass corresponding to the provided base classes. Previously, only :class:`type` instances were returned. + The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. + which may result in incomplete initialization. + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it + will be no longer allowed. + .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, NULL)``. @@ -327,6 +343,12 @@ The following functions and structs are used to create base classes provided in *Py_tp_base[s]* slots. Previously, only :class:`type` instances were returned. + The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. + which may result in incomplete initialization. + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it + will be no longer allowed. + .. c:type:: PyType_Spec Structure defining a type's behavior. diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 3381ce7b6b0d56..63db5d3f89be19 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1320,6 +1320,21 @@ Porting to Python 3.12 available on debug builds. If you happen to be using it then you'll need to start using ``_Py_GetGlobalRefTotal()``. +* The following functions now select an appropriate metaclass for the newly + created type: + + * :c:func:`PyType_FromSpec` + * :c:func:`PyType_FromSpecWithBases` + * :c:func:`PyType_FromModuleAndSpec` + + Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` + is deprecated, and in Python 3.14+ it will be disallowed. + Note that these functions ignore ``tp_new`` of the metaclass, possibly + allowing incomplete initialization. + + Note that :c:func:`PyType_FromMetaclass` (added in Python 3.12) + already disallows creating classes whose metaclass overrides ``tp_new``. + Deprecated ---------- @@ -1396,6 +1411,11 @@ Deprecated * ``_PyErr_ChainExceptions`` is deprecated. Use ``_PyErr_ChainExceptions1`` instead. (Contributed by Irit Katriel in :gh:`102192`.) +* Using :c:func:`PyType_FromSpec`, :c:func:`PyType_FromSpecWithBases` + or :c:func:`PyType_FromModuleAndSpec` to create a class whose metaclass + overrides :c:member:`~PyTypeObject.tp_new` is deprecated. + Call the metaclass instead. + Removed ------- diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 9d5d1ca6e7dce2..1d426d0f8f825c 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -681,6 +681,20 @@ def test_heaptype_with_custom_metaclass(self): with self.assertRaisesRegex(TypeError, msg): t = _testcapi.pytype_fromspec_meta(_testcapi.HeapCTypeMetaclassCustomNew) + def test_heaptype_with_custom_metaclass_deprecation(self): + # gh-103968: a metaclass with custom tp_new is deprecated, but still + # allowed for functions that existed in 3.11 + # (PyType_FromSpecWithBases is used here). + class Base(metaclass=_testcapi.HeapCTypeMetaclassCustomNew): + pass + + with warnings_helper.check_warnings( + ('.*custom tp_new.*in Python 3.14.*', DeprecationWarning), + ): + sub = _testcapi.make_type_with_base(Base) + self.assertTrue(issubclass(sub, Base)) + self.assertIsInstance(sub, _testcapi.HeapCTypeMetaclassCustomNew) + def test_multiple_inheritance_ctypes_with_weakref_or_dict(self): with self.assertRaises(TypeError): diff --git a/Misc/NEWS.d/next/C API/2023-04-28-18-04-38.gh-issue-103968.EnVvOx.rst b/Misc/NEWS.d/next/C API/2023-04-28-18-04-38.gh-issue-103968.EnVvOx.rst new file mode 100644 index 00000000000000..5e4270f82afd84 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-04-28-18-04-38.gh-issue-103968.EnVvOx.rst @@ -0,0 +1,4 @@ +:c:func:`PyType_FromSpec` and its variants now allow creating classes whose +metaclass overrides :c:member:`~PyTypeObject.tp_new`. The ``tp_new`` is +ignored. This behavior is deprecated and will be disallowed in 3.14+. The +new :c:func:`PyType_FromMetaclass` already disallows it. diff --git a/Modules/_testcapi/heaptype.c b/Modules/_testcapi/heaptype.c index 209cc182c0698d..6384fbc485fbdf 100644 --- a/Modules/_testcapi/heaptype.c +++ b/Modules/_testcapi/heaptype.c @@ -22,7 +22,7 @@ static PyObject *pytype_fromspec_meta(PyObject* self, PyObject *meta) "_testcapi.HeapCTypeViaMetaclass", sizeof(PyObject), 0, - Py_TPFLAGS_DEFAULT, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, HeapCTypeViaMetaclass_slots }; @@ -385,6 +385,19 @@ make_immutable_type_with_base(PyObject *self, PyObject *base) return PyType_FromSpecWithBases(&ImmutableSubclass_spec, base); } +static PyObject * +make_type_with_base(PyObject *self, PyObject *base) +{ + assert(PyType_Check(base)); + PyType_Spec ImmutableSubclass_spec = { + .name = "_testcapi.Subclass", + .basicsize = (int)((PyTypeObject*)base)->tp_basicsize, + .slots = empty_type_slots, + .flags = Py_TPFLAGS_DEFAULT, + }; + return PyType_FromSpecWithBases(&ImmutableSubclass_spec, base); +} + static PyMethodDef TestMethods[] = { {"pytype_fromspec_meta", pytype_fromspec_meta, METH_O}, @@ -397,6 +410,7 @@ static PyMethodDef TestMethods[] = { test_from_spec_invalid_metatype_inheritance, METH_NOARGS}, {"make_immutable_type_with_base", make_immutable_type_with_base, METH_O}, + {"make_type_with_base", make_type_with_base, METH_O}, {NULL}, }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index cf0efe199b2828..4ced04b0bde966 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3950,9 +3950,10 @@ check_basicsize_includes_size_and_offsets(PyTypeObject* type) return 1; } -PyObject * -PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, - PyType_Spec *spec, PyObject *bases_in) +static PyObject * +_PyType_FromMetaclass_impl( + PyTypeObject *metaclass, PyObject *module, + PyType_Spec *spec, PyObject *bases_in, int _allow_tp_new) { /* Invariant: A non-NULL value in one of these means this function holds * a strong reference or owns allocated memory. @@ -4127,9 +4128,21 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, goto finally; } if (metaclass->tp_new != PyType_Type.tp_new) { - PyErr_SetString(PyExc_TypeError, - "Metaclasses with custom tp_new are not supported."); - goto finally; + if (_allow_tp_new) { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "Using PyType_Spec with metaclasses that have custom " + "tp_new is deprecated and will no longer be allowed in " + "Python 3.14.") < 0) { + goto finally; + } + } + else { + PyErr_SetString( + PyExc_TypeError, + "Metaclasses with custom tp_new are not supported."); + goto finally; + } } /* Calculate best base, and check that all bases are type objects */ @@ -4316,22 +4329,29 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, return (PyObject*)res; } +PyObject * +PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, + PyType_Spec *spec, PyObject *bases_in) +{ + return _PyType_FromMetaclass_impl(metaclass, module, spec, bases_in, 0); +} + PyObject * PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) { - return PyType_FromMetaclass(NULL, module, spec, bases); + return _PyType_FromMetaclass_impl(NULL, module, spec, bases, 1); } PyObject * PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) { - return PyType_FromMetaclass(NULL, NULL, spec, bases); + return _PyType_FromMetaclass_impl(NULL, NULL, spec, bases, 1); } PyObject * PyType_FromSpec(PyType_Spec *spec) { - return PyType_FromMetaclass(NULL, NULL, spec, NULL); + return _PyType_FromMetaclass_impl(NULL, NULL, spec, NULL, 1); } PyObject * From 0fc58c66bafbd20f02c206c801cf9ab939853164 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Wed, 3 May 2023 07:04:50 -0700 Subject: [PATCH 017/204] gh-103693: Add convenience variable feature to `pdb` (#103694) --- Doc/library/pdb.rst | 15 ++++ Doc/whatsnew/3.12.rst | 8 ++ Lib/pdb.py | 17 ++++ Lib/test/test_pdb.py | 78 +++++++++++++++++++ ...-04-22-21-34-13.gh-issue-103693.SBtuLQ.rst | 1 + 5 files changed, 119 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-04-22-21-34-13.gh-issue-103693.SBtuLQ.rst diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 5bc48a6d5f77fd..8a386aa77368f2 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -263,6 +263,21 @@ the commands; the input is split at the first ``;;`` pair, even if it is in the middle of a quoted string. A workaround for strings with double semicolons is to use implicit string concatenation ``';'';'`` or ``";"";"``. +To set a temporary global variable, use a *convenience variable*. A *convenience +variable* is a variable whose name starts with ``$``. For example, ``$foo = 1`` +sets a global variable ``$foo`` which you can use in the debugger session. The +*convenience variables* are cleared when the program resumes execution so it's +less likely to interfere with your program compared to using normal variables +like ``foo = 1``. + +There are three preset *convenience variables*: + +* ``$_frame``: the current frame you are debugging +* ``$_retval``: the return value if the frame is returning +* ``$_exception``: the exception if the frame is raising an exception + +.. versionadded:: 3.12 + .. index:: pair: .pdbrc; file triple: debugger; configuration; file diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 63db5d3f89be19..94b44ed9ce0953 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -408,6 +408,14 @@ os.path * Add :func:`os.path.splitroot` to split a path into a triad ``(drive, root, tail)``. (Contributed by Barney Gale in :gh:`101000`.) +pdb +--- + +* Add convenience variables to hold values temporarily for debug session + and provide quick access to values like the current frame or the return + value. + (Contributed by Tian Gao in :gh:`103693`.) + shutil ------ diff --git a/Lib/pdb.py b/Lib/pdb.py index 645cbf518e58e3..b3dc5a455e56b9 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -270,6 +270,8 @@ def forget(self): self.lineno = None self.stack = [] self.curindex = 0 + if hasattr(self, 'curframe') and self.curframe: + self.curframe.f_globals.pop('__pdb_convenience_variables', None) self.curframe = None self.tb_lineno.clear() @@ -288,6 +290,7 @@ def setup(self, f, tb): # locals whenever the .f_locals accessor is called, so we # cache it here to ensure that modifications are not overwritten. self.curframe_locals = self.curframe.f_locals + self.set_convenience_variable(self.curframe, '_frame', self.curframe) return self.execRcLines() # Can be executed earlier than 'setup' if desired @@ -359,6 +362,7 @@ def user_return(self, frame, return_value): if self._wait_for_mainpyfile: return frame.f_locals['__return__'] = return_value + self.set_convenience_variable(frame, '_retval', return_value) self.message('--Return--') self.interaction(frame, None) @@ -369,6 +373,7 @@ def user_exception(self, frame, exc_info): return exc_type, exc_value, exc_traceback = exc_info frame.f_locals['__exception__'] = exc_type, exc_value + self.set_convenience_variable(frame, '_exception', exc_value) # An 'Internal StopIteration' exception is an exception debug event # issued by the interpreter when handling a subgenerator run with @@ -394,6 +399,7 @@ def _cmdloop(self): self.message('--KeyboardInterrupt--') # Called before loop, handles display expressions + # Set up convenience variable containers def preloop(self): displaying = self.displaying.get(self.curframe) if displaying: @@ -477,6 +483,9 @@ def precmd(self, line): next = line[marker+2:].lstrip() self.cmdqueue.append(next) line = line[:marker].rstrip() + + # Replace all the convenience variables + line = re.sub(r'\$([a-zA-Z_][a-zA-Z0-9_]*)', r'__pdb_convenience_variables["\1"]', line) return line def onecmd(self, line): @@ -527,6 +536,13 @@ def message(self, msg): def error(self, msg): print('***', msg, file=self.stdout) + # convenience variables + + def set_convenience_variable(self, frame, name, value): + if '__pdb_convenience_variables' not in frame.f_globals: + frame.f_globals['__pdb_convenience_variables'] = {} + frame.f_globals['__pdb_convenience_variables'][name] = value + # Generic completion functions. Individual complete_foo methods can be # assigned below to one of these functions. @@ -1018,6 +1034,7 @@ def _select_frame(self, number): self.curindex = number self.curframe = self.stack[self.curindex][0] self.curframe_locals = self.curframe.f_locals + self.set_convenience_variable(self.curframe, '_frame', self.curframe) self.print_stack_entry(self.stack[self.curindex]) self.lineno = None diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 2f712a10257984..482c92dbf1f6a0 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -746,6 +746,84 @@ def test_pdb_where_command(): (Pdb) continue """ +def test_convenience_variables(): + """Test convenience variables + + >>> def util_function(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... try: + ... raise Exception('test') + ... except: + ... pass + ... return 1 + + >>> def test_function(): + ... util_function() + + >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... '$_frame.f_lineno', # Check frame convenience variable + ... '$a = 10', # Set a convenience variable + ... '$a', # Print its value + ... 'p $a + 2', # Do some calculation + ... 'u', # Switch frame + ... '$_frame.f_lineno', # Make sure the frame changed + ... '$a', # Make sure the value persists + ... 'd', # Go back to the original frame + ... 'next', + ... '$a', # The value should be gone + ... 'next', + ... '$_exception', # Check exception convenience variable + ... 'next', + ... '$_exception', # Exception should be gone + ... 'return', + ... '$_retval', # Check return convenience variable + ... 'continue', + ... ]): + ... test_function() + > (3)util_function() + -> try: + (Pdb) $_frame.f_lineno + 3 + (Pdb) $a = 10 + (Pdb) $a + 10 + (Pdb) p $a + 2 + 12 + (Pdb) u + > (2)test_function() + -> util_function() + (Pdb) $_frame.f_lineno + 2 + (Pdb) $a + 10 + (Pdb) d + > (3)util_function() + -> try: + (Pdb) next + > (4)util_function() + -> raise Exception('test') + (Pdb) $a + *** KeyError: 'a' + (Pdb) next + Exception: test + > (4)util_function() + -> raise Exception('test') + (Pdb) $_exception + Exception('test') + (Pdb) next + > (5)util_function() + -> except: + (Pdb) $_exception + *** KeyError: '_exception' + (Pdb) return + --Return-- + > (7)util_function()->1 + -> return 1 + (Pdb) $_retval + 1 + (Pdb) continue + """ + def test_post_mortem(): """Test post mortem traceback debugging. diff --git a/Misc/NEWS.d/next/Library/2023-04-22-21-34-13.gh-issue-103693.SBtuLQ.rst b/Misc/NEWS.d/next/Library/2023-04-22-21-34-13.gh-issue-103693.SBtuLQ.rst new file mode 100644 index 00000000000000..52c68bfc9ceea4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-22-21-34-13.gh-issue-103693.SBtuLQ.rst @@ -0,0 +1 @@ +Add convenience variable feature to :mod:`pdb` From 38dc3f28dde92b4fa6284a57f4e554991a3d9276 Mon Sep 17 00:00:00 2001 From: Paul Ganssle <1377457+pganssle@users.noreply.github.com> Date: Wed, 3 May 2023 11:17:27 -0400 Subject: [PATCH 018/204] GH-103944: Remove last use of `utcfromtimestamp` (#103995) * Remove last use of `utcfromtimestamp` This was a weirdly valid use of `utcfromtimestamp` in the sense that the "timestamps" in TZif files are not epoch times, but actually something more properly thought of as "number of seconds since 1970 in the local time zone", so even though we didn't want UTC time, `utcfromtimestamp` was still a good way to get the thing we wanted. Since we're deprecating `utcfromtimestamp`, it's just as valid to use `timedelta` arithmetic here. We may be able to avoid the question entirely by switching these tests over to using `ZoneInfo` in the future. * Fix a few missing DeprecationWarnings in tests In one test, we simply turn off DeprecationWarning rather than asserting about it, because whether the error condition happens before or after the warning seems to differ between the Python and C versions. --- Lib/test/datetimetester.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index fb07d2a5ad9b06..55e061950ff280 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -2,18 +2,19 @@ See https://www.zope.dev/Members/fdrake/DateTimeWiki/TestCases """ -import io -import itertools import bisect import copy import decimal -import sys +import io +import itertools import os import pickle import random import re import struct +import sys import unittest +import warnings from array import array @@ -51,11 +52,12 @@ for proto in range(pickle.HIGHEST_PROTOCOL + 1)] assert len(pickle_choices) == pickle.HIGHEST_PROTOCOL + 1 +EPOCH_NAIVE = datetime(1970, 1, 1, 0, 0) # For calculating transitions + # An arbitrary collection of objects of non-datetime types, for testing # mixed-type comparisons. OTHERSTUFF = (10, 34.5, "abc", {}, [], ()) - # XXX Copied from test_float. INF = float("inf") NAN = float("nan") @@ -2626,9 +2628,10 @@ def test_utcfromtimestamp_limits(self): for test_name, ts in test_cases: with self.subTest(test_name, ts=ts): with self.assertRaises((ValueError, OverflowError)): - # converting a Python int to C time_t can raise a - # OverflowError, especially on 32-bit platforms. - self.theclass.utcfromtimestamp(ts) + with self.assertWarns(DeprecationWarning): + # converting a Python int to C time_t can raise a + # OverflowError, especially on 32-bit platforms. + self.theclass.utcfromtimestamp(ts) def test_insane_fromtimestamp(self): # It's possible that some platform maps time_t to double, @@ -2645,8 +2648,9 @@ def test_insane_utcfromtimestamp(self): # exempt such platforms (provided they return reasonable # results!). for insane in -1e200, 1e200: - self.assertRaises(OverflowError, self.theclass.utcfromtimestamp, - insane) + with self.assertWarns(DeprecationWarning): + self.assertRaises(OverflowError, self.theclass.utcfromtimestamp, + insane) @unittest.skipIf(sys.platform == "win32", "Windows doesn't accept negative timestamps") def test_negative_float_fromtimestamp(self): @@ -3005,7 +3009,7 @@ def __new__(cls, *args, **kwargs): for name, meth_name, kwargs in test_cases: with self.subTest(name): constr = getattr(DateTimeSubclass, meth_name) - if constr == "utcnow": + if meth_name == "utcnow": with self.assertWarns(DeprecationWarning): dt = constr(**kwargs) else: @@ -4733,8 +4737,10 @@ def test_tzinfo_utcfromtimestamp(self): # Try with and without naming the keyword; for whatever reason, # utcfromtimestamp() doesn't accept a tzinfo argument. off42 = FixedOffset(42, "42") - self.assertRaises(TypeError, meth, ts, off42) - self.assertRaises(TypeError, meth, ts, tzinfo=off42) + with warnings.catch_warnings(category=DeprecationWarning): + warnings.simplefilter("ignore", category=DeprecationWarning) + self.assertRaises(TypeError, meth, ts, off42) + self.assertRaises(TypeError, meth, ts, tzinfo=off42) def test_tzinfo_timetuple(self): # TestDateTime tested most of this. datetime adds a twist to the @@ -6102,15 +6108,14 @@ def stats(cls, start_year=1): def transitions(self): for (_, prev_ti), (t, ti) in pairs(zip(self.ut, self.ti)): shift = ti[0] - prev_ti[0] - # TODO: Remove this use of utcfromtimestamp - yield datetime.utcfromtimestamp(t), shift + yield (EPOCH_NAIVE + timedelta(seconds=t)), shift def nondst_folds(self): """Find all folds with the same value of isdst on both sides of the transition.""" for (_, prev_ti), (t, ti) in pairs(zip(self.ut, self.ti)): shift = ti[0] - prev_ti[0] if shift < ZERO and ti[1] == prev_ti[1]: - yield datetime.utcfromtimestamp(t), -shift, prev_ti[2], ti[2] + yield _utcfromtimestamp(datetime, t,), -shift, prev_ti[2], ti[2] @classmethod def print_all_nondst_folds(cls, same_abbr=False, start_year=1): From da1980afcb8820ffaa0574df735bc39b1a276a76 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Wed, 3 May 2023 17:28:44 +0100 Subject: [PATCH 019/204] GH-104114: Fix `pathlib.WindowsPath.glob()` use of literal pattern segment case (GH-104116) We now use `_WildcardSelector` to evaluate literal pattern segments, which allows us to retrieve the real filesystem case. This change is necessary in order to implement a *case_sensitive* argument (see GH-81079) and a *follow_symlinks* argument (see GH-77609). --- Lib/pathlib.py | 52 +++++-------------- Lib/test/test_pathlib.py | 4 +- ...-05-03-03-14-33.gh-issue-104114.RG26RD.rst | 3 ++ 3 files changed, 18 insertions(+), 41 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-03-03-14-33.gh-issue-104114.RG26RD.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index dee19d1f89ad84..8a1651c23d7f4d 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -54,11 +54,6 @@ def _ignore_error(exception): getattr(exception, 'winerror', None) in _IGNORED_WINERRORS) -def _is_wildcard_pattern(pat): - # Whether this pattern needs actual matching using fnmatch, or can - # be looked up directly as a file. - return "*" in pat or "?" in pat or "[" in pat - def _is_case_sensitive(flavour): return flavour.normcase('Aa') == 'Aa' @@ -78,10 +73,8 @@ def _make_selector(pattern_parts, flavour): cls = _ParentSelector elif '**' in pat: raise ValueError("Invalid pattern: '**' can only be an entire path component") - elif _is_wildcard_pattern(pat): - cls = _WildcardSelector else: - cls = _PreciseSelector + cls = _WildcardSelector return cls(pat, child_parts, flavour) @@ -102,17 +95,15 @@ def select_from(self, parent_path): """Iterate over all child paths of `parent_path` matched by this selector. This can contain parent_path itself.""" path_cls = type(parent_path) - is_dir = path_cls.is_dir - exists = path_cls.exists scandir = path_cls._scandir - if not is_dir(parent_path): + if not parent_path.is_dir(): return iter([]) - return self._select_from(parent_path, is_dir, exists, scandir) + return self._select_from(parent_path, scandir) class _TerminatingSelector: - def _select_from(self, parent_path, is_dir, exists, scandir): + def _select_from(self, parent_path, scandir): yield parent_path @@ -120,29 +111,12 @@ class _ParentSelector(_Selector): def __init__(self, name, child_parts, flavour): _Selector.__init__(self, child_parts, flavour) - def _select_from(self, parent_path, is_dir, exists, scandir): + def _select_from(self, parent_path, scandir): path = parent_path._make_child_relpath('..') - for p in self.successor._select_from(path, is_dir, exists, scandir): + for p in self.successor._select_from(path, scandir): yield p -class _PreciseSelector(_Selector): - - def __init__(self, name, child_parts, flavour): - self.name = name - _Selector.__init__(self, child_parts, flavour) - - def _select_from(self, parent_path, is_dir, exists, scandir): - try: - path = parent_path._make_child_relpath(self.name) - follow = is_dir(path) if self.dironly else exists(path, follow_symlinks=False) - if follow: - for p in self.successor._select_from(path, is_dir, exists, scandir): - yield p - except PermissionError: - return - - class _WildcardSelector(_Selector): def __init__(self, pat, child_parts, flavour): @@ -150,7 +124,7 @@ def __init__(self, pat, child_parts, flavour): self.match = re.compile(fnmatch.translate(pat), flags=flags).fullmatch _Selector.__init__(self, child_parts, flavour) - def _select_from(self, parent_path, is_dir, exists, scandir): + def _select_from(self, parent_path, scandir): try: # We must close the scandir() object before proceeding to # avoid exhausting file descriptors when globbing deep trees. @@ -171,7 +145,7 @@ def _select_from(self, parent_path, is_dir, exists, scandir): name = entry.name if self.match(name): path = parent_path._make_child_relpath(name) - for p in self.successor._select_from(path, is_dir, exists, scandir): + for p in self.successor._select_from(path, scandir): yield p except PermissionError: return @@ -182,7 +156,7 @@ class _RecursiveWildcardSelector(_Selector): def __init__(self, pat, child_parts, flavour): _Selector.__init__(self, child_parts, flavour) - def _iterate_directories(self, parent_path, is_dir, scandir): + def _iterate_directories(self, parent_path, scandir): yield parent_path try: # We must close the scandir() object before proceeding to @@ -198,18 +172,18 @@ def _iterate_directories(self, parent_path, is_dir, scandir): raise if entry_is_dir and not entry.is_symlink(): path = parent_path._make_child_relpath(entry.name) - for p in self._iterate_directories(path, is_dir, scandir): + for p in self._iterate_directories(path, scandir): yield p except PermissionError: return - def _select_from(self, parent_path, is_dir, exists, scandir): + def _select_from(self, parent_path, scandir): try: yielded = set() try: successor_select = self.successor._select_from - for starting_point in self._iterate_directories(parent_path, is_dir, scandir): - for p in successor_select(starting_point, is_dir, exists, scandir): + for starting_point in self._iterate_directories(parent_path, scandir): + for p in successor_select(starting_point, scandir): if p not in yielded: yield p yielded.add(p) diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 620d480e37e2ba..424bb92a87d112 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -3122,7 +3122,7 @@ def test_glob(self): self.assertEqual(set(p.glob("FILEa")), { P(BASE, "fileA") }) self.assertEqual(set(p.glob("*a\\")), { P(BASE, "dirA") }) self.assertEqual(set(p.glob("F*a")), { P(BASE, "fileA") }) - self.assertEqual(set(map(str, p.glob("FILEa"))), {f"{p}\\FILEa"}) + self.assertEqual(set(map(str, p.glob("FILEa"))), {f"{p}\\fileA"}) self.assertEqual(set(map(str, p.glob("F*a"))), {f"{p}\\fileA"}) def test_rglob(self): @@ -3130,7 +3130,7 @@ def test_rglob(self): p = P(BASE, "dirC") self.assertEqual(set(p.rglob("FILEd")), { P(BASE, "dirC/dirD/fileD") }) self.assertEqual(set(p.rglob("*\\")), { P(BASE, "dirC/dirD") }) - self.assertEqual(set(map(str, p.rglob("FILEd"))), {f"{p}\\dirD\\FILEd"}) + self.assertEqual(set(map(str, p.rglob("FILEd"))), {f"{p}\\dirD\\fileD"}) def test_expanduser(self): P = self.cls diff --git a/Misc/NEWS.d/next/Library/2023-05-03-03-14-33.gh-issue-104114.RG26RD.rst b/Misc/NEWS.d/next/Library/2023-05-03-03-14-33.gh-issue-104114.RG26RD.rst new file mode 100644 index 00000000000000..e705fea8326e7a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-03-03-14-33.gh-issue-104114.RG26RD.rst @@ -0,0 +1,3 @@ +Fix issue where :meth:`pathlib.Path.glob` returns paths using the case of +non-wildcard segments for corresponding path segments, rather than the real +filesystem case. From 9f9e001ab2ab67acdb2e0383a25ab4c164608a47 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 3 May 2023 21:55:19 +0100 Subject: [PATCH 020/204] gh-103590: mention that the change is included in 3.11.4 and clarify except* documentation (#104095) --- Doc/reference/compound_stmts.rst | 6 ++++-- Doc/whatsnew/3.12.rst | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index f0a8936c35bf4a..0731589e4cadb8 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -362,8 +362,10 @@ one :keyword:`!except*` clause, the first that matches it. :: Any remaining exceptions that were not handled by any :keyword:`!except*` -clause are re-raised at the end, combined into an exception group along with -all exceptions that were raised from within :keyword:`!except*` clauses. +clause are re-raised at the end, along with all exceptions that were +raised from within the :keyword:`!except*` clauses. If this list contains +more than one exception to reraise, they are combined into an exception +group. If the raised exception is not an exception group and its type matches one of the :keyword:`!except*` clauses, it is caught and wrapped by an diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 94b44ed9ce0953..49f9bd28248dac 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -239,7 +239,8 @@ Other Language Changes * When a ``try-except*`` construct handles the entire :exc:`ExceptionGroup` and raises one other exception, that exception is no longer wrapped in an - :exc:`ExceptionGroup`. (Contributed by Irit Katriel in :gh:`103590`.) + :exc:`ExceptionGroup`. Also changed in version 3.11.4. (Contributed by Irit + Katriel in :gh:`103590`.) New Modules From d6e83fbf30fb25996b547d8a2444814437e228e5 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 4 May 2023 02:11:54 +0300 Subject: [PATCH 021/204] gh-97850: Deprecate `find_loader` and `get_loader` in `pkgutil` (GH-98520) Co-authored-by: C.A.M. Gerlach Co-authored-by: Brett Cannon Co-authored-by: Oleg Iarygin --- Doc/library/pkgutil.rst | 7 ++++ Doc/whatsnew/3.12.rst | 5 +++ Lib/pkgutil.py | 8 ++++ Lib/test/test_pkgutil.py | 39 ++++++++++--------- ...2-10-21-16-23-31.gh-issue-97850.N46coo.rst | 2 + 5 files changed, 43 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-10-21-16-23-31.gh-issue-97850.N46coo.rst diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index 98e6e294af0c5c..891a867d1ceb68 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -64,6 +64,10 @@ support. .. versionchanged:: 3.4 Updated to be based on :pep:`451` + .. deprecated-removed:: 3.12 3.14 + Use :func:`importlib.util.find_spec` instead. + + .. function:: get_importer(path_item) Retrieve a :term:`finder` for the given *path_item*. @@ -96,6 +100,9 @@ support. .. versionchanged:: 3.4 Updated to be based on :pep:`451` + .. deprecated-removed:: 3.12 3.14 + Use :func:`importlib.util.find_spec` instead. + .. function:: iter_importers(fullname='') diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 49f9bd28248dac..3dfd787e3df837 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -813,6 +813,11 @@ Pending Removal in Python 3.14 * The *onerror* argument of :func:`shutil.rmtree` is deprecated in 3.12, and will be removed in 3.14. +* :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` + now raise :exc:`DeprecationWarning`; + use :func:`importlib.util.find_spec` instead. + (Contributed by Nikita Sobolev in :gh:`97850`.) + Pending Removal in Future Versions ---------------------------------- diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index f62eccb974d6f5..dccbec52aa731e 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -270,6 +270,10 @@ def get_loader(module_or_name): If the named module is not already imported, its containing package (if any) is imported, in order to establish the package __path__. """ + warnings._deprecated("pkgutil.get_loader", + f"{warnings._DEPRECATED_MSG}; " + "use importlib.util.find_spec() instead", + remove=(3, 14)) if module_or_name in sys.modules: module_or_name = sys.modules[module_or_name] if module_or_name is None: @@ -294,6 +298,10 @@ def find_loader(fullname): importlib.util.find_spec that converts most failures to ImportError and only returns the loader rather than the full spec """ + warnings._deprecated("pkgutil.find_loader", + f"{warnings._DEPRECATED_MSG}; " + "use importlib.util.find_spec() instead", + remove=(3, 14)) if fullname.startswith('.'): msg = "Relative module name {!r} not supported".format(fullname) raise ImportError(msg) diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index 902627088fc291..6fcd726345eeac 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -1,6 +1,6 @@ from pathlib import Path from test.support.import_helper import unload, CleanImport -from test.support.warnings_helper import check_warnings +from test.support.warnings_helper import check_warnings, ignore_warnings import unittest import sys import importlib @@ -535,25 +535,18 @@ class ImportlibMigrationTests(unittest.TestCase): # PEP 302 emulation in this module is in the process of being # deprecated in favour of importlib proper - def test_get_loader_avoids_emulation(self): - with check_warnings() as w: - self.assertIsNotNone(pkgutil.get_loader("sys")) - self.assertIsNotNone(pkgutil.get_loader("os")) - self.assertIsNotNone(pkgutil.get_loader("test.support")) - self.assertEqual(len(w.warnings), 0) - @unittest.skipIf(__name__ == '__main__', 'not compatible with __main__') + @ignore_warnings(category=DeprecationWarning) def test_get_loader_handles_missing_loader_attribute(self): global __loader__ this_loader = __loader__ del __loader__ try: - with check_warnings() as w: - self.assertIsNotNone(pkgutil.get_loader(__name__)) - self.assertEqual(len(w.warnings), 0) + self.assertIsNotNone(pkgutil.get_loader(__name__)) finally: __loader__ = this_loader + @ignore_warnings(category=DeprecationWarning) def test_get_loader_handles_missing_spec_attribute(self): name = 'spam' mod = type(sys)(name) @@ -563,6 +556,7 @@ def test_get_loader_handles_missing_spec_attribute(self): loader = pkgutil.get_loader(name) self.assertIsNone(loader) + @ignore_warnings(category=DeprecationWarning) def test_get_loader_handles_spec_attribute_none(self): name = 'spam' mod = type(sys)(name) @@ -572,6 +566,7 @@ def test_get_loader_handles_spec_attribute_none(self): loader = pkgutil.get_loader(name) self.assertIsNone(loader) + @ignore_warnings(category=DeprecationWarning) def test_get_loader_None_in_sys_modules(self): name = 'totally bogus' sys.modules[name] = None @@ -581,18 +576,26 @@ def test_get_loader_None_in_sys_modules(self): del sys.modules[name] self.assertIsNone(loader) + def test_get_loader_is_deprecated(self): + with check_warnings( + (r".*\bpkgutil.get_loader\b.*", DeprecationWarning), + ): + res = pkgutil.get_loader("sys") + self.assertIsNotNone(res) + + def test_find_loader_is_deprecated(self): + with check_warnings( + (r".*\bpkgutil.find_loader\b.*", DeprecationWarning), + ): + res = pkgutil.find_loader("sys") + self.assertIsNotNone(res) + + @ignore_warnings(category=DeprecationWarning) def test_find_loader_missing_module(self): name = 'totally bogus' loader = pkgutil.find_loader(name) self.assertIsNone(loader) - def test_find_loader_avoids_emulation(self): - with check_warnings() as w: - self.assertIsNotNone(pkgutil.find_loader("sys")) - self.assertIsNotNone(pkgutil.find_loader("os")) - self.assertIsNotNone(pkgutil.find_loader("test.support")) - self.assertEqual(len(w.warnings), 0) - def test_get_importer_avoids_emulation(self): # We use an illegal path so *none* of the path hooks should fire with check_warnings() as w: diff --git a/Misc/NEWS.d/next/Library/2022-10-21-16-23-31.gh-issue-97850.N46coo.rst b/Misc/NEWS.d/next/Library/2022-10-21-16-23-31.gh-issue-97850.N46coo.rst new file mode 100644 index 00000000000000..e3297d164fff6d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-21-16-23-31.gh-issue-97850.N46coo.rst @@ -0,0 +1,2 @@ +Deprecate :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` +in favor of :func:`importlib.util.find_spec`. From e95dd40aff35775efce4c03bec7d82f03711310b Mon Sep 17 00:00:00 2001 From: Yuxin Wu Date: Thu, 4 May 2023 07:26:39 +0800 Subject: [PATCH 022/204] pydoc.safeimport: Use importlib.import_module instead of __import__ (GH-103118) --- Lib/pydoc.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Lib/pydoc.py b/Lib/pydoc.py index b10a5da99402b5..84e673a7f87f90 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -448,7 +448,7 @@ def safeimport(path, forceload=0, cache={}): # Prevent garbage collection. cache[key] = sys.modules[key] del sys.modules[key] - module = __import__(path) + module = importlib.import_module(path) except BaseException as err: # Did the error occur before or after the module was found? if path in sys.modules: @@ -463,9 +463,6 @@ def safeimport(path, forceload=0, cache={}): else: # Some other error occurred during the importing process. raise ErrorDuringImport(path, err) - for part in path.split('.')[1:]: - try: module = getattr(module, part) - except AttributeError: return None return module # ---------------------------------------------------- formatter base class From 9885677b0494e9be3eb8d7d69bebca0e79d8abcc Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 4 May 2023 02:11:29 +0200 Subject: [PATCH 023/204] gh-104146: Remove unused 'second_pass_replacements' from clinic.py (#104147) The code that manipulated 'second_pass_replacements' was removed in 2015 with commit 0759f84 (gh-67688, bpo-23500). --- Tools/clinic/clinic.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index b8b2b75c749152..d3e120c0e7a8ee 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -2112,8 +2112,6 @@ def parse(self, input): traceback.format_exc().rstrip()) printer.print_block(block) - second_pass_replacements = {} - # these are destinations not buffers for name, destination in self.destinations.items(): if destination.type == 'suppress': @@ -2155,23 +2153,8 @@ def parse(self, input): printer_2.print_block(block, core_includes=True) write_file(destination.filename, printer_2.f.getvalue()) continue - text = printer.f.getvalue() - - if second_pass_replacements: - printer_2 = BlockPrinter(self.language) - parser_2 = BlockParser(text, self.language) - changed = False - for block in parser_2: - if block.dsl_name: - for id, replacement in second_pass_replacements.items(): - if id in block.output: - changed = True - block.output = block.output.replace(id, replacement) - printer_2.print_block(block) - if changed: - text = printer_2.f.getvalue() - - return text + + return printer.f.getvalue() def _module_and_class(self, fields): From 35d273825abc319d0ecbd69110e847f6040d0cd7 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 4 May 2023 08:11:09 +0100 Subject: [PATCH 024/204] GH-97950: Allow translation of index directive content (#104000) --- Doc/conf.py | 5 +++++ Doc/tools/extensions/pyspecific.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/Doc/conf.py b/Doc/conf.py index 6a3c01cd91a77b..485c0bdf84df2e 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -91,6 +91,11 @@ # Avoid a warning with Sphinx >= 2.0 master_doc = 'contents' +# Allow translation of index directives +gettext_additional_targets = [ + 'index', +] + # Options for HTML output # ----------------------- diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index d659a4a54b9d11..39c7c42e66f9a0 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -674,6 +674,34 @@ def process_audit_events(app, doctree, fromdocname): node.replace_self(table) +def patch_pairindextypes(app) -> None: + if app.builder.name != 'gettext': + return + + # allow translating deprecated index entries + try: + from sphinx.domains.python import pairindextypes + except ImportError: + pass + else: + # Sphinx checks if a 'pair' type entry on an index directive is one of + # the Sphinx-translated pairindextypes values. As we intend to move + # away from this, we need Sphinx to believe that these values don't + # exist, by deleting them when using the gettext builder. + + # pairindextypes.pop('module', None) + # pairindextypes.pop('keyword', None) + # pairindextypes.pop('operator', None) + # pairindextypes.pop('object', None) + # pairindextypes.pop('exception', None) + # pairindextypes.pop('statement', None) + # pairindextypes.pop('builtin', None) + + # there needs to be at least one statement in this block, will be + # removed when the first of the below is uncommented. + pass + + def setup(app): app.add_role('issue', issue_role) app.add_role('gh', gh_issue_role) @@ -695,6 +723,7 @@ def setup(app): app.add_directive_to_domain('py', 'awaitablemethod', PyAwaitableMethod) app.add_directive_to_domain('py', 'abstractmethod', PyAbstractMethod) app.add_directive('miscnews', MiscNews) + app.connect('builder-inited', patch_pairindextypes) app.connect('doctree-resolved', process_audit_events) app.connect('env-merge-info', audit_events_merge) app.connect('env-purge-doc', audit_events_purge) From cd9a56c2b0e14f56f2e83dd4db43c5c69a74b232 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 4 May 2023 09:56:53 +0200 Subject: [PATCH 025/204] gh-103509: PEP 697 -- Limited C API for Extending Opaque Types (GH-103511) Co-authored-by: Oleg Iarygin Co-authored-by: Erlend E. Aasland --- Doc/c-api/object.rst | 39 ++ Doc/c-api/structures.rst | 16 + Doc/c-api/type.rst | 48 ++- Doc/c-api/typeobj.rst | 20 + Doc/data/stable_abi.dat | 2 + Doc/whatsnew/3.12.rst | 15 + Include/cpython/object.h | 1 + Include/descrobject.h | 1 + Include/internal/pycore_object.h | 5 - Include/object.h | 5 + Include/pyport.h | 11 + Lib/test/test_capi/test_misc.py | 158 +++++++- Lib/test/test_stable_abi_ctypes.py | 2 + ...-04-13-16-54-00.gh-issue-103509.A26Qu8.rst | 5 + Misc/stable_abi.toml | 9 + Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/heaptype.c | 131 ++++++- Modules/_testcapi/heaptype_relative.c | 343 ++++++++++++++++++ Modules/_testcapi/parts.h | 1 + Modules/_testcapimodule.c | 3 + Objects/descrobject.c | 6 + Objects/typeobject.c | 107 +++++- PC/pyconfig.h | 2 + PC/python3dll.c | 2 + PCbuild/_testcapi.vcxproj | 1 + PCbuild/_testcapi.vcxproj.filters | 3 + Python/structmember.c | 12 + configure | 35 ++ configure.ac | 1 + pyconfig.h.in | 3 + 30 files changed, 970 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-04-13-16-54-00.gh-issue-103509.A26Qu8.rst create mode 100644 Modules/_testcapi/heaptype_relative.c diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 0a12bb9e8c54f0..a0c3194ab0fb90 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -395,3 +395,42 @@ Object Protocol returns ``NULL`` if the object cannot be iterated. .. versionadded:: 3.10 + +.. c:function:: void *PyObject_GetTypeData(PyObject *o, PyTypeObject *cls) + + Get a pointer to subclass-specific data reserved for *cls*. + + The object *o* must be an instance of *cls*, and *cls* must have been + created using negative :c:member:`PyType_Spec.basicsize`. + Python does not check this. + + On error, set an exception and return ``NULL``. + + .. versionadded:: 3.12 + +.. c:function:: Py_ssize_t PyType_GetTypeDataSize(PyTypeObject *cls) + + Return the size of the instance memory space reserved for *cls*, i.e. the size of the + memory :c:func:`PyObject_GetTypeData` returns. + + This may be larger than requested using :c:member:`-PyType_Spec.basicsize `; + it is safe to use this larger size (e.g. with :c:func:`!memset`). + + The type *cls* **must** have been created using + negative :c:member:`PyType_Spec.basicsize`. + Python does not check this. + + On error, set an exception and return a negative value. + + .. versionadded:: 3.12 + +.. c:function:: void *PyObject_GetItemData(PyObject *o) + + Get a pointer to per-item data for a class with + :const:`Py_TPFLAGS_ITEMS_AT_END`. + + On error, set an exception and return ``NULL``. + :py:exc:`TypeError` is raised if *o* does not have + :const:`Py_TPFLAGS_ITEMS_AT_END` set. + + .. versionadded:: 3.12 diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 9618a0cf676972..338db6378d240e 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -486,6 +486,22 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: Emit an ``object.__getattr__`` :ref:`audit event ` before reading. +.. c:macro:: Py_RELATIVE_OFFSET + + Indicates that the :c:member:`~PyMemberDef.offset` of this ``PyMemberDef`` + entry indicates an offset from the subclass-specific data, rather than + from ``PyObject``. + + Can only be used as part of :c:member:`Py_tp_members ` + :c:type:`slot ` when creating a class using negative + :c:member:`~PyTypeDef.basicsize`. + It is mandatory in that case. + + This flag is only used in :c:type:`PyTypeSlot`. + When setting :c:member:`~PyTypeObject.tp_members` during + class creation, Python clears it and sets + :c:member:`PyMemberDef.offset` to the offset from the ``PyObject`` struct. + .. index:: single: READ_RESTRICTED single: WRITE_RESTRICTED diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 9fd40e1008c4a8..c21fd9255d28ad 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -353,25 +353,57 @@ The following functions and structs are used to create Structure defining a type's behavior. - .. c:member:: const char* PyType_Spec.name + .. c:member:: const char* name Name of the type, used to set :c:member:`PyTypeObject.tp_name`. - .. c:member:: int PyType_Spec.basicsize - .. c:member:: int PyType_Spec.itemsize + .. c:member:: int basicsize - Size of the instance in bytes, used to set - :c:member:`PyTypeObject.tp_basicsize` and - :c:member:`PyTypeObject.tp_itemsize`. + If positive, specifies the size of the instance in bytes. + It is used to set :c:member:`PyTypeObject.tp_basicsize`. - .. c:member:: int PyType_Spec.flags + If zero, specifies that :c:member:`~PyTypeObject.tp_basicsize` + should be inherited. + + If negative, the absolute value specifies how much space instances of the + class need *in addition* to the superclass. + Use :c:func:`PyObject_GetTypeData` to get a pointer to subclass-specific + memory reserved this way. + + .. versionchanged:: 3.12 + + Previously, this field could not be negative. + + .. c:member:: int itemsize + + Size of one element of a variable-size type, in bytes. + Used to set :c:member:`PyTypeObject.tp_itemsize`. + See ``tp_itemsize`` documentation for caveats. + + If zero, :c:member:`~PyTypeObject.tp_itemsize` is inherited. + Extending arbitrary variable-sized classes is dangerous, + since some types use a fixed offset for variable-sized memory, + which can then overlap fixed-sized memory used by a subclass. + To help prevent mistakes, inheriting ``itemsize`` is only possible + in the following situations: + + - The base is not variable-sized (its + :c:member:`~PyTypeObject.tp_itemsize`). + - The requested :c:member:`PyType_Spec.basicsize` is positive, + suggesting that the memory layout of the base class is known. + - The requested :c:member:`PyType_Spec.basicsize` is zero, + suggesting that the subclass does not access the instance's memory + directly. + - With the :const:`Py_TPFLAGS_ITEMS_AT_END` flag. + + .. c:member:: unsigned int flags Type flags, used to set :c:member:`PyTypeObject.tp_flags`. If the ``Py_TPFLAGS_HEAPTYPE`` flag is not set, :c:func:`PyType_FromSpecWithBases` sets it automatically. - .. c:member:: PyType_Slot *PyType_Spec.slots + .. c:member:: PyType_Slot *slots Array of :c:type:`PyType_Slot` structures. Terminated by the special slot value ``{0, NULL}``. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index e963b90628aa49..e13db3fb2211e2 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1171,6 +1171,26 @@ and :c:type:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_weaklistoffset` field is set in a superclass. + .. data:: Py_TPFLAGS_ITEMS_AT_END + + Only usable with variable-size types, i.e. ones with non-zero + :c:member:`~PyObject.tp_itemsize`. + + Indicates that the variable-sized portion of an instance of this type is + at the end of the instance's memory area, at an offset of + :c:expr:`Py_TYPE(obj)->tp_basicsize` (which may be different in each + subclass). + + When setting this flag, be sure that all superclasses either + use this memory layout, or are not variable-sized. + Python does not check this. + + .. versionadded:: 3.12 + + **Inheritance:** + + This flag is inherited. + .. XXX Document more flags here? diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 4cc06d22baaa93..f112d268129fd1 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -521,6 +521,7 @@ function,PyObject_GetAttrString,3.2,, function,PyObject_GetBuffer,3.11,, function,PyObject_GetItem,3.2,, function,PyObject_GetIter,3.2,, +function,PyObject_GetTypeData,3.12,, function,PyObject_HasAttr,3.2,, function,PyObject_HasAttrString,3.2,, function,PyObject_Hash,3.2,, @@ -675,6 +676,7 @@ function,PyType_GetModuleState,3.10,, function,PyType_GetName,3.11,, function,PyType_GetQualName,3.11,, function,PyType_GetSlot,3.4,, +function,PyType_GetTypeDataSize,3.12,, function,PyType_IsSubtype,3.2,, function,PyType_Modified,3.2,, function,PyType_Ready,3.2,, diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 3dfd787e3df837..edbf921467553c 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1159,6 +1159,21 @@ New Features (Contributed by Petr Viktorin in :gh:`101101`.) +* :pep:`697`: Added API for extending types whose instance memory layout is + opaque: + + - :c:member:`PyType_Spec.basicsize` can be zero or negative to specify + inheriting or extending the base class size. + - :c:func:`PyObject_GetTypeData` and :c:func:`PyType_GetTypeDataSize` + added to allow access to subclass-specific instance data. + - :const:`Py_TPFLAGS_ITEMS_AT_END` and :c:func:`PyObject_GetItemData` + added to allow safely extending certain variable-sized types, including + :c:var:`PyType_Type`. + - :c:macro:`Py_RELATIVE_OFFSET` added to allow defining + :c:type:`members ` in terms of a subclass-specific struct. + + (Contributed by Petr Viktorin in :gh:`103509`.) + * Added the new limited C API function :c:func:`PyType_FromMetaclass`, which generalizes the existing :c:func:`PyType_FromModuleAndSpec` using an additional metaclass argument. diff --git a/Include/cpython/object.h b/Include/cpython/object.h index ce4d13cd9c28fe..d8eff691039d24 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -553,6 +553,7 @@ Py_DEPRECATED(3.11) typedef int UsingDeprecatedTrashcanMacro; Py_TRASHCAN_END; \ } while(0); +PyAPI_FUNC(void *) PyObject_GetItemData(PyObject *obj); PyAPI_FUNC(int) _PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg); PyAPI_FUNC(void) _PyObject_ClearManagedDict(PyObject *obj); diff --git a/Include/descrobject.h b/Include/descrobject.h index 0a420b865dfd1b..fd66d17b497a31 100644 --- a/Include/descrobject.h +++ b/Include/descrobject.h @@ -83,6 +83,7 @@ struct PyMemberDef { #define Py_READONLY 1 #define Py_AUDIT_READ 2 // Added in 3.10, harmless no-op before that #define _Py_WRITE_RESTRICTED 4 // Deprecated, no-op. Do not reuse the value. +#define Py_RELATIVE_OFFSET 8 PyAPI_FUNC(PyObject *) PyMember_GetOne(const char *, PyMemberDef *); PyAPI_FUNC(int) PyMember_SetOne(char *, PyMemberDef *, PyObject *); diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 91853ad0525b55..b9e700ea280cbf 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -389,11 +389,6 @@ extern PyObject ** _PyObject_ComputedDictPointer(PyObject *); extern void _PyObject_FreeInstanceAttributes(PyObject *obj); extern int _PyObject_IsInstanceDictEmpty(PyObject *); -// Access macro to the members which are floating "behind" the object -static inline PyMemberDef* _PyHeapType_GET_MEMBERS(PyHeapTypeObject *etype) { - return (PyMemberDef*)((char*)etype + Py_TYPE(etype)->tp_basicsize); -} - PyAPI_FUNC(PyObject *) _PyObject_LookupSpecial(PyObject *, PyObject *); /* C function call trampolines to mitigate bad function pointer casts. diff --git a/Include/object.h b/Include/object.h index 66c3df0d7f780a..81aeb2d8bd5a69 100644 --- a/Include/object.h +++ b/Include/object.h @@ -355,6 +355,8 @@ PyAPI_FUNC(PyObject *) PyType_GetQualName(PyTypeObject *); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030C0000 PyAPI_FUNC(PyObject *) PyType_FromMetaclass(PyTypeObject*, PyObject*, PyType_Spec*, PyObject*); +PyAPI_FUNC(void *) PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls); +PyAPI_FUNC(Py_ssize_t) PyType_GetTypeDataSize(PyTypeObject *cls); #endif /* Generic type check */ @@ -521,6 +523,9 @@ given type object has a specified feature. // subject itself (rather than a mapped attribute on it): #define _Py_TPFLAGS_MATCH_SELF (1UL << 22) +/* Items (ob_size*tp_itemsize) are found at the end of an instance's memory */ +#define Py_TPFLAGS_ITEMS_AT_END (1UL << 23) + /* These flags are used to determine if a type is a subclass. */ #define Py_TPFLAGS_LONG_SUBCLASS (1UL << 24) #define Py_TPFLAGS_LIST_SUBCLASS (1UL << 25) diff --git a/Include/pyport.h b/Include/pyport.h index bd0ba6d0681b21..d7c6ae64f2bf2f 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -765,4 +765,15 @@ extern char * _getpty(int *, int, mode_t, int); #undef __bool__ #endif +// Make sure we have maximum alignment, even if the current compiler +// does not support max_align_t. Note that: +// - Autoconf reports alignment of unknown types to 0. +// - 'long double' has maximum alignment on *most* platforms, +// looks like the best we can do for pre-C11 compilers. +// - The value is tested, see test_alignof_max_align_t +#if !defined(ALIGNOF_MAX_ALIGN_T) || ALIGNOF_MAX_ALIGN_T == 0 +# undef ALIGNOF_MAX_ALIGN_T +# define ALIGNOF_MAX_ALIGN_T _Alignof(long double) +#endif + #endif /* Py_PYPORT_H */ diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 1d426d0f8f825c..22be3c0814278e 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -16,11 +16,13 @@ import unittest import warnings import weakref +import operator from test import support from test.support import MISSING_C_DOCSTRINGS from test.support import import_helper from test.support import threading_helper from test.support import warnings_helper +from test.support import requires_limited_api from test.support.script_helper import assert_python_failure, assert_python_ok, run_python_until_end try: import _posixsubprocess @@ -770,7 +772,6 @@ def meth(self): MutableBase.meth = lambda self: 'changed' self.assertEqual(instance.meth(), 'changed') - def test_pynumber_tobase(self): from _testcapi import pynumber_tobase small_number = 123 @@ -1072,6 +1073,161 @@ class Data(_testcapi.ObjExtraData): self.assertIsNone(d.extra) +@requires_limited_api +class TestHeapTypeRelative(unittest.TestCase): + """Test API for extending opaque types (PEP 697)""" + + @requires_limited_api + def test_heaptype_relative_sizes(self): + # Test subclassing using "relative" basicsize, see PEP 697 + def check(extra_base_size, extra_size): + Base, Sub, instance, data_ptr, data_offset, data_size = ( + _testcapi.make_sized_heaptypes( + extra_base_size, -extra_size)) + + # no alignment shenanigans when inheriting directly + if extra_size == 0: + self.assertEqual(Base.__basicsize__, Sub.__basicsize__) + self.assertEqual(data_size, 0) + + else: + # The following offsets should be in increasing order: + offsets = [ + (0, 'start of object'), + (Base.__basicsize__, 'end of base data'), + (data_offset, 'subclass data'), + (data_offset + extra_size, 'end of requested subcls data'), + (data_offset + data_size, 'end of reserved subcls data'), + (Sub.__basicsize__, 'end of object'), + ] + ordered_offsets = sorted(offsets, key=operator.itemgetter(0)) + self.assertEqual( + offsets, ordered_offsets, + msg=f'Offsets not in expected order, got: {ordered_offsets}') + + # end of reserved subcls data == end of object + self.assertEqual(Sub.__basicsize__, data_offset + data_size) + + # we don't reserve (requested + alignment) or more data + self.assertLess(data_size - extra_size, + _testcapi.ALIGNOF_MAX_ALIGN_T) + + # The offsets/sizes we calculated should be aligned. + self.assertEqual(data_offset % _testcapi.ALIGNOF_MAX_ALIGN_T, 0) + self.assertEqual(data_size % _testcapi.ALIGNOF_MAX_ALIGN_T, 0) + + sizes = sorted({0, 1, 2, 3, 4, 7, 8, 123, + object.__basicsize__, + object.__basicsize__-1, + object.__basicsize__+1}) + for extra_base_size in sizes: + for extra_size in sizes: + args = dict(extra_base_size=extra_base_size, + extra_size=extra_size) + with self.subTest(**args): + check(**args) + + def test_HeapCCollection(self): + """Make sure HeapCCollection works properly by itself""" + collection = _testcapi.HeapCCollection(1, 2, 3) + self.assertEqual(list(collection), [1, 2, 3]) + + def test_heaptype_inherit_itemsize(self): + """Test HeapCCollection subclasses work properly""" + sizes = sorted({0, 1, 2, 3, 4, 7, 8, 123, + object.__basicsize__, + object.__basicsize__-1, + object.__basicsize__+1}) + for extra_size in sizes: + with self.subTest(extra_size=extra_size): + Sub = _testcapi.subclass_var_heaptype( + _testcapi.HeapCCollection, -extra_size, 0, 0) + collection = Sub(1, 2, 3) + collection.set_data_to_3s() + + self.assertEqual(list(collection), [1, 2, 3]) + mem = collection.get_data() + self.assertGreaterEqual(len(mem), extra_size) + self.assertTrue(set(mem) <= {3}, f'got {mem!r}') + + def test_heaptype_invalid_inheritance(self): + with self.assertRaises(SystemError, + msg="Cannot extend variable-size class without " + + "Py_TPFLAGS_ITEMS_AT_END"): + _testcapi.subclass_heaptype(int, -8, 0) + + def test_heaptype_relative_members(self): + """Test HeapCCollection subclasses work properly""" + sizes = sorted({0, 1, 2, 3, 4, 7, 8, 123, + object.__basicsize__, + object.__basicsize__-1, + object.__basicsize__+1}) + for extra_base_size in sizes: + for extra_size in sizes: + for offset in sizes: + with self.subTest(extra_base_size=extra_base_size, extra_size=extra_size, offset=offset): + if offset < extra_size: + Sub = _testcapi.make_heaptype_with_member( + extra_base_size, -extra_size, offset, True) + Base = Sub.mro()[1] + instance = Sub() + self.assertEqual(instance.memb, instance.get_memb()) + instance.set_memb(13) + self.assertEqual(instance.memb, instance.get_memb()) + self.assertEqual(instance.get_memb(), 13) + instance.memb = 14 + self.assertEqual(instance.memb, instance.get_memb()) + self.assertEqual(instance.get_memb(), 14) + self.assertGreaterEqual(instance.get_memb_offset(), Base.__basicsize__) + self.assertLess(instance.get_memb_offset(), Sub.__basicsize__) + with self.assertRaises(SystemError): + instance.get_memb_relative() + with self.assertRaises(SystemError): + instance.set_memb_relative(0) + else: + with self.assertRaises(SystemError): + Sub = _testcapi.make_heaptype_with_member( + extra_base_size, -extra_size, offset, True) + with self.assertRaises(SystemError): + Sub = _testcapi.make_heaptype_with_member( + extra_base_size, extra_size, offset, True) + with self.subTest(extra_base_size=extra_base_size, extra_size=extra_size): + with self.assertRaises(SystemError): + Sub = _testcapi.make_heaptype_with_member( + extra_base_size, -extra_size, -1, True) + + def test_heaptype_relative_members_errors(self): + with self.assertRaisesRegex( + SystemError, + r"With Py_RELATIVE_OFFSET, basicsize must be negative"): + _testcapi.make_heaptype_with_member(0, 1234, 0, True) + with self.assertRaisesRegex( + SystemError, r"Member offset out of range \(0\.\.-basicsize\)"): + _testcapi.make_heaptype_with_member(0, -8, 1234, True) + with self.assertRaisesRegex( + SystemError, r"Member offset out of range \(0\.\.-basicsize\)"): + _testcapi.make_heaptype_with_member(0, -8, -1, True) + + Sub = _testcapi.make_heaptype_with_member(0, -8, 0, True) + instance = Sub() + with self.assertRaisesRegex( + SystemError, r"PyMember_GetOne used with Py_RELATIVE_OFFSET"): + instance.get_memb_relative() + with self.assertRaisesRegex( + SystemError, r"PyMember_SetOne used with Py_RELATIVE_OFFSET"): + instance.set_memb_relative(0) + + def test_pyobject_getitemdata_error(self): + """Test PyObject_GetItemData fails on unsupported types""" + with self.assertRaises(TypeError): + # None is not variable-length + _testcapi.pyobject_getitemdata(None) + with self.assertRaises(TypeError): + # int is variable-length, but doesn't have the + # Py_TPFLAGS_ITEMS_AT_END layout (and flag) + _testcapi.pyobject_getitemdata(0) + + class TestPendingCalls(unittest.TestCase): def pendingcalls_submit(self, l, n): diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 2feaaf8603b831..4ca39d85e5460c 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -529,6 +529,7 @@ def test_windows_feature_macros(self): "PyObject_GetBuffer", "PyObject_GetItem", "PyObject_GetIter", + "PyObject_GetTypeData", "PyObject_HasAttr", "PyObject_HasAttrString", "PyObject_Hash", @@ -679,6 +680,7 @@ def test_windows_feature_macros(self): "PyType_GetName", "PyType_GetQualName", "PyType_GetSlot", + "PyType_GetTypeDataSize", "PyType_IsSubtype", "PyType_Modified", "PyType_Ready", diff --git a/Misc/NEWS.d/next/C API/2023-04-13-16-54-00.gh-issue-103509.A26Qu8.rst b/Misc/NEWS.d/next/C API/2023-04-13-16-54-00.gh-issue-103509.A26Qu8.rst new file mode 100644 index 00000000000000..af630c3aafa940 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-04-13-16-54-00.gh-issue-103509.A26Qu8.rst @@ -0,0 +1,5 @@ +Added C API for extending types whose instance memory layout is opaque: +:c:member:`PyType_Spec.basicsize` can now be zero or negative, +:c:func:`PyObject_GetTypeData` can be used to get subclass-specific data, +and :c:macro:`Py_TPFLAGS_ITEMS_AT_END` can be used to safely extend +variable-size objects. See :pep:`697` for details. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 23baeeeae79193..48299e9b35ff97 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2397,3 +2397,12 @@ added = '3.12' # Before 3.12, available in "structmember.h" w/o Py_ prefix [const.Py_AUDIT_READ] added = '3.12' # Before 3.12, available in "structmember.h" + +[function.PyObject_GetTypeData] + added = '3.12' +[function.PyType_GetTypeDataSize] + added = '3.12' +[const.Py_RELATIVE_OFFSET] + added = '3.12' +[const.Py_TPFLAGS_ITEMS_AT_END] + added = '3.12' diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index a90c1e96ef0231..6b483341953711 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -169,7 +169,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/pyos.c _testcapi/immortal.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c # Some testing modules MUST be built as shared libraries. diff --git a/Modules/_testcapi/heaptype.c b/Modules/_testcapi/heaptype.c index 6384fbc485fbdf..3488e35922c5ac 100644 --- a/Modules/_testcapi/heaptype.c +++ b/Modules/_testcapi/heaptype.c @@ -371,7 +371,6 @@ create_type_from_repeated_slots(PyObject *self, PyObject *variant_obj) } - static PyObject * make_immutable_type_with_base(PyObject *self, PyObject *base) { @@ -399,6 +398,17 @@ make_type_with_base(PyObject *self, PyObject *base) } +static PyObject * +pyobject_getitemdata(PyObject *self, PyObject *o) +{ + void *pointer = PyObject_GetItemData(o); + if (pointer == NULL) { + return NULL; + } + return PyLong_FromVoidPtr(pointer); +} + + static PyMethodDef TestMethods[] = { {"pytype_fromspec_meta", pytype_fromspec_meta, METH_O}, {"test_type_from_ephemeral_spec", test_type_from_ephemeral_spec, METH_NOARGS}, @@ -411,6 +421,7 @@ static PyMethodDef TestMethods[] = { METH_NOARGS}, {"make_immutable_type_with_base", make_immutable_type_with_base, METH_O}, {"make_type_with_base", make_type_with_base, METH_O}, + {"pyobject_getitemdata", pyobject_getitemdata, METH_O}, {NULL}, }; @@ -987,6 +998,113 @@ static PyType_Spec HeapCTypeSetattr_spec = { HeapCTypeSetattr_slots }; +PyDoc_STRVAR(HeapCCollection_doc, +"Tuple-like heap type that uses PyObject_GetItemData for items."); + +static PyObject* +HeapCCollection_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) +{ + PyObject *self = NULL; + PyObject *result = NULL; + + Py_ssize_t size = PyTuple_GET_SIZE(args); + self = subtype->tp_alloc(subtype, size); + if (!self) { + goto finally; + } + PyObject **data = PyObject_GetItemData(self); + if (!data) { + goto finally; + } + + for (Py_ssize_t i = 0; i < size; i++) { + data[i] = Py_NewRef(PyTuple_GET_ITEM(args, i)); + } + + result = self; + self = NULL; + finally: + Py_XDECREF(self); + return result; +} + +static Py_ssize_t +HeapCCollection_length(PyVarObject *self) +{ + return Py_SIZE(self); +} + +static PyObject* +HeapCCollection_item(PyObject *self, Py_ssize_t i) +{ + if (i < 0 || i >= Py_SIZE(self)) { + return PyErr_Format(PyExc_IndexError, "index %zd out of range", i); + } + PyObject **data = PyObject_GetItemData(self); + if (!data) { + return NULL; + } + return Py_NewRef(data[i]); +} + +static int +HeapCCollection_traverse(PyObject *self, visitproc visit, void *arg) +{ + PyObject **data = PyObject_GetItemData(self); + if (!data) { + return -1; + } + for (Py_ssize_t i = 0; i < Py_SIZE(self); i++) { + Py_VISIT(data[i]); + } + return 0; +} + +static int +HeapCCollection_clear(PyObject *self) +{ + PyObject **data = PyObject_GetItemData(self); + if (!data) { + return -1; + } + Py_ssize_t size = Py_SIZE(self); + Py_SET_SIZE(self, 0); + for (Py_ssize_t i = 0; i < size; i++) { + Py_CLEAR(data[i]); + } + return 0; +} + +static void +HeapCCollection_dealloc(PyObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + HeapCCollection_clear(self); + PyObject_GC_UnTrack(self); + tp->tp_free(self); + Py_DECREF(tp); +} + +static PyType_Slot HeapCCollection_slots[] = { + {Py_tp_new, HeapCCollection_new}, + {Py_sq_length, HeapCCollection_length}, + {Py_sq_item, HeapCCollection_item}, + {Py_tp_traverse, HeapCCollection_traverse}, + {Py_tp_clear, HeapCCollection_clear}, + {Py_tp_dealloc, HeapCCollection_dealloc}, + {Py_tp_doc, (void *)HeapCCollection_doc}, + {0, 0}, +}; + +static PyType_Spec HeapCCollection_spec = { + .name = "_testcapi.HeapCCollection", + .basicsize = sizeof(PyVarObject), + .itemsize = sizeof(PyObject*), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_ITEMS_AT_END), + .slots = HeapCCollection_slots, +}; + int _PyTestCapi_Init_Heaptype(PyObject *m) { _testcapimodule = PyModule_GetDef(m); @@ -1110,5 +1228,16 @@ _PyTestCapi_Init_Heaptype(PyObject *m) { } PyModule_AddObject(m, "HeapCTypeMetaclassCustomNew", HeapCTypeMetaclassCustomNew); + PyObject *HeapCCollection = PyType_FromMetaclass( + NULL, m, &HeapCCollection_spec, NULL); + if (HeapCCollection == NULL) { + return -1; + } + int rc = PyModule_AddType(m, (PyTypeObject *)HeapCCollection); + Py_DECREF(HeapCCollection); + if (rc < 0) { + return -1; + } + return 0; } diff --git a/Modules/_testcapi/heaptype_relative.c b/Modules/_testcapi/heaptype_relative.c new file mode 100644 index 00000000000000..c247ca33b33708 --- /dev/null +++ b/Modules/_testcapi/heaptype_relative.c @@ -0,0 +1,343 @@ +#define Py_LIMITED_API 0x030c0000 // 3.12 +#include "parts.h" +#include // max_align_t +#include // memset + +#ifdef LIMITED_API_AVAILABLE + +static PyType_Slot empty_slots[] = { + {0, NULL}, +}; + +static PyObject * +make_sized_heaptypes(PyObject *module, PyObject *args) +{ + PyObject *base = NULL; + PyObject *sub = NULL; + PyObject *instance = NULL; + PyObject *result = NULL; + + int extra_base_size, basicsize; + + int r = PyArg_ParseTuple(args, "ii", &extra_base_size, &basicsize); + if (!r) { + goto finally; + } + + PyType_Spec base_spec = { + .name = "_testcapi.Base", + .basicsize = sizeof(PyObject) + extra_base_size, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = empty_slots, + }; + PyType_Spec sub_spec = { + .name = "_testcapi.Sub", + .basicsize = basicsize, + .flags = Py_TPFLAGS_DEFAULT, + .slots = empty_slots, + }; + + base = PyType_FromMetaclass(NULL, module, &base_spec, NULL); + if (!base) { + goto finally; + } + sub = PyType_FromMetaclass(NULL, module, &sub_spec, base); + if (!sub) { + goto finally; + } + instance = PyObject_CallNoArgs(sub); + if (!instance) { + goto finally; + } + char *data_ptr = PyObject_GetTypeData(instance, (PyTypeObject *)sub); + if (!data_ptr) { + goto finally; + } + Py_ssize_t data_size = PyType_GetTypeDataSize((PyTypeObject *)sub); + if (data_size < 0) { + goto finally; + } + + result = Py_BuildValue("OOOKnn", base, sub, instance, + (unsigned long long)data_ptr, + (Py_ssize_t)(data_ptr - (char*)instance), + data_size); + finally: + Py_XDECREF(base); + Py_XDECREF(sub); + Py_XDECREF(instance); + return result; +} + +static PyObject * +var_heaptype_set_data_to_3s( + PyObject *self, PyTypeObject *defining_class, + PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +{ + void *data_ptr = PyObject_GetTypeData(self, defining_class); + if (!data_ptr) { + return NULL; + } + Py_ssize_t data_size = PyType_GetTypeDataSize(defining_class); + if (data_size < 0) { + return NULL; + } + memset(data_ptr, 3, data_size); + Py_RETURN_NONE; +} + +static PyObject * +var_heaptype_get_data(PyObject *self, PyTypeObject *defining_class, + PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +{ + void *data_ptr = PyObject_GetTypeData(self, defining_class); + if (!data_ptr) { + return NULL; + } + Py_ssize_t data_size = PyType_GetTypeDataSize(defining_class); + if (data_size < 0) { + return NULL; + } + return PyBytes_FromStringAndSize(data_ptr, data_size); +} + +static PyMethodDef var_heaptype_methods[] = { + {"set_data_to_3s", _PyCFunction_CAST(var_heaptype_set_data_to_3s), + METH_METHOD | METH_FASTCALL | METH_KEYWORDS}, + {"get_data", _PyCFunction_CAST(var_heaptype_get_data), + METH_METHOD | METH_FASTCALL | METH_KEYWORDS}, + {NULL}, +}; + +static PyObject * +subclass_var_heaptype(PyObject *module, PyObject *args) +{ + PyObject *result = NULL; + + PyObject *base; // borrowed from args + int basicsize, itemsize; + long pfunc; + + int r = PyArg_ParseTuple(args, "Oiil", &base, &basicsize, &itemsize, &pfunc); + if (!r) { + goto finally; + } + + PyType_Slot slots[] = { + {Py_tp_methods, var_heaptype_methods}, + {0, NULL}, + }; + + PyType_Spec sub_spec = { + .name = "_testcapi.Sub", + .basicsize = basicsize, + .itemsize = itemsize, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_ITEMS_AT_END, + .slots = slots, + }; + + result = PyType_FromMetaclass(NULL, module, &sub_spec, base); + finally: + return result; +} + +static PyObject * +subclass_heaptype(PyObject *module, PyObject *args) +{ + PyObject *result = NULL; + + PyObject *base; // borrowed from args + int basicsize, itemsize; + + int r = PyArg_ParseTuple(args, "Oii", &base, &basicsize, &itemsize); + if (!r) { + goto finally; + } + + PyType_Slot slots[] = { + {Py_tp_methods, var_heaptype_methods}, + {0, NULL}, + }; + + PyType_Spec sub_spec = { + .name = "_testcapi.Sub", + .basicsize = basicsize, + .itemsize = itemsize, + .flags = Py_TPFLAGS_DEFAULT, + .slots = slots, + }; + + result = PyType_FromMetaclass(NULL, module, &sub_spec, base); + finally: + return result; +} + +static PyMemberDef * +heaptype_with_member_extract_and_check_memb(PyObject *self) +{ + PyMemberDef *def = PyType_GetSlot(Py_TYPE(self), Py_tp_members); + if (!def) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, "tp_members is NULL"); + } + return NULL; + } + if (!def[0].name) { + PyErr_SetString(PyExc_ValueError, "tp_members[0] is NULL"); + return NULL; + } + if (def[1].name) { + PyErr_SetString(PyExc_ValueError, "tp_members[1] is not NULL"); + return NULL; + } + if (strcmp(def[0].name, "memb")) { + PyErr_SetString(PyExc_ValueError, "tp_members[0] is not for `memb`"); + return NULL; + } + if (def[0].flags) { + PyErr_SetString(PyExc_ValueError, "tp_members[0] has flags set"); + return NULL; + } + return def; +} + +static PyObject * +heaptype_with_member_get_memb(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyMemberDef *def = heaptype_with_member_extract_and_check_memb(self); + return PyMember_GetOne((const char *)self, def); +} + +static PyObject * +heaptype_with_member_set_memb(PyObject *self, PyObject *value) +{ + PyMemberDef *def = heaptype_with_member_extract_and_check_memb(self); + int r = PyMember_SetOne((char *)self, def, value); + if (r < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +get_memb_offset(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyMemberDef *def = heaptype_with_member_extract_and_check_memb(self); + return PyLong_FromSsize_t(def->offset); +} + +static PyObject * +heaptype_with_member_get_memb_relative(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyMemberDef def = {"memb", Py_T_BYTE, sizeof(PyObject), Py_RELATIVE_OFFSET}; + return PyMember_GetOne((const char *)self, &def); +} + +static PyObject * +heaptype_with_member_set_memb_relative(PyObject *self, PyObject *value) +{ + PyMemberDef def = {"memb", Py_T_BYTE, sizeof(PyObject), Py_RELATIVE_OFFSET}; + int r = PyMember_SetOne((char *)self, &def, value); + if (r < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyMethodDef heaptype_with_member_methods[] = { + {"get_memb", heaptype_with_member_get_memb, METH_NOARGS}, + {"set_memb", heaptype_with_member_set_memb, METH_O}, + {"get_memb_offset", get_memb_offset, METH_NOARGS}, + {"get_memb_relative", heaptype_with_member_get_memb_relative, METH_NOARGS}, + {"set_memb_relative", heaptype_with_member_set_memb_relative, METH_O}, + {NULL}, +}; + +static PyObject * +make_heaptype_with_member(PyObject *module, PyObject *args) +{ + PyObject *base = NULL; + PyObject *result = NULL; + + int extra_base_size, basicsize, offset, add_flag; + + int r = PyArg_ParseTuple(args, "iiip", &extra_base_size, &basicsize, &offset, &add_flag); + if (!r) { + goto finally; + } + + PyType_Spec base_spec = { + .name = "_testcapi.Base", + .basicsize = sizeof(PyObject) + extra_base_size, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = empty_slots, + }; + base = PyType_FromMetaclass(NULL, module, &base_spec, NULL); + if (!base) { + goto finally; + } + + PyMemberDef members[] = { + {"memb", Py_T_BYTE, offset, add_flag ? Py_RELATIVE_OFFSET : 0}, + {0}, + }; + PyType_Slot slots[] = { + {Py_tp_members, members}, + {Py_tp_methods, heaptype_with_member_methods}, + {0, NULL}, + }; + + PyType_Spec sub_spec = { + .name = "_testcapi.Sub", + .basicsize = basicsize, + .flags = Py_TPFLAGS_DEFAULT, + .slots = slots, + }; + + result = PyType_FromMetaclass(NULL, module, &sub_spec, base); + finally: + Py_XDECREF(base); + return result; +} + + +static PyObject * +test_alignof_max_align_t(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + // We define ALIGNOF_MAX_ALIGN_T even if the compiler doesn't support + // max_align_t. Double-check that it's correct. + assert(ALIGNOF_MAX_ALIGN_T > 0); + assert(ALIGNOF_MAX_ALIGN_T >= _Alignof(long long)); + assert(ALIGNOF_MAX_ALIGN_T >= _Alignof(long double)); + assert(ALIGNOF_MAX_ALIGN_T >= _Alignof(void*)); + assert(ALIGNOF_MAX_ALIGN_T >= _Alignof(void (*)(void))); + + // Ensure it's a power of two + assert((ALIGNOF_MAX_ALIGN_T & (ALIGNOF_MAX_ALIGN_T - 1)) == 0); + + Py_RETURN_NONE; +} + +static PyMethodDef TestMethods[] = { + {"make_sized_heaptypes", make_sized_heaptypes, METH_VARARGS}, + {"subclass_var_heaptype", subclass_var_heaptype, METH_VARARGS}, + {"subclass_heaptype", subclass_heaptype, METH_VARARGS}, + {"make_heaptype_with_member", make_heaptype_with_member, METH_VARARGS}, + {"test_alignof_max_align_t", test_alignof_max_align_t, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_HeaptypeRelative(PyObject *m) { + if (PyModule_AddFunctions(m, TestMethods) < 0) { + return -1; + } + + if (PyModule_AddIntMacro(m, ALIGNOF_MAX_ALIGN_T) < 0) { + return -1; + } + + return 0; +} + +#endif // LIMITED_API_AVAILABLE diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 4d2d6832a827ae..d75412d5116063 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -43,6 +43,7 @@ int _PyTestCapi_Init_Immortal(PyObject *module); #ifdef LIMITED_API_AVAILABLE int _PyTestCapi_Init_VectorcallLimited(PyObject *module); +int _PyTestCapi_Init_HeaptypeRelative(PyObject *module); #endif // LIMITED_API_AVAILABLE #endif diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 47e0ed9be8e709..1ecc442058088a 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4324,6 +4324,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_VectorcallLimited(m) < 0) { return NULL; } + if (_PyTestCapi_Init_HeaptypeRelative(m) < 0) { + return NULL; + } #endif PyState_AddModule(m, &_testcapimodule); diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 334be75e8df9df..17c0c85a06c4b8 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -978,6 +978,12 @@ PyDescr_NewMember(PyTypeObject *type, PyMemberDef *member) { PyMemberDescrObject *descr; + if (member->flags & Py_RELATIVE_OFFSET) { + PyErr_SetString( + PyExc_SystemError, + "PyDescr_NewMember used with Py_RELATIVE_OFFSET"); + return NULL; + } descr = (PyMemberDescrObject *)descr_new(&PyMemberDescr_Type, type, member->name); if (descr != NULL) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 4ced04b0bde966..171c76a59a5541 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -18,6 +18,7 @@ #include "structmember.h" // PyMemberDef #include +#include // ptrdiff_t /*[clinic input] class type "PyTypeObject *" "&PyType_Type" @@ -1686,6 +1687,12 @@ PyType_GenericNew(PyTypeObject *type, PyObject *args, PyObject *kwds) /* Helpers for subtyping */ +static inline PyMemberDef * +_PyHeapType_GET_MEMBERS(PyHeapTypeObject* type) +{ + return PyObject_GetItemData((PyObject *)type); +} + static int traverse_slots(PyTypeObject *type, PyObject *self, visitproc visit, void *arg) { @@ -3873,6 +3880,15 @@ static const PySlot_Offset pyslot_offsets[] = { #include "typeslots.inc" }; +/* Align up to the nearest multiple of alignof(max_align_t) + * (like _Py_ALIGN_UP, but for a size rather than pointer) + */ +static Py_ssize_t +_align_up(Py_ssize_t size) +{ + return (size + ALIGNOF_MAX_ALIGN_T - 1) & ~(ALIGNOF_MAX_ALIGN_T - 1); +} + /* Given a PyType_FromMetaclass `bases` argument (NULL, type, or tuple of * types), return a tuple of types. */ @@ -4013,6 +4029,20 @@ _PyType_FromMetaclass_impl( assert(memb->flags == READONLY); vectorcalloffset = memb->offset; } + if (memb->flags & Py_RELATIVE_OFFSET) { + if (spec->basicsize > 0) { + PyErr_SetString( + PyExc_SystemError, + "With Py_RELATIVE_OFFSET, basicsize must be negative."); + goto finally; + } + if (memb->offset < 0 || memb->offset >= -spec->basicsize) { + PyErr_SetString( + PyExc_SystemError, + "Member offset out of range (0..-basicsize)"); + goto finally; + } + } } break; case Py_tp_doc: @@ -4154,6 +4184,32 @@ _PyType_FromMetaclass_impl( // here we just check its work assert(_PyType_HasFeature(base, Py_TPFLAGS_BASETYPE)); + /* Calculate sizes */ + + Py_ssize_t basicsize = spec->basicsize; + Py_ssize_t type_data_offset = spec->basicsize; + if (basicsize == 0) { + /* Inherit */ + basicsize = base->tp_basicsize; + } + else if (basicsize < 0) { + /* Extend */ + type_data_offset = _align_up(base->tp_basicsize); + basicsize = type_data_offset + _align_up(-spec->basicsize); + + /* Inheriting variable-sized types is limited */ + if (base->tp_itemsize + && !((base->tp_flags | spec->flags) & Py_TPFLAGS_ITEMS_AT_END)) + { + PyErr_SetString( + PyExc_SystemError, + "Cannot extend variable-size class without Py_TPFLAGS_ITEMS_AT_END."); + goto finally; + } + } + + Py_ssize_t itemsize = spec->itemsize; + /* Allocate the new type * * Between here and PyType_Ready, we should limit: @@ -4201,8 +4257,8 @@ _PyType_FromMetaclass_impl( /* Copy the sizes */ - type->tp_basicsize = spec->basicsize; - type->tp_itemsize = spec->itemsize; + type->tp_basicsize = basicsize; + type->tp_itemsize = itemsize; /* Copy all the ordinary slots */ @@ -4219,6 +4275,16 @@ _PyType_FromMetaclass_impl( size_t len = Py_TYPE(type)->tp_itemsize * nmembers; memcpy(_PyHeapType_GET_MEMBERS(res), slot->pfunc, len); type->tp_members = _PyHeapType_GET_MEMBERS(res); + PyMemberDef *memb; + Py_ssize_t i; + for (memb = _PyHeapType_GET_MEMBERS(res), i = nmembers; + i > 0; ++memb, --i) + { + if (memb->flags & Py_RELATIVE_OFFSET) { + memb->flags &= ~Py_RELATIVE_OFFSET; + memb->offset += type_data_offset; + } + } } break; default: @@ -4227,6 +4293,7 @@ _PyType_FromMetaclass_impl( PySlot_Offset slotoffsets = pyslot_offsets[slot->slot]; short slot_offset = slotoffsets.slot_offset; if (slotoffsets.subslot_offset == -1) { + /* Set a slot in the main PyTypeObject */ *(void**)((char*)res_start + slot_offset) = slot->pfunc; } else { @@ -4461,6 +4528,34 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) return NULL; } +void * +PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls) +{ + assert(PyObject_TypeCheck(obj, cls)); + return (char *)obj + _align_up(cls->tp_base->tp_basicsize); +} + +Py_ssize_t +PyType_GetTypeDataSize(PyTypeObject *cls) +{ + ptrdiff_t result = cls->tp_basicsize - _align_up(cls->tp_base->tp_basicsize); + if (result < 0) { + return 0; + } + return result; +} + +void * +PyObject_GetItemData(PyObject *obj) +{ + if (!PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_ITEMS_AT_END)) { + PyErr_Format(PyExc_TypeError, + "type '%s' does not have Py_TPFLAGS_ITEMS_AT_END", + Py_TYPE(obj)->tp_name); + return NULL; + } + return (char *)obj + Py_TYPE(obj)->tp_basicsize; +} /* Internal API to look for a name through the MRO, bypassing the method cache. This returns a borrowed reference, and might set an exception. @@ -5158,7 +5253,8 @@ PyTypeObject PyType_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_TYPE_SUBCLASS | - Py_TPFLAGS_HAVE_VECTORCALL, /* tp_flags */ + Py_TPFLAGS_HAVE_VECTORCALL | + Py_TPFLAGS_ITEMS_AT_END, /* tp_flags */ type_doc, /* tp_doc */ (traverseproc)type_traverse, /* tp_traverse */ (inquiry)type_clear, /* tp_clear */ @@ -6572,9 +6668,14 @@ inherit_special(PyTypeObject *type, PyTypeObject *base) else if (PyType_IsSubtype(base, &PyDict_Type)) { type->tp_flags |= Py_TPFLAGS_DICT_SUBCLASS; } + + /* Setup some inheritable flags */ if (PyType_HasFeature(base, _Py_TPFLAGS_MATCH_SELF)) { type->tp_flags |= _Py_TPFLAGS_MATCH_SELF; } + if (PyType_HasFeature(base, Py_TPFLAGS_ITEMS_AT_END)) { + type->tp_flags |= Py_TPFLAGS_ITEMS_AT_END; + } } static int diff --git a/PC/pyconfig.h b/PC/pyconfig.h index 8a3bf8968ce29d..3415efe2dea117 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -330,6 +330,7 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ # define SIZEOF_HKEY 8 # define SIZEOF_SIZE_T 8 # define ALIGNOF_SIZE_T 8 +# define ALIGNOF_MAX_ALIGN_T 8 /* configure.ac defines HAVE_LARGEFILE_SUPPORT iff sizeof(off_t) > sizeof(long), and sizeof(long long) >= sizeof(off_t). On Win64 the second condition is not true, but if fpos_t replaces off_t @@ -351,6 +352,7 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ # else # define SIZEOF_TIME_T 4 # endif +# define ALIGNOF_MAX_ALIGN_T 8 #endif #ifdef _DEBUG diff --git a/PC/python3dll.c b/PC/python3dll.c index 706affa18351b3..7e848abccfd1fa 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -467,6 +467,7 @@ EXPORT_FUNC(PyObject_GetAttrString) EXPORT_FUNC(PyObject_GetBuffer) EXPORT_FUNC(PyObject_GetItem) EXPORT_FUNC(PyObject_GetIter) +EXPORT_FUNC(PyObject_GetTypeData) EXPORT_FUNC(PyObject_HasAttr) EXPORT_FUNC(PyObject_HasAttrString) EXPORT_FUNC(PyObject_Hash) @@ -618,6 +619,7 @@ EXPORT_FUNC(PyType_GetModuleState) EXPORT_FUNC(PyType_GetName) EXPORT_FUNC(PyType_GetQualName) EXPORT_FUNC(PyType_GetSlot) +EXPORT_FUNC(PyType_GetTypeDataSize) EXPORT_FUNC(PyType_IsSubtype) EXPORT_FUNC(PyType_Modified) EXPORT_FUNC(PyType_Ready) diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 21941247eb9692..56448b6ee7d492 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -98,6 +98,7 @@ + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 0e42e4982c21ff..297c9ce799bea1 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -24,6 +24,9 @@ Source Files + + Source Files + Source Files diff --git a/Python/structmember.c b/Python/structmember.c index 1b8be28dcf2eb2..19a75224a0f32e 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -8,6 +8,12 @@ PyObject * PyMember_GetOne(const char *obj_addr, PyMemberDef *l) { PyObject *v; + if (l->flags & Py_RELATIVE_OFFSET) { + PyErr_SetString( + PyExc_SystemError, + "PyMember_GetOne used with Py_RELATIVE_OFFSET"); + return NULL; + } const char* addr = obj_addr + l->offset; switch (l->type) { @@ -103,6 +109,12 @@ int PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) { PyObject *oldv; + if (l->flags & Py_RELATIVE_OFFSET) { + PyErr_SetString( + PyExc_SystemError, + "PyMember_SetOne used with Py_RELATIVE_OFFSET"); + return -1; + } addr += l->offset; diff --git a/configure b/configure index 8133d47f61355b..b8fa9d66e735dd 100755 --- a/configure +++ b/configure @@ -10729,6 +10729,41 @@ cat >>confdefs.h <<_ACEOF _ACEOF +# The cast to long int works around a bug in the HP C Compiler, +# see AC_CHECK_SIZEOF for more information. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking alignment of max_align_t" >&5 +$as_echo_n "checking alignment of max_align_t... " >&6; } +if ${ac_cv_alignof_max_align_t+:} false; then : + $as_echo_n "(cached) " >&6 +else + if ac_fn_c_compute_int "$LINENO" "(long int) offsetof (ac__type_alignof_, y)" "ac_cv_alignof_max_align_t" "$ac_includes_default +#ifndef offsetof +# define offsetof(type, member) ((char *) &((type *) 0)->member - (char *) 0) +#endif +typedef struct { char x; max_align_t y; } ac__type_alignof_;"; then : + +else + if test "$ac_cv_type_max_align_t" = yes; then + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error 77 "cannot compute alignment of max_align_t +See \`config.log' for more details" "$LINENO" 5; } + else + ac_cv_alignof_max_align_t=0 + fi +fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_alignof_max_align_t" >&5 +$as_echo "$ac_cv_alignof_max_align_t" >&6; } + + + +cat >>confdefs.h <<_ACEOF +#define ALIGNOF_MAX_ALIGN_T $ac_cv_alignof_max_align_t +_ACEOF + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for long double" >&5 diff --git a/configure.ac b/configure.ac index 3f20d8980d8abc..0940b93c25f7bf 100644 --- a/configure.ac +++ b/configure.ac @@ -2921,6 +2921,7 @@ AC_CHECK_SIZEOF(size_t, 4) AC_CHECK_ALIGNOF(size_t) AC_CHECK_SIZEOF(pid_t, 4) AC_CHECK_SIZEOF(uintptr_t) +AC_CHECK_ALIGNOF(max_align_t) AC_TYPE_LONG_DOUBLE AC_CHECK_SIZEOF(long double, 16) diff --git a/pyconfig.h.in b/pyconfig.h.in index 236cee6588d49b..2c22b27af65ea3 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -19,6 +19,9 @@ /* The normal alignment of `long', in bytes. */ #undef ALIGNOF_LONG +/* The normal alignment of `max_align_t', in bytes. */ +#undef ALIGNOF_MAX_ALIGN_T + /* The normal alignment of `size_t', in bytes. */ #undef ALIGNOF_SIZE_T From d0122372f2acb4cc56b89ab8c577ff9039d17d89 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 4 May 2023 09:17:12 +0100 Subject: [PATCH 026/204] GH-97950: Use new-style index directive ('module') (#103996) * Use new-style index directive ('module') - C API * Use new-style index directive ('module') - Library * Use new-style index directive ('module') - Reference * Use new-style index directive ('module') - Tutorial * Uncomment module removal in pairindextypes * Use new-style index directive ('module') - C API * Use new-style index directive ('module') - Library * Use new-style index directive ('module') - Reference --- Doc/c-api/exceptions.rst | 6 +++--- Doc/c-api/init.rst | 14 +++++++------- Doc/c-api/intro.rst | 6 +++--- Doc/library/_thread.rst | 2 +- Doc/library/binascii.rst | 4 ++-- Doc/library/cmath.rst | 2 +- Doc/library/copy.rst | 2 +- Doc/library/copyreg.rst | 4 ++-- Doc/library/exceptions.rst | 2 +- Doc/library/fnmatch.rst | 4 ++-- Doc/library/functions.rst | 4 ++-- Doc/library/http.client.rst | 2 +- Doc/library/internet.rst | 2 +- Doc/library/locale.rst | 4 ++-- Doc/library/marshal.rst | 4 ++-- Doc/library/os.path.rst | 2 +- Doc/library/os.rst | 4 ++-- Doc/library/pdb.rst | 4 ++-- Doc/library/posix.rst | 2 +- Doc/library/pwd.rst | 2 +- Doc/library/pyexpat.rst | 2 +- Doc/library/runpy.rst | 4 ++-- Doc/library/shelve.rst | 6 +++--- Doc/library/site.rst | 4 ++-- Doc/library/socket.rst | 2 +- Doc/library/stdtypes.rst | 8 ++++---- Doc/reference/compound_stmts.rst | 2 +- Doc/reference/datamodel.rst | 8 ++++---- Doc/reference/executionmodel.rst | 2 +- Doc/reference/toplevel_components.rst | 8 ++++---- Doc/tools/extensions/pyspecific.py | 6 +----- Doc/tutorial/inputoutput.rst | 2 +- Doc/tutorial/modules.rst | 4 ++-- 33 files changed, 65 insertions(+), 69 deletions(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 49d2f18d4573b0..4ed96f01dbbc3e 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -602,7 +602,7 @@ Signal Handling .. c:function:: int PyErr_CheckSignals() .. index:: - module: signal + pair: module; signal single: SIGINT single: KeyboardInterrupt (built-in exception) @@ -633,7 +633,7 @@ Signal Handling .. c:function:: void PyErr_SetInterrupt() .. index:: - module: signal + pair: module; signal single: SIGINT single: KeyboardInterrupt (built-in exception) @@ -648,7 +648,7 @@ Signal Handling .. c:function:: int PyErr_SetInterruptEx(int signum) .. index:: - module: signal + pair: module; signal single: KeyboardInterrupt (built-in exception) Simulate the effect of a signal arriving. The next time diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 38e324fb6409bc..26762969ef8eba 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -336,9 +336,9 @@ Initializing and finalizing the interpreter single: PyEval_InitThreads() single: modules (in module sys) single: path (in module sys) - module: builtins - module: __main__ - module: sys + pair: module; builtins + pair: module; __main__ + pair: module; sys triple: module; search; path single: PySys_SetArgv() single: PySys_SetArgvEx() @@ -1051,7 +1051,7 @@ code, or when embedding the Python interpreter: .. deprecated:: 3.9 - .. index:: module: _thread + .. index:: pair: module; _thread .. c:function:: int PyEval_ThreadsInitialized() @@ -1494,9 +1494,9 @@ function. You can create and destroy them using the following functions: .. c:function:: PyThreadState* Py_NewInterpreter() .. index:: - module: builtins - module: __main__ - module: sys + pair: module; builtins + pair: module; __main__ + pair: module; sys single: stdout (in module sys) single: stderr (in module sys) single: stdin (in module sys) diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index acd4e033dfbc4b..1a200b919f0e75 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -705,9 +705,9 @@ interpreter can only be used after the interpreter has been initialized. .. index:: single: Py_Initialize() - module: builtins - module: __main__ - module: sys + pair: module; builtins + pair: module; __main__ + pair: module; sys triple: module; search; path single: path (in module sys) diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index 122692a428594f..ba9314e46ab6ea 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -208,7 +208,7 @@ In addition to these methods, lock objects can also be used via the **Caveats:** - .. index:: module: signal + .. index:: pair: module; signal * Threads interact strangely with interrupts: the :exc:`KeyboardInterrupt` exception will be received by an arbitrary thread. (When the :mod:`signal` diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 5a0815faa38eac..21960cb7972e6e 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -6,8 +6,8 @@ representations. .. index:: - module: uu - module: base64 + pair: module; uu + pair: module; base64 -------------- diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index 5ed7a09b3e9db2..b17d58e1cc0ce1 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -301,7 +301,7 @@ Constants .. versionadded:: 3.6 -.. index:: module: math +.. index:: pair: module; math Note that the selection of functions is similar, but not identical, to that in module :mod:`math`. The reason for having two modules is that some users aren't diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index a8bc2fa55ea8c3..8f32477ed508c3 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -68,7 +68,7 @@ Shallow copies of dictionaries can be made using :meth:`dict.copy`, and of lists by assigning a slice of the entire list, for example, ``copied_list = original_list[:]``. -.. index:: module: pickle +.. index:: pair: module; pickle Classes can use the same interfaces to control copying that they use to control pickling. See the description of module :mod:`pickle` for information on these diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst index afc3e66f0bf7ac..2a28c043f80723 100644 --- a/Doc/library/copyreg.rst +++ b/Doc/library/copyreg.rst @@ -7,8 +7,8 @@ **Source code:** :source:`Lib/copyreg.py` .. index:: - module: pickle - module: copy + pair: module; pickle + pair: module; copy -------------- diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 18c3f47dddc079..aee1cb5cc6a475 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -318,7 +318,7 @@ The following exceptions are the exceptions that are usually raised. .. exception:: OSError([arg]) OSError(errno, strerror[, filename[, winerror[, filename2]]]) - .. index:: module: errno + .. index:: pair: module; errno This exception is raised when a system function returns a system-related error, including I/O failures such as "file not found" or "disk full" diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index 46bf0fc2848058..aed8991d44772f 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -8,7 +8,7 @@ .. index:: single: filenames; wildcard expansion -.. index:: module: re +.. index:: pair: module; re -------------- @@ -38,7 +38,7 @@ special characters used in shell-style wildcards are: For a literal match, wrap the meta-characters in brackets. For example, ``'[?]'`` matches the character ``'?'``. -.. index:: module: glob +.. index:: pair: module; glob Note that the filename separator (``'/'`` on Unix) is *not* special to this module. See module :mod:`glob` for pathname expansion (:mod:`glob` uses diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 47e3880129595e..ddb918de115fcf 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1340,7 +1340,7 @@ are always available. They are listed here in alphabetical order. single: I/O control; buffering single: binary mode single: text mode - module: sys + pair: module; sys See also the file handling modules, such as :mod:`fileinput`, :mod:`io` (where :func:`open` is declared), :mod:`os`, :mod:`os.path`, :mod:`tempfile`, @@ -1987,7 +1987,7 @@ are always available. They are listed here in alphabetical order. .. index:: statement: import - module: builtins + pair: module; builtins .. note:: diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index abdc6b447a8b61..eb8c1e198e2b09 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -10,7 +10,7 @@ pair: HTTP; protocol single: HTTP; http.client (standard module) -.. index:: module: urllib.request +.. index:: pair: module; urllib.request -------------- diff --git a/Doc/library/internet.rst b/Doc/library/internet.rst index ff58dcf4d89c36..681769a4820dba 100644 --- a/Doc/library/internet.rst +++ b/Doc/library/internet.rst @@ -9,7 +9,7 @@ Internet Protocols and Support single: Internet single: World Wide Web -.. index:: module: socket +.. index:: pair: module; socket The modules described in this chapter implement internet protocols and support for related technology. They are all implemented in Python. Most of these diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index f726f8397c9648..f2abb3638a141f 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -16,7 +16,7 @@ functionality. The POSIX locale mechanism allows programmers to deal with certain cultural issues in an application, without requiring the programmer to know all the specifics of each country where the software is executed. -.. index:: module: _locale +.. index:: pair: module; _locale The :mod:`locale` module is implemented on top of the :mod:`_locale` module, which in turn uses an ANSI C locale implementation if available. @@ -464,7 +464,7 @@ The :mod:`locale` module defines the following exception and functions: .. data:: LC_CTYPE - .. index:: module: string + .. index:: pair: module; string Locale category for the character type functions. Depending on the settings of this category, the functions of module :mod:`string` dealing with case change diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index 24f9dc1689da4a..0556f19699dc15 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -15,8 +15,8 @@ undocumented on purpose; it may change between Python versions (although it rarely does). [#]_ .. index:: - module: pickle - module: shelve + pair: module; pickle + pair: module; shelve This is not a general "persistence" module. For general persistence and transfer of Python objects through RPC calls, see the modules :mod:`pickle` and diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 96bcb48ad7d126..7881c52db87090 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -159,7 +159,7 @@ the :mod:`glob` module.) On Unix and Windows, return the argument with an initial component of ``~`` or ``~user`` replaced by that *user*'s home directory. - .. index:: module: pwd + .. index:: pair: module; pwd On Unix, an initial ``~`` is replaced by the environment variable :envvar:`HOME` if it is set; otherwise the current user's home directory is looked up in the diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 76623c6305432a..641e289e77c518 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1284,7 +1284,7 @@ or `the MSDN `_ on Windo .. function:: openpty() - .. index:: module: pty + .. index:: pair: module; pty Open a new pseudo-terminal pair. Return a pair of file descriptors ``(master, slave)`` for the pty and the tty, respectively. The new file @@ -2890,7 +2890,7 @@ features: possible and call :func:`lstat` on the result. This does not apply to dangling symlinks or junction points, which will raise the usual exceptions. - .. index:: module: stat + .. index:: pair: module; stat Example:: diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 8a386aa77368f2..74bffef5562ae1 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -20,8 +20,8 @@ supports post-mortem debugging and can be called under program control. .. index:: single: Pdb (class in pdb) - module: bdb - module: cmd + pair: module; bdb + pair: module; cmd The debugger is extensible -- it is actually defined as the class :class:`Pdb`. This is currently undocumented but easily understood by reading the source. The diff --git a/Doc/library/posix.rst b/Doc/library/posix.rst index ec04b0dcfc162f..0413f9d02a8d57 100644 --- a/Doc/library/posix.rst +++ b/Doc/library/posix.rst @@ -11,7 +11,7 @@ This module provides access to operating system functionality that is standardized by the C Standard and the POSIX standard (a thinly disguised Unix interface). -.. index:: module: os +.. index:: pair: module; os **Do not import this module directly.** Instead, import the module :mod:`os`, which provides a *portable* version of this interface. On Unix, the :mod:`os` diff --git a/Doc/library/pwd.rst b/Doc/library/pwd.rst index 98f3c45e29cbcb..7cafc66fd7e93c 100644 --- a/Doc/library/pwd.rst +++ b/Doc/library/pwd.rst @@ -39,7 +39,7 @@ raised if the entry asked for cannot be found. .. note:: - .. index:: module: crypt + .. index:: pair: module; crypt In traditional Unix the field ``pw_passwd`` usually contains a password encrypted with a DES derived algorithm (see module :mod:`crypt`). However most diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index d6581e21b01c0e..935e872480efda 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -33,7 +33,7 @@ can be set to handler functions. When an XML document is then fed to the parser, the handler functions are called for the character data and markup in the XML document. -.. index:: module: pyexpat +.. index:: pair: module; pyexpat This module uses the :mod:`pyexpat` module to provide access to the Expat parser. Direct use of the :mod:`pyexpat` module is deprecated. diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index 501f4ddf5a3e3f..42ed8c253b8027 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -30,7 +30,7 @@ The :mod:`runpy` module provides two functions: .. function:: run_module(mod_name, init_globals=None, run_name=None, alter_sys=False) .. index:: - module: __main__ + pair: module; __main__ Execute the code of the specified module and return the resulting module globals dictionary. The module's code is first located using the standard @@ -101,7 +101,7 @@ The :mod:`runpy` module provides two functions: .. function:: run_path(path_name, init_globals=None, run_name=None) .. index:: - module: __main__ + pair: module; __main__ Execute the code at the named filesystem location and return the resulting module globals dictionary. As with a script name supplied to the CPython diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index a50fc6f0bf77b2..dc87af398ed757 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -6,7 +6,7 @@ **Source code:** :source:`Lib/shelve.py` -.. index:: module: pickle +.. index:: pair: module; pickle -------------- @@ -95,8 +95,8 @@ Restrictions ------------ .. index:: - module: dbm.ndbm - module: dbm.gnu + pair: module; dbm.ndbm + pair: module; dbm.gnu * The choice of which database package will be used (such as :mod:`dbm.ndbm` or :mod:`dbm.gnu`) depends on which interface is available. Therefore it is not diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 4a88013f1d6ed2..a4efefd587a609 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -109,7 +109,7 @@ directory precedes the :file:`foo` directory because :file:`bar.pth` comes alphabetically before :file:`foo.pth`; and :file:`spam` is omitted because it is not mentioned in either path configuration file. -.. index:: module: sitecustomize +.. index:: pair: module; sitecustomize After these path manipulations, an attempt is made to import a module named :mod:`sitecustomize`, which can perform arbitrary site-specific customizations. @@ -121,7 +121,7 @@ with :file:`pythonw.exe` on Windows (which is used by default to start IDLE), attempted output from :mod:`sitecustomize` is ignored. Any other exception causes a silent and perhaps mysterious failure of the process. -.. index:: module: usercustomize +.. index:: pair: module; usercustomize After this, an attempt is made to import a module named :mod:`usercustomize`, which can perform arbitrary user-specific customizations, if diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index c8ca555700a3c9..150c1f93afd967 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -1916,7 +1916,7 @@ to sockets. .. method:: socket.setsockopt(level, optname, None, optlen: int) :noindex: - .. index:: module: struct + .. index:: pair: module; struct Set the value of the given socket option (see the Unix manual page :manpage:`setsockopt(2)`). The needed symbolic constants are defined in the diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index f6662b4336c276..4c0a213dcbea26 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -330,7 +330,7 @@ Notes: (3) .. index:: - module: math + pair: module; math single: floor() (in module math) single: ceil() (in module math) single: trunc() (in module math) @@ -1600,7 +1600,7 @@ String Methods -------------- .. index:: - module: re + pair: module; re Strings implement all of the :ref:`common ` sequence operations, along with the additional methods described below. @@ -2511,7 +2511,7 @@ Binary Sequence Types --- :class:`bytes`, :class:`bytearray`, :class:`memoryview object: bytes object: bytearray object: memoryview - module: array + pair: module; array The core built-in types for manipulating binary data are :class:`bytes` and :class:`bytearray`. They are supported by :class:`memoryview` which uses @@ -5378,7 +5378,7 @@ Type Objects .. index:: builtin: type - module: types + pair: module; types Type objects represent the various object types. An object's type is accessed by the built-in function :func:`type`. There are no special operations on diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 0731589e4cadb8..4bdecbf243650f 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -297,7 +297,7 @@ traceback attached to them, they form a reference cycle with the stack frame, keeping all locals in that frame alive until the next garbage collection occurs. .. index:: - module: sys + pair: module; sys object: traceback Before an :keyword:`!except` clause's suite is executed, diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index c35bf4016a28d2..9d3ea3f78de16f 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -377,7 +377,7 @@ Sequences (and hence unhashable), byte arrays otherwise provide the same interface and functionality as immutable :class:`bytes` objects. - .. index:: module: array + .. index:: pair: module; array The extension module :mod:`array` provides an additional example of a mutable sequence type, as does the :mod:`collections` module. @@ -451,8 +451,8 @@ Mappings section :ref:`dict`). .. index:: - module: dbm.ndbm - module: dbm.gnu + pair: module; dbm.ndbm + pair: module; dbm.gnu The extension modules :mod:`dbm.ndbm` and :mod:`dbm.gnu` provide additional examples of mapping types, as does the :mod:`collections` @@ -909,7 +909,7 @@ Class instances I/O objects (also known as file objects) .. index:: builtin: open - module: io + pair: module; io single: popen() (in module os) single: makefile() (socket method) single: sys.stdin diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index a264015cbf4049..8917243999d399 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -151,7 +151,7 @@ to previously bound variables in the nearest enclosing function scope. :exc:`SyntaxError` is raised at compile time if the given name does not exist in any enclosing function scope. -.. index:: module: __main__ +.. index:: pair: module; __main__ The namespace for a module is automatically created the first time a module is imported. The main module for a script is always called :mod:`__main__`. diff --git a/Doc/reference/toplevel_components.rst b/Doc/reference/toplevel_components.rst index 319c9de484241e..ee472ace6e2129 100644 --- a/Doc/reference/toplevel_components.rst +++ b/Doc/reference/toplevel_components.rst @@ -21,9 +21,9 @@ Complete Python programs .. index:: single: program .. index:: - module: sys - module: __main__ - module: builtins + pair: module; sys + pair: module; __main__ + pair: module; builtins While a language specification need not prescribe how the language interpreter is invoked, it is useful to have a notion of a complete Python program. A @@ -38,7 +38,7 @@ the next section. .. index:: single: interactive mode - module: __main__ + pair: module; __main__ The interpreter may also be invoked in interactive mode; in this case, it does not read and execute a complete program but reads and executes one statement diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 39c7c42e66f9a0..12d3e382c87318 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -689,7 +689,7 @@ def patch_pairindextypes(app) -> None: # away from this, we need Sphinx to believe that these values don't # exist, by deleting them when using the gettext builder. - # pairindextypes.pop('module', None) + pairindextypes.pop('module', None) # pairindextypes.pop('keyword', None) # pairindextypes.pop('operator', None) # pairindextypes.pop('object', None) @@ -697,10 +697,6 @@ def patch_pairindextypes(app) -> None: # pairindextypes.pop('statement', None) # pairindextypes.pop('builtin', None) - # there needs to be at least one statement in this block, will be - # removed when the first of the below is uncommented. - pass - def setup(app): app.add_role('issue', issue_role) diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index 3581b3727a53ea..aeacc1559a228c 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -466,7 +466,7 @@ Reference for a complete guide to file objects. Saving structured data with :mod:`json` --------------------------------------- -.. index:: module: json +.. index:: pair: module; json Strings can easily be written to and read from a file. Numbers take a bit more effort, since the :meth:`read` method only returns strings, which will have to diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index 4daafa49a34d2e..3bd034bcc9703f 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -264,7 +264,7 @@ Some tips for experts: Standard Modules ================ -.. index:: module: sys +.. index:: pair: module; sys Python comes with a library of standard modules, described in a separate document, the Python Library Reference ("Library Reference" hereafter). Some @@ -345,7 +345,7 @@ Without arguments, :func:`dir` lists the names you have defined currently:: Note that it lists all types of names: variables, modules, functions, etc. -.. index:: module: builtins +.. index:: pair: module; builtins :func:`dir` does not list the names of built-in functions and variables. If you want a list of those, they are defined in the standard module From 33ca322c50baa5152afe388fb3b8b7d63dc5a9b9 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 4 May 2023 09:44:12 +0100 Subject: [PATCH 027/204] GH-97950: Use new-style index directive ('keyword') (#104153) * Uncomment keyword removal in pairindextypes * Use new-style index directive ('keyword') - Reference --- Doc/reference/compound_stmts.rst | 36 +++++++++++++++--------------- Doc/reference/expressions.rst | 6 ++--- Doc/reference/simple_stmts.rst | 12 +++++----- Doc/tools/extensions/pyspecific.py | 2 +- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 4bdecbf243650f..1f03180f1565ba 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -85,8 +85,8 @@ The :keyword:`!if` statement .. index:: ! statement: if - keyword: elif - keyword: else + pair: keyword; elif + pair: keyword; else single: : (colon); compound statement The :keyword:`if` statement is used for conditional execution: @@ -110,7 +110,7 @@ The :keyword:`!while` statement .. index:: ! statement: while - keyword: else + pair: keyword; else pair: loop; statement single: : (colon); compound statement @@ -143,8 +143,8 @@ The :keyword:`!for` statement .. index:: ! statement: for - keyword: in - keyword: else + pair: keyword; in + pair: keyword; else pair: target; list pair: loop; statement object: sequence @@ -206,10 +206,10 @@ The :keyword:`!try` statement .. index:: ! statement: try - keyword: except - keyword: finally - keyword: else - keyword: as + pair: keyword; except + pair: keyword; finally + pair: keyword; else + pair: keyword; as single: : (colon); compound statement The :keyword:`!try` statement specifies exception handlers and/or cleanup code @@ -326,7 +326,7 @@ stored in the :mod:`sys` module is reset to its previous value:: .. index:: - keyword: except_star + pair: keyword; except_star .. _except_star: @@ -387,7 +387,7 @@ cannot appear in an :keyword:`!except*` clause. .. index:: - keyword: else + pair: keyword; else statement: return statement: break statement: continue @@ -404,7 +404,7 @@ the :keyword:`!else` clause are not handled by the preceding :keyword:`except` clauses. -.. index:: keyword: finally +.. index:: pair: keyword; finally .. _finally: @@ -469,7 +469,7 @@ The :keyword:`!with` statement .. index:: ! statement: with - keyword: as + pair: keyword; as single: as; with statement single: , (comma); with statement single: : (colon); compound statement @@ -586,10 +586,10 @@ The :keyword:`!match` statement .. index:: ! statement: match - ! keyword: case + ! pair: keyword; case ! single: pattern matching - keyword: if - keyword: as + pair: keyword; if + pair: keyword; as pair: match; case single: as; match statement single: : (colon); compound statement @@ -1474,8 +1474,8 @@ Coroutine function definition : ["->" `expression`] ":" `suite` .. index:: - keyword: async - keyword: await + pair: keyword; async + pair: keyword; await Execution of Python coroutines can be suspended and resumed at many points (see :term:`coroutine`). :keyword:`await` expressions, :keyword:`async for` and diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 1e4a13fbd6a3ce..fe2b090b23c614 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -415,8 +415,8 @@ Yield expressions ----------------- .. index:: - keyword: yield - keyword: from + pair: keyword; yield + pair: keyword; from pair: yield; expression pair: generator; function @@ -1149,7 +1149,7 @@ a class instance: if that method was called. -.. index:: keyword: await +.. index:: pair: keyword; await .. _await: Await expression diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index c98ac81e415b72..5c7b5d7efaa2b5 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -495,7 +495,7 @@ If an expression list is present, it is evaluated, else ``None`` is substituted. :keyword:`return` leaves the current function call with the expression list (or ``None``) as return value. -.. index:: keyword: finally +.. index:: pair: keyword; finally When :keyword:`return` passes control out of a :keyword:`try` statement with a :keyword:`finally` clause, that :keyword:`!finally` clause is executed before @@ -679,7 +679,7 @@ The :keyword:`!break` statement :keyword:`while` loop, but not nested in a function or class definition within that loop. -.. index:: keyword: else +.. index:: pair: keyword; else pair: loop control; target It terminates the nearest enclosing loop, skipping the optional :keyword:`!else` @@ -688,7 +688,7 @@ clause if the loop has one. If a :keyword:`for` loop is terminated by :keyword:`break`, the loop control target keeps its current value. -.. index:: keyword: finally +.. index:: pair: keyword; finally When :keyword:`break` passes control out of a :keyword:`try` statement with a :keyword:`finally` clause, that :keyword:`!finally` clause is executed before @@ -705,7 +705,7 @@ The :keyword:`!continue` statement statement: for statement: while pair: loop; statement - keyword: finally + pair: keyword; finally .. productionlist:: python-grammar continue_stmt: "continue" @@ -729,8 +729,8 @@ The :keyword:`!import` statement ! statement: import single: module; importing pair: name; binding - keyword: from - keyword: as + pair: keyword; from + pair: keyword; as exception: ImportError single: , (comma); import statement diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 12d3e382c87318..c347a327e0910e 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -690,7 +690,7 @@ def patch_pairindextypes(app) -> None: # exist, by deleting them when using the gettext builder. pairindextypes.pop('module', None) - # pairindextypes.pop('keyword', None) + pairindextypes.pop('keyword', None) # pairindextypes.pop('operator', None) # pairindextypes.pop('object', None) # pairindextypes.pop('exception', None) From b0ce2db1184124575d5934d205344a9cf5debb79 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 4 May 2023 10:03:11 +0100 Subject: [PATCH 028/204] GH-97950: Use new-style index directive ('operator') (#104156) --- Doc/library/stdtypes.rst | 56 +++++++++++++++--------------- Doc/reference/expressions.rst | 50 +++++++++++++------------- Doc/tools/extensions/pyspecific.py | 2 +- 3 files changed, 54 insertions(+), 54 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 4c0a213dcbea26..46b0389adaa88f 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -61,8 +61,8 @@ objects considered false: ``range(0)`` .. index:: - operator: or - operator: and + pair: operator; or + pair: operator; and single: False single: True @@ -95,9 +95,9 @@ These are the Boolean operations, ordered by ascending priority: +-------------+---------------------------------+-------+ .. index:: - operator: and - operator: or - operator: not + pair: operator; and + pair: operator; or + pair: operator; not Notes: @@ -122,14 +122,14 @@ Comparisons .. index:: pair: chaining; comparisons pair: operator; comparison - operator: == - operator: < (less) - operator: <= - operator: > (greater) - operator: >= - operator: != - operator: is - operator: is not + pair: operator; == + pair: operator; < (less) + pair: operator; <= + pair: operator; > (greater) + pair: operator; >= + pair: operator; != + pair: operator; is + pair: operator; is not There are eight comparison operations in Python. They all have the same priority (which is higher than that of the Boolean operations). Comparisons can @@ -192,8 +192,8 @@ customized; also they can be applied to any two objects and never raise an exception. .. index:: - operator: in - operator: not in + pair: operator; in + pair: operator; not in Two more operations with the same syntactic priority, :keyword:`in` and :keyword:`not in`, are supported by types that are :term:`iterable` or @@ -253,11 +253,11 @@ and imaginary parts. single: operator; - (minus) single: - (minus); unary operator single: - (minus); binary operator - operator: * (asterisk) - operator: / (slash) - operator: // - operator: % (percent) - operator: ** + pair: operator; * (asterisk) + pair: operator; / (slash) + pair: operator; // + pair: operator; % (percent) + pair: operator; ** Python fully supports mixed arithmetic: when a binary arithmetic operator has operands of different numeric types, the operand with the "narrower" type is @@ -392,12 +392,12 @@ Bitwise Operations on Integer Types pair: bitwise; operations pair: shifting; operations pair: masking; operations - operator: | (vertical bar) - operator: ^ (caret) - operator: & (ampersand) - operator: << - operator: >> - operator: ~ (tilde) + pair: operator; | (vertical bar) + pair: operator; ^ (caret) + pair: operator; & (ampersand) + pair: operator; << + pair: operator; >> + pair: operator; ~ (tilde) Bitwise operations only make sense for integers. The result of bitwise operations is calculated as though carried out in two's complement with an @@ -952,8 +952,8 @@ operations have the same priority as the corresponding numeric operations. [3]_ pair: repetition; operation pair: subscript; operation pair: slice; operation - operator: in - operator: not in + pair: operator; in + pair: operator; not in single: count() (sequence method) single: index() (sequence method) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index fe2b090b23c614..b1bb162f62e1ec 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -1171,7 +1171,7 @@ The power operator .. index:: pair: power; operation - operator: ** + pair: operator; ** The power operator binds more tightly than unary operators on its left; it binds less tightly than unary operators on its right. The syntax is: @@ -1232,7 +1232,7 @@ operation can be overridden with the :meth:`__pos__` special method. .. index:: single: inversion - operator: ~ (tilde) + pair: operator; ~ (tilde) The unary ``~`` (invert) operator yields the bitwise inversion of its integer argument. The bitwise inversion of ``x`` is defined as ``-(x+1)``. It only @@ -1267,7 +1267,7 @@ operators and one for additive operators: .. index:: single: multiplication - operator: * (asterisk) + pair: operator; * (asterisk) The ``*`` (multiplication) operator yields the product of its arguments. The arguments must either both be numbers, or one argument must be an integer and @@ -1280,7 +1280,7 @@ This operation can be customized using the special :meth:`__mul__` and .. index:: single: matrix multiplication - operator: @ (at) + pair: operator; @ (at) The ``@`` (at) operator is intended to be used for matrix multiplication. No builtin Python types implement this operator. @@ -1290,8 +1290,8 @@ builtin Python types implement this operator. .. index:: exception: ZeroDivisionError single: division - operator: / (slash) - operator: // + pair: operator; / (slash) + pair: operator; // The ``/`` (division) and ``//`` (floor division) operators yield the quotient of their arguments. The numeric arguments are first converted to a common type. @@ -1305,7 +1305,7 @@ This operation can be customized using the special :meth:`__truediv__` and .. index:: single: modulo - operator: % (percent) + pair: operator; % (percent) The ``%`` (modulo) operator yields the remainder from the division of the first argument by the second. The numeric arguments are first converted to a common @@ -1363,8 +1363,8 @@ Shifting operations .. index:: pair: shifting; operation - operator: << - operator: >> + pair: operator; << + pair: operator; >> The shifting operations have lower priority than the arithmetic operations: @@ -1399,7 +1399,7 @@ Each of the three bitwise operations has a different priority level: .. index:: pair: bitwise; and - operator: & (ampersand) + pair: operator; & (ampersand) The ``&`` operator yields the bitwise AND of its arguments, which must be integers or one of them must be a custom object overriding :meth:`__and__` or @@ -1408,7 +1408,7 @@ integers or one of them must be a custom object overriding :meth:`__and__` or .. index:: pair: bitwise; xor pair: exclusive; or - operator: ^ (caret) + pair: operator; ^ (caret) The ``^`` operator yields the bitwise XOR (exclusive OR) of its arguments, which must be integers or one of them must be a custom object overriding :meth:`__xor__` or @@ -1417,7 +1417,7 @@ must be integers or one of them must be a custom object overriding :meth:`__xor_ .. index:: pair: bitwise; or pair: inclusive; or - operator: | (vertical bar) + pair: operator; | (vertical bar) The ``|`` operator yields the bitwise (inclusive) OR of its arguments, which must be integers or one of them must be a custom object overriding :meth:`__or__` or @@ -1432,12 +1432,12 @@ Comparisons .. index:: single: comparison pair: C; language - operator: < (less) - operator: > (greater) - operator: <= - operator: >= - operator: == - operator: != + pair: operator; < (less) + pair: operator; > (greater) + pair: operator; <= + pair: operator; >= + pair: operator; == + pair: operator; != Unlike C, all comparison operations in Python have the same priority, which is lower than that of any arithmetic, shifting or bitwise operation. Also unlike @@ -1669,8 +1669,8 @@ raises the :exc:`IndexError` exception. (If any other exception is raised, it i if :keyword:`in` raised that exception). .. index:: - operator: in - operator: not in + pair: operator; in + pair: operator; not in pair: membership; test object: sequence @@ -1678,8 +1678,8 @@ The operator :keyword:`not in` is defined to have the inverse truth value of :keyword:`in`. .. index:: - operator: is - operator: is not + pair: operator; is + pair: operator; is not pair: identity; test @@ -1719,17 +1719,17 @@ control flow statements, the following values are interpreted as false: other values are interpreted as true. User-defined objects can customize their truth value by providing a :meth:`__bool__` method. -.. index:: operator: not +.. index:: pair: operator; not The operator :keyword:`not` yields ``True`` if its argument is false, ``False`` otherwise. -.. index:: operator: and +.. index:: pair: operator; and The expression ``x and y`` first evaluates *x*; if *x* is false, its value is returned; otherwise, *y* is evaluated and the resulting value is returned. -.. index:: operator: or +.. index:: pair: operator; or The expression ``x or y`` first evaluates *x*; if *x* is true, its value is returned; otherwise, *y* is evaluated and the resulting value is returned. diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index c347a327e0910e..8ca3ef04523a82 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -691,7 +691,7 @@ def patch_pairindextypes(app) -> None: pairindextypes.pop('module', None) pairindextypes.pop('keyword', None) - # pairindextypes.pop('operator', None) + pairindextypes.pop('operator', None) # pairindextypes.pop('object', None) # pairindextypes.pop('exception', None) # pairindextypes.pop('statement', None) From 6ab463684b9d79880d98cd1f1406aa86af65985e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 4 May 2023 11:04:41 +0100 Subject: [PATCH 029/204] GH-97950: Use new-style index directive ('object') (#104158) * Uncomment object removal in pairindextypes * Use new-style index directive ('object') - C API * Use new-style index directive ('object') - Library * Use new-style index directive ('object') - Reference * Use new-style index directive ('object') - Tutorial --- Doc/c-api/bytearray.rst | 2 +- Doc/c-api/bytes.rst | 2 +- Doc/c-api/capsule.rst | 2 +- Doc/c-api/complex.rst | 2 +- Doc/c-api/concrete.rst | 6 +- Doc/c-api/dict.rst | 2 +- Doc/c-api/file.rst | 2 +- Doc/c-api/float.rst | 2 +- Doc/c-api/function.rst | 2 +- Doc/c-api/intro.rst | 2 +- Doc/c-api/list.rst | 2 +- Doc/c-api/long.rst | 4 +- Doc/c-api/memoryview.rst | 2 +- Doc/c-api/method.rst | 4 +- Doc/c-api/module.rst | 2 +- Doc/c-api/none.rst | 2 +- Doc/c-api/set.rst | 4 +- Doc/c-api/tuple.rst | 2 +- Doc/c-api/type.rst | 2 +- Doc/library/functions.rst | 2 +- Doc/library/socket.rst | 2 +- Doc/library/stdtypes.rst | 50 ++++++++--------- Doc/library/sys.rst | 2 +- Doc/library/traceback.rst | 2 +- Doc/reference/compound_stmts.rst | 10 ++-- Doc/reference/datamodel.rst | 88 +++++++++++++++--------------- Doc/reference/expressions.rst | 60 ++++++++++---------- Doc/reference/simple_stmts.rst | 16 +++--- Doc/tools/extensions/pyspecific.py | 2 +- Doc/tutorial/classes.rst | 2 +- Doc/tutorial/inputoutput.rst | 2 +- 31 files changed, 143 insertions(+), 143 deletions(-) diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index 4bf3cfe100cd01..456f7d89bca03c 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -5,7 +5,7 @@ Byte Array Objects ------------------ -.. index:: object: bytearray +.. index:: pair: object; bytearray .. c:type:: PyByteArrayObject diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index d62962cab45f6b..9f48f2ffafe170 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -8,7 +8,7 @@ Bytes Objects These functions raise :exc:`TypeError` when expecting a bytes parameter and called with a non-bytes parameter. -.. index:: object: bytes +.. index:: pair: object; bytes .. c:type:: PyBytesObject diff --git a/Doc/c-api/capsule.rst b/Doc/c-api/capsule.rst index 1c8f432505ef68..427ed959c58568 100644 --- a/Doc/c-api/capsule.rst +++ b/Doc/c-api/capsule.rst @@ -5,7 +5,7 @@ Capsules -------- -.. index:: object: Capsule +.. index:: pair: object; Capsule Refer to :ref:`using-capsules` for more information on using these objects. diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index 9228ce85200023..344da903da4c1a 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -5,7 +5,7 @@ Complex Number Objects ---------------------- -.. index:: object: complex number +.. index:: pair: object; complex number Python's complex number objects are implemented as two distinct types when viewed from the C API: one is the Python object exposed to Python programs, and diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst index 8d3124a12fa9d2..880f7b15ce68e8 100644 --- a/Doc/c-api/concrete.rst +++ b/Doc/c-api/concrete.rst @@ -40,7 +40,7 @@ This section describes Python type objects and the singleton object ``None``. Numeric Objects =============== -.. index:: object: numeric +.. index:: pair: object; numeric .. toctree:: @@ -55,7 +55,7 @@ Numeric Objects Sequence Objects ================ -.. index:: object: sequence +.. index:: pair: object; sequence Generic operations on sequence objects were discussed in the previous chapter; this section deals with the specific kinds of sequence objects that are @@ -77,7 +77,7 @@ intrinsic to the Python language. Container Objects ================= -.. index:: object: mapping +.. index:: pair: object; mapping .. toctree:: diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index b9f84cea785644..f02abb01f0220b 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -5,7 +5,7 @@ Dictionary Objects ------------------ -.. index:: object: dictionary +.. index:: pair: object; dictionary .. c:type:: PyDictObject diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index 58ed58e5466859..f32ecba9f27029 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -5,7 +5,7 @@ File Objects ------------ -.. index:: object: file +.. index:: pair: object; file These APIs are a minimal emulation of the Python 2 C API for built-in file objects, which used to rely on the buffered I/O (:c:expr:`FILE*`) support diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 023b12c20b7c83..05b2d100d575cb 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -5,7 +5,7 @@ Floating Point Objects ---------------------- -.. index:: object: floating point +.. index:: pair: object; floating point .. c:type:: PyFloatObject diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 947ed70404081b..5857dba82c11c6 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -5,7 +5,7 @@ Function Objects ---------------- -.. index:: object: function +.. index:: pair: object; function There are a few functions specific to Python functions. diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 1a200b919f0e75..8de76e55cd0586 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -261,7 +261,7 @@ complete listing. Objects, Types and Reference Counts =================================== -.. index:: object: type +.. index:: pair: object; type Most Python/C API functions have one or more arguments as well as a return value of type :c:expr:`PyObject*`. This type is a pointer to an opaque data type diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index f9e65354a259f4..317421f0db8453 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -5,7 +5,7 @@ List Objects ------------ -.. index:: object: list +.. index:: pair: object; list .. c:type:: PyListObject diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 41b5632d23003f..4a71c89ad85d31 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -5,8 +5,8 @@ Integer Objects --------------- -.. index:: object: long integer - object: integer +.. index:: pair: object; long integer + pair: object; integer All integers are implemented as "long" integer objects of arbitrary size. diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst index ebd5c7760437bf..2aa43318e7a455 100644 --- a/Doc/c-api/memoryview.rst +++ b/Doc/c-api/memoryview.rst @@ -3,7 +3,7 @@ .. _memoryview-objects: .. index:: - object: memoryview + pair: object; memoryview MemoryView objects ------------------ diff --git a/Doc/c-api/method.rst b/Doc/c-api/method.rst index 6e7e1e21aa93f2..93ad30cd4f7a8d 100644 --- a/Doc/c-api/method.rst +++ b/Doc/c-api/method.rst @@ -5,7 +5,7 @@ Instance Method Objects ----------------------- -.. index:: object: instancemethod +.. index:: pair: object; instancemethod An instance method is a wrapper for a :c:data:`PyCFunction` and the new way to bind a :c:data:`PyCFunction` to a class object. It replaces the former call @@ -47,7 +47,7 @@ to bind a :c:data:`PyCFunction` to a class object. It replaces the former call Method Objects -------------- -.. index:: object: method +.. index:: pair: object; method Methods are bound function objects. Methods are always bound to an instance of a user-defined class. Unbound methods (methods bound to a class object) are diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index c0351c8a6c72aa..230b471d473be7 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -5,7 +5,7 @@ Module Objects -------------- -.. index:: object: module +.. index:: pair: object; module .. c:var:: PyTypeObject PyModule_Type diff --git a/Doc/c-api/none.rst b/Doc/c-api/none.rst index 26d2b7aab201ba..b84a16a28ead56 100644 --- a/Doc/c-api/none.rst +++ b/Doc/c-api/none.rst @@ -5,7 +5,7 @@ The ``None`` Object ------------------- -.. index:: object: None +.. index:: pair: object; None Note that the :c:type:`PyTypeObject` for ``None`` is not directly exposed in the Python/C API. Since ``None`` is a singleton, testing for object identity (using diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index f0d905bae8ae44..8e8af60252250e 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -9,8 +9,8 @@ Set Objects .. index:: - object: set - object: frozenset + pair: object; set + pair: object; frozenset This section details the public API for :class:`set` and :class:`frozenset` objects. Any functionality not listed below is best accessed using either diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index 5acddf7849aa33..ac62058676eeeb 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -5,7 +5,7 @@ Tuple Objects ------------- -.. index:: object: tuple +.. index:: pair: object; tuple .. c:type:: PyTupleObject diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index c21fd9255d28ad..fb38935e003336 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -5,7 +5,7 @@ Type Objects ------------ -.. index:: object: type +.. index:: pair: object; type .. c:type:: PyTypeObject diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index ddb918de115fcf..5a022edb75b7d1 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1830,7 +1830,7 @@ are always available. They are listed here in alphabetical order. .. class:: type(object) type(name, bases, dict, **kwds) - .. index:: object: type + .. index:: pair: object; type With one argument, return the type of an *object*. The return value is a type object and generally the same object as returned by diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 150c1f93afd967..4ee0897db94063 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -19,7 +19,7 @@ all modern Unix systems, Windows, MacOS, and probably additional platforms. .. include:: ../includes/wasm-notavail.rst -.. index:: object: socket +.. index:: pair: object; socket The Python interface is a straightforward transliteration of the Unix system call and library interface for sockets to Python's object-oriented style: the diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 46b0389adaa88f..2f549f2e97c1de 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -205,11 +205,11 @@ Numeric Types --- :class:`int`, :class:`float`, :class:`complex` ================================================================ .. index:: - object: numeric - object: Boolean - object: integer - object: floating point - object: complex number + pair: object; numeric + pair: object; Boolean + pair: object; integer + pair: object; floating point + pair: object; complex number pair: C; language There are three distinct numeric types: :dfn:`integers`, :dfn:`floating @@ -927,7 +927,7 @@ described in dedicated sections. Common Sequence Operations -------------------------- -.. index:: object: sequence +.. index:: pair: object; sequence The operations in the following table are supported by most sequence types, both mutable and immutable. The :class:`collections.abc.Sequence` ABC is @@ -1112,7 +1112,7 @@ Immutable Sequence Types .. index:: triple: immutable; sequence; types - object: tuple + pair: object; tuple builtin: hash The only operation that immutable sequence types generally implement that is @@ -1134,8 +1134,8 @@ Mutable Sequence Types .. index:: triple: mutable; sequence; types - object: list - object: bytearray + pair: object; list + pair: object; bytearray The operations in the following table are defined on mutable sequence types. The :class:`collections.abc.MutableSequence` ABC is provided to make it @@ -1252,7 +1252,7 @@ Notes: Lists ----- -.. index:: object: list +.. index:: pair: object; list Lists are mutable sequences, typically used to store collections of homogeneous items (where the precise degree of similarity will vary by @@ -1331,7 +1331,7 @@ application). Tuples ------ -.. index:: object: tuple +.. index:: pair: object; tuple Tuples are immutable sequences, typically used to store collections of heterogeneous data (such as the 2-tuples produced by the :func:`enumerate` @@ -1375,7 +1375,7 @@ choice than a simple tuple object. Ranges ------ -.. index:: object: range +.. index:: pair: object; range The :class:`range` type represents an immutable sequence of numbers and is commonly used for looping a specific number of times in :keyword:`for` @@ -1500,7 +1500,7 @@ objects that compare equal might have different :attr:`~range.start`, .. index:: single: string; text sequence type single: str (built-in class); (see also string) - object: string + pair: object; string .. _textseq: @@ -1534,7 +1534,7 @@ Since there is no separate "character" type, indexing a string produces strings of length 1. That is, for a non-empty string *s*, ``s[0] == s[0:1]``. .. index:: - object: io.StringIO + pair: object; io.StringIO There is also no mutable string type, but :meth:`str.join` or :class:`io.StringIO` can be used to efficiently construct strings from @@ -2508,9 +2508,9 @@ Binary Sequence Types --- :class:`bytes`, :class:`bytearray`, :class:`memoryview ================================================================================= .. index:: - object: bytes - object: bytearray - object: memoryview + pair: object; bytes + pair: object; bytearray + pair: object; memoryview pair: module; array The core built-in types for manipulating binary data are :class:`bytes` and @@ -2526,7 +2526,7 @@ The :mod:`array` module supports efficient storage of basic data types like Bytes Objects ------------- -.. index:: object: bytes +.. index:: pair: object; bytes Bytes objects are immutable sequences of single bytes. Since many major binary protocols are based on the ASCII text encoding, bytes objects offer @@ -2633,7 +2633,7 @@ always convert a bytes object into a list of integers using ``list(b)``. Bytearray Objects ----------------- -.. index:: object: bytearray +.. index:: pair: object; bytearray :class:`bytearray` objects are a mutable counterpart to :class:`bytes` objects. @@ -4212,7 +4212,7 @@ copying. Set Types --- :class:`set`, :class:`frozenset` ============================================== -.. index:: object: set +.. index:: pair: object; set A :dfn:`set` object is an unordered collection of distinct :term:`hashable` objects. Common uses include membership testing, removing duplicates from a sequence, and @@ -4414,8 +4414,8 @@ Mapping Types --- :class:`dict` =============================== .. index:: - object: mapping - object: dictionary + pair: object; mapping + pair: object; dictionary triple: operations on; mapping; types triple: operations on; dictionary; type statement: del @@ -4889,7 +4889,7 @@ Generic Alias Type ------------------ .. index:: - object: GenericAlias + pair: object; GenericAlias pair: Generic; Alias ``GenericAlias`` objects are generally created by @@ -5144,7 +5144,7 @@ Union Type ---------- .. index:: - object: Union + pair: object; Union pair: union; type A union object holds the value of the ``|`` (bitwise or) operation on @@ -5301,7 +5301,7 @@ See :ref:`function` for more information. Methods ------- -.. index:: object: method +.. index:: pair: object; method Methods are functions that are called using the attribute notation. There are two flavors: built-in methods (such as :meth:`append` on lists) and class diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 57a0d0a1258c2a..bacf8ceac5041e 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -444,7 +444,7 @@ always available. object ` which typically encapsulates the call stack at the point where the exception last occurred. - .. index:: object: traceback + .. index:: pair: object; traceback If no exception is being handled anywhere on the stack, this function return a tuple containing three ``None`` values. diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 561c85290463ef..5c0e261b90763c 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -14,7 +14,7 @@ interpreter when it prints a stack trace. This is useful when you want to print stack traces under program control, such as in a "wrapper" around the interpreter. -.. index:: object: traceback +.. index:: pair: object; traceback The module uses traceback objects --- these are objects of type :class:`types.TracebackType`, which are assigned to the ``__traceback__`` field of :class:`BaseException` instances. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 1f03180f1565ba..5750d3149c8e95 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -147,7 +147,7 @@ The :keyword:`!for` statement pair: keyword; else pair: target; list pair: loop; statement - object: sequence + pair: object; sequence single: : (colon); compound statement The :keyword:`for` statement is used to iterate over the elements of a sequence @@ -298,7 +298,7 @@ keeping all locals in that frame alive until the next garbage collection occurs. .. index:: pair: module; sys - object: traceback + pair: object; traceback Before an :keyword:`!except` clause's suite is executed, the exception is stored in the :mod:`sys` module, where it can be accessed @@ -1194,8 +1194,8 @@ Function definitions pair: function; definition pair: function; name pair: name; binding - object: user-defined function - object: function + pair: object; user-defined function + pair: object; function pair: function; name pair: name; binding single: () (parentheses); function definition @@ -1363,7 +1363,7 @@ Class definitions ================= .. index:: - object: class + pair: object; class statement: class pair: class; definition pair: class; name diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 9d3ea3f78de16f..18906a005b2761 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -142,7 +142,7 @@ attributes.' These are attributes that provide access to the implementation and are not intended for general use. Their definition may change in the future. None - .. index:: object: None + .. index:: pair: object; None This type has a single value. There is a single object with this value. This object is accessed through the built-in name ``None``. It is used to signify the @@ -150,7 +150,7 @@ None don't explicitly return anything. Its truth value is false. NotImplemented - .. index:: object: NotImplemented + .. index:: pair: object; NotImplemented This type has a single value. There is a single object with this value. This object is accessed through the built-in name ``NotImplemented``. Numeric methods @@ -171,7 +171,7 @@ NotImplemented Ellipsis .. index:: - object: Ellipsis + pair: object; Ellipsis single: ...; ellipsis literal This type has a single value. There is a single object with this value. This @@ -179,7 +179,7 @@ Ellipsis ``Ellipsis``. Its truth value is true. :class:`numbers.Number` - .. index:: object: numeric + .. index:: pair: object; numeric These are created by numeric literals and returned as results by arithmetic operators and arithmetic built-in functions. Numeric objects are immutable; @@ -209,7 +209,7 @@ Ellipsis numbers: :class:`numbers.Integral` - .. index:: object: integer + .. index:: pair: object; integer These represent elements from the mathematical set of integers (positive and negative). @@ -225,7 +225,7 @@ Ellipsis Booleans (:class:`bool`) .. index:: - object: Boolean + pair: object; Boolean single: False single: True @@ -242,7 +242,7 @@ Ellipsis :class:`numbers.Real` (:class:`float`) .. index:: - object: floating point + pair: object; floating point pair: floating point; number pair: C; language pair: Java; language @@ -257,7 +257,7 @@ Ellipsis :class:`numbers.Complex` (:class:`complex`) .. index:: - object: complex + pair: object; complex pair: complex; number These represent complex numbers as a pair of machine-level double precision @@ -268,7 +268,7 @@ Ellipsis Sequences .. index:: builtin: len - object: sequence + pair: object; sequence single: index operation single: item selection single: subscription @@ -293,8 +293,8 @@ Sequences Immutable sequences .. index:: - object: immutable sequence - object: immutable + pair: object; immutable sequence + pair: object; immutable An object of an immutable sequence type cannot change once it is created. (If the object contains references to other objects, these other objects may be @@ -328,7 +328,7 @@ Sequences Tuples .. index:: - object: tuple + pair: object; tuple pair: singleton; tuple pair: empty; tuple @@ -350,8 +350,8 @@ Sequences Mutable sequences .. index:: - object: mutable sequence - object: mutable + pair: object; mutable sequence + pair: object; mutable pair: assignment; statement single: subscription single: slicing @@ -363,7 +363,7 @@ Sequences There are currently two intrinsic mutable sequence types: Lists - .. index:: object: list + .. index:: pair: object; list The items of a list are arbitrary Python objects. Lists are formed by placing a comma-separated list of expressions in square brackets. (Note @@ -385,7 +385,7 @@ Sequences Set types .. index:: builtin: len - object: set type + pair: object; set type These represent unordered, finite sets of unique, immutable objects. As such, they cannot be indexed by any subscript. However, they can be iterated over, and @@ -402,14 +402,14 @@ Set types There are currently two intrinsic set types: Sets - .. index:: object: set + .. index:: pair: object; set These represent a mutable set. They are created by the built-in :func:`set` constructor and can be modified afterwards by several methods, such as :meth:`~set.add`. Frozen sets - .. index:: object: frozenset + .. index:: pair: object; frozenset These represent an immutable set. They are created by the built-in :func:`frozenset` constructor. As a frozenset is immutable and @@ -420,7 +420,7 @@ Mappings .. index:: builtin: len single: subscription - object: mapping + pair: object; mapping These represent finite sets of objects indexed by arbitrary index sets. The subscript notation ``a[k]`` selects the item indexed by ``k`` from the mapping @@ -431,7 +431,7 @@ Mappings There is currently a single intrinsic mapping type: Dictionaries - .. index:: object: dictionary + .. index:: pair: object; dictionary These represent finite sets of objects indexed by nearly arbitrary values. The only types of values not acceptable as keys are values containing lists or @@ -465,7 +465,7 @@ Mappings Callable types .. index:: - object: callable + pair: object; callable pair: function; call single: invocation pair: function; argument @@ -476,8 +476,8 @@ Callable types User-defined functions .. index:: pair: user-defined; function - object: function - object: user-defined function + pair: object; function + pair: object; user-defined function A user-defined function object is created by a function definition (see section :ref:`function`). It should be called with an argument list @@ -580,8 +580,8 @@ Callable types Instance methods .. index:: - object: method - object: user-defined method + pair: object; method + pair: object; user-defined method pair: user-defined; method An instance method object combines a class, a class instance and any @@ -688,8 +688,8 @@ Callable types Built-in functions .. index:: - object: built-in function - object: function + pair: object; built-in function + pair: object; function pair: C; language A built-in function object is a wrapper around a C function. Examples of @@ -703,8 +703,8 @@ Callable types Built-in methods .. index:: - object: built-in method - object: method + pair: object; built-in method + pair: object; method pair: built-in; method This is really a different disguise of a built-in function, this time containing @@ -728,7 +728,7 @@ Callable types Modules .. index:: statement: import - object: module + pair: object; module Modules are a basic organizational unit of Python code, and are created by the :ref:`import system ` as invoked either by the @@ -805,12 +805,12 @@ Custom classes .. XXX: Could we add that MRO doc as an appendix to the language ref? .. index:: - object: class - object: class instance - object: instance + pair: object; class + pair: object; class instance + pair: object; instance pair: class object; call single: container - object: dictionary + pair: object; dictionary pair: class; attribute When a class attribute reference (for class :class:`C`, say) would yield a @@ -865,8 +865,8 @@ Custom classes Class instances .. index:: - object: class instance - object: instance + pair: object; class instance + pair: object; instance pair: class; instance pair: class instance; attribute @@ -892,9 +892,9 @@ Class instances dictionary directly. .. index:: - object: numeric - object: sequence - object: mapping + pair: object; numeric + pair: object; sequence + pair: object; mapping Class instances can pretend to be numbers, sequences, or mappings if they have methods with certain special names. See section :ref:`specialnames`. @@ -996,7 +996,7 @@ Internal types required stack size; :attr:`co_flags` is an integer encoding a number of flags for the interpreter. - .. index:: object: generator + .. index:: pair: object; generator The following flag bits are defined for :attr:`co_flags`: bit ``0x04`` is set if the function uses the ``*arguments`` syntax to accept an arbitrary number of @@ -1053,7 +1053,7 @@ Internal types .. _frame-objects: Frame objects - .. index:: object: frame + .. index:: pair: object; frame Frame objects represent execution frames. They may occur in traceback objects (see below), and are also passed to registered trace functions. @@ -1116,7 +1116,7 @@ Internal types Traceback objects .. index:: - object: traceback + pair: object; traceback pair: stack; trace pair: exception; handler pair: execution; stack @@ -1498,7 +1498,7 @@ Basic customization .. method:: object.__hash__(self) .. index:: - object: dictionary + pair: object; dictionary builtin: hash Called by built-in function :func:`hash` and for operations on members of @@ -2506,7 +2506,7 @@ through the object's keys; for sequences, it should iterate through the values. .. versionadded:: 3.4 -.. index:: object: slice +.. index:: pair: object; slice .. note:: diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index b1bb162f62e1ec..78185657bf822e 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -245,7 +245,7 @@ List displays pair: list; display pair: list; comprehensions pair: empty; list - object: list + pair: object; list single: [] (square brackets); list expression single: , (comma); expression list @@ -270,7 +270,7 @@ Set displays .. index:: pair: set; display pair: set; comprehensions - object: set + pair: object; set single: {} (curly brackets); set expression single: , (comma); expression list @@ -299,7 +299,7 @@ Dictionary displays pair: dictionary; display pair: dictionary; comprehensions key, datum, key/datum pair - object: dictionary + pair: object; dictionary single: {} (curly brackets); dictionary expression single: : (colon); in dictionary expressions single: , (comma); in dictionary displays @@ -361,7 +361,7 @@ Generator expressions .. index:: pair: generator; expression - object: generator + pair: object; generator single: () (parentheses); generator expression A generator expression is a compact generator notation in parentheses: @@ -522,7 +522,7 @@ on the right hand side of an assignment statement. The proposal that expanded on :pep:`492` by adding generator capabilities to coroutine functions. -.. index:: object: generator +.. index:: pair: object; generator .. _generator-methods: Generator-iterator methods @@ -701,7 +701,7 @@ of a *finalizer* method see the implementation of The expression ``yield from `` is a syntax error when used in an asynchronous generator function. -.. index:: object: asynchronous-generator +.. index:: pair: object; asynchronous-generator .. _asynchronous-generator-methods: Asynchronous generator-iterator methods @@ -811,8 +811,8 @@ An attribute reference is a primary followed by a period and a name: .. index:: exception: AttributeError - object: module - object: list + pair: object; module + pair: object; list The primary must evaluate to an object of a type that supports attribute references, which most objects do. This object is then asked to produce the @@ -833,12 +833,12 @@ Subscriptions single: [] (square brackets); subscription .. index:: - object: sequence - object: mapping - object: string - object: tuple - object: list - object: dictionary + pair: object; sequence + pair: object; mapping + pair: object; string + pair: object; tuple + pair: object; list + pair: object; dictionary pair: sequence; item The subscription of an instance of a :ref:`container class ` @@ -906,10 +906,10 @@ Slicings single: , (comma); slicing .. index:: - object: sequence - object: string - object: tuple - object: list + pair: object; sequence + pair: object; string + pair: object; tuple + pair: object; list A slicing selects a range of items in a sequence object (e.g., a string, tuple or list). Slicings may be used as expressions or as targets in assignment or @@ -950,7 +950,7 @@ substituting ``None`` for missing expressions. .. index:: - object: callable + pair: object; callable single: call single: argument; call semantics single: () (parentheses); call @@ -1100,8 +1100,8 @@ a user-defined function: .. index:: pair: function; call triple: user-defined; function; call - object: user-defined function - object: function + pair: object; user-defined function + pair: object; function The code block for the function is executed, passing it the argument list. The first thing the code block will do is bind the formal parameters to the @@ -1115,25 +1115,25 @@ a built-in function or method: pair: built-in function; call pair: method; call pair: built-in method; call - object: built-in method - object: built-in function - object: method - object: function + pair: object; built-in method + pair: object; built-in function + pair: object; method + pair: object; function The result is up to the interpreter; see :ref:`built-in-funcs` for the descriptions of built-in functions and methods. a class object: .. index:: - object: class + pair: object; class pair: class object; call A new instance of that class is returned. a class instance method: .. index:: - object: class instance - object: instance + pair: object; class instance + pair: object; instance pair: class instance; call The corresponding user-defined function is called, with an argument list that is @@ -1672,7 +1672,7 @@ if :keyword:`in` raised that exception). pair: operator; in pair: operator; not in pair: membership; test - object: sequence + pair: object; sequence The operator :keyword:`not in` is defined to have the inverse truth value of :keyword:`in`. @@ -1854,7 +1854,7 @@ Expression lists starred_expression: `expression` | (`starred_item` ",")* [`starred_item`] starred_item: `assignment_expression` | "*" `or_expr` -.. index:: object: tuple +.. index:: pair: object; tuple Except when part of a list or set display, an expression list containing at least one comma yields a tuple. The length of diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 5c7b5d7efaa2b5..acc3998e09be22 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -54,7 +54,7 @@ expression). .. index:: builtin: repr - object: None + pair: object; None pair: string; conversion single: output pair: standard; output @@ -76,7 +76,7 @@ Assignment statements pair: assignment; statement pair: binding; name pair: rebinding; name - object: mutable + pair: object; mutable pair: attribute; assignment Assignment statements are used to (re)bind names to values and to modify @@ -185,7 +185,7 @@ Assignment of an object to a single target is recursively defined as follows. .. index:: pair: subscription; assignment - object: mutable + pair: object; mutable * If the target is a subscription: The primary expression in the reference is evaluated. It should yield either a mutable sequence object (such as a list) @@ -193,8 +193,8 @@ Assignment of an object to a single target is recursively defined as follows. evaluated. .. index:: - object: sequence - object: list + pair: object; sequence + pair: object; list If the primary is a mutable sequence object (such as a list), the subscript must yield an integer. If it is negative, the sequence's length is added to @@ -204,8 +204,8 @@ Assignment of an object to a single target is recursively defined as follows. raised (assignment to a subscripted sequence cannot add new items to a list). .. index:: - object: mapping - object: dictionary + pair: object; mapping + pair: object; dictionary If the primary is a mapping object (such as a dictionary), the subscript must have a type compatible with the mapping's key type, and the mapping is then @@ -574,7 +574,7 @@ instantiating the class with no arguments. The :dfn:`type` of the exception is the exception instance's class, the :dfn:`value` is the instance itself. -.. index:: object: traceback +.. index:: pair: object; traceback A traceback object is normally created automatically when an exception is raised and attached to it as the :attr:`__traceback__` attribute, which is writable. diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 8ca3ef04523a82..41016113d25732 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -692,7 +692,7 @@ def patch_pairindextypes(app) -> None: pairindextypes.pop('module', None) pairindextypes.pop('keyword', None) pairindextypes.pop('operator', None) - # pairindextypes.pop('object', None) + pairindextypes.pop('object', None) # pairindextypes.pop('exception', None) # pairindextypes.pop('statement', None) # pairindextypes.pop('builtin', None) diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 116801177a3add..06445e000c1ef6 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -344,7 +344,7 @@ list objects have methods called append, insert, remove, sort, and so on. However, in the following discussion, we'll use the term method exclusively to mean methods of class instance objects, unless explicitly stated otherwise.) -.. index:: object: method +.. index:: pair: object; method Valid method names of an instance object depend on its class. By definition, all attributes of a class that are function objects define corresponding diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index aeacc1559a228c..3fcf4e3f43a300 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -286,7 +286,7 @@ Reading and Writing Files .. index:: builtin: open - object: file + pair: object; file :func:`open` returns a :term:`file object`, and is most commonly used with two positional arguments and one keyword argument: From 83751bbd142c23ca3f6af34ec617630dc3173b2a Mon Sep 17 00:00:00 2001 From: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> Date: Thu, 4 May 2023 18:20:20 +0800 Subject: [PATCH 030/204] gh-104089: catch DeprecationWarning in `test_fstring` (#104137) Co-authored-by: Alex Waygood --- Lib/test/test_fstring.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index be71fde5aaba54..58e2550715cecf 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -980,11 +980,18 @@ def test_roundtrip_raw_quotes(self): self.assertEqual(fr'\"\'\"\'', '\\"\\\'\\"\\\'') def test_fstring_backslash_before_double_bracket(self): - self.assertEqual(f'\{{\}}', '\\{\\}') - self.assertEqual(f'\{{', '\\{') - self.assertEqual(f'\{{{1+1}', '\\{2') - self.assertEqual(f'\}}{1+1}', '\\}2') - self.assertEqual(f'{1+1}\}}', '2\\}') + deprecated_cases = [ + (r"f'\{{\}}'", '\\{\\}'), + (r"f'\{{'", '\\{'), + (r"f'\{{{1+1}'", '\\{2'), + (r"f'\}}{1+1}'", '\\}2'), + (r"f'{1+1}\}}'", '2\\}') + ] + for case, expected_result in deprecated_cases: + with self.subTest(case=case, expected_result=expected_result): + with self.assertWarns(DeprecationWarning): + result = eval(case) + self.assertEqual(result, expected_result) self.assertEqual(fr'\{{\}}', '\\{\\}') self.assertEqual(fr'\{{', '\\{') self.assertEqual(fr'\{{{1+1}', '\\{2') From 3e7995ca119d4253d05a520f35ab3efca793ee49 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 4 May 2023 11:30:24 +0100 Subject: [PATCH 031/204] GH-97950: Use new-style index directive ('exception') (#104160) --- Doc/reference/expressions.rst | 18 +++++++++--------- Doc/reference/simple_stmts.rst | 6 +++--- Doc/tools/extensions/pyspecific.py | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 78185657bf822e..b97a08f25d92a2 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -71,7 +71,7 @@ An identifier occurring as an atom is a name. See section :ref:`identifiers` for lexical definition and section :ref:`naming` for documentation of naming and binding. -.. index:: exception: NameError +.. index:: pair: exception; NameError When the name is bound to an object, evaluation of the atom yields that object. When a name is not bound, an attempt to evaluate it raises a :exc:`NameError` @@ -534,7 +534,7 @@ be used to control the execution of a generator function. Note that calling any of the generator methods below when the generator is already executing raises a :exc:`ValueError` exception. -.. index:: exception: StopIteration +.. index:: pair: exception; StopIteration .. method:: generator.__next__() @@ -589,7 +589,7 @@ is already executing raises a :exc:`ValueError` exception. The second signature \(type\[, value\[, traceback\]\]\) is deprecated and may be removed in a future version of Python. -.. index:: exception: GeneratorExit +.. index:: pair: exception; GeneratorExit .. method:: generator.close() @@ -711,7 +711,7 @@ This subsection describes the methods of an asynchronous generator iterator, which are used to control the execution of a generator function. -.. index:: exception: StopAsyncIteration +.. index:: pair: exception; StopAsyncIteration .. coroutinemethod:: agen.__anext__() @@ -763,7 +763,7 @@ which are used to control the execution of a generator function. The second signature \(type\[, value\[, traceback\]\]\) is deprecated and may be removed in a future version of Python. -.. index:: exception: GeneratorExit +.. index:: pair: exception; GeneratorExit .. coroutinemethod:: agen.aclose() @@ -810,7 +810,7 @@ An attribute reference is a primary followed by a period and a name: attributeref: `primary` "." `identifier` .. index:: - exception: AttributeError + pair: exception; AttributeError pair: object; module pair: object; list @@ -1241,7 +1241,7 @@ applies to integral numbers or to custom objects that override the -.. index:: exception: TypeError +.. index:: pair: exception; TypeError In all three cases, if the argument does not have the proper type, a :exc:`TypeError` exception is raised. @@ -1288,7 +1288,7 @@ builtin Python types implement this operator. .. versionadded:: 3.5 .. index:: - exception: ZeroDivisionError + pair: exception; ZeroDivisionError single: division pair: operator; / (slash) pair: operator; // @@ -1377,7 +1377,7 @@ the left or right by the number of bits given by the second argument. This operation can be customized using the special :meth:`__lshift__` and :meth:`__rshift__` methods. -.. index:: exception: ValueError +.. index:: pair: exception; ValueError A right shift by *n* bits is defined as floor division by ``pow(2,n)``. A left shift by *n* bits is defined as multiplication with ``pow(2,n)``. diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index acc3998e09be22..7cf096019631da 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -398,7 +398,7 @@ The extended form, ``assert expression1, expression2``, is equivalent to :: .. index:: single: __debug__ - exception: AssertionError + pair: exception; AssertionError These equivalences assume that :const:`__debug__` and :exc:`AssertionError` refer to the built-in variables with those names. In the current implementation, the @@ -521,7 +521,7 @@ The :keyword:`!yield` statement single: generator; function single: generator; iterator single: function; generator - exception: StopIteration + pair: exception; StopIteration .. productionlist:: python-grammar yield_stmt: `yield_expression` @@ -731,7 +731,7 @@ The :keyword:`!import` statement pair: name; binding pair: keyword; from pair: keyword; as - exception: ImportError + pair: exception; ImportError single: , (comma); import statement .. productionlist:: python-grammar diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 41016113d25732..9a62ae748e0744 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -693,7 +693,7 @@ def patch_pairindextypes(app) -> None: pairindextypes.pop('keyword', None) pairindextypes.pop('operator', None) pairindextypes.pop('object', None) - # pairindextypes.pop('exception', None) + pairindextypes.pop('exception', None) # pairindextypes.pop('statement', None) # pairindextypes.pop('builtin', None) From 55d50d147c953fab37b273bca9ab010f40e067d3 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 4 May 2023 11:48:45 +0100 Subject: [PATCH 032/204] GH-97950: Use new-style index directive ('statement') (#104162) --- Doc/library/exceptions.rst | 8 +++--- Doc/library/functions.rst | 2 +- Doc/library/site.rst | 2 +- Doc/library/stdtypes.rst | 8 +++--- Doc/reference/compound_stmts.rst | 42 +++++++++++++++--------------- Doc/reference/datamodel.rst | 8 +++--- Doc/reference/simple_stmts.rst | 32 +++++++++++------------ Doc/tools/extensions/pyspecific.py | 2 +- Doc/tutorial/controlflow.rst | 2 +- 9 files changed, 53 insertions(+), 53 deletions(-) diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index aee1cb5cc6a475..4c84e5f855431a 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -4,8 +4,8 @@ Built-in Exceptions =================== .. index:: - statement: try - statement: except + pair: statement; try + pair: statement; except In Python, all exceptions must be instances of a class that derives from :class:`BaseException`. In a :keyword:`try` statement with an :keyword:`except` @@ -14,7 +14,7 @@ classes derived from that class (but not exception classes from which *it* is derived). Two exception classes that are not related via subclassing are never equivalent, even if they have the same name. -.. index:: statement: raise +.. index:: pair: statement; raise The built-in exceptions listed below can be generated by the interpreter or built-in functions. Except where mentioned, they have an "associated value" @@ -175,7 +175,7 @@ The following exceptions are the exceptions that are usually raised. .. exception:: AssertionError - .. index:: statement: assert + .. index:: pair: statement; assert Raised when an :keyword:`assert` statement fails. diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 5a022edb75b7d1..80b7b8b4f4ed07 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1986,7 +1986,7 @@ are always available. They are listed here in alphabetical order. .. function:: __import__(name, globals=None, locals=None, fromlist=(), level=0) .. index:: - statement: import + pair: statement; import pair: module; builtins .. note:: diff --git a/Doc/library/site.rst b/Doc/library/site.rst index a4efefd587a609..44f90a3b9e496f 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -51,7 +51,7 @@ searched for site-packages; otherwise they will. .. index:: single: # (hash); comment - statement: import + pair: statement; import A path configuration file is a file whose name has the form :file:`{name}.pth` and exists in one of the four directories mentioned above; its contents are diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 2f549f2e97c1de..c739485c5564da 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -32,8 +32,8 @@ Truth Value Testing =================== .. index:: - statement: if - statement: while + pair: statement; if + pair: statement; while pair: truth; value pair: Boolean; operations single: false @@ -1152,7 +1152,7 @@ accepts integers that meet the value restriction ``0 <= x <= 255``). triple: operations on; list; type pair: subscript; assignment pair: slice; assignment - statement: del + pair: statement; del single: append() (sequence method) single: clear() (sequence method) single: copy() (sequence method) @@ -4418,7 +4418,7 @@ Mapping Types --- :class:`dict` pair: object; dictionary triple: operations on; mapping; types triple: operations on; dictionary; type - statement: del + pair: statement; del builtin: len A :term:`mapping` object maps :term:`hashable` values to arbitrary objects. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 5750d3149c8e95..e9c1c493ae4278 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -84,7 +84,7 @@ The :keyword:`!if` statement ============================ .. index:: - ! statement: if + ! pair: statement; if pair: keyword; elif pair: keyword; else single: : (colon); compound statement @@ -109,7 +109,7 @@ The :keyword:`!while` statement =============================== .. index:: - ! statement: while + ! pair: statement; while pair: keyword; else pair: loop; statement single: : (colon); compound statement @@ -127,8 +127,8 @@ suite of the :keyword:`!else` clause, if present, is executed and the loop terminates. .. index:: - statement: break - statement: continue + pair: statement; break + pair: statement; continue A :keyword:`break` statement executed in the first suite terminates the loop without executing the :keyword:`!else` clause's suite. A :keyword:`continue` @@ -142,7 +142,7 @@ The :keyword:`!for` statement ============================= .. index:: - ! statement: for + ! pair: statement; for pair: keyword; in pair: keyword; else pair: target; list @@ -167,8 +167,8 @@ the suite in the :keyword:`!else` clause, if present, is executed, and the loop terminates. .. index:: - statement: break - statement: continue + pair: statement; break + pair: statement; continue A :keyword:`break` statement executed in the first suite terminates the loop without executing the :keyword:`!else` clause's suite. A :keyword:`continue` @@ -205,7 +205,7 @@ The :keyword:`!try` statement ============================= .. index:: - ! statement: try + ! pair: statement; try pair: keyword; except pair: keyword; finally pair: keyword; else @@ -388,9 +388,9 @@ cannot appear in an :keyword:`!except*` clause. .. index:: pair: keyword; else - statement: return - statement: break - statement: continue + pair: statement; return + pair: statement; break + pair: statement; continue .. _except_else: @@ -434,9 +434,9 @@ The exception information is not available to the program during execution of the :keyword:`!finally` clause. .. index:: - statement: return - statement: break - statement: continue + pair: statement; return + pair: statement; break + pair: statement; continue When a :keyword:`return`, :keyword:`break` or :keyword:`continue` statement is executed in the :keyword:`try` suite of a :keyword:`!try`...\ :keyword:`!finally` @@ -468,7 +468,7 @@ The :keyword:`!with` statement ============================== .. index:: - ! statement: with + ! pair: statement; with pair: keyword; as single: as; with statement single: , (comma); with statement @@ -585,7 +585,7 @@ The :keyword:`!match` statement =============================== .. index:: - ! statement: match + ! pair: statement; match ! pair: keyword; case ! single: pattern matching pair: keyword; if @@ -1190,7 +1190,7 @@ Function definitions ==================== .. index:: - statement: def + pair: statement; def pair: function; definition pair: function; name pair: name; binding @@ -1364,7 +1364,7 @@ Class definitions .. index:: pair: object; class - statement: class + pair: statement; class pair: class; definition pair: class; name pair: name; binding @@ -1463,7 +1463,7 @@ Coroutines .. versionadded:: 3.5 -.. index:: statement: async def +.. index:: pair: statement; async def .. _`async def`: Coroutine function definition @@ -1497,7 +1497,7 @@ An example of a coroutine function:: ``await`` and ``async`` are now keywords; previously they were only treated as such inside the body of a coroutine function. -.. index:: statement: async for +.. index:: pair: statement; async for .. _`async for`: The :keyword:`!async for` statement @@ -1542,7 +1542,7 @@ It is a :exc:`SyntaxError` to use an ``async for`` statement outside the body of a coroutine function. -.. index:: statement: async with +.. index:: pair: statement; async with .. _`async with`: The :keyword:`!async with` statement diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 18906a005b2761..1294d683e268a0 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -727,7 +727,7 @@ Callable types Modules .. index:: - statement: import + pair: statement; import pair: object; module Modules are a basic organizational unit of Python code, and are created by @@ -1151,7 +1151,7 @@ Internal types single: tb_frame (traceback attribute) single: tb_lineno (traceback attribute) single: tb_lasti (traceback attribute) - statement: try + pair: statement; try Special read-only attributes: :attr:`tb_frame` points to the execution frame of the current level; @@ -1310,7 +1310,7 @@ Basic customization .. index:: single: destructor single: finalizer - statement: del + pair: statement; del Called when the instance is about to be destroyed. This is also called a finalizer or (improperly) a destructor. If a base class has a @@ -2796,7 +2796,7 @@ execution of the block of code. Context managers are normally invoked using the used by directly invoking their methods. .. index:: - statement: with + pair: statement; with single: context manager Typical uses of context managers include saving and restoring various kinds of diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 7cf096019631da..ca1aa744389bda 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -376,7 +376,7 @@ The :keyword:`!assert` statement ================================ .. index:: - ! statement: assert + ! pair: statement; assert pair: debugging; assertions single: , (comma); expression list @@ -419,7 +419,7 @@ The :keyword:`!pass` statement ============================== .. index:: - statement: pass + pair: statement; pass pair: null; operation pair: null; operation @@ -441,7 +441,7 @@ The :keyword:`!del` statement ============================= .. index:: - ! statement: del + ! pair: statement; del pair: deletion; target triple: deletion; target; list @@ -454,7 +454,7 @@ Rather than spelling it out in full details, here are some hints. Deletion of a target list recursively deletes each target, from left to right. .. index:: - statement: global + pair: statement; global pair: unbinding; name Deletion of a name removes the binding of that name from the local or global @@ -480,7 +480,7 @@ The :keyword:`!return` statement ================================ .. index:: - ! statement: return + ! pair: statement; return pair: function; definition pair: class; definition @@ -517,7 +517,7 @@ The :keyword:`!yield` statement =============================== .. index:: - statement: yield + pair: statement; yield single: generator; function single: generator; iterator single: function; generator @@ -553,7 +553,7 @@ The :keyword:`!raise` statement =============================== .. index:: - ! statement: raise + ! pair: statement; raise single: exception pair: raising; exception single: __traceback__ (exception attribute) @@ -667,9 +667,9 @@ The :keyword:`!break` statement =============================== .. index:: - ! statement: break - statement: for - statement: while + ! pair: statement; break + pair: statement; for + pair: statement; while pair: loop; statement .. productionlist:: python-grammar @@ -701,9 +701,9 @@ The :keyword:`!continue` statement ================================== .. index:: - ! statement: continue - statement: for - statement: while + ! pair: statement; continue + pair: statement; for + pair: statement; while pair: loop; statement pair: keyword; finally @@ -726,7 +726,7 @@ The :keyword:`!import` statement ================================ .. index:: - ! statement: import + ! pair: statement; import single: module; importing pair: name; binding pair: keyword; from @@ -942,7 +942,7 @@ The :keyword:`!global` statement ================================ .. index:: - ! statement: global + ! pair: statement; global triple: global; name; binding single: , (comma); identifier list @@ -988,7 +988,7 @@ call. The same applies to the :func:`eval` and :func:`compile` functions. The :keyword:`!nonlocal` statement ================================== -.. index:: statement: nonlocal +.. index:: pair: statement; nonlocal single: , (comma); identifier list .. productionlist:: python-grammar diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 9a62ae748e0744..8fc44cab44455a 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -694,7 +694,7 @@ def patch_pairindextypes(app) -> None: pairindextypes.pop('operator', None) pairindextypes.pop('object', None) pairindextypes.pop('exception', None) - # pairindextypes.pop('statement', None) + pairindextypes.pop('statement', None) # pairindextypes.pop('builtin', None) diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 52db51e84cd5fc..c9b3d982c31c9a 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -46,7 +46,7 @@ details see :ref:`tut-match`. ========================== .. index:: - statement: for + pair: statement; for The :keyword:`for` statement in Python differs a bit from what you may be used to in C or Pascal. Rather than always iterating over an arithmetic progression From ef0df5284f929719b2ef3955b1b569ade0a5193c Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 4 May 2023 14:26:23 +0200 Subject: [PATCH 033/204] gh-97556: Raise null bytes syntax error upon null in multiline string (GH-104136) --- Lib/test/test_cmd_line_script.py | 13 +++++++++++++ Parser/tokenizer.c | 9 ++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index d98e23855e0c19..8bf299382e9ca4 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -669,6 +669,19 @@ def test_syntaxerror_null_bytes(self): ], ) + def test_syntaxerror_null_bytes_in_multiline_string(self): + scripts = ["\n'''\nmultilinestring\0\n'''", "\nf'''\nmultilinestring\0\n'''"] # Both normal and f-strings + with os_helper.temp_dir() as script_dir: + for script in scripts: + script_name = _make_test_script(script_dir, 'script', script) + _, _, stderr = assert_python_failure(script_name) + self.assertEqual( + stderr.splitlines()[-2:], + [ b" multilinestring", + b'SyntaxError: source code cannot contain null bytes' + ] + ) + def test_consistent_sys_path_for_direct_execution(self): # This test case ensures that the following all give the same # sys.path configuration: diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index d2f9fee110ebf5..7c07d2011fda61 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -2301,8 +2301,12 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t /* Get rest of string */ while (end_quote_size != quote_size) { c = tok_nextc(tok); - if (tok->done == E_DECODE) + if (tok->done == E_ERROR) { + return MAKE_TOKEN(ERRORTOKEN); + } + if (tok->done == E_DECODE) { break; + } if (c == EOF || (quote_size == 1 && c == '\n')) { assert(tok->multi_line_start != NULL); // shift the tok_state's location into @@ -2554,6 +2558,9 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct while (end_quote_size != current_tok->f_string_quote_size) { int c = tok_nextc(tok); + if (tok->done == E_ERROR) { + return MAKE_TOKEN(ERRORTOKEN); + } if (c == EOF || (current_tok->f_string_quote_size == 1 && c == '\n')) { if (tok->decoding_erred) { return MAKE_TOKEN(ERRORTOKEN); From 4f524da4843fb71b3ba43d89cd0265e4ecb29798 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 4 May 2023 15:03:24 +0200 Subject: [PATCH 034/204] gh-103092: Port _ctypes.COMError to heap type (#104020) --- Modules/_ctypes/_ctypes.c | 97 +++++++++++++++++--------------------- Modules/_ctypes/callproc.c | 3 +- Modules/_ctypes/ctypes.h | 7 ++- 3 files changed, 47 insertions(+), 60 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index c7ed6bd2229c79..f6cda45eaeac27 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -139,10 +139,6 @@ static PyTypeObject Simple_Type; strong reference to _ctypes._unpickle() function */ static PyObject *_unpickle; -#ifdef MS_WIN32 -PyObject *ComError; // Borrowed reference to: &PyComError_Type -#endif - /****************************************************************/ @@ -5480,46 +5476,38 @@ comerror_init(PyObject *self, PyObject *args, PyObject *kwds) return 0; } -static PyTypeObject PyComError_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.COMError", /* tp_name */ - sizeof(PyBaseExceptionObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR(comerror_doc), /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)comerror_init, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ +static int +comerror_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + +static void +comerror_dealloc(PyObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); + tp->tp_free(self); + Py_DECREF(tp); +} + +static PyType_Slot comerror_slots[] = { + {Py_tp_doc, (void *)PyDoc_STR(comerror_doc)}, + {Py_tp_init, comerror_init}, + {Py_tp_traverse, comerror_traverse}, + {Py_tp_dealloc, comerror_dealloc}, + {0, NULL}, }; + +static PyType_Spec comerror_spec = { + .name = "_ctypes.COMError", + .basicsize = sizeof(PyBaseExceptionObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE), + .slots = comerror_slots, +}; + #endif // MS_WIN32 static PyObject * @@ -5661,8 +5649,9 @@ _ctypes_add_types(PyObject *mod) } \ } while (0) -#define CREATE_TYPE(MOD, TP, SPEC) do { \ - PyObject *type = PyType_FromMetaclass(NULL, MOD, SPEC, NULL); \ +#define CREATE_TYPE(MOD, TP, SPEC, BASE) do { \ + PyObject *type = PyType_FromMetaclass(NULL, MOD, SPEC, \ + (PyObject *)BASE); \ if (type == NULL) { \ return -1; \ } \ @@ -5675,8 +5664,8 @@ _ctypes_add_types(PyObject *mod) ob_type is the metatype (the 'type'), defaults to PyType_Type, tp_base is the base type, defaults to 'object' aka PyBaseObject_Type. */ - CREATE_TYPE(mod, st->PyCArg_Type, &carg_spec); - CREATE_TYPE(mod, st->PyCThunk_Type, &cthunk_spec); + CREATE_TYPE(mod, st->PyCArg_Type, &carg_spec, NULL); + CREATE_TYPE(mod, st->PyCThunk_Type, &cthunk_spec, NULL); TYPE_READY(&PyCData_Type); /* StgDict is derived from PyDict_Type */ TYPE_READY_BASE(&PyCStgDict_Type, &PyDict_Type); @@ -5709,18 +5698,18 @@ _ctypes_add_types(PyObject *mod) * Simple classes */ - CREATE_TYPE(mod, st->PyCField_Type, &cfield_spec); + CREATE_TYPE(mod, st->PyCField_Type, &cfield_spec, NULL); /************************************************* * * Other stuff */ - CREATE_TYPE(mod, st->DictRemover_Type, &dictremover_spec); - CREATE_TYPE(mod, st->StructParam_Type, &structparam_spec); + CREATE_TYPE(mod, st->DictRemover_Type, &dictremover_spec, NULL); + CREATE_TYPE(mod, st->StructParam_Type, &structparam_spec, NULL); #ifdef MS_WIN32 - TYPE_READY_BASE(&PyComError_Type, (PyTypeObject*)PyExc_Exception); + CREATE_TYPE(mod, st->PyComError_Type, &comerror_spec, PyExc_Exception); #endif #undef TYPE_READY @@ -5750,7 +5739,8 @@ _ctypes_add_objects(PyObject *mod) MOD_ADD("_pointer_type_cache", Py_NewRef(_ctypes_ptrtype_cache)); #ifdef MS_WIN32 - MOD_ADD("COMError", Py_NewRef(ComError)); + ctypes_state *st = GLOBAL_STATE(); + MOD_ADD("COMError", Py_NewRef(st->PyComError_Type)); MOD_ADD("FUNCFLAG_HRESULT", PyLong_FromLong(FUNCFLAG_HRESULT)); MOD_ADD("FUNCFLAG_STDCALL", PyLong_FromLong(FUNCFLAG_STDCALL)); #endif @@ -5807,9 +5797,6 @@ _ctypes_mod_exec(PyObject *mod) if (_ctypes_add_types(mod) < 0) { return -1; } -#ifdef MS_WIN32 - ComError = (PyObject*)&PyComError_Type; -#endif if (_ctypes_add_objects(mod) < 0) { return -1; diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index f10cf58216acf7..d2fe525dd4d396 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1115,7 +1115,8 @@ GetComError(HRESULT errcode, GUID *riid, IUnknown *pIunk) descr, source, helpfile, helpcontext, progid); if (obj) { - PyErr_SetObject(ComError, obj); + ctypes_state *st = GLOBAL_STATE(); + PyErr_SetObject((PyObject *)st->PyComError_Type, obj); Py_DECREF(obj); } LocalFree(text); diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 252d9da7dbb56d..8891a0a741de7b 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -37,6 +37,9 @@ typedef struct { PyTypeObject *PyCArg_Type; PyTypeObject *PyCField_Type; PyTypeObject *PyCThunk_Type; +#ifdef MS_WIN32 + PyTypeObject *PyComError_Type; +#endif PyTypeObject *StructParam_Type; } ctypes_state; @@ -392,10 +395,6 @@ extern int _ctypes_simple_instance(PyObject *obj); extern PyObject *_ctypes_ptrtype_cache; PyObject *_ctypes_get_errobj(int **pspace); -#ifdef MS_WIN32 -extern PyObject *ComError; -#endif - #ifdef USING_MALLOC_CLOSURE_DOT_C void Py_ffi_closure_free(void *p); void *Py_ffi_closure_alloc(size_t size, void** codeloc); From e2ef5015d1b6cb56f1a7988583f2fb8c0e6d65fc Mon Sep 17 00:00:00 2001 From: Christopher Chavez Date: Thu, 4 May 2023 08:03:52 -0500 Subject: [PATCH 035/204] =?UTF-8?q?Fix=20typo=20"involvin"=20=E2=86=92=20"?= =?UTF-8?q?involving"=20(#104166)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Lib/test/test_idle.py | 2 +- Lib/test/test_tix.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py index b94b18a541a701..90cff9002b75b2 100644 --- a/Lib/test/test_idle.py +++ b/Lib/test/test_idle.py @@ -3,7 +3,7 @@ from test.support import check_sanitizer if check_sanitizer(address=True, memory=True): - raise unittest.SkipTest("Tests involvin libX11 can SEGFAULT on ASAN/MSAN builds") + raise unittest.SkipTest("Tests involving libX11 can SEGFAULT on ASAN/MSAN builds") # Skip test_idle if _tkinter wasn't built, if tkinter is missing, # if tcl/tk is not the 8.5+ needed for ttk widgets, diff --git a/Lib/test/test_tix.py b/Lib/test/test_tix.py index 454baeb38a9342..d0d2a164ad2c67 100644 --- a/Lib/test/test_tix.py +++ b/Lib/test/test_tix.py @@ -5,7 +5,7 @@ from test.support import check_sanitizer if check_sanitizer(address=True, memory=True): - raise unittest.SkipTest("Tests involvin libX11 can SEGFAULT on ASAN/MSAN builds") + raise unittest.SkipTest("Tests involving libX11 can SEGFAULT on ASAN/MSAN builds") # Skip this test if the _tkinter module wasn't built. From eba64d2afb4c429e80d863dc0dd7808bdbef30d3 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 4 May 2023 15:15:26 +0100 Subject: [PATCH 036/204] gh-104169: Ensure the tokenizer doesn't overwrite previous errors (#104170) --- Parser/tokenizer.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 7c07d2011fda61..52d0d9a534cb6a 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1277,6 +1277,12 @@ _syntaxerror_range(struct tok_state *tok, const char *format, int col_offset, int end_col_offset, va_list vargs) { + // In release builds, we don't want to overwrite a previous error, but in debug builds we + // want to fail if we are not doing it so we can fix it. + assert(tok->done != E_ERROR); + if (tok->done == E_ERROR) { + return ERRORTOKEN; + } PyObject *errmsg, *errtext, *args; errmsg = PyUnicode_FromFormatV(format, vargs); if (!errmsg) { From 45398ad51220b63b8df08fb5551c6b736205daed Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 4 May 2023 16:21:01 +0200 Subject: [PATCH 037/204] gh-103323: Remove PyRuntimeState_GetThreadState() (#104171) This function no longer makes sense, since its runtime parameter is no longer used. Use directly _PyThreadState_GET() and _PyInterpreterState_GET() instead. --- Include/internal/pycore_pystate.h | 10 ++-------- Python/ceval_gil.c | 5 ++--- Python/pylifecycle.c | 7 +++---- Python/pystate.c | 2 +- Python/sysmodule.c | 2 +- 5 files changed, 9 insertions(+), 17 deletions(-) diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 180ea676bc22eb..daa40cf4bcd855 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -68,7 +68,7 @@ _Py_ThreadCanHandlePendingCalls(void) } -/* Variable and macro for in-line access to current thread +/* Variable and static inline functions for in-line access to current thread and interpreter state */ #if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) @@ -93,12 +93,6 @@ _PyThreadState_GET(void) #endif } -static inline PyThreadState* -_PyRuntimeState_GetThreadState(_PyRuntimeState *Py_UNUSED(runtime)) -{ - return _PyThreadState_GET(); -} - static inline void _Py_EnsureFuncTstateNotNULL(const char *func, PyThreadState *tstate) @@ -118,7 +112,7 @@ _Py_EnsureFuncTstateNotNULL(const char *func, PyThreadState *tstate) /* Get the current interpreter state. - The macro is unsafe: it does not check for error and it can return NULL. + The function is unsafe: it does not check for error and it can return NULL. The caller must hold the GIL. diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 29796be4b80e93..4d501c89661743 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -546,8 +546,7 @@ _PyEval_Fini(void) void PyEval_AcquireLock(void) { - _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + PyThreadState *tstate = _PyThreadState_GET(); _Py_EnsureTstateNotNULL(tstate); take_gil(tstate); @@ -557,7 +556,7 @@ void PyEval_ReleaseLock(void) { _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + PyThreadState *tstate = _PyThreadState_GET(); /* This function must succeed when the current thread state is NULL. We therefore avoid PyThreadState_Get() which dumps a fatal error in debug mode. */ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index b9add89b9c6c52..97957d3f17e6db 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1303,8 +1303,7 @@ _Py_InitializeMain(void) if (_PyStatus_EXCEPTION(status)) { return status; } - _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + PyThreadState *tstate = _PyThreadState_GET(); return pyinit_main(tstate); } @@ -1755,7 +1754,7 @@ Py_FinalizeEx(void) } /* Get current thread state and interpreter pointer */ - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + PyThreadState *tstate = _PyThreadState_GET(); // XXX assert(_Py_IsMainInterpreter(tstate->interp)); // XXX assert(_Py_IsMainThread()); @@ -2800,7 +2799,7 @@ fatal_error(int fd, int header, const char *prefix, const char *msg, tss_tstate != tstate if the current Python thread does not hold the GIL. */ - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + PyThreadState *tstate = _PyThreadState_GET(); PyInterpreterState *interp = NULL; PyThreadState *tss_tstate = PyGILState_GetThisThreadState(); if (tstate != NULL) { diff --git a/Python/pystate.c b/Python/pystate.c index f103a059f0f369..f09d37657f9c27 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1809,7 +1809,7 @@ int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) { _PyRuntimeState *runtime = &_PyRuntime; - PyInterpreterState *interp = _PyRuntimeState_GetThreadState(runtime)->interp; + PyInterpreterState *interp = _PyInterpreterState_GET(); /* Although the GIL is held, a few C API functions can be called * without the GIL held, and in particular some that create and diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 781588b0df4ead..894a3e8a98fd8a 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -365,7 +365,7 @@ PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) _PyRuntimeState *runtime = &_PyRuntime; PyThreadState *tstate; if (runtime->initialized) { - tstate = _PyRuntimeState_GetThreadState(runtime); + tstate = _PyThreadState_GET(); } else { tstate = NULL; From c9ecd3ee75b472bb0a7538e0288c5cfea146da83 Mon Sep 17 00:00:00 2001 From: Kevin Krakauer Date: Thu, 4 May 2023 07:38:20 -0700 Subject: [PATCH 038/204] gh-102795: Fix use of poll in test_epoll's test_control_and_wait (#102796) This test can fail unnecessarily. In the test we wait for events on two file descriptors. This is done in a single call to select.epoll's poll() function. However, it is valid for the OS to return only one event via poll() and the next via a subsequent call to poll(). This rarely happens, but it can cause the test to fail despite properly functioning polling. Instead, we poll a second time when necessary. --- Lib/test/test_epoll.py | 15 +++++++++++---- ...2023-03-17-22-00-47.gh-issue-102795.z21EoC.rst | 1 + 2 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-03-17-22-00-47.gh-issue-102795.z21EoC.rst diff --git a/Lib/test/test_epoll.py b/Lib/test/test_epoll.py index b623852f9eb4ee..c94946a6ae6b7c 100644 --- a/Lib/test/test_epoll.py +++ b/Lib/test/test_epoll.py @@ -27,6 +27,7 @@ import socket import time import unittest +from test import support if not hasattr(select, "epoll"): raise unittest.SkipTest("test works only on Linux 2.6") @@ -186,10 +187,16 @@ def test_control_and_wait(self): client.sendall(b"Hello!") server.sendall(b"world!!!") - now = time.monotonic() - events = ep.poll(1.0, 4) - then = time.monotonic() - self.assertFalse(then - now > 0.01) + # we might receive events one at a time, necessitating multiple calls to + # poll + events = [] + for _ in support.busy_retry(support.SHORT_TIMEOUT): + now = time.monotonic() + events += ep.poll(1.0, 4) + then = time.monotonic() + self.assertFalse(then - now > 0.01) + if len(events) >= 2: + break expected = [(client.fileno(), select.EPOLLIN | select.EPOLLOUT), (server.fileno(), select.EPOLLIN | select.EPOLLOUT)] diff --git a/Misc/NEWS.d/next/Tests/2023-03-17-22-00-47.gh-issue-102795.z21EoC.rst b/Misc/NEWS.d/next/Tests/2023-03-17-22-00-47.gh-issue-102795.z21EoC.rst new file mode 100644 index 00000000000000..fe2afff91ece7a --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-03-17-22-00-47.gh-issue-102795.z21EoC.rst @@ -0,0 +1 @@ +fix use of poll in test_epoll's test_control_and_wait From fdcb49c36b2ed8347d8d9f2dcd7052cc90207beb Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Thu, 4 May 2023 07:50:26 -0700 Subject: [PATCH 039/204] gh-104066: Improve performance of hasattr for module objects (#104063) --- Include/internal/pycore_moduleobject.h | 3 + ...-05-01-14-48-29.gh-issue-104066.pzoUZQ.rst | 2 + Objects/moduleobject.c | 91 +++++++++++++------ Objects/object.c | 11 +++ 4 files changed, 78 insertions(+), 29 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-48-29.gh-issue-104066.pzoUZQ.rst diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index 76361b8dff113a..15a1bcb6ae5163 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -36,6 +36,9 @@ static inline PyObject* _PyModule_GetDict(PyObject *mod) { return dict; } +PyObject* _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress); +PyObject* _Py_module_getattro(PyModuleObject *m, PyObject *name); + #ifdef __cplusplus } #endif diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-48-29.gh-issue-104066.pzoUZQ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-48-29.gh-issue-104066.pzoUZQ.rst new file mode 100644 index 00000000000000..97e0c01689cb6f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-48-29.gh-issue-104066.pzoUZQ.rst @@ -0,0 +1,2 @@ +Improve the performance of :func:`hasattr` for module objects with a missing +attribute. diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index a0be19a3ca8ac8..ffcd90ed122e77 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -702,7 +702,11 @@ int _PyModuleSpec_IsInitializing(PyObject *spec) { if (spec != NULL) { - PyObject *value = PyObject_GetAttr(spec, &_Py_ID(_initializing)); + PyObject *value; + int ok = _PyObject_LookupAttr(spec, &_Py_ID(_initializing), &value); + if (ok == 0) { + return 0; + } if (value != NULL) { int initializing = PyObject_IsTrue(value); Py_DECREF(value); @@ -738,19 +742,37 @@ _PyModuleSpec_IsUninitializedSubmodule(PyObject *spec, PyObject *name) return is_uninitialized; } -static PyObject* -module_getattro(PyModuleObject *m, PyObject *name) +PyObject* +_Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) { + // When suppress=1, this function suppresses AttributeError. PyObject *attr, *mod_name, *getattr; - attr = PyObject_GenericGetAttr((PyObject *)m, name); - if (attr || !PyErr_ExceptionMatches(PyExc_AttributeError)) { + attr = _PyObject_GenericGetAttrWithDict((PyObject *)m, name, NULL, suppress); + if (attr) { return attr; } - PyErr_Clear(); + if (suppress == 1) { + if (PyErr_Occurred()) { + // pass up non-AttributeError exception + return NULL; + } + } + else { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + // pass up non-AttributeError exception + return NULL; + } + PyErr_Clear(); + } assert(m->md_dict != NULL); getattr = PyDict_GetItemWithError(m->md_dict, &_Py_ID(__getattr__)); if (getattr) { - return PyObject_CallOneArg(getattr, name); + PyObject *result = PyObject_CallOneArg(getattr, name); + if (result == NULL && suppress == 1 && PyErr_ExceptionMatches(PyExc_AttributeError)) { + // suppress AttributeError + PyErr_Clear(); + } + return result; } if (PyErr_Occurred()) { return NULL; @@ -763,37 +785,48 @@ module_getattro(PyModuleObject *m, PyObject *name) Py_DECREF(mod_name); return NULL; } - Py_XINCREF(spec); - if (_PyModuleSpec_IsInitializing(spec)) { - PyErr_Format(PyExc_AttributeError, - "partially initialized " - "module '%U' has no attribute '%U' " - "(most likely due to a circular import)", - mod_name, name); - } - else if (_PyModuleSpec_IsUninitializedSubmodule(spec, name)) { - PyErr_Format(PyExc_AttributeError, - "cannot access submodule '%U' of module '%U' " - "(most likely due to a circular import)", - name, mod_name); - } - else { - PyErr_Format(PyExc_AttributeError, - "module '%U' has no attribute '%U'", - mod_name, name); + if (suppress != 1) { + Py_XINCREF(spec); + if (_PyModuleSpec_IsInitializing(spec)) { + PyErr_Format(PyExc_AttributeError, + "partially initialized " + "module '%U' has no attribute '%U' " + "(most likely due to a circular import)", + mod_name, name); + } + else if (_PyModuleSpec_IsUninitializedSubmodule(spec, name)) { + PyErr_Format(PyExc_AttributeError, + "cannot access submodule '%U' of module '%U' " + "(most likely due to a circular import)", + name, mod_name); + } + else { + PyErr_Format(PyExc_AttributeError, + "module '%U' has no attribute '%U'", + mod_name, name); + } + Py_XDECREF(spec); } - Py_XDECREF(spec); Py_DECREF(mod_name); return NULL; } else if (PyErr_Occurred()) { return NULL; } - PyErr_Format(PyExc_AttributeError, - "module has no attribute '%U'", name); + if (suppress != 1) { + PyErr_Format(PyExc_AttributeError, + "module has no attribute '%U'", name); + } return NULL; } + +PyObject* +_Py_module_getattro(PyModuleObject *m, PyObject *name) +{ + return _Py_module_getattro_impl(m, name, 0); +} + static int module_traverse(PyModuleObject *m, visitproc visit, void *arg) { @@ -951,7 +984,7 @@ PyTypeObject PyModule_Type = { 0, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ - (getattrofunc)module_getattro, /* tp_getattro */ + (getattrofunc)_Py_module_getattro, /* tp_getattro */ PyObject_GenericSetAttr, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | diff --git a/Objects/object.c b/Objects/object.c index c6ef5928164824..41c52e21045a21 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1085,6 +1085,17 @@ _PyObject_LookupAttr(PyObject *v, PyObject *name, PyObject **result) return 0; } } + else if (tp->tp_getattro == (getattrofunc)_Py_module_getattro) { + // optimization: suppress attribute error from module getattro method + *result = _Py_module_getattro_impl((PyModuleObject*)v, name, 1); + if (*result != NULL) { + return 1; + } + if (PyErr_Occurred()) { + return -1; + } + return 0; + } else if (tp->tp_getattro != NULL) { *result = (*tp->tp_getattro)(v, name); } From b17d32c1142d16a5fea0c95bce185bf9be696491 Mon Sep 17 00:00:00 2001 From: Gleb Smirnoff Date: Thu, 4 May 2023 07:57:05 -0700 Subject: [PATCH 040/204] gh-96534: socketmodule: support FreeBSD divert(4) socket (#96536) --- Doc/library/socket.rst | 11 +++++++++++ .../2022-09-03-09-24-02.gh-issue-96534.EU4Oxv.rst | 1 + Modules/socketmodule.c | 13 +++++++++++++ 3 files changed, 25 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2022-09-03-09-24-02.gh-issue-96534.EU4Oxv.rst diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 4ee0897db94063..13a82cf82d5908 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -509,6 +509,17 @@ Constants .. versionadded:: 3.9 +.. data:: AF_DIVERT + PF_DIVERT + + These two constants, documented in the FreeBSD divert(4) manual page, are + also defined in the socket module. + + .. availability:: FreeBSD >= 14.0. + + .. versionadded:: 3.12 + + .. data:: AF_PACKET PF_PACKET PACKET_* diff --git a/Misc/NEWS.d/next/Library/2022-09-03-09-24-02.gh-issue-96534.EU4Oxv.rst b/Misc/NEWS.d/next/Library/2022-09-03-09-24-02.gh-issue-96534.EU4Oxv.rst new file mode 100644 index 00000000000000..0497d9eb69163e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-09-03-09-24-02.gh-issue-96534.EU4Oxv.rst @@ -0,0 +1 @@ +Support divert(4) added in FreeBSD 14. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index f11d4b1a6e0591..e5478382e11f89 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -1903,6 +1903,11 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, /* RDS sockets use sockaddr_in: fall-through */ #endif /* AF_RDS */ +#ifdef AF_DIVERT + case AF_DIVERT: + /* FreeBSD divert(4) sockets use sockaddr_in: fall-through */ +#endif /* AF_DIVERT */ + case AF_INET: { struct maybe_idna host = {NULL, NULL}; @@ -7683,6 +7688,14 @@ socket_exec(PyObject *m) ADD_INT_MACRO(m, AF_SYSTEM); #endif +/* FreeBSD divert(4) */ +#ifdef PF_DIVERT + PyModule_AddIntMacro(m, PF_DIVERT); +#endif +#ifdef AF_DIVERT + PyModule_AddIntMacro(m, AF_DIVERT); +#endif + #ifdef AF_PACKET ADD_INT_MACRO(m, AF_PACKET); #endif From 04f673327530f47f002e784459037231de478412 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 4 May 2023 07:59:46 -0700 Subject: [PATCH 041/204] gh-102500: Implement PEP 688 (#102521) Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> --- .../pycore_global_objects_fini_generated.h | 3 + Include/internal/pycore_global_strings.h | 3 + Include/internal/pycore_memoryobject.h | 17 ++ .../internal/pycore_runtime_init_generated.h | 3 + Include/internal/pycore_typeobject.h | 2 + .../internal/pycore_unicodeobject_generated.h | 9 + Include/pybuffer.h | 2 +- Lib/_collections_abc.py | 17 +- Lib/inspect.py | 23 ++ Lib/test/test_buffer.py | 142 +++++++++++++ Lib/test/test_collections.py | 11 +- Lib/test/test_doctest.py | 2 +- ...-03-07-17-37-00.gh-issue-102500.RUSQhz.rst | 3 + Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/buffer.c | 102 +++++++++ Modules/_testcapi/parts.h | 1 + Modules/_testcapimodule.c | 4 +- Objects/clinic/memoryobject.c.h | 62 +++++- Objects/memoryobject.c | 40 +++- Objects/object.c | 2 + Objects/typeobject.c | 197 +++++++++++++++++- PCbuild/_testcapi.vcxproj | 1 + PCbuild/_testcapi.vcxproj.filters | 3 + Tools/build/generate_global_objects.py | 2 + Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 + Tools/c-analyzer/cpython/ignored.tsv | 1 + 26 files changed, 640 insertions(+), 15 deletions(-) create mode 100644 Include/internal/pycore_memoryobject.h create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst create mode 100644 Modules/_testcapi/buffer.c diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index f0740b68dd1114..9377fd8526e3a2 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -593,6 +593,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__await__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__bases__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__bool__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__buffer__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__build_class__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__builtins__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__bytes__)); @@ -692,6 +693,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__rdivmod__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__reduce__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__reduce_ex__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__release_buffer__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__repr__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__reversed__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__rfloordiv__)); @@ -1122,6 +1124,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reducer_override)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(registry)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(rel_tol)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(release)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reload)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(repl)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(replace)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 234d5e2a09892c..ed9b2bb44ddffc 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -81,6 +81,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__await__) STRUCT_FOR_ID(__bases__) STRUCT_FOR_ID(__bool__) + STRUCT_FOR_ID(__buffer__) STRUCT_FOR_ID(__build_class__) STRUCT_FOR_ID(__builtins__) STRUCT_FOR_ID(__bytes__) @@ -180,6 +181,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__rdivmod__) STRUCT_FOR_ID(__reduce__) STRUCT_FOR_ID(__reduce_ex__) + STRUCT_FOR_ID(__release_buffer__) STRUCT_FOR_ID(__repr__) STRUCT_FOR_ID(__reversed__) STRUCT_FOR_ID(__rfloordiv__) @@ -610,6 +612,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(reducer_override) STRUCT_FOR_ID(registry) STRUCT_FOR_ID(rel_tol) + STRUCT_FOR_ID(release) STRUCT_FOR_ID(reload) STRUCT_FOR_ID(repl) STRUCT_FOR_ID(replace) diff --git a/Include/internal/pycore_memoryobject.h b/Include/internal/pycore_memoryobject.h new file mode 100644 index 00000000000000..acc12c9275172c --- /dev/null +++ b/Include/internal/pycore_memoryobject.h @@ -0,0 +1,17 @@ +#ifndef Py_INTERNAL_MEMORYOBJECT_H +#define Py_INTERNAL_MEMORYOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +PyObject * +PyMemoryView_FromObjectAndFlags(PyObject *v, int flags); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_MEMORYOBJECT_H */ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 16f2147aa8e93b..6ade8fb6eade03 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -587,6 +587,7 @@ extern "C" { INIT_ID(__await__), \ INIT_ID(__bases__), \ INIT_ID(__bool__), \ + INIT_ID(__buffer__), \ INIT_ID(__build_class__), \ INIT_ID(__builtins__), \ INIT_ID(__bytes__), \ @@ -686,6 +687,7 @@ extern "C" { INIT_ID(__rdivmod__), \ INIT_ID(__reduce__), \ INIT_ID(__reduce_ex__), \ + INIT_ID(__release_buffer__), \ INIT_ID(__repr__), \ INIT_ID(__reversed__), \ INIT_ID(__rfloordiv__), \ @@ -1116,6 +1118,7 @@ extern "C" { INIT_ID(reducer_override), \ INIT_ID(registry), \ INIT_ID(rel_tol), \ + INIT_ID(release), \ INIT_ID(reload), \ INIT_ID(repl), \ INIT_ID(replace), \ diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 6a5ab7e63f85c3..f42f8f62de2c1d 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -138,6 +138,8 @@ _Py_type_getattro(PyTypeObject *type, PyObject *name); PyObject *_Py_slot_tp_getattro(PyObject *self, PyObject *name); PyObject *_Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); +PyAPI_DATA(PyTypeObject) _PyBufferWrapper_Type; + PyObject * _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, PyObject *name, int *meth_found); PyObject * diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index cd41b731537f0e..0b33ea187e60ff 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -96,6 +96,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(__bool__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(__buffer__); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(__build_class__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -393,6 +396,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(__reduce_ex__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(__release_buffer__); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(__repr__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1683,6 +1689,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(rel_tol); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(release); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(reload); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Include/pybuffer.h b/Include/pybuffer.h index bbac60972f5127..ca1c6058d9052c 100644 --- a/Include/pybuffer.h +++ b/Include/pybuffer.h @@ -104,7 +104,7 @@ PyAPI_FUNC(void) PyBuffer_Release(Py_buffer *view); /* Maximum number of dimensions */ #define PyBUF_MAX_NDIM 64 -/* Flags for getting buffers */ +/* Flags for getting buffers. Keep these in sync with inspect.BufferFlags. */ #define PyBUF_SIMPLE 0 #define PyBUF_WRITABLE 0x0001 diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 9d7724c33474cc..2117190cf8b64a 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -49,7 +49,7 @@ def _f(): pass "Mapping", "MutableMapping", "MappingView", "KeysView", "ItemsView", "ValuesView", "Sequence", "MutableSequence", - "ByteString", + "ByteString", "Buffer", ] # This module has been renamed from collections.abc to _collections_abc to @@ -439,6 +439,21 @@ def __subclasshook__(cls, C): return NotImplemented +class Buffer(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __buffer__(self, flags: int, /) -> memoryview: + raise NotImplementedError + + @classmethod + def __subclasshook__(cls, C): + if cls is Buffer: + return _check_methods(C, "__buffer__") + return NotImplemented + + class _CallableGenericAlias(GenericAlias): """ Represent `Callable[argtypes, resulttype]`. diff --git a/Lib/inspect.py b/Lib/inspect.py index 92c2675cfd7d32..95da7fb71a3997 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -43,6 +43,7 @@ "Attribute", "BlockFinder", "BoundArguments", + "BufferFlags", "CORO_CLOSED", "CORO_CREATED", "CORO_RUNNING", @@ -3312,6 +3313,28 @@ def signature(obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=F globals=globals, locals=locals, eval_str=eval_str) +class BufferFlags(enum.IntFlag): + SIMPLE = 0x0 + WRITABLE = 0x1 + FORMAT = 0x4 + ND = 0x8 + STRIDES = 0x10 | ND + C_CONTIGUOUS = 0x20 | STRIDES + F_CONTIGUOUS = 0x40 | STRIDES + ANY_CONTIGUOUS = 0x80 | STRIDES + INDIRECT = 0x100 | STRIDES + CONTIG = ND | WRITABLE + CONTIG_RO = ND + STRIDED = STRIDES | WRITABLE + STRIDED_RO = STRIDES + RECORDS = STRIDES | WRITABLE | FORMAT + RECORDS_RO = STRIDES | FORMAT + FULL = INDIRECT | WRITABLE | FORMAT + FULL_RO = INDIRECT | FORMAT + READ = 0x100 + WRITE = 0x200 + + def _main(): """ Logic for inspecting an object given at command line """ import argparse diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 098d2d999643cb..b6e82ad4db266a 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -17,6 +17,7 @@ import unittest from test import support from test.support import os_helper +import inspect from itertools import permutations, product from random import randrange, sample, choice import warnings @@ -4438,5 +4439,146 @@ def test_pybuffer_size_from_format(self): struct.calcsize(format)) +class TestPythonBufferProtocol(unittest.TestCase): + def test_basic(self): + class MyBuffer: + def __buffer__(self, flags): + return memoryview(b"hello") + + mv = memoryview(MyBuffer()) + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(bytes(MyBuffer()), b"hello") + + def test_bad_buffer_method(self): + class MustReturnMV: + def __buffer__(self, flags): + return 42 + + self.assertRaises(TypeError, memoryview, MustReturnMV()) + + class NoBytesEither: + def __buffer__(self, flags): + return b"hello" + + self.assertRaises(TypeError, memoryview, NoBytesEither()) + + class WrongArity: + def __buffer__(self): + return memoryview(b"hello") + + self.assertRaises(TypeError, memoryview, WrongArity()) + + def test_release_buffer(self): + class WhatToRelease: + def __init__(self): + self.held = False + self.ba = bytearray(b"hello") + + def __buffer__(self, flags): + if self.held: + raise TypeError("already held") + self.held = True + return memoryview(self.ba) + + def __release_buffer__(self, buffer): + self.held = False + + wr = WhatToRelease() + self.assertFalse(wr.held) + with memoryview(wr) as mv: + self.assertTrue(wr.held) + self.assertEqual(mv.tobytes(), b"hello") + self.assertFalse(wr.held) + + def test_same_buffer_returned(self): + class WhatToRelease: + def __init__(self): + self.held = False + self.ba = bytearray(b"hello") + self.created_mv = None + + def __buffer__(self, flags): + if self.held: + raise TypeError("already held") + self.held = True + self.created_mv = memoryview(self.ba) + return self.created_mv + + def __release_buffer__(self, buffer): + assert buffer is self.created_mv + self.held = False + + wr = WhatToRelease() + self.assertFalse(wr.held) + with memoryview(wr) as mv: + self.assertTrue(wr.held) + self.assertEqual(mv.tobytes(), b"hello") + self.assertFalse(wr.held) + + def test_buffer_flags(self): + class PossiblyMutable: + def __init__(self, data, mutable) -> None: + self._data = bytearray(data) + self._mutable = mutable + + def __buffer__(self, flags): + if flags & inspect.BufferFlags.WRITABLE: + if not self._mutable: + raise RuntimeError("not mutable") + return memoryview(self._data) + else: + return memoryview(bytes(self._data)) + + mutable = PossiblyMutable(b"hello", True) + immutable = PossiblyMutable(b"hello", False) + with memoryview._from_flags(mutable, inspect.BufferFlags.WRITABLE) as mv: + self.assertEqual(mv.tobytes(), b"hello") + mv[0] = ord(b'x') + self.assertEqual(mv.tobytes(), b"xello") + with memoryview._from_flags(mutable, inspect.BufferFlags.SIMPLE) as mv: + self.assertEqual(mv.tobytes(), b"xello") + with self.assertRaises(TypeError): + mv[0] = ord(b'h') + self.assertEqual(mv.tobytes(), b"xello") + with memoryview._from_flags(immutable, inspect.BufferFlags.SIMPLE) as mv: + self.assertEqual(mv.tobytes(), b"hello") + with self.assertRaises(TypeError): + mv[0] = ord(b'x') + self.assertEqual(mv.tobytes(), b"hello") + + with self.assertRaises(RuntimeError): + memoryview._from_flags(immutable, inspect.BufferFlags.WRITABLE) + with memoryview(immutable) as mv: + self.assertEqual(mv.tobytes(), b"hello") + with self.assertRaises(TypeError): + mv[0] = ord(b'x') + self.assertEqual(mv.tobytes(), b"hello") + + def test_call_builtins(self): + ba = bytearray(b"hello") + mv = ba.__buffer__(0) + self.assertEqual(mv.tobytes(), b"hello") + ba.__release_buffer__(mv) + with self.assertRaises(OverflowError): + ba.__buffer__(sys.maxsize + 1) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_buffer(self): + buf = _testcapi.testBuf() + self.assertEqual(buf.references, 0) + mv = buf.__buffer__(0) + self.assertIsInstance(mv, memoryview) + self.assertEqual(mv.tobytes(), b"test") + self.assertEqual(buf.references, 1) + buf.__release_buffer__(mv) + self.assertEqual(buf.references, 0) + with self.assertRaises(ValueError): + mv.tobytes() + # Calling it again doesn't cause issues + with self.assertRaises(ValueError): + buf.__release_buffer__(mv) + self.assertEqual(buf.references, 0) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index fb568a48396498..8fc28a6bf98e8f 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -25,7 +25,7 @@ from collections.abc import Set, MutableSet from collections.abc import Mapping, MutableMapping, KeysView, ItemsView, ValuesView from collections.abc import Sequence, MutableSequence -from collections.abc import ByteString +from collections.abc import ByteString, Buffer class TestUserObjects(unittest.TestCase): @@ -1949,6 +1949,15 @@ def test_ByteString(self): self.assertFalse(issubclass(memoryview, ByteString)) self.validate_abstract_methods(ByteString, '__getitem__', '__len__') + def test_Buffer(self): + for sample in [bytes, bytearray, memoryview]: + self.assertIsInstance(sample(b"x"), Buffer) + self.assertTrue(issubclass(sample, Buffer)) + for sample in [str, list, tuple]: + self.assertNotIsInstance(sample(), Buffer) + self.assertFalse(issubclass(sample, Buffer)) + self.validate_abstract_methods(Buffer, '__buffer__') + def test_MutableSequence(self): for sample in [tuple, str, bytes]: self.assertNotIsInstance(sample(), MutableSequence) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 3491d4cdb1c18b..542fcdb5cf6f66 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -707,7 +707,7 @@ def non_Python_modules(): r""" >>> import builtins >>> tests = doctest.DocTestFinder().find(builtins) - >>> 830 < len(tests) < 850 # approximate number of objects with docstrings + >>> 830 < len(tests) < 860 # approximate number of objects with docstrings True >>> real_tests = [t for t in tests if len(t.examples) > 0] >>> len(real_tests) # objects that actually have doctests diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst new file mode 100644 index 00000000000000..e03113ba05cd7d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst @@ -0,0 +1,3 @@ +Make the buffer protocol accessible in Python code using the new +``__buffer__`` and ``__release_buffer__`` magic methods. See :pep:`688` for +details. Patch by Jelle Zijlstra. diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 6b483341953711..a7803cf7c00e68 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -169,7 +169,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c # Some testing modules MUST be built as shared libraries. diff --git a/Modules/_testcapi/buffer.c b/Modules/_testcapi/buffer.c new file mode 100644 index 00000000000000..aff9a477eff57e --- /dev/null +++ b/Modules/_testcapi/buffer.c @@ -0,0 +1,102 @@ +/* Test PEP 688 - Buffers */ + +#include "parts.h" + +#include "structmember.h" // PyMemberDef +#include // offsetof + +typedef struct { + PyObject_HEAD + PyObject *obj; + Py_ssize_t references; +} testBufObject; + +static PyObject * +testbuf_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyObject *obj = PyBytes_FromString("test"); + if (obj == NULL) { + return NULL; + } + testBufObject *self = (testBufObject *)type->tp_alloc(type, 0); + if (self == NULL) { + Py_DECREF(obj); + return NULL; + } + self->obj = obj; + self->references = 0; + return (PyObject *)self; +} + +static int +testbuf_traverse(testBufObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->obj); + return 0; +} + +static int +testbuf_clear(testBufObject *self) +{ + Py_CLEAR(self->obj); + return 0; +} + +static void +testbuf_dealloc(testBufObject *self) +{ + PyObject_GC_UnTrack(self); + Py_XDECREF(self->obj); + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static int +testbuf_getbuf(testBufObject *self, Py_buffer *view, int flags) +{ + int buf = PyObject_GetBuffer(self->obj, view, flags); + Py_SETREF(view->obj, Py_NewRef(self)); + self->references++; + return buf; +} + +static void +testbuf_releasebuf(testBufObject *self, Py_buffer *view) +{ + self->references--; + assert(self->references >= 0); +} + +static PyBufferProcs testbuf_as_buffer = { + .bf_getbuffer = (getbufferproc) testbuf_getbuf, + .bf_releasebuffer = (releasebufferproc) testbuf_releasebuf, +}; + +static struct PyMemberDef testbuf_members[] = { + {"references", T_PYSSIZET, offsetof(testBufObject, references), READONLY}, + {NULL}, +}; + +static PyTypeObject testBufType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "testBufType", + .tp_basicsize = sizeof(testBufObject), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_new = testbuf_new, + .tp_dealloc = (destructor) testbuf_dealloc, + .tp_traverse = (traverseproc) testbuf_traverse, + .tp_clear = (inquiry) testbuf_clear, + .tp_as_buffer = &testbuf_as_buffer, + .tp_members = testbuf_members +}; + +int +_PyTestCapi_Init_Buffer(PyObject *m) { + if (PyType_Ready(&testBufType) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "testBuf", (PyObject *)&testBufType)) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index d75412d5116063..663d4f2255de88 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -38,6 +38,7 @@ int _PyTestCapi_Init_Float(PyObject *module); int _PyTestCapi_Init_Structmember(PyObject *module); int _PyTestCapi_Init_Exceptions(PyObject *module); int _PyTestCapi_Init_Code(PyObject *module); +int _PyTestCapi_Init_Buffer(PyObject *module); int _PyTestCapi_Init_PyOS(PyObject *module); int _PyTestCapi_Init_Immortal(PyObject *module); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 1ecc442058088a..38f4758e6575d8 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3959,7 +3959,6 @@ static PyTypeObject MyList_Type = { MyList_new, /* tp_new */ }; - /* Test PEP 560 */ typedef struct { @@ -4310,6 +4309,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Code(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Buffer(m) < 0) { + return NULL; + } if (_PyTestCapi_Init_PyOS(m) < 0) { return NULL; } diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index ff7b50bb114b05..25a22341185903 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -62,6 +62,66 @@ memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) return return_value; } +PyDoc_STRVAR(memoryview__from_flags__doc__, +"_from_flags($type, /, object, flags)\n" +"--\n" +"\n" +"Create a new memoryview object which references the given object."); + +#define MEMORYVIEW__FROM_FLAGS_METHODDEF \ + {"_from_flags", _PyCFunction_CAST(memoryview__from_flags), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, memoryview__from_flags__doc__}, + +static PyObject * +memoryview__from_flags_impl(PyTypeObject *type, PyObject *object, int flags); + +static PyObject * +memoryview__from_flags(PyTypeObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(object), &_Py_ID(flags), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"object", "flags", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_from_flags", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *object; + int flags; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + object = args[0]; + flags = _PyLong_AsInt(args[1]); + if (flags == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = memoryview__from_flags_impl(type, object, flags); + +exit: + return return_value; +} + PyDoc_STRVAR(memoryview_release__doc__, "release($self, /)\n" "--\n" @@ -356,4 +416,4 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=a832f2fc44e4794c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=01613814112cedd7 input=a9049054013a1b77]*/ diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 34cc797b404cda..f008a8cc3e0474 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -85,7 +85,7 @@ mbuf_alloc(void) } static PyObject * -_PyManagedBuffer_FromObject(PyObject *base) +_PyManagedBuffer_FromObject(PyObject *base, int flags) { _PyManagedBufferObject *mbuf; @@ -93,7 +93,7 @@ _PyManagedBuffer_FromObject(PyObject *base) if (mbuf == NULL) return NULL; - if (PyObject_GetBuffer(base, &mbuf->master, PyBUF_FULL_RO) < 0) { + if (PyObject_GetBuffer(base, &mbuf->master, flags) < 0) { mbuf->master.obj = NULL; Py_DECREF(mbuf); return NULL; @@ -777,11 +777,12 @@ PyMemoryView_FromBuffer(const Py_buffer *info) return mv; } -/* Create a memoryview from an object that implements the buffer protocol. +/* Create a memoryview from an object that implements the buffer protocol, + using the given flags. If the object is a memoryview, the new memoryview must be registered with the same managed buffer. Otherwise, a new managed buffer is created. */ PyObject * -PyMemoryView_FromObject(PyObject *v) +PyMemoryView_FromObjectAndFlags(PyObject *v, int flags) { _PyManagedBufferObject *mbuf; @@ -792,7 +793,7 @@ PyMemoryView_FromObject(PyObject *v) } else if (PyObject_CheckBuffer(v)) { PyObject *ret; - mbuf = (_PyManagedBufferObject *)_PyManagedBuffer_FromObject(v); + mbuf = (_PyManagedBufferObject *)_PyManagedBuffer_FromObject(v, flags); if (mbuf == NULL) return NULL; ret = mbuf_add_view(mbuf, NULL); @@ -805,6 +806,14 @@ PyMemoryView_FromObject(PyObject *v) Py_TYPE(v)->tp_name); return NULL; } +/* Create a memoryview from an object that implements the buffer protocol. + If the object is a memoryview, the new memoryview must be registered + with the same managed buffer. Otherwise, a new managed buffer is created. */ +PyObject * +PyMemoryView_FromObject(PyObject *v) +{ + return PyMemoryView_FromObjectAndFlags(v, PyBUF_FULL_RO); +} /* Copy the format string from a base object that might vanish. */ static int @@ -851,7 +860,7 @@ memory_from_contiguous_copy(const Py_buffer *src, char order) if (bytes == NULL) return NULL; - mbuf = (_PyManagedBufferObject *)_PyManagedBuffer_FromObject(bytes); + mbuf = (_PyManagedBufferObject *)_PyManagedBuffer_FromObject(bytes, PyBUF_FULL_RO); Py_DECREF(bytes); if (mbuf == NULL) return NULL; @@ -968,6 +977,24 @@ memoryview_impl(PyTypeObject *type, PyObject *object) } +/*[clinic input] +@classmethod +memoryview._from_flags + + object: object + flags: int + +Create a new memoryview object which references the given object. +[clinic start generated code]*/ + +static PyObject * +memoryview__from_flags_impl(PyTypeObject *type, PyObject *object, int flags) +/*[clinic end generated code: output=bf71f9906c266ee2 input=f5f82fd0e744356b]*/ +{ + return PyMemoryView_FromObjectAndFlags(object, flags); +} + + /****************************************************************************/ /* Previously in abstract.c */ /****************************************************************************/ @@ -3184,6 +3211,7 @@ static PyMethodDef memory_methods[] = { MEMORYVIEW_TOLIST_METHODDEF MEMORYVIEW_CAST_METHODDEF MEMORYVIEW_TOREADONLY_METHODDEF + MEMORYVIEW__FROM_FLAGS_METHODDEF {"__enter__", memory_enter, METH_NOARGS, NULL}, {"__exit__", memory_exit, METH_VARARGS, NULL}, {NULL, NULL} diff --git a/Objects/object.c b/Objects/object.c index 41c52e21045a21..a7c79c673d5fd3 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -14,6 +14,7 @@ #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_symtable.h" // PySTEntry_Type +#include "pycore_typeobject.h" // _PyBufferWrapper_Type #include "pycore_unionobject.h" // _PyUnion_Type #include "pycore_interpreteridobject.h" // _PyInterpreterID_Type @@ -2084,6 +2085,7 @@ static PyTypeObject* static_types[] = { &_PyAsyncGenASend_Type, &_PyAsyncGenAThrow_Type, &_PyAsyncGenWrappedValue_Type, + &_PyBufferWrapper_Type, &_PyContextTokenMissing_Type, &_PyCoroWrapper_Type, &_Py_GenericAliasIterType, diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 171c76a59a5541..456b10ee01d6bc 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6,6 +6,7 @@ #include "pycore_symtable.h" // _Py_Mangle() #include "pycore_dict.h" // _PyDict_KeysSize() #include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_memoryobject.h" // PyMemoryView_FromObjectAndFlags() #include "pycore_moduleobject.h" // _PyModule_GetDef() #include "pycore_object.h" // _PyType_HasFeature() #include "pycore_long.h" // _PyLong_IsNegative() @@ -8059,6 +8060,58 @@ wrap_descr_delete(PyObject *self, PyObject *args, void *wrapped) Py_RETURN_NONE; } +static PyObject * +wrap_buffer(PyObject *self, PyObject *args, void *wrapped) +{ + PyObject *arg = NULL; + + if (!PyArg_UnpackTuple(args, "", 1, 1, &arg)) { + return NULL; + } + Py_ssize_t flags = PyNumber_AsSsize_t(arg, PyExc_OverflowError); + if (flags == -1 && PyErr_Occurred()) { + return NULL; + } + if (flags > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "buffer flags too large"); + return NULL; + } + + return PyMemoryView_FromObjectAndFlags(self, Py_SAFE_DOWNCAST(flags, Py_ssize_t, int)); +} + +static PyObject * +wrap_releasebuffer(PyObject *self, PyObject *args, void *wrapped) +{ + PyObject *arg = NULL; + if (!PyArg_UnpackTuple(args, "", 1, 1, &arg)) { + return NULL; + } + if (!PyMemoryView_Check(arg)) { + PyErr_SetString(PyExc_TypeError, + "expected a memoryview object"); + return NULL; + } + PyMemoryViewObject *mview = (PyMemoryViewObject *)arg; + if (mview->view.obj != self) { + PyErr_SetString(PyExc_ValueError, + "memoryview's buffer is not this object"); + return NULL; + } + if (mview->flags & _Py_MEMORYVIEW_RELEASED) { + PyErr_SetString(PyExc_ValueError, + "memoryview's buffer has already been released"); + return NULL; + } + PyObject *res = PyObject_CallMethodNoArgs((PyObject *)mview, &_Py_ID(release)); + if (res == NULL) { + return NULL; + } + Py_DECREF(res); + Py_RETURN_NONE; +} + static PyObject * wrap_init(PyObject *self, PyObject *args, void *wrapped, PyObject *kwds) { @@ -8895,6 +8948,132 @@ slot_tp_finalize(PyObject *self) PyErr_SetRaisedException(exc); } +typedef struct _PyBufferWrapper { + PyObject_HEAD + PyObject *mv; + PyObject *obj; +} PyBufferWrapper; + +static int +bufferwrapper_traverse(PyBufferWrapper *self, visitproc visit, void *arg) +{ + Py_VISIT(self->mv); + Py_VISIT(self->obj); + return 0; +} + +static void +bufferwrapper_dealloc(PyObject *self) +{ + PyBufferWrapper *bw = (PyBufferWrapper *)self; + + _PyObject_GC_UNTRACK(self); + Py_XDECREF(bw->mv); + Py_XDECREF(bw->obj); + Py_TYPE(self)->tp_free(self); +} + +static void +bufferwrapper_releasebuf(PyObject *self, Py_buffer *view) +{ + PyBufferWrapper *bw = (PyBufferWrapper *)self; + + assert(PyMemoryView_Check(bw->mv)); + Py_TYPE(bw->mv)->tp_as_buffer->bf_releasebuffer(bw->mv, view); + if (Py_TYPE(bw->obj)->tp_as_buffer != NULL + && Py_TYPE(bw->obj)->tp_as_buffer->bf_releasebuffer != NULL) { + Py_TYPE(bw->obj)->tp_as_buffer->bf_releasebuffer(bw->obj, view); + } +} + +static PyBufferProcs bufferwrapper_as_buffer = { + .bf_releasebuffer = bufferwrapper_releasebuf, +}; + + +PyTypeObject _PyBufferWrapper_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "_buffer_wrapper", + .tp_basicsize = sizeof(PyBufferWrapper), + .tp_alloc = PyType_GenericAlloc, + .tp_free = PyObject_GC_Del, + .tp_traverse = (traverseproc)bufferwrapper_traverse, + .tp_dealloc = bufferwrapper_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_as_buffer = &bufferwrapper_as_buffer, +}; + +static int +slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) +{ + PyObject *flags_obj = PyLong_FromLong(flags); + if (flags_obj == NULL) { + return -1; + } + PyBufferWrapper *wrapper = NULL; + PyObject *stack[2] = {self, flags_obj}; + PyObject *ret = vectorcall_method(&_Py_ID(__buffer__), stack, 2); + if (ret == NULL) { + goto fail; + } + if (!PyMemoryView_Check(ret)) { + PyErr_Format(PyExc_TypeError, + "__buffer__ returned non-memoryview object"); + goto fail; + } + + if (PyObject_GetBuffer(ret, buffer, flags) < 0) { + goto fail; + } + assert(buffer->obj == ret); + + wrapper = PyObject_GC_New(PyBufferWrapper, &_PyBufferWrapper_Type); + if (wrapper == NULL) { + goto fail; + } + wrapper->mv = ret; + wrapper->obj = Py_NewRef(self); + _PyObject_GC_TRACK(wrapper); + + buffer->obj = (PyObject *)wrapper; + Py_DECREF(ret); + Py_DECREF(flags_obj); + return 0; + +fail: + Py_XDECREF(wrapper); + Py_XDECREF(ret); + Py_DECREF(flags_obj); + return -1; +} + +static void +slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) +{ + PyObject *mv; + if (Py_TYPE(buffer->obj) == &_PyBufferWrapper_Type) { + // Make sure we pass the same memoryview to + // __release_buffer__() that __buffer__() returned. + mv = Py_NewRef(((PyBufferWrapper *)buffer->obj)->mv); + } + else { + mv = PyMemoryView_FromBuffer(buffer); + if (mv == NULL) { + PyErr_WriteUnraisable(self); + return; + } + } + PyObject *stack[2] = {self, mv}; + PyObject *ret = vectorcall_method(&_Py_ID(__release_buffer__), stack, 2); + Py_DECREF(mv); + if (ret == NULL) { + PyErr_WriteUnraisable(self); + } + else { + Py_DECREF(ret); + } +} + static PyObject * slot_am_await(PyObject *self) { @@ -8962,6 +9141,7 @@ an all-zero entry. #undef TPSLOT #undef FLSLOT +#undef BUFSLOT #undef AMSLOT #undef ETSLOT #undef SQSLOT @@ -8981,6 +9161,8 @@ an all-zero entry. #define ETSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \ {#NAME, offsetof(PyHeapTypeObject, SLOT), (void *)(FUNCTION), WRAPPER, \ PyDoc_STR(DOC), .name_strobj = &_Py_ID(NAME) } +#define BUFSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \ + ETSLOT(NAME, as_buffer.SLOT, FUNCTION, WRAPPER, DOC) #define AMSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \ ETSLOT(NAME, as_async.SLOT, FUNCTION, WRAPPER, DOC) #define SQSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \ @@ -9062,6 +9244,13 @@ static pytype_slotdef slotdefs[] = { "Create and return new object. See help(type) for accurate signature."), TPSLOT(__del__, tp_finalize, slot_tp_finalize, (wrapperfunc)wrap_del, ""), + BUFSLOT(__buffer__, bf_getbuffer, slot_bf_getbuffer, wrap_buffer, + "__buffer__($self, flags, /)\n--\n\n" + "Return a buffer object that exposes the underlying memory of the object."), + BUFSLOT(__release_buffer__, bf_releasebuffer, slot_bf_releasebuffer, wrap_releasebuffer, + "__release_buffer__($self, /)\n--\n\n" + "Release the buffer object that exposes the underlying memory of the object."), + AMSLOT(__await__, am_await, slot_am_await, wrap_unaryfunc, "__await__($self, /)\n--\n\nReturn an iterator to be used in await expression."), AMSLOT(__aiter__, am_aiter, slot_am_aiter, wrap_unaryfunc, @@ -9208,8 +9397,12 @@ slotptr(PyTypeObject *type, int ioffset) /* Note: this depends on the order of the members of PyHeapTypeObject! */ assert(offset >= 0); - assert((size_t)offset < offsetof(PyHeapTypeObject, as_buffer)); - if ((size_t)offset >= offsetof(PyHeapTypeObject, as_sequence)) { + assert((size_t)offset < offsetof(PyHeapTypeObject, ht_name)); + if ((size_t)offset >= offsetof(PyHeapTypeObject, as_buffer)) { + ptr = (char *)type->tp_as_buffer; + offset -= offsetof(PyHeapTypeObject, as_buffer); + } + else if ((size_t)offset >= offsetof(PyHeapTypeObject, as_sequence)) { ptr = (char *)type->tp_as_sequence; offset -= offsetof(PyHeapTypeObject, as_sequence); } diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 56448b6ee7d492..350f97f8ff41aa 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -110,6 +110,7 @@ + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 297c9ce799bea1..af80f1eebb3c4d 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -60,6 +60,9 @@ Source Files + + Source Files + Source Files diff --git a/Tools/build/generate_global_objects.py b/Tools/build/generate_global_objects.py index c27817702bf97d..ded19ee489e79b 100644 --- a/Tools/build/generate_global_objects.py +++ b/Tools/build/generate_global_objects.py @@ -121,6 +121,8 @@ '__xor__', '__divmod__', '__rdivmod__', + '__buffer__', + '__release_buffer__', ] NON_GENERATED_IMMORTAL_OBJECTS = [ diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 4dfbbe72df56a0..165bd74587d7c7 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -86,6 +86,7 @@ Objects/sliceobject.c - PyEllipsis_Type - Objects/sliceobject.c - PySlice_Type - Objects/tupleobject.c - PyTupleIter_Type - Objects/tupleobject.c - PyTuple_Type - +Objects/typeobject.c - _PyBufferWrapper_Type - Objects/typeobject.c - PyBaseObject_Type - Objects/typeobject.c - PySuper_Type - Objects/typeobject.c - PyType_Type - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 7a5d7d45f5184b..fee493ff7f1666 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -404,6 +404,7 @@ Modules/_testbuffer.c ndarray_memoryview_from_buffer strides - Modules/_testbuffer.c ndarray_memoryview_from_buffer suboffsets - Modules/_testbuffer.c ndarray_push kwlist - Modules/_testbuffer.c staticarray_init kwlist - +Modules/_testcapi/buffer.c - testBufType - Modules/_testcapi/code.c get_code_extra_index key - Modules/_testcapi/datetime.c - test_run_counter - Modules/_testcapi/exceptions.c - PyRecursingInfinitelyError_Type - From b7a0a521960a6e9ea46b79b51cbcc3c4ffcc7057 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 4 May 2023 08:23:40 -0700 Subject: [PATCH 042/204] gh-102500: Document PEP 688 (#102571) Co-authored-by: Hugo van Kemenade Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- Doc/library/collections.abc.rst | 8 +++++++ Doc/library/inspect.rst | 33 ++++++++++++++++++++++++++ Doc/reference/datamodel.rst | 41 +++++++++++++++++++++++++++++++++ Doc/whatsnew/3.12.rst | 14 ++++++++++- 4 files changed, 95 insertions(+), 1 deletion(-) diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 1ada0d352a0cc6..669b7345499a78 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -177,6 +177,7 @@ ABC Inherits from Abstract Methods Mi :class:`AsyncIterable` [1]_ ``__aiter__`` :class:`AsyncIterator` [1]_ :class:`AsyncIterable` ``__anext__`` ``__aiter__`` :class:`AsyncGenerator` [1]_ :class:`AsyncIterator` ``asend``, ``athrow`` ``aclose``, ``__aiter__``, ``__anext__`` +:class:`Buffer` [1]_ ``__buffer__`` ============================== ====================== ======================= ==================================================== @@ -346,6 +347,13 @@ Collections Abstract Base Classes -- Detailed Descriptions .. versionadded:: 3.6 +.. class:: Buffer + + ABC for classes that provide the :meth:`~object.__buffer__` method, + implementing the :ref:`buffer protocol `. See :pep:`688`. + + .. versionadded:: 3.12 + Examples and Recipes -------------------- diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 88f843c03b1d5a..7884308a333020 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -1603,6 +1603,39 @@ the following flags: for any introspection needs. +Buffer flags +------------ + +.. class:: BufferFlags + + This is an :class:`enum.IntFlag` that represents the flags that + can be passed to the :meth:`~object.__buffer__` method of objects + implementing the :ref:`buffer protocol `. + + The meaning of the flags is explained at :ref:`buffer-request-types`. + + .. attribute:: BufferFlags.SIMPLE + .. attribute:: BufferFlags.WRITABLE + .. attribute:: BufferFlags.FORMAT + .. attribute:: BufferFlags.ND + .. attribute:: BufferFlags.STRIDES + .. attribute:: BufferFlags.C_CONTIGUOUS + .. attribute:: BufferFlags.F_CONTIGUOUS + .. attribute:: BufferFlags.ANY_CONTIGUOUS + .. attribute:: BufferFlags.INDIRECT + .. attribute:: BufferFlags.CONTIG + .. attribute:: BufferFlags.CONTIG_RO + .. attribute:: BufferFlags.STRIDED + .. attribute:: BufferFlags.STRIDED_RO + .. attribute:: BufferFlags.RECORDS + .. attribute:: BufferFlags.RECORDS_RO + .. attribute:: BufferFlags.FULL + .. attribute:: BufferFlags.FULL_RO + .. attribute:: BufferFlags.READ + .. attribute:: BufferFlags.WRITE + + .. versionadded:: 3.12 + .. _inspect-module-cli: Command Line Interface diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 1294d683e268a0..0a9cabc158b9e4 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2865,6 +2865,47 @@ a :exc:`TypeError`. The specification for the Python ``match`` statement. +.. _python-buffer-protocol: + +Emulating buffer types +---------------------- + +The :ref:`buffer protocol ` provides a way for Python +objects to expose efficient access to a low-level memory array. This protocol +is implemented by builtin types such as :class:`bytes` and :class:`memoryview`, +and third-party libraries may define additional buffer types. + +While buffer types are usually implemented in C, it is also possible to +implement the protocol in Python. + +.. method:: object.__buffer__(self, flags) + + Called when a buffer is requested from *self* (for example, by the + :class:`memoryview` constructor). The *flags* argument is an integer + representing the kind of buffer requested, affecting for example whether + the returned buffer is read-only or writable. :class:`inspect.BufferFlags` + provides a convenient way to interpret the flags. The method must return + a :class:`memoryview` object. + +.. method:: object.__release_buffer__(self, buffer) + + Called when a buffer is no longer needed. The *buffer* argument is a + :class:`memoryview` object that was previously returned by + :meth:`~object.__buffer__`. The method must release any resources associated + with the buffer. This method should return ``None``. + Buffer objects that do not need to perform any cleanup are not required + to implement this method. + +.. versionadded:: 3.12 + +.. seealso:: + + :pep:`688` - Making the buffer protocol accessible in Python + Introduces the Python ``__buffer__`` and ``__release_buffer__`` methods. + + :class:`collections.abc.Buffer` + ABC for buffer types. + .. _special-lookup: Special method lookup diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index edbf921467553c..3fe3310a26e59b 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -149,6 +149,19 @@ New Features In Python 3.14, the default will switch to ``'data'``. (Contributed by Petr Viktorin in :pep:`706`.) +PEP 688: Making the buffer protocol accessible in Python +-------------------------------------------------------- + +:pep:`688` introduces a way to use the :ref:`buffer protocol ` +from Python code. Classes that implement the :meth:`~object.__buffer__` method +are now usable as buffer types. + +The new :class:`collections.abc.Buffer` ABC provides a standard +way to represent buffer objects, for example in type annotations. +The new :class:`inspect.BufferFlags` enum represents the flags that +can be used to customize buffer creation. +(Contributed by Jelle Zijlstra in :gh:`102500`.) + New Features Related to Type Hints ================================== @@ -179,7 +192,6 @@ See :pep:`692` for more details. (PEP written by Franek Magiera) - Other Language Changes ====================== From 2ba931ff727395cf89b290ed313a8e15db0bfcf1 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 4 May 2023 18:25:09 +0300 Subject: [PATCH 043/204] gh-99593: Add tests for Unicode C API (part 2) (#99868) Add tests for lower-level functions. Co-authored-by: Oleg Iarygin --- Lib/test/test_capi/test_unicode.py | 497 ++++++++++++++++++++++++++-- Modules/_testcapi/unicode.c | 506 +++++++++++++++++++++++++++-- Modules/_testcapimodule.c | 1 + 3 files changed, 944 insertions(+), 60 deletions(-) diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 857579f758386f..00807d968a7c43 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -17,6 +17,287 @@ class Str(str): class CAPITest(unittest.TestCase): + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_new(self): + """Test PyUnicode_New()""" + from _testcapi import unicode_new as new + + for maxchar in 0, 0x61, 0xa1, 0x4f60, 0x1f600, 0x10ffff: + self.assertEqual(new(0, maxchar), '') + self.assertEqual(new(5, maxchar), chr(maxchar)*5) + self.assertEqual(new(0, 0x110000), '') + self.assertRaises(SystemError, new, 5, 0x110000) + self.assertRaises(SystemError, new, -1, 0) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_fill(self): + """Test PyUnicode_Fill()""" + from _testcapi import unicode_fill as fill + + strings = [ + # all strings have exactly 5 characters + 'abcde', '\xa1\xa2\xa3\xa4\xa5', + '\u4f60\u597d\u4e16\u754c\uff01', + '\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604' + ] + chars = [0x78, 0xa9, 0x20ac, 0x1f638] + + for idx, fill_char in enumerate(chars): + # wide -> narrow: exceed maxchar limitation + for to in strings[:idx]: + self.assertRaises(ValueError, fill, to, 0, 0, fill_char) + for to in strings[idx:]: + for start in range(7): + for length in range(-1, 7 - start): + filled = max(min(length, 5 - start), 0) + if filled == 5 and to != strings[idx]: + # narrow -> wide + # Tests omitted since this creates invalid strings. + continue + expected = to[:start] + chr(fill_char) * filled + to[start + filled:] + self.assertEqual(fill(to, start, length, fill_char), + (expected, filled)) + + s = strings[0] + self.assertRaises(IndexError, fill, s, -1, 0, 0x78) + self.assertRaises(ValueError, fill, s, 0, 0, 0x110000) + self.assertRaises(SystemError, fill, b'abc', 0, 0, 0x78) + self.assertRaises(SystemError, fill, [], 0, 0, 0x78) + # CRASHES fill(s, 0, NULL, 0, 0) + # CRASHES fill(NULL, 0, 0, 0x78) + # TODO: Test PyUnicode_Fill() with non-modifiable unicode. + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_writechar(self): + """Test PyUnicode_ReadChar()""" + from _testcapi import unicode_writechar as writechar + + strings = [ + # one string for every kind + 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', + '\U0001f600\U0001f601\U0001f602' + ] + # one character for every kind + out of range code + chars = [0x78, 0xa9, 0x20ac, 0x1f638, 0x110000] + for i, s in enumerate(strings): + for j, c in enumerate(chars): + if j <= i: + self.assertEqual(writechar(s, 1, c), + (s[:1] + chr(c) + s[2:], 0)) + else: + self.assertRaises(ValueError, writechar, s, 1, c) + + self.assertRaises(IndexError, writechar, 'abc', 3, 0x78) + self.assertRaises(IndexError, writechar, 'abc', -1, 0x78) + self.assertRaises(TypeError, writechar, b'abc', 0, 0x78) + self.assertRaises(TypeError, writechar, [], 0, 0x78) + # CRASHES writechar(NULL, 0, 0x78) + # TODO: Test PyUnicode_CopyCharacters() with non-modifiable and legacy + # unicode. + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_resize(self): + """Test PyUnicode_Resize()""" + from _testcapi import unicode_resize as resize + + strings = [ + # all strings have exactly 3 characters + 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', + '\U0001f600\U0001f601\U0001f602' + ] + for s in strings: + self.assertEqual(resize(s, 3), (s, 0)) + self.assertEqual(resize(s, 2), (s[:2], 0)) + self.assertEqual(resize(s, 4), (s + '\0', 0)) + self.assertEqual(resize(s, 0), ('', 0)) + self.assertRaises(SystemError, resize, b'abc', 0) + self.assertRaises(SystemError, resize, [], 0) + self.assertRaises(SystemError, resize, NULL, 0) + # TODO: Test PyUnicode_Resize() with non-modifiable and legacy unicode + # and with NULL as the address. + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_append(self): + """Test PyUnicode_Append()""" + from _testcapi import unicode_append as append + + strings = [ + 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', + '\U0001f600\U0001f601\U0001f602' + ] + for left in strings: + left = left[::-1] + for right in strings: + expected = left + right + self.assertEqual(append(left, right), expected) + + self.assertRaises(SystemError, append, 'abc', b'abc') + self.assertRaises(SystemError, append, b'abc', 'abc') + self.assertRaises(SystemError, append, b'abc', b'abc') + self.assertRaises(SystemError, append, 'abc', []) + self.assertRaises(SystemError, append, [], 'abc') + self.assertRaises(SystemError, append, [], []) + self.assertRaises(SystemError, append, NULL, 'abc') + self.assertRaises(SystemError, append, 'abc', NULL) + # TODO: Test PyUnicode_Append() with modifiable unicode + # and with NULL as the address. + # TODO: Check reference counts. + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_appendanddel(self): + """Test PyUnicode_AppendAndDel()""" + from _testcapi import unicode_appendanddel as appendanddel + + strings = [ + 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', + '\U0001f600\U0001f601\U0001f602' + ] + for left in strings: + left = left[::-1] + for right in strings: + self.assertEqual(appendanddel(left, right), left + right) + + self.assertRaises(SystemError, appendanddel, 'abc', b'abc') + self.assertRaises(SystemError, appendanddel, b'abc', 'abc') + self.assertRaises(SystemError, appendanddel, b'abc', b'abc') + self.assertRaises(SystemError, appendanddel, 'abc', []) + self.assertRaises(SystemError, appendanddel, [], 'abc') + self.assertRaises(SystemError, appendanddel, [], []) + self.assertRaises(SystemError, appendanddel, NULL, 'abc') + self.assertRaises(SystemError, appendanddel, 'abc', NULL) + # TODO: Test PyUnicode_AppendAndDel() with modifiable unicode + # and with NULL as the address. + # TODO: Check reference counts. + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_fromstringandsize(self): + """Test PyUnicode_FromStringAndSize()""" + from _testcapi import unicode_fromstringandsize as fromstringandsize + + self.assertEqual(fromstringandsize(b'abc'), 'abc') + self.assertEqual(fromstringandsize(b'abc', 2), 'ab') + self.assertEqual(fromstringandsize(b'abc\0def'), 'abc\0def') + self.assertEqual(fromstringandsize(b'\xc2\xa1\xc2\xa2'), '\xa1\xa2') + self.assertEqual(fromstringandsize(b'\xe4\xbd\xa0'), '\u4f60') + self.assertEqual(fromstringandsize(b'\xf0\x9f\x98\x80'), '\U0001f600') + self.assertRaises(UnicodeDecodeError, fromstringandsize, b'\xc2\xa1', 1) + self.assertRaises(UnicodeDecodeError, fromstringandsize, b'\xa1', 1) + self.assertEqual(fromstringandsize(b'', 0), '') + self.assertEqual(fromstringandsize(NULL, 0), '') + + self.assertRaises(SystemError, fromstringandsize, b'abc', -1) + # TODO: Test PyUnicode_FromStringAndSize(NULL, size) for size != 0 + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_fromstring(self): + """Test PyUnicode_FromString()""" + from _testcapi import unicode_fromstring as fromstring + + self.assertEqual(fromstring(b'abc'), 'abc') + self.assertEqual(fromstring(b'\xc2\xa1\xc2\xa2'), '\xa1\xa2') + self.assertEqual(fromstring(b'\xe4\xbd\xa0'), '\u4f60') + self.assertEqual(fromstring(b'\xf0\x9f\x98\x80'), '\U0001f600') + self.assertRaises(UnicodeDecodeError, fromstring, b'\xc2') + self.assertRaises(UnicodeDecodeError, fromstring, b'\xa1') + self.assertEqual(fromstring(b''), '') + + # CRASHES fromstring(NULL) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_fromkindanddata(self): + """Test PyUnicode_FromKindAndData()""" + from _testcapi import unicode_fromkindanddata as fromkindanddata + + strings = [ + 'abcde', '\xa1\xa2\xa3\xa4\xa5', + '\u4f60\u597d\u4e16\u754c\uff01', + '\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604' + ] + enc1 = 'latin1' + for s in strings[:2]: + self.assertEqual(fromkindanddata(1, s.encode(enc1)), s) + enc2 = 'utf-16le' if sys.byteorder == 'little' else 'utf-16be' + for s in strings[:3]: + self.assertEqual(fromkindanddata(2, s.encode(enc2)), s) + enc4 = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be' + for s in strings: + self.assertEqual(fromkindanddata(4, s.encode(enc4)), s) + self.assertEqual(fromkindanddata(2, '\U0001f600'.encode(enc2)), + '\ud83d\ude00') + for kind in 1, 2, 4: + self.assertEqual(fromkindanddata(kind, b''), '') + self.assertEqual(fromkindanddata(kind, b'\0'*kind), '\0') + self.assertEqual(fromkindanddata(kind, NULL, 0), '') + + for kind in -1, 0, 3, 5, 8: + self.assertRaises(SystemError, fromkindanddata, kind, b'') + self.assertRaises(ValueError, fromkindanddata, 1, b'abc', -1) + self.assertRaises(ValueError, fromkindanddata, 1, NULL, -1) + # CRASHES fromkindanddata(1, NULL, 1) + # CRASHES fromkindanddata(4, b'\xff\xff\xff\xff') + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_substring(self): + """Test PyUnicode_Substring()""" + from _testcapi import unicode_substring as substring + + strings = [ + 'ab', 'ab\xa1\xa2', + 'ab\xa1\xa2\u4f60\u597d', + 'ab\xa1\xa2\u4f60\u597d\U0001f600\U0001f601' + ] + for s in strings: + for start in range(0, len(s) + 2): + for end in range(max(start-1, 0), len(s) + 2): + self.assertEqual(substring(s, start, end), s[start:end]) + + self.assertRaises(IndexError, substring, 'abc', -1, 0) + self.assertRaises(IndexError, substring, 'abc', 0, -1) + # CRASHES substring(b'abc', 0, 0) + # CRASHES substring([], 0, 0) + # CRASHES substring(NULL, 0, 0) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_getlength(self): + """Test PyUnicode_GetLength()""" + from _testcapi import unicode_getlength as getlength + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', + 'a\ud800b\udfffc', '\ud834\udd1e']: + self.assertEqual(getlength(s), len(s)) + + self.assertRaises(TypeError, getlength, b'abc') + self.assertRaises(TypeError, getlength, []) + # CRASHES getlength(NULL) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_readchar(self): + """Test PyUnicode_ReadChar()""" + from _testcapi import unicode_readchar as readchar + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', + 'a\ud800b\udfffc', '\ud834\udd1e']: + for i, c in enumerate(s): + self.assertEqual(readchar(s, i), ord(c)) + self.assertRaises(IndexError, readchar, s, len(s)) + self.assertRaises(IndexError, readchar, s, -1) + + self.assertRaises(TypeError, readchar, b'abc', 0) + self.assertRaises(TypeError, readchar, [], 0) + # CRASHES readchar(NULL, 0) + @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_fromobject(self): @@ -293,13 +574,70 @@ def check_format(expected, format, *args): self.assertRaisesRegex(SystemError, 'invalid format string', PyUnicode_FromFormat, b'%+i', c_int(10)) + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_interninplace(self): + """Test PyUnicode_InternInPlace()""" + from _testcapi import unicode_interninplace as interninplace + + s = b'abc'.decode() + r = interninplace(s) + self.assertEqual(r, 'abc') + + # CRASHES interninplace(b'abc') + # CRASHES interninplace(NULL) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_internfromstring(self): + """Test PyUnicode_InternFromString()""" + from _testcapi import unicode_internfromstring as internfromstring + + self.assertEqual(internfromstring(b'abc'), 'abc') + self.assertEqual(internfromstring(b'\xf0\x9f\x98\x80'), '\U0001f600') + self.assertRaises(UnicodeDecodeError, internfromstring, b'\xc2') + self.assertRaises(UnicodeDecodeError, internfromstring, b'\xa1') + self.assertEqual(internfromstring(b''), '') + + # CRASHES internfromstring(NULL) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_fromwidechar(self): + """Test PyUnicode_FromWideChar()""" + from _testcapi import unicode_fromwidechar as fromwidechar + from _testcapi import SIZEOF_WCHAR_T + + if SIZEOF_WCHAR_T == 2: + encoding = 'utf-16le' if sys.byteorder == 'little' else 'utf-16be' + elif SIZEOF_WCHAR_T == 4: + encoding = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be' + + for s in '', 'abc', '\xa1\xa2', '\u4f60', '\U0001f600': + b = s.encode(encoding) + self.assertEqual(fromwidechar(b), s) + self.assertEqual(fromwidechar(b + b'\0'*SIZEOF_WCHAR_T, -1), s) + for s in '\ud83d', '\ude00': + b = s.encode(encoding, 'surrogatepass') + self.assertEqual(fromwidechar(b), s) + self.assertEqual(fromwidechar(b + b'\0'*SIZEOF_WCHAR_T, -1), s) + + self.assertEqual(fromwidechar('abc'.encode(encoding), 2), 'ab') + if SIZEOF_WCHAR_T == 2: + self.assertEqual(fromwidechar('a\U0001f600'.encode(encoding), 2), 'a\ud83d') + + self.assertRaises(SystemError, fromwidechar, b'\0'*SIZEOF_WCHAR_T, -2) + self.assertEqual(fromwidechar(NULL, 0), '') + self.assertRaises(SystemError, fromwidechar, NULL, 1) + self.assertRaises(SystemError, fromwidechar, NULL, -1) + @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_aswidechar(self): """Test PyUnicode_AsWideChar()""" from _testcapi import unicode_aswidechar - import_helper.import_module('ctypes') - from ctypes import c_wchar, sizeof + from _testcapi import unicode_aswidechar_null + from _testcapi import SIZEOF_WCHAR_T wchar, size = unicode_aswidechar('abcdef', 2) self.assertEqual(size, 2) @@ -308,6 +646,8 @@ def test_aswidechar(self): wchar, size = unicode_aswidechar('abc', 3) self.assertEqual(size, 3) self.assertEqual(wchar, 'abc') + self.assertEqual(unicode_aswidechar_null('abc', 10), 4) + self.assertEqual(unicode_aswidechar_null('abc', 0), 4) wchar, size = unicode_aswidechar('abc', 4) self.assertEqual(size, 3) @@ -320,60 +660,113 @@ def test_aswidechar(self): wchar, size = unicode_aswidechar('abc\0def', 20) self.assertEqual(size, 7) self.assertEqual(wchar, 'abc\0def\0') + self.assertEqual(unicode_aswidechar_null('abc\0def', 20), 8) nonbmp = chr(0x10ffff) - if sizeof(c_wchar) == 2: - buflen = 3 + if SIZEOF_WCHAR_T == 2: nchar = 2 - else: # sizeof(c_wchar) == 4 - buflen = 2 + else: # SIZEOF_WCHAR_T == 4 nchar = 1 - wchar, size = unicode_aswidechar(nonbmp, buflen) + wchar, size = unicode_aswidechar(nonbmp, 10) self.assertEqual(size, nchar) self.assertEqual(wchar, nonbmp + '\0') + self.assertEqual(unicode_aswidechar_null(nonbmp, 10), nchar + 1) + + self.assertRaises(TypeError, unicode_aswidechar, b'abc', 10) + self.assertRaises(TypeError, unicode_aswidechar, [], 10) + self.assertRaises(SystemError, unicode_aswidechar, NULL, 10) + self.assertRaises(TypeError, unicode_aswidechar_null, b'abc', 10) + self.assertRaises(TypeError, unicode_aswidechar_null, [], 10) + self.assertRaises(SystemError, unicode_aswidechar_null, NULL, 10) @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_aswidecharstring(self): """Test PyUnicode_AsWideCharString()""" from _testcapi import unicode_aswidecharstring - import_helper.import_module('ctypes') - from ctypes import c_wchar, sizeof + from _testcapi import unicode_aswidecharstring_null + from _testcapi import SIZEOF_WCHAR_T wchar, size = unicode_aswidecharstring('abc') self.assertEqual(size, 3) self.assertEqual(wchar, 'abc\0') + self.assertEqual(unicode_aswidecharstring_null('abc'), 'abc') wchar, size = unicode_aswidecharstring('abc\0def') self.assertEqual(size, 7) self.assertEqual(wchar, 'abc\0def\0') + self.assertRaises(ValueError, unicode_aswidecharstring_null, 'abc\0def') nonbmp = chr(0x10ffff) - if sizeof(c_wchar) == 2: + if SIZEOF_WCHAR_T == 2: nchar = 2 - else: # sizeof(c_wchar) == 4 + else: # SIZEOF_WCHAR_T == 4 nchar = 1 wchar, size = unicode_aswidecharstring(nonbmp) self.assertEqual(size, nchar) self.assertEqual(wchar, nonbmp + '\0') + self.assertEqual(unicode_aswidecharstring_null(nonbmp), nonbmp) + + self.assertRaises(TypeError, unicode_aswidecharstring, b'abc') + self.assertRaises(TypeError, unicode_aswidecharstring, []) + self.assertRaises(SystemError, unicode_aswidecharstring, NULL) + self.assertRaises(TypeError, unicode_aswidecharstring_null, b'abc') + self.assertRaises(TypeError, unicode_aswidecharstring_null, []) + self.assertRaises(SystemError, unicode_aswidecharstring_null, NULL) @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_asucs4(self): """Test PyUnicode_AsUCS4()""" from _testcapi import unicode_asucs4 + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', 'a\ud800b\udfffc', '\ud834\udd1e']: l = len(s) - self.assertEqual(unicode_asucs4(s, l, True), s+'\0') - self.assertEqual(unicode_asucs4(s, l, False), s+'\uffff') - self.assertEqual(unicode_asucs4(s, l+1, True), s+'\0\uffff') - self.assertEqual(unicode_asucs4(s, l+1, False), s+'\0\uffff') - self.assertRaises(SystemError, unicode_asucs4, s, l-1, True) - self.assertRaises(SystemError, unicode_asucs4, s, l-2, False) + self.assertEqual(unicode_asucs4(s, l, 1), s+'\0') + self.assertEqual(unicode_asucs4(s, l, 0), s+'\uffff') + self.assertEqual(unicode_asucs4(s, l+1, 1), s+'\0\uffff') + self.assertEqual(unicode_asucs4(s, l+1, 0), s+'\0\uffff') + self.assertRaises(SystemError, unicode_asucs4, s, l-1, 1) + self.assertRaises(SystemError, unicode_asucs4, s, l-2, 0) + s = '\0'.join([s, s]) + self.assertEqual(unicode_asucs4(s, len(s), 1), s+'\0') + self.assertEqual(unicode_asucs4(s, len(s), 0), s+'\uffff') + + # CRASHES unicode_asucs4(b'abc', 1, 0) + # CRASHES unicode_asucs4(b'abc', 1, 1) + # CRASHES unicode_asucs4([], 1, 1) + # CRASHES unicode_asucs4(NULL, 1, 0) + # CRASHES unicode_asucs4(NULL, 1, 1) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_asucs4copy(self): + """Test PyUnicode_AsUCS4Copy()""" + from _testcapi import unicode_asucs4copy as asucs4copy + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', + 'a\ud800b\udfffc', '\ud834\udd1e']: + self.assertEqual(asucs4copy(s), s+'\0') s = '\0'.join([s, s]) - self.assertEqual(unicode_asucs4(s, len(s), True), s+'\0') - self.assertEqual(unicode_asucs4(s, len(s), False), s+'\uffff') + self.assertEqual(asucs4copy(s), s+'\0') + + # CRASHES asucs4copy(b'abc') + # CRASHES asucs4copy([]) + # CRASHES asucs4copy(NULL) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_fromordinal(self): + """Test PyUnicode_FromOrdinal()""" + from _testcapi import unicode_fromordinal as fromordinal + + self.assertEqual(fromordinal(0x61), 'a') + self.assertEqual(fromordinal(0x20ac), '\u20ac') + self.assertEqual(fromordinal(0x1f600), '\U0001f600') + + self.assertRaises(ValueError, fromordinal, 0x110000) + self.assertRaises(ValueError, fromordinal, -1) @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') @@ -381,29 +774,62 @@ def test_asutf8(self): """Test PyUnicode_AsUTF8()""" from _testcapi import unicode_asutf8 - bmp = '\u0100' - bmp2 = '\uffff' - nonbmp = chr(0x10ffff) + self.assertEqual(unicode_asutf8('abc', 4), b'abc\0') + self.assertEqual(unicode_asutf8('абв', 7), b'\xd0\xb0\xd0\xb1\xd0\xb2\0') + self.assertEqual(unicode_asutf8('\U0001f600', 5), b'\xf0\x9f\x98\x80\0') + self.assertEqual(unicode_asutf8('abc\0def', 8), b'abc\0def\0') - self.assertEqual(unicode_asutf8(bmp), b'\xc4\x80') - self.assertEqual(unicode_asutf8(bmp2), b'\xef\xbf\xbf') - self.assertEqual(unicode_asutf8(nonbmp), b'\xf4\x8f\xbf\xbf') - self.assertRaises(UnicodeEncodeError, unicode_asutf8, 'a\ud800b\udfffc') + self.assertRaises(UnicodeEncodeError, unicode_asutf8, '\ud8ff', 0) + self.assertRaises(TypeError, unicode_asutf8, b'abc', 0) + self.assertRaises(TypeError, unicode_asutf8, [], 0) + # CRASHES unicode_asutf8(NULL, 0) @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_asutf8andsize(self): """Test PyUnicode_AsUTF8AndSize()""" from _testcapi import unicode_asutf8andsize + from _testcapi import unicode_asutf8andsize_null - bmp = '\u0100' - bmp2 = '\uffff' - nonbmp = chr(0x10ffff) + self.assertEqual(unicode_asutf8andsize('abc', 4), (b'abc\0', 3)) + self.assertEqual(unicode_asutf8andsize('абв', 7), (b'\xd0\xb0\xd0\xb1\xd0\xb2\0', 6)) + self.assertEqual(unicode_asutf8andsize('\U0001f600', 5), (b'\xf0\x9f\x98\x80\0', 4)) + self.assertEqual(unicode_asutf8andsize('abc\0def', 8), (b'abc\0def\0', 7)) + self.assertEqual(unicode_asutf8andsize_null('abc', 4), b'abc\0') + self.assertEqual(unicode_asutf8andsize_null('abc\0def', 8), b'abc\0def\0') - self.assertEqual(unicode_asutf8andsize(bmp), (b'\xc4\x80', 2)) - self.assertEqual(unicode_asutf8andsize(bmp2), (b'\xef\xbf\xbf', 3)) - self.assertEqual(unicode_asutf8andsize(nonbmp), (b'\xf4\x8f\xbf\xbf', 4)) - self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize, 'a\ud800b\udfffc') + self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize, '\ud8ff', 0) + self.assertRaises(TypeError, unicode_asutf8andsize, b'abc', 0) + self.assertRaises(TypeError, unicode_asutf8andsize, [], 0) + # CRASHES unicode_asutf8andsize(NULL, 0) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_getdefaultencoding(self): + """Test PyUnicode_GetDefaultEncoding()""" + from _testcapi import unicode_getdefaultencoding as getdefaultencoding + + self.assertEqual(getdefaultencoding(), b'utf-8') + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_transform_decimal_and_space(self): + """Test _PyUnicode_TransformDecimalAndSpaceToASCII()""" + from _testcapi import unicode_transformdecimalandspacetoascii as transform_decimal + + self.assertEqual(transform_decimal('123'), + '123') + self.assertEqual(transform_decimal('\u0663.\u0661\u0664'), + '3.14') + self.assertEqual(transform_decimal("\N{EM SPACE}3.14\N{EN SPACE}"), + " 3.14 ") + self.assertEqual(transform_decimal('12\u20ac3'), + '12?') + self.assertEqual(transform_decimal(''), '') + + self.assertRaises(SystemError, transform_decimal, b'123') + self.assertRaises(SystemError, transform_decimal, []) + # CRASHES transform_decimal(NULL) @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') @@ -858,6 +1284,7 @@ def test_copycharacters(self): from _testcapi import unicode_copycharacters strings = [ + # all strings have exactly 5 characters 'abcde', '\xa1\xa2\xa3\xa4\xa5', '\u4f60\u597d\u4e16\u754c\uff01', '\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604' @@ -894,6 +1321,10 @@ def test_copycharacters(self): self.assertRaises(SystemError, unicode_copycharacters, s, 1, s, 0, 5) self.assertRaises(SystemError, unicode_copycharacters, s, 0, s, 0, -1) self.assertRaises(SystemError, unicode_copycharacters, s, 0, b'', 0, 0) + self.assertRaises(SystemError, unicode_copycharacters, s, 0, [], 0, 0) + # CRASHES unicode_copycharacters(s, 0, NULL, 0, 0) + # TODO: Test PyUnicode_CopyCharacters() with non-unicode and + # non-modifiable unicode as "to". @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index 2d23993ce420b3..7dd3b9c0c03e50 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -102,6 +102,278 @@ test_widechar(PyObject *self, PyObject *Py_UNUSED(ignored)) #define NULLABLE(x) do { if (x == Py_None) x = NULL; } while (0); +static PyObject * +unicode_copy(PyObject *unicode) +{ + PyObject *copy; + + if (!unicode) { + return NULL; + } + if (!PyUnicode_Check(unicode)) { + Py_INCREF(unicode); + return unicode; + } + + copy = PyUnicode_New(PyUnicode_GET_LENGTH(unicode), + PyUnicode_MAX_CHAR_VALUE(unicode)); + if (!copy) { + return NULL; + } + if (PyUnicode_CopyCharacters(copy, 0, unicode, + 0, PyUnicode_GET_LENGTH(unicode)) < 0) + { + Py_DECREF(copy); + return NULL; + } + return copy; +} + +/* Test PyUnicode_New() */ +static PyObject * +unicode_new(PyObject *self, PyObject *args) +{ + Py_ssize_t size; + unsigned int maxchar; + PyObject *result; + + if (!PyArg_ParseTuple(args, "nI", &size, &maxchar)) { + return NULL; + } + + result = PyUnicode_New(size, (Py_UCS4)maxchar); + if (!result) { + return NULL; + } + if (size > 0 && maxchar <= 0x10ffff && + PyUnicode_Fill(result, 0, size, (Py_UCS4)maxchar) < 0) + { + Py_DECREF(result); + return NULL; + } + return result; +} + +/* Test PyUnicode_Fill() */ +static PyObject * +unicode_fill(PyObject *self, PyObject *args) +{ + PyObject *to, *to_copy; + Py_ssize_t start, length, filled; + unsigned int fill_char; + + if (!PyArg_ParseTuple(args, "OnnI", &to, &start, &length, &fill_char)) { + return NULL; + } + + NULLABLE(to); + if (!(to_copy = unicode_copy(to)) && to) { + return NULL; + } + + filled = PyUnicode_Fill(to_copy, start, length, (Py_UCS4)fill_char); + if (filled == -1 && PyErr_Occurred()) { + Py_DECREF(to_copy); + return NULL; + } + return Py_BuildValue("(Nn)", to_copy, filled); +} + +/* Test PyUnicode_WriteChar() */ +static PyObject * +unicode_writechar(PyObject *self, PyObject *args) +{ + PyObject *to, *to_copy; + Py_ssize_t index; + unsigned int character; + int result; + + if (!PyArg_ParseTuple(args, "OnI", &to, &index, &character)) { + return NULL; + } + + NULLABLE(to); + if (!(to_copy = unicode_copy(to)) && to) { + return NULL; + } + + result = PyUnicode_WriteChar(to_copy, index, (Py_UCS4)character); + if (result == -1 && PyErr_Occurred()) { + Py_DECREF(to_copy); + return NULL; + } + return Py_BuildValue("(Ni)", to_copy, result); +} + +/* Test PyUnicode_Resize() */ +static PyObject * +unicode_resize(PyObject *self, PyObject *args) +{ + PyObject *obj, *copy; + Py_ssize_t length; + int result; + + if (!PyArg_ParseTuple(args, "On", &obj, &length)) { + return NULL; + } + + NULLABLE(obj); + if (!(copy = unicode_copy(obj)) && obj) { + return NULL; + } + result = PyUnicode_Resize(©, length); + if (result == -1 && PyErr_Occurred()) { + Py_XDECREF(copy); + return NULL; + } + if (obj && PyUnicode_Check(obj) && length > PyUnicode_GET_LENGTH(obj)) { + if (PyUnicode_Fill(copy, PyUnicode_GET_LENGTH(obj), length, 0U) < 0) { + Py_DECREF(copy); + return NULL; + } + } + return Py_BuildValue("(Ni)", copy, result); +} + +/* Test PyUnicode_Append() */ +static PyObject * +unicode_append(PyObject *self, PyObject *args) +{ + PyObject *left, *right, *left_copy; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + if (!(left_copy = unicode_copy(left)) && left) { + return NULL; + } + PyUnicode_Append(&left_copy, right); + return left_copy; +} + +/* Test PyUnicode_AppendAndDel() */ +static PyObject * +unicode_appendanddel(PyObject *self, PyObject *args) +{ + PyObject *left, *right, *left_copy; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + if (!(left_copy = unicode_copy(left)) && left) { + return NULL; + } + Py_XINCREF(right); + PyUnicode_AppendAndDel(&left_copy, right); + return left_copy; +} + +/* Test PyUnicode_FromStringAndSize() */ +static PyObject * +unicode_fromstringandsize(PyObject *self, PyObject *args) +{ + const char *s; + Py_ssize_t bsize; + Py_ssize_t size = -100; + + if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { + return NULL; + } + + if (size == -100) { + size = bsize; + } + return PyUnicode_FromStringAndSize(s, size); +} + +/* Test PyUnicode_FromString() */ +static PyObject * +unicode_fromstring(PyObject *self, PyObject *arg) +{ + const char *s; + Py_ssize_t size; + + if (!PyArg_Parse(arg, "z#", &s, &size)) { + return NULL; + } + return PyUnicode_FromString(s); +} + +/* Test PyUnicode_FromKindAndData() */ +static PyObject * +unicode_fromkindanddata(PyObject *self, PyObject *args) +{ + int kind; + void *buffer; + Py_ssize_t bsize; + Py_ssize_t size = -100; + + if (!PyArg_ParseTuple(args, "iz#|n", &kind, &buffer, &bsize, &size)) { + return NULL; + } + + if (size == -100) { + size = bsize; + } + if (kind && size % kind) { + PyErr_SetString(PyExc_AssertionError, + "invalid size in unicode_fromkindanddata()"); + return NULL; + } + return PyUnicode_FromKindAndData(kind, buffer, kind ? size / kind : 0); +} + +/* Test PyUnicode_Substring() */ +static PyObject * +unicode_substring(PyObject *self, PyObject *args) +{ + PyObject *str; + Py_ssize_t start, end; + + if (!PyArg_ParseTuple(args, "Onn", &str, &start, &end)) { + return NULL; + } + + NULLABLE(str); + return PyUnicode_Substring(str, start, end); +} + +/* Test PyUnicode_GetLength() */ +static PyObject * +unicode_getlength(PyObject *self, PyObject *arg) +{ + Py_ssize_t result; + + NULLABLE(arg); + result = PyUnicode_GetLength(arg); + if (result == -1) + return NULL; + return PyLong_FromSsize_t(result); +} + +/* Test PyUnicode_ReadChar() */ +static PyObject * +unicode_readchar(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t index; + Py_UCS4 result; + + if (!PyArg_ParseTuple(args, "On", &unicode, &index)) { + return NULL; + } + + NULLABLE(unicode); + result = PyUnicode_ReadChar(unicode, index); + if (result == (Py_UCS4)-1) + return NULL; + return PyLong_FromUnsignedLong(result); +} + /* Test PyUnicode_FromObject() */ static PyObject * unicode_fromobject(PyObject *self, PyObject *arg) @@ -110,6 +382,51 @@ unicode_fromobject(PyObject *self, PyObject *arg) return PyUnicode_FromObject(arg); } +/* Test PyUnicode_InternInPlace() */ +static PyObject * +unicode_interninplace(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + Py_XINCREF(arg); + PyUnicode_InternInPlace(&arg); + return arg; +} + +/* Test PyUnicode_InternFromString() */ +static PyObject * +unicode_internfromstring(PyObject *self, PyObject *arg) +{ + const char *s; + Py_ssize_t size; + + if (!PyArg_Parse(arg, "z#", &s, &size)) { + return NULL; + } + return PyUnicode_InternFromString(s); +} + +/* Test PyUnicode_FromWideChar() */ +static PyObject * +unicode_fromwidechar(PyObject *self, PyObject *args) +{ + const char *s; + Py_ssize_t bsize; + Py_ssize_t size = -100; + + if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { + return NULL; + } + if (size == -100) { + if (bsize % SIZEOF_WCHAR_T) { + PyErr_SetString(PyExc_AssertionError, + "invalid size in unicode_fromwidechar()"); + return NULL; + } + size = bsize / SIZEOF_WCHAR_T; + } + return PyUnicode_FromWideChar((const wchar_t *)s, size); +} + /* Test PyUnicode_AsWideChar() */ static PyObject * unicode_aswidechar(PyObject *self, PyObject *args) @@ -118,8 +435,9 @@ unicode_aswidechar(PyObject *self, PyObject *args) Py_ssize_t buflen, size; wchar_t *buffer; - if (!PyArg_ParseTuple(args, "Un", &unicode, &buflen)) + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) return NULL; + NULLABLE(unicode); buffer = PyMem_New(wchar_t, buflen); if (buffer == NULL) return PyErr_NoMemory(); @@ -142,17 +460,35 @@ unicode_aswidechar(PyObject *self, PyObject *args) return Py_BuildValue("(Nn)", result, size); } +/* Test PyUnicode_AsWideCharString() with NULL as buffer */ +static PyObject * +unicode_aswidechar_null(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t buflen, size; + + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) + return NULL; + NULLABLE(unicode); + size = PyUnicode_AsWideChar(unicode, NULL, buflen); + if (size == -1) { + return NULL; + } + return PyLong_FromSsize_t(size); +} + /* Test PyUnicode_AsWideCharString() */ static PyObject * unicode_aswidecharstring(PyObject *self, PyObject *args) { PyObject *unicode, *result; - Py_ssize_t size; + Py_ssize_t size = 100; wchar_t *buffer; - if (!PyArg_ParseTuple(args, "U", &unicode)) + if (!PyArg_ParseTuple(args, "O", &unicode)) return NULL; + NULLABLE(unicode); buffer = PyUnicode_AsWideCharString(unicode, &size); if (buffer == NULL) return NULL; @@ -164,6 +500,28 @@ unicode_aswidecharstring(PyObject *self, PyObject *args) return Py_BuildValue("(Nn)", result, size); } +/* Test PyUnicode_AsWideCharString() with NULL as the size address */ +static PyObject * +unicode_aswidecharstring_null(PyObject *self, PyObject *args) +{ + PyObject *unicode, *result; + wchar_t *buffer; + + if (!PyArg_ParseTuple(args, "O", &unicode)) + return NULL; + + NULLABLE(unicode); + buffer = PyUnicode_AsWideCharString(unicode, NULL); + if (buffer == NULL) + return NULL; + + result = PyUnicode_FromWideChar(buffer, -1); + PyMem_Free(buffer); + if (result == NULL) + return NULL; + return result; +} + /* Test PyUnicode_AsUCS4() */ static PyObject * unicode_asucs4(PyObject *self, PyObject *args) @@ -173,10 +531,11 @@ unicode_asucs4(PyObject *self, PyObject *args) int copy_null; Py_ssize_t str_len, buf_len; - if (!PyArg_ParseTuple(args, "Unp:unicode_asucs4", &unicode, &str_len, ©_null)) { + if (!PyArg_ParseTuple(args, "Onp:unicode_asucs4", &unicode, &str_len, ©_null)) { return NULL; } + NULLABLE(unicode); buf_len = str_len + 1; buffer = PyMem_NEW(Py_UCS4, buf_len); if (buffer == NULL) { @@ -195,48 +554,117 @@ unicode_asucs4(PyObject *self, PyObject *args) return result; } -/* Test PyUnicode_AsUTF8() */ +/* Test PyUnicode_AsUCS4Copy() */ static PyObject * -unicode_asutf8(PyObject *self, PyObject *args) +unicode_asucs4copy(PyObject *self, PyObject *args) { PyObject *unicode; - const char *buffer; + Py_UCS4 *buffer; + PyObject *result; - if (!PyArg_ParseTuple(args, "U", &unicode)) { + if (!PyArg_ParseTuple(args, "O", &unicode)) { return NULL; } - buffer = PyUnicode_AsUTF8(unicode); + NULLABLE(unicode); + buffer = PyUnicode_AsUCS4Copy(unicode); if (buffer == NULL) { return NULL; } + result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buffer, + PyUnicode_GET_LENGTH(unicode) + 1); + PyMem_FREE(buffer); + return result; +} + +/* Test PyUnicode_FromOrdinal() */ +static PyObject * +unicode_fromordinal(PyObject *self, PyObject *args) +{ + int ordinal; + + if (!PyArg_ParseTuple(args, "i", &ordinal)) + return NULL; - return PyBytes_FromString(buffer); + return PyUnicode_FromOrdinal(ordinal); +} + +/* Test PyUnicode_AsUTF8() */ +static PyObject * +unicode_asutf8(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t buflen; + const char *s; + + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) + return NULL; + + NULLABLE(unicode); + s = PyUnicode_AsUTF8(unicode); + if (s == NULL) + return NULL; + + return PyBytes_FromStringAndSize(s, buflen); } /* Test PyUnicode_AsUTF8AndSize() */ static PyObject * unicode_asutf8andsize(PyObject *self, PyObject *args) { - PyObject *unicode, *result; - const char *buffer; - Py_ssize_t utf8_len; + PyObject *unicode; + Py_ssize_t buflen; + const char *s; + Py_ssize_t size = -100; - if(!PyArg_ParseTuple(args, "U", &unicode)) { + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) return NULL; - } - buffer = PyUnicode_AsUTF8AndSize(unicode, &utf8_len); - if (buffer == NULL) { + NULLABLE(unicode); + s = PyUnicode_AsUTF8AndSize(unicode, &size); + if (s == NULL) return NULL; - } - result = PyBytes_FromString(buffer); - if (result == NULL) { + return Py_BuildValue("(y#n)", s, buflen, size); +} + +/* Test PyUnicode_AsUTF8AndSize() with NULL as the size address */ +static PyObject * +unicode_asutf8andsize_null(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t buflen; + const char *s; + + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) return NULL; - } - return Py_BuildValue("(Nn)", result, utf8_len); + NULLABLE(unicode); + s = PyUnicode_AsUTF8AndSize(unicode, NULL); + if (s == NULL) + return NULL; + + return PyBytes_FromStringAndSize(s, buflen); +} + +/* Test PyUnicode_GetDefaultEncoding() */ +static PyObject * +unicode_getdefaultencoding(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + const char *s = PyUnicode_GetDefaultEncoding(); + if (s == NULL) + return NULL; + + return PyBytes_FromString(s); +} + +/* Test _PyUnicode_TransformDecimalAndSpaceToASCII() */ +static PyObject * +unicode_transformdecimalandspacetoascii(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return _PyUnicode_TransformDecimalAndSpaceToASCII(arg); } /* Test PyUnicode_DecodeUTF8() */ @@ -470,11 +898,11 @@ unicode_findchar(PyObject *self, PyObject *args) Py_ssize_t result; Py_ssize_t start, end; - if (!PyArg_ParseTuple(args, "UInni:unicode_findchar", &str, &ch, + if (!PyArg_ParseTuple(args, "OInni:unicode_findchar", &str, &ch, &start, &end, &direction)) { return NULL; } - + NULLABLE(str); result = PyUnicode_FindChar(str, (Py_UCS4)ch, start, end, direction); if (result == -2) return NULL; @@ -612,11 +1040,12 @@ unicode_copycharacters(PyObject *self, PyObject *args) PyObject *from, *to, *to_copy; Py_ssize_t from_start, to_start, how_many, copied; - if (!PyArg_ParseTuple(args, "UnOnn:unicode_copycharacters", &to, &to_start, + if (!PyArg_ParseTuple(args, "UnOnn", &to, &to_start, &from, &from_start, &how_many)) { return NULL; } + NULLABLE(from); if (!(to_copy = PyUnicode_New(PyUnicode_GET_LENGTH(to), PyUnicode_MAX_CHAR_VALUE(to)))) { return NULL; @@ -626,8 +1055,9 @@ unicode_copycharacters(PyObject *self, PyObject *args) return NULL; } - if ((copied = PyUnicode_CopyCharacters(to_copy, to_start, from, - from_start, how_many)) < 0) { + copied = PyUnicode_CopyCharacters(to_copy, to_start, from, + from_start, how_many); + if (copied == -1 && PyErr_Occurred()) { Py_DECREF(to_copy); return NULL; } @@ -1053,14 +1483,36 @@ static PyMethodDef TestMethods[] = { test_unicode_compare_with_ascii, METH_NOARGS}, {"test_string_from_format", test_string_from_format, METH_NOARGS}, {"test_widechar", test_widechar, METH_NOARGS}, + {"unicode_new", unicode_new, METH_VARARGS}, + {"unicode_fill", unicode_fill, METH_VARARGS}, + {"unicode_writechar", unicode_writechar, METH_VARARGS}, + {"unicode_resize", unicode_resize, METH_VARARGS}, + {"unicode_append", unicode_append, METH_VARARGS}, + {"unicode_appendanddel", unicode_appendanddel, METH_VARARGS}, + {"unicode_fromstringandsize",unicode_fromstringandsize, METH_VARARGS}, + {"unicode_fromstring", unicode_fromstring, METH_O}, + {"unicode_fromkindanddata", unicode_fromkindanddata, METH_VARARGS}, + {"unicode_substring", unicode_substring, METH_VARARGS}, + {"unicode_getlength", unicode_getlength, METH_O}, + {"unicode_readchar", unicode_readchar, METH_VARARGS}, {"unicode_fromobject", unicode_fromobject, METH_O}, + {"unicode_interninplace", unicode_interninplace, METH_O}, + {"unicode_internfromstring", unicode_internfromstring, METH_O}, + {"unicode_fromwidechar", unicode_fromwidechar, METH_VARARGS}, {"unicode_aswidechar", unicode_aswidechar, METH_VARARGS}, + {"unicode_aswidechar_null", unicode_aswidechar_null, METH_VARARGS}, {"unicode_aswidecharstring", unicode_aswidecharstring, METH_VARARGS}, + {"unicode_aswidecharstring_null",unicode_aswidecharstring_null,METH_VARARGS}, {"unicode_asucs4", unicode_asucs4, METH_VARARGS}, + {"unicode_asucs4copy", unicode_asucs4copy, METH_VARARGS}, + {"unicode_fromordinal", unicode_fromordinal, METH_VARARGS}, {"unicode_asutf8", unicode_asutf8, METH_VARARGS}, {"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS}, + {"unicode_asutf8andsize_null",unicode_asutf8andsize_null, METH_VARARGS}, {"unicode_decodeutf8", unicode_decodeutf8, METH_VARARGS}, {"unicode_decodeutf8stateful",unicode_decodeutf8stateful, METH_VARARGS}, + {"unicode_getdefaultencoding",unicode_getdefaultencoding, METH_NOARGS}, + {"unicode_transformdecimalandspacetoascii", unicode_transformdecimalandspacetoascii, METH_O}, {"unicode_concat", unicode_concat, METH_VARARGS}, {"unicode_splitlines", unicode_splitlines, METH_VARARGS}, {"unicode_split", unicode_split, METH_VARARGS}, diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 38f4758e6575d8..ae70a33eb4d1ed 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4247,6 +4247,7 @@ PyInit__testcapi(void) PyModule_AddObject(m, "ULLONG_MAX", PyLong_FromUnsignedLongLong(ULLONG_MAX)); PyModule_AddObject(m, "PY_SSIZE_T_MAX", PyLong_FromSsize_t(PY_SSIZE_T_MAX)); PyModule_AddObject(m, "PY_SSIZE_T_MIN", PyLong_FromSsize_t(PY_SSIZE_T_MIN)); + PyModule_AddObject(m, "SIZEOF_WCHAR_T", PyLong_FromSsize_t(sizeof(wchar_t))); PyModule_AddObject(m, "SIZEOF_TIME_T", PyLong_FromSsize_t(sizeof(time_t))); PyModule_AddObject(m, "Py_Version", PyLong_FromUnsignedLong(Py_Version)); Py_INCREF(&PyInstanceMethod_Type); From 09b7695f12f2b44d2df6898407d7e68dd9493ed4 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 4 May 2023 09:39:33 -0700 Subject: [PATCH 044/204] gh-91896: Deprecate collections.abc.ByteString (#102096) Co-authored-by: Alex Waygood Co-authored-by: Hugo van Kemenade --- Doc/library/collections.abc.rst | 6 +++++ Doc/library/typing.rst | 3 +-- Doc/whatsnew/3.12.rst | 5 ++++ Lib/_collections_abc.py | 23 +++++++++++++++++-- Lib/test/test_collections.py | 19 +++++++++++---- ...3-03-08-02-45-46.gh-issue-91896.kgON_a.rst | 1 + 6 files changed, 49 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-08-02-45-46.gh-issue-91896.kgON_a.rst diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 669b7345499a78..43a3286ba832cf 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -273,6 +273,12 @@ Collections Abstract Base Classes -- Detailed Descriptions The index() method added support for *stop* and *start* arguments. + .. deprecated-removed:: 3.12 3.14 + The :class:`ByteString` ABC has been deprecated. + For use in typing, prefer a union, like ``bytes | bytearray``, or + :class:`collections.abc.Buffer`. + For use as an ABC, prefer :class:`Sequence` or :class:`collections.abc.Buffer`. + .. class:: Set MutableSet diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index c22fc0b28a50d0..162041fc7a846e 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2139,8 +2139,7 @@ Corresponding to collections in :mod:`collections.abc` annotate arguments of any of the types mentioned above. .. deprecated:: 3.9 - :class:`collections.abc.ByteString` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + Prefer :class:`collections.abc.Buffer`, or a union like ``bytes | bytearray | memoryview``. .. class:: Collection(Sized, Iterable[T_co], Container[T_co]) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 3fe3310a26e59b..4a988bf4b993da 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -792,6 +792,11 @@ Pending Removal in Python 3.14 (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) +* Deprecated :class:`collections.abc.ByteString`. + Prefer :class:`Sequence` or :class:`collections.abc.Buffer`. + For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. + (Contributed by Shantanu Jain in :gh:`91896`.) + * Creating immutable types (:data:`Py_TPFLAGS_IMMUTABLETYPE`) with mutable bases using the C API. diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 2117190cf8b64a..601107d2d86771 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -1071,8 +1071,27 @@ def count(self, value): Sequence.register(range) Sequence.register(memoryview) - -class ByteString(Sequence): +class _DeprecateByteStringMeta(ABCMeta): + def __new__(cls, name, bases, namespace, **kwargs): + if name != "ByteString": + import warnings + + warnings._deprecated( + "collections.abc.ByteString", + remove=(3, 14), + ) + return super().__new__(cls, name, bases, namespace, **kwargs) + + def __instancecheck__(cls, instance): + import warnings + + warnings._deprecated( + "collections.abc.ByteString", + remove=(3, 14), + ) + return super().__instancecheck__(instance) + +class ByteString(Sequence, metaclass=_DeprecateByteStringMeta): """This unifies bytes and bytearray. XXX Should add all their methods. diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 8fc28a6bf98e8f..bb8b352518ef3e 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -1940,14 +1940,25 @@ def assert_index_same(seq1, seq2, index_args): def test_ByteString(self): for sample in [bytes, bytearray]: - self.assertIsInstance(sample(), ByteString) + with self.assertWarns(DeprecationWarning): + self.assertIsInstance(sample(), ByteString) self.assertTrue(issubclass(sample, ByteString)) for sample in [str, list, tuple]: - self.assertNotIsInstance(sample(), ByteString) + with self.assertWarns(DeprecationWarning): + self.assertNotIsInstance(sample(), ByteString) self.assertFalse(issubclass(sample, ByteString)) - self.assertNotIsInstance(memoryview(b""), ByteString) + with self.assertWarns(DeprecationWarning): + self.assertNotIsInstance(memoryview(b""), ByteString) self.assertFalse(issubclass(memoryview, ByteString)) - self.validate_abstract_methods(ByteString, '__getitem__', '__len__') + with self.assertWarns(DeprecationWarning): + self.validate_abstract_methods(ByteString, '__getitem__', '__len__') + + with self.assertWarns(DeprecationWarning): + class X(ByteString): pass + + with self.assertWarns(DeprecationWarning): + # No metaclass conflict + class Z(ByteString, Awaitable): pass def test_Buffer(self): for sample in [bytes, bytearray, memoryview]: diff --git a/Misc/NEWS.d/next/Library/2023-03-08-02-45-46.gh-issue-91896.kgON_a.rst b/Misc/NEWS.d/next/Library/2023-03-08-02-45-46.gh-issue-91896.kgON_a.rst new file mode 100644 index 00000000000000..b5282d3d612916 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-08-02-45-46.gh-issue-91896.kgON_a.rst @@ -0,0 +1 @@ +Deprecate :class:`collections.abc.ByteString` From 8100be5535073a5442c2b8c68dcb2093ee69433d Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Thu, 4 May 2023 17:44:36 +0100 Subject: [PATCH 045/204] GH-81079: Add case_sensitive argument to `pathlib.Path.glob()` (GH-102710) This argument allows case-sensitive matching to be enabled on Windows, and case-insensitive matching to be enabled on Posix. Co-authored-by: Steve Dower --- Doc/library/pathlib.rst | 20 +++++++++-- Lib/pathlib.py | 34 +++++++++++-------- Lib/test/test_pathlib.py | 12 +++++++ ...3-03-15-00-37-43.gh-issue-81079.heTAod.rst | 2 ++ 4 files changed, 51 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-15-00-37-43.gh-issue-81079.heTAod.rst diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 4847ac24c77513..14118127835bbe 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -855,7 +855,7 @@ call fails (for example because the path doesn't exist). .. versionadded:: 3.5 -.. method:: Path.glob(pattern) +.. method:: Path.glob(pattern, *, case_sensitive=None) Glob the given relative *pattern* in the directory represented by this path, yielding all matching files (of any kind):: @@ -876,6 +876,11 @@ call fails (for example because the path doesn't exist). PosixPath('setup.py'), PosixPath('test_pathlib.py')] + By default, or when the *case_sensitive* keyword-only argument is set to + ``None``, this method matches paths using platform-specific casing rules: + typically, case-sensitive on POSIX, and case-insensitive on Windows. + Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. + .. note:: Using the "``**``" pattern in large directory trees may consume an inordinate amount of time. @@ -886,6 +891,9 @@ call fails (for example because the path doesn't exist). Return only directories if *pattern* ends with a pathname components separator (:data:`~os.sep` or :data:`~os.altsep`). + .. versionadded:: 3.12 + The *case_sensitive* argument. + .. method:: Path.group() Return the name of the group owning the file. :exc:`KeyError` is raised @@ -1271,7 +1279,7 @@ call fails (for example because the path doesn't exist). .. versionadded:: 3.6 The *strict* argument (pre-3.6 behavior is strict). -.. method:: Path.rglob(pattern) +.. method:: Path.rglob(pattern, *, case_sensitive=None) Glob the given relative *pattern* recursively. This is like calling :func:`Path.glob` with "``**/``" added in front of the *pattern*, where @@ -1284,12 +1292,20 @@ call fails (for example because the path doesn't exist). PosixPath('setup.py'), PosixPath('test_pathlib.py')] + By default, or when the *case_sensitive* keyword-only argument is set to + ``None``, this method matches paths using platform-specific casing rules: + typically, case-sensitive on POSIX, and case-insensitive on Windows. + Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. + .. audit-event:: pathlib.Path.rglob self,pattern pathlib.Path.rglob .. versionchanged:: 3.11 Return only directories if *pattern* ends with a pathname components separator (:data:`~os.sep` or :data:`~os.altsep`). + .. versionadded:: 3.12 + The *case_sensitive* argument. + .. method:: Path.rmdir() Remove this directory. The directory must be empty. diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 8a1651c23d7f4d..f32e1e2d822834 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -62,7 +62,7 @@ def _is_case_sensitive(flavour): # @functools.lru_cache() -def _make_selector(pattern_parts, flavour): +def _make_selector(pattern_parts, flavour, case_sensitive): pat = pattern_parts[0] child_parts = pattern_parts[1:] if not pat: @@ -75,17 +75,17 @@ def _make_selector(pattern_parts, flavour): raise ValueError("Invalid pattern: '**' can only be an entire path component") else: cls = _WildcardSelector - return cls(pat, child_parts, flavour) + return cls(pat, child_parts, flavour, case_sensitive) class _Selector: """A selector matches a specific glob pattern part against the children of a given path.""" - def __init__(self, child_parts, flavour): + def __init__(self, child_parts, flavour, case_sensitive): self.child_parts = child_parts if child_parts: - self.successor = _make_selector(child_parts, flavour) + self.successor = _make_selector(child_parts, flavour, case_sensitive) self.dironly = True else: self.successor = _TerminatingSelector() @@ -108,8 +108,9 @@ def _select_from(self, parent_path, scandir): class _ParentSelector(_Selector): - def __init__(self, name, child_parts, flavour): - _Selector.__init__(self, child_parts, flavour) + + def __init__(self, name, child_parts, flavour, case_sensitive): + _Selector.__init__(self, child_parts, flavour, case_sensitive) def _select_from(self, parent_path, scandir): path = parent_path._make_child_relpath('..') @@ -119,10 +120,13 @@ def _select_from(self, parent_path, scandir): class _WildcardSelector(_Selector): - def __init__(self, pat, child_parts, flavour): - flags = re.NOFLAG if _is_case_sensitive(flavour) else re.IGNORECASE + def __init__(self, pat, child_parts, flavour, case_sensitive): + _Selector.__init__(self, child_parts, flavour, case_sensitive) + if case_sensitive is None: + # TODO: evaluate case-sensitivity of each directory in _select_from() + case_sensitive = _is_case_sensitive(flavour) + flags = re.NOFLAG if case_sensitive else re.IGNORECASE self.match = re.compile(fnmatch.translate(pat), flags=flags).fullmatch - _Selector.__init__(self, child_parts, flavour) def _select_from(self, parent_path, scandir): try: @@ -153,8 +157,8 @@ def _select_from(self, parent_path, scandir): class _RecursiveWildcardSelector(_Selector): - def __init__(self, pat, child_parts, flavour): - _Selector.__init__(self, child_parts, flavour) + def __init__(self, pat, child_parts, flavour, case_sensitive): + _Selector.__init__(self, child_parts, flavour, case_sensitive) def _iterate_directories(self, parent_path, scandir): yield parent_path @@ -819,7 +823,7 @@ def _scandir(self): # includes scandir(), which is used to implement glob(). return os.scandir(self) - def glob(self, pattern): + def glob(self, pattern, *, case_sensitive=None): """Iterate over this subtree and yield all existing files (of any kind, including directories) matching the given relative pattern. """ @@ -831,11 +835,11 @@ def glob(self, pattern): raise NotImplementedError("Non-relative patterns are unsupported") if pattern[-1] in (self._flavour.sep, self._flavour.altsep): pattern_parts.append('') - selector = _make_selector(tuple(pattern_parts), self._flavour) + selector = _make_selector(tuple(pattern_parts), self._flavour, case_sensitive) for p in selector.select_from(self): yield p - def rglob(self, pattern): + def rglob(self, pattern, *, case_sensitive=None): """Recursively yield all existing files (of any kind, including directories) matching the given relative pattern, anywhere in this subtree. @@ -846,7 +850,7 @@ def rglob(self, pattern): raise NotImplementedError("Non-relative patterns are unsupported") if pattern and pattern[-1] in (self._flavour.sep, self._flavour.altsep): pattern_parts.append('') - selector = _make_selector(("**",) + tuple(pattern_parts), self._flavour) + selector = _make_selector(("**",) + tuple(pattern_parts), self._flavour, case_sensitive) for p in selector.select_from(self): yield p diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 424bb92a87d112..a932e03df4236d 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1816,6 +1816,18 @@ def _check(glob, expected): else: _check(p.glob("*/"), ["dirA", "dirB", "dirC", "dirE", "linkB"]) + def test_glob_case_sensitive(self): + P = self.cls + def _check(path, pattern, case_sensitive, expected): + actual = {str(q) for q in path.glob(pattern, case_sensitive=case_sensitive)} + expected = {str(P(BASE, q)) for q in expected} + self.assertEqual(actual, expected) + path = P(BASE) + _check(path, "DIRB/FILE*", True, []) + _check(path, "DIRB/FILE*", False, ["dirB/fileB"]) + _check(path, "dirb/file*", True, []) + _check(path, "dirb/file*", False, ["dirB/fileB"]) + def test_rglob_common(self): def _check(glob, expected): self.assertEqual(set(glob), { P(BASE, q) for q in expected }) diff --git a/Misc/NEWS.d/next/Library/2023-03-15-00-37-43.gh-issue-81079.heTAod.rst b/Misc/NEWS.d/next/Library/2023-03-15-00-37-43.gh-issue-81079.heTAod.rst new file mode 100644 index 00000000000000..ef5690533985d5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-15-00-37-43.gh-issue-81079.heTAod.rst @@ -0,0 +1,2 @@ +Add *case_sensitive* keyword-only argument to :meth:`pathlib.Path.glob` and +:meth:`~pathlib.Path.rglob`. From d47cddfa27cbcbeb4dddb9263976d85c41241672 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 4 May 2023 19:55:48 +0300 Subject: [PATCH 046/204] gh-102500: Add PEP 688 and 698 to the 3.12 release highlights (#104174) --- Doc/whatsnew/3.12.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 4a988bf4b993da..4f952e2a37ef2f 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -68,13 +68,17 @@ Summary -- Release highlights New typing features: +* :pep:`688`: Making the buffer protocol accessible in Python + * :ref:`whatsnew312-pep692` +* :pep:`698`: Override Decorator for Static Typing + Important deprecations, removals or restrictions: -* :pep:`623`, Remove wstr from Unicode +* :pep:`623`: Remove wstr from Unicode -* :pep:`632`, Remove the ``distutils`` package. +* :pep:`632`: Remove the ``distutils`` package Improved Error Messages ======================= From f5c38382f9c40f0017cef086896a8160e313ac9e Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Thu, 4 May 2023 11:45:56 -0600 Subject: [PATCH 047/204] gh-103963: fix 'make regen-opcode' in out-of-tree builds (#104177) --- Makefile.pre.in | 4 +++- PCbuild/regen.targets | 2 +- Tools/build/generate_opcode_h.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index b285ef9e832db5..736a520d0e8fb6 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1376,9 +1376,11 @@ regen-opcode: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_opcode_h.py \ $(srcdir)/Lib/opcode.py \ $(srcdir)/Include/opcode.h.new \ - $(srcdir)/Include/internal/pycore_opcode.h.new + $(srcdir)/Include/internal/pycore_opcode.h.new \ + $(srcdir)/Include/internal/pycore_intrinsics.h.new $(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode.h $(srcdir)/Include/internal/pycore_opcode.h.new + $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_intrinsics.h $(srcdir)/Include/internal/pycore_intrinsics.h.new .PHONY: regen-token regen-token: diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index aeb7e2e185d9f8..107066817ba6b0 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -59,7 +59,7 @@ Inputs="@(_OpcodeSources)" Outputs="@(_OpcodeOutputs)" DependsOnTargets="FindPythonForBuild"> - diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index adcbaf2b8e08bf..5be981005725bf 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -233,4 +233,4 @@ def main(opcode_py, outfile='Include/opcode.h', if __name__ == '__main__': - main(sys.argv[1], sys.argv[2], sys.argv[3]) + main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]) From 7d35c3121ade679dd6e8b4a0bac7b3702aee6921 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 4 May 2023 15:07:42 -0700 Subject: [PATCH 048/204] GH-103899: Provide a hint when accidentally calling a module (GH-103900) --- Lib/test/test_call.py | 32 ++++++++++++++ ...-04-26-15-14-23.gh-issue-103899.1pqKPF.rst | 3 ++ Objects/call.c | 44 ++++++++++++++++--- 3 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-26-15-14-23.gh-issue-103899.1pqKPF.rst diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index aab7b1580eaf35..12759c53bb662c 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -10,6 +10,7 @@ import gc import contextlib import sys +import types class BadStr(str): @@ -202,6 +203,37 @@ def test_oldargs1_2_kw(self): msg = r"count\(\) takes no keyword arguments" self.assertRaisesRegex(TypeError, msg, [].count, x=2, y=2) + def test_object_not_callable(self): + msg = r"^'object' object is not callable$" + self.assertRaisesRegex(TypeError, msg, object()) + + def test_module_not_callable_no_suggestion_0(self): + msg = r"^'module' object is not callable$" + self.assertRaisesRegex(TypeError, msg, types.ModuleType("mod")) + + def test_module_not_callable_no_suggestion_1(self): + msg = r"^'module' object is not callable$" + mod = types.ModuleType("mod") + mod.mod = 42 + self.assertRaisesRegex(TypeError, msg, mod) + + def test_module_not_callable_no_suggestion_2(self): + msg = r"^'module' object is not callable$" + mod = types.ModuleType("mod") + del mod.__name__ + self.assertRaisesRegex(TypeError, msg, mod) + + def test_module_not_callable_no_suggestion_3(self): + msg = r"^'module' object is not callable$" + mod = types.ModuleType("mod") + mod.__name__ = 42 + self.assertRaisesRegex(TypeError, msg, mod) + + def test_module_not_callable_suggestion(self): + msg = r"^'module' object is not callable\. Did you mean: 'mod\.mod\(\.\.\.\)'\?$" + mod = types.ModuleType("mod") + mod.mod = lambda: ... + self.assertRaisesRegex(TypeError, msg, mod) class TestCallingConventions(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-26-15-14-23.gh-issue-103899.1pqKPF.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-26-15-14-23.gh-issue-103899.1pqKPF.rst new file mode 100644 index 00000000000000..c12a6b9cb841f2 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-04-26-15-14-23.gh-issue-103899.1pqKPF.rst @@ -0,0 +1,3 @@ +Provide a helpful hint in the :exc:`TypeError` message when accidentally +calling a :term:`module` object that has a callable attribute of the same +name (such as :func:`dis.dis` or :class:`datetime.datetime`). diff --git a/Objects/call.c b/Objects/call.c index cf6e357a990441..0d548dcd5e1aed 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -157,6 +157,42 @@ PyObject_VectorcallDict(PyObject *callable, PyObject *const *args, return _PyObject_FastCallDictTstate(tstate, callable, args, nargsf, kwargs); } +static void +object_is_not_callable(PyThreadState *tstate, PyObject *callable) +{ + if (Py_IS_TYPE(callable, &PyModule_Type)) { + // >>> import pprint + // >>> pprint(thing) + // Traceback (most recent call last): + // File "", line 1, in + // TypeError: 'module' object is not callable. Did you mean: 'pprint.pprint(...)'? + PyObject *name = PyModule_GetNameObject(callable); + if (name == NULL) { + _PyErr_Clear(tstate); + goto basic_type_error; + } + PyObject *attr; + int res = _PyObject_LookupAttr(callable, name, &attr); + if (res < 0) { + _PyErr_Clear(tstate); + } + else if (res > 0 && PyCallable_Check(attr)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object is not callable. " + "Did you mean: '%U.%U(...)'?", + Py_TYPE(callable)->tp_name, name, name); + Py_DECREF(attr); + Py_DECREF(name); + return; + } + Py_XDECREF(attr); + Py_DECREF(name); + } +basic_type_error: + _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not callable", + Py_TYPE(callable)->tp_name); +} + PyObject * _PyObject_MakeTpCall(PyThreadState *tstate, PyObject *callable, @@ -171,9 +207,7 @@ _PyObject_MakeTpCall(PyThreadState *tstate, PyObject *callable, * temporary dictionary for keyword arguments (if any) */ ternaryfunc call = Py_TYPE(callable)->tp_call; if (call == NULL) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not callable", - Py_TYPE(callable)->tp_name); + object_is_not_callable(tstate, callable); return NULL; } @@ -322,9 +356,7 @@ _PyObject_Call(PyThreadState *tstate, PyObject *callable, else { call = Py_TYPE(callable)->tp_call; if (call == NULL) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not callable", - Py_TYPE(callable)->tp_name); + object_is_not_callable(tstate, callable); return NULL; } From 46361bb84332800bc3632688e6ef3b4dd4a48723 Mon Sep 17 00:00:00 2001 From: Michael Blahay Date: Thu, 4 May 2023 18:37:17 -0400 Subject: [PATCH 049/204] gh-68968: Correcting message display issue with assertEqual (#103937) --- Lib/test/test_unittest/test_assertions.py | 6 +- Lib/test/test_unittest/test_case.py | 60 +++++++++++++++++++ Lib/unittest/case.py | 31 +++++++--- ...3-04-27-18-46-31.gh-issue-68968.E3tnhy.rst | 1 + 4 files changed, 87 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-27-18-46-31.gh-issue-68968.E3tnhy.rst diff --git a/Lib/test/test_unittest/test_assertions.py b/Lib/test/test_unittest/test_assertions.py index 6557104b81fc0f..5c1a28ecda5b49 100644 --- a/Lib/test/test_unittest/test_assertions.py +++ b/Lib/test/test_unittest/test_assertions.py @@ -273,9 +273,9 @@ def testAssertDictEqual(self): def testAssertMultiLineEqual(self): self.assertMessages('assertMultiLineEqual', ("", "foo"), - [r"\+ foo$", "^oops$", - r"\+ foo$", - r"\+ foo : oops$"]) + [r"\+ foo\n$", "^oops$", + r"\+ foo\n$", + r"\+ foo\n : oops$"]) def testAssertLess(self): self.assertMessages('assertLess', (2, 1), diff --git a/Lib/test/test_unittest/test_case.py b/Lib/test/test_unittest/test_case.py index dd5ff6d553e61d..ed5eb5609a5dd1 100644 --- a/Lib/test/test_unittest/test_case.py +++ b/Lib/test/test_unittest/test_case.py @@ -1149,6 +1149,66 @@ def testAssertEqualSingleLine(self): error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + def testAssertEqualwithEmptyString(self): + '''Verify when there is an empty string involved, the diff output + does not treat the empty string as a single empty line. It should + instead be handled as a non-line. + ''' + sample_text = '' + revised_sample_text = 'unladen swallows fly quickly' + sample_text_error = '''\ ++ unladen swallows fly quickly +''' + try: + self.assertEqual(sample_text, revised_sample_text) + except self.failureException as e: + # need to remove the first line of the error message + error = str(e).split('\n', 1)[1] + self.assertEqual(sample_text_error, error) + + def testAssertEqualMultipleLinesMissingNewlineTerminator(self): + '''Verifying format of diff output from assertEqual involving strings + with multiple lines, but missing the terminating newline on both. + ''' + sample_text = 'laden swallows\nfly sloely' + revised_sample_text = 'laden swallows\nfly slowly' + sample_text_error = '''\ + laden swallows +- fly sloely +? ^ ++ fly slowly +? ^ +''' + try: + self.assertEqual(sample_text, revised_sample_text) + except self.failureException as e: + # need to remove the first line of the error message + error = str(e).split('\n', 1)[1] + self.assertEqual(sample_text_error, error) + + def testAssertEqualMultipleLinesMismatchedNewlinesTerminators(self): + '''Verifying format of diff output from assertEqual involving strings + with multiple lines and mismatched newlines. The output should + include a - on it's own line to indicate the newline difference + between the two strings + ''' + sample_text = 'laden swallows\nfly sloely\n' + revised_sample_text = 'laden swallows\nfly slowly' + sample_text_error = '''\ + laden swallows +- fly sloely +? ^ ++ fly slowly +? ^ +-\x20 +''' + try: + self.assertEqual(sample_text, revised_sample_text) + except self.failureException as e: + # need to remove the first line of the error message + error = str(e).split('\n', 1)[1] + self.assertEqual(sample_text_error, error) + def testEqualityBytesWarning(self): if sys.flags.bytes_warning: def bytes_warning(): diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index 018f22e7ce0c73..001b640dc43ad6 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -1217,19 +1217,34 @@ def assertCountEqual(self, first, second, msg=None): def assertMultiLineEqual(self, first, second, msg=None): """Assert that two multi-line strings are equal.""" - self.assertIsInstance(first, str, 'First argument is not a string') - self.assertIsInstance(second, str, 'Second argument is not a string') + self.assertIsInstance(first, str, "First argument is not a string") + self.assertIsInstance(second, str, "Second argument is not a string") if first != second: - # don't use difflib if the strings are too long + # Don't use difflib if the strings are too long if (len(first) > self._diffThreshold or len(second) > self._diffThreshold): self._baseAssertEqual(first, second, msg) - firstlines = first.splitlines(keepends=True) - secondlines = second.splitlines(keepends=True) - if len(firstlines) == 1 and first.strip('\r\n') == first: - firstlines = [first + '\n'] - secondlines = [second + '\n'] + + # Append \n to both strings if either is missing the \n. + # This allows the final ndiff to show the \n difference. The + # exception here is if the string is empty, in which case no + # \n should be added + first_presplit = first + second_presplit = second + if first and second: + if first[-1] != '\n' or second[-1] != '\n': + first_presplit += '\n' + second_presplit += '\n' + elif second and second[-1] != '\n': + second_presplit += '\n' + elif first and first[-1] != '\n': + first_presplit += '\n' + + firstlines = first_presplit.splitlines(keepends=True) + secondlines = second_presplit.splitlines(keepends=True) + + # Generate the message and diff, then raise the exception standardMsg = '%s != %s' % _common_shorten_repr(first, second) diff = '\n' + ''.join(difflib.ndiff(firstlines, secondlines)) standardMsg = self._truncateMessage(standardMsg, diff) diff --git a/Misc/NEWS.d/next/Library/2023-04-27-18-46-31.gh-issue-68968.E3tnhy.rst b/Misc/NEWS.d/next/Library/2023-04-27-18-46-31.gh-issue-68968.E3tnhy.rst new file mode 100644 index 00000000000000..bf29b64793b933 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-27-18-46-31.gh-issue-68968.E3tnhy.rst @@ -0,0 +1 @@ +Fixed garbled output of :meth:`~unittest.TestCase.assertEqual` when an input lacks final newline. From fa86a77589a06661fcebb806d36f3a7450e2aecf Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Thu, 4 May 2023 16:38:53 -0600 Subject: [PATCH 050/204] gh-104112: link from cached_property docs to method-caching FAQ (#104113) Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- Doc/library/functools.rst | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 29cbc87bf66d12..40f43f8b3519cd 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -110,18 +110,10 @@ The :mod:`functools` module defines the following functions: ``__slots__`` without including ``__dict__`` as one of the defined slots (as such classes don't provide a ``__dict__`` attribute at all). - If a mutable mapping is not available or if space-efficient key sharing - is desired, an effect similar to :func:`cached_property` can be achieved - by a stacking :func:`property` on top of :func:`cache`:: - - class DataSet: - def __init__(self, sequence_of_numbers): - self._data = sequence_of_numbers - - @property - @cache - def stdev(self): - return statistics.stdev(self._data) + If a mutable mapping is not available or if space-efficient key sharing is + desired, an effect similar to :func:`cached_property` can also be achieved by + stacking :func:`property` on top of :func:`lru_cache`. See + :ref:`faq-cache-method-calls` for more details on how this differs from :func:`cached_property`. .. versionadded:: 3.8 From ce871fdc3a02e8441ad73b13f9fced308a9d9ad1 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 4 May 2023 17:00:07 -0700 Subject: [PATCH 051/204] GH-104142: Fix _Py_RefcntAdd to respect immortality (GH-104143) --- Include/internal/pycore_object.h | 3 ++ Lib/test/test_builtin.py | 43 +++++++++++-------- ...-05-02-18-29-49.gh-issue-104142._5Et6I.rst | 2 + 3 files changed, 30 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-02-18-29-49.gh-issue-104142._5Et6I.rst diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index b9e700ea280cbf..500b3eece68055 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -58,6 +58,9 @@ extern void _Py_DecRefTotal(PyInterpreterState *); // Increment reference count by n static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) { + if (_Py_IsImmortal(op)) { + return; + } #ifdef Py_REF_DEBUG _Py_AddRefTotal(_PyInterpreterState_GET(), n); #endif diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 04dd8ff3070c99..821710a7fa3286 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2372,24 +2372,31 @@ def __del__(self): @cpython_only class ImmortalTests(unittest.TestCase): - def test_immortal(self): - none_refcount = sys.getrefcount(None) - true_refcount = sys.getrefcount(True) - false_refcount = sys.getrefcount(False) - smallint_refcount = sys.getrefcount(100) - - # Assert that all of these immortal instances have large ref counts. - self.assertGreater(none_refcount, 2 ** 15) - self.assertGreater(true_refcount, 2 ** 15) - self.assertGreater(false_refcount, 2 ** 15) - self.assertGreater(smallint_refcount, 2 ** 15) - - # Confirm that the refcount doesn't change even with a new ref to them. - l = [None, True, False, 100] - self.assertEqual(sys.getrefcount(None), none_refcount) - self.assertEqual(sys.getrefcount(True), true_refcount) - self.assertEqual(sys.getrefcount(False), false_refcount) - self.assertEqual(sys.getrefcount(100), smallint_refcount) + + if sys.maxsize < (1 << 32): + IMMORTAL_REFCOUNT = (1 << 30) - 1 + else: + IMMORTAL_REFCOUNT = (1 << 32) - 1 + + IMMORTALS = (None, True, False, Ellipsis, NotImplemented, *range(-5, 257)) + + def assert_immortal(self, immortal): + with self.subTest(immortal): + self.assertEqual(sys.getrefcount(immortal), self.IMMORTAL_REFCOUNT) + + def test_immortals(self): + for immortal in self.IMMORTALS: + self.assert_immortal(immortal) + + def test_list_repeat_respect_immortality(self): + refs = list(self.IMMORTALS) * 42 + for immortal in self.IMMORTALS: + self.assert_immortal(immortal) + + def test_tuple_repeat_respect_immortality(self): + refs = tuple(self.IMMORTALS) * 42 + for immortal in self.IMMORTALS: + self.assert_immortal(immortal) class TestType(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-02-18-29-49.gh-issue-104142._5Et6I.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-02-18-29-49.gh-issue-104142._5Et6I.rst new file mode 100644 index 00000000000000..6a19ae84057f4c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-02-18-29-49.gh-issue-104142._5Et6I.rst @@ -0,0 +1,2 @@ +Fix an issue where :class:`list` or :class:`tuple` repetition could fail to +respect :pep:`683`. From e5b8b19d99861c73ab76ee0175a685acf6082d7e Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Fri, 5 May 2023 12:27:25 +0900 Subject: [PATCH 052/204] gh-104106: Add gcc fallback of mkfifoat/mknodat for macOS (gh-104129) --- ...023-05-04-10-56-14.gh-issue-104106.-W9BJS.rst | 1 + Modules/posixmodule.c | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2023-05-04-10-56-14.gh-issue-104106.-W9BJS.rst diff --git a/Misc/NEWS.d/next/Build/2023-05-04-10-56-14.gh-issue-104106.-W9BJS.rst b/Misc/NEWS.d/next/Build/2023-05-04-10-56-14.gh-issue-104106.-W9BJS.rst new file mode 100644 index 00000000000000..900e5bd61d6033 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-05-04-10-56-14.gh-issue-104106.-W9BJS.rst @@ -0,0 +1 @@ +Add gcc fallback of mkfifoat/mknodat for macOS. Patch by Dong-hee Na. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index dcb5e7a0e0408c..b395c265c72d0c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -175,6 +175,14 @@ # define HAVE_PWRITEV_RUNTIME (pwritev != NULL) # endif +# ifdef HAVE_MKFIFOAT +# define HAVE_MKFIFOAT_RUNTIME (mkfifoat != NULL) +# endif + +# ifdef HAVE_MKNODAT +# define HAVE_MKNODAT_RUNTIME (mknodat != NULL) +# endif + #endif #ifdef HAVE_FUTIMESAT @@ -4802,7 +4810,7 @@ os__path_isdir_impl(PyObject *module, PyObject *path) } Py_BEGIN_ALLOW_THREADS - if (_path.wide) { + if (_path.wide) { if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, &statInfo, sizeof(statInfo))) { if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { @@ -4899,7 +4907,7 @@ os__path_isfile_impl(PyObject *module, PyObject *path) } Py_BEGIN_ALLOW_THREADS - if (_path.wide) { + if (_path.wide) { if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, &statInfo, sizeof(statInfo))) { if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { @@ -4995,7 +5003,7 @@ os__path_exists_impl(PyObject *module, PyObject *path) } Py_BEGIN_ALLOW_THREADS - if (_path.wide) { + if (_path.wide) { if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, &statInfo, sizeof(statInfo))) { if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { @@ -5082,7 +5090,7 @@ os__path_islink_impl(PyObject *module, PyObject *path) } Py_BEGIN_ALLOW_THREADS - if (_path.wide) { + if (_path.wide) { if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, &statInfo, sizeof(statInfo))) { slow_path = FALSE; From 163034515a81f137d1dd7d289dc048eb0f1cd424 Mon Sep 17 00:00:00 2001 From: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> Date: Fri, 5 May 2023 17:54:03 +0800 Subject: [PATCH 053/204] gh-104190: fix ubsan crash (#104191) --- Modules/_testcapimodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index ae70a33eb4d1ed..376f04f23e324a 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4221,7 +4221,7 @@ PyInit__testcapi(void) return NULL; } int ret = PyModule_AddType(m, (PyTypeObject*)ObjExtraData_Type); - Py_DECREF(&ObjExtraData_Type); + Py_DECREF(ObjExtraData_Type); if (ret < 0) { return NULL; } From 81fc135f263e9ec5df3d967290b9665d7e100c7f Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Fri, 5 May 2023 11:34:13 +0100 Subject: [PATCH 054/204] gh-104051: fix crash in test_xxtestfuzz with -We (#104052) --- Modules/_xxtestfuzz/fuzzer.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Modules/_xxtestfuzz/fuzzer.c b/Modules/_xxtestfuzz/fuzzer.c index fb0c191d2c494d..37d402824853f0 100644 --- a/Modules/_xxtestfuzz/fuzzer.c +++ b/Modules/_xxtestfuzz/fuzzer.c @@ -526,13 +526,20 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { #if !defined(_Py_FUZZ_ONE) || defined(_Py_FUZZ_fuzz_sre_compile) static int SRE_COMPILE_INITIALIZED = 0; if (!SRE_COMPILE_INITIALIZED && !init_sre_compile()) { - PyErr_Print(); - abort(); + if (!PyErr_ExceptionMatches(PyExc_DeprecationWarning)) { + PyErr_Print(); + abort(); + } + else { + PyErr_Clear(); + } } else { SRE_COMPILE_INITIALIZED = 1; } - rv |= _run_fuzz(data, size, fuzz_sre_compile); + if (SRE_COMPILE_INITIALIZED) { + rv |= _run_fuzz(data, size, fuzz_sre_compile); + } #endif #if !defined(_Py_FUZZ_ONE) || defined(_Py_FUZZ_fuzz_sre_match) static int SRE_MATCH_INITIALIZED = 0; From 2318bedb3645d2dfb56944553f01d6c434904c4b Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Fri, 5 May 2023 16:42:58 +0530 Subject: [PATCH 055/204] GH-103092: port `_asyncio` freelist to module state (#104196) --- Modules/_asynciomodule.c | 52 +++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 8b1a29b6d33e8b..fef34d65523870 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -18,6 +18,10 @@ module _asyncio /*[clinic end generated code: output=da39a3ee5e6b4b0d input=8fd17862aa989c69]*/ +#define FI_FREELIST_MAXLEN 255 + +typedef struct futureiterobject futureiterobject; + /* State of the _asyncio module */ typedef struct { PyTypeObject *FutureIterType; @@ -67,6 +71,9 @@ typedef struct { /* Counter for autogenerated Task names */ uint64_t task_name_counter; + + futureiterobject *fi_freelist; + Py_ssize_t fi_freelist_len; } asyncio_state; static inline asyncio_state * @@ -1574,28 +1581,24 @@ FutureObj_dealloc(PyObject *self) /*********************** Future Iterator **************************/ -typedef struct { +typedef struct futureiterobject { PyObject_HEAD FutureObj *future; } futureiterobject; -#define FI_FREELIST_MAXLEN 255 -static futureiterobject *fi_freelist = NULL; -static Py_ssize_t fi_freelist_len = 0; - - static void FutureIter_dealloc(futureiterobject *it) { PyTypeObject *tp = Py_TYPE(it); + asyncio_state *state = get_asyncio_state_by_def((PyObject *)it); PyObject_GC_UnTrack(it); tp->tp_clear((PyObject *)it); - if (fi_freelist_len < FI_FREELIST_MAXLEN) { - fi_freelist_len++; - it->future = (FutureObj*) fi_freelist; - fi_freelist = it; + if (state->fi_freelist_len < FI_FREELIST_MAXLEN) { + state->fi_freelist_len++; + it->future = (FutureObj*) state->fi_freelist; + state->fi_freelist = it; } else { PyObject_GC_Del(it); @@ -1799,17 +1802,12 @@ future_new_iter(PyObject *fut) futureiterobject *it; asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); - if (!Future_Check(state, fut)) { - PyErr_BadInternalCall(); - return NULL; - } - ENSURE_FUTURE_ALIVE(state, fut) - if (fi_freelist_len) { - fi_freelist_len--; - it = fi_freelist; - fi_freelist = (futureiterobject*) it->future; + if (state->fi_freelist_len) { + state->fi_freelist_len--; + it = state->fi_freelist; + state->fi_freelist = (futureiterobject*) it->future; it->future = NULL; _Py_NewReference((PyObject*) it); } @@ -3556,22 +3554,22 @@ _asyncio_current_task_impl(PyObject *module, PyObject *loop) static void -module_free_freelists(void) +module_free_freelists(asyncio_state *state) { PyObject *next; PyObject *current; - next = (PyObject*) fi_freelist; + next = (PyObject*) state->fi_freelist; while (next != NULL) { - assert(fi_freelist_len > 0); - fi_freelist_len--; + assert(state->fi_freelist_len > 0); + state->fi_freelist_len--; current = next; next = (PyObject*) ((futureiterobject*) current)->future; PyObject_GC_Del(current); } - assert(fi_freelist_len == 0); - fi_freelist = NULL; + assert(state->fi_freelist_len == 0); + state->fi_freelist = NULL; } static int @@ -3603,7 +3601,7 @@ module_traverse(PyObject *mod, visitproc visit, void *arg) Py_VISIT(state->context_kwname); // Visit freelist. - PyObject *next = (PyObject*) fi_freelist; + PyObject *next = (PyObject*) state->fi_freelist; while (next != NULL) { PyObject *current = next; Py_VISIT(current); @@ -3640,7 +3638,7 @@ module_clear(PyObject *mod) Py_CLEAR(state->context_kwname); - module_free_freelists(); + module_free_freelists(state); return 0; } From 721a78395d07c68351625aedb827a47dd4f48525 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 5 May 2023 13:32:00 +0200 Subject: [PATCH 056/204] gh-64658: Expand Argument Clinic return converter docs (#104175) --- Doc/howto/clinic.rst | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 8a10fe327358c0..6ebc2d9b0a71a9 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1033,19 +1033,36 @@ you're not permitted to use: Using a return converter ------------------------ -By default the impl function Argument Clinic generates for you returns ``PyObject *``. -But your C function often computes some C type, then converts it into the ``PyObject *`` +By default, the impl function Argument Clinic generates for you returns +:c:type:`PyObject * `. +But your C function often computes some C type, +then converts it into the :c:type:`!PyObject *` at the last moment. Argument Clinic handles converting your inputs from Python types into native C types—why not have it convert your return value from a native C type into a Python type too? That's what a "return converter" does. It changes your impl function to return some C type, then adds code to the generated (non-impl) function to handle converting -that value into the appropriate ``PyObject *``. +that value into the appropriate :c:type:`!PyObject *`. The syntax for return converters is similar to that of parameter converters. You specify the return converter like it was a return annotation on the -function itself. Return converters behave much the same as parameter converters; +function itself, using ``->`` notation. + +For example: + +.. code-block:: c + + /*[clinic input] + add -> int + + a: int + b: int + / + + [clinic start generated code]*/ + +Return converters behave much the same as parameter converters; they take arguments, the arguments are all keyword-only, and if you're not changing any of the default arguments you can omit the parentheses. @@ -1066,19 +1083,17 @@ Currently Argument Clinic supports only a few return converters: .. code-block:: none bool + double + float int - unsigned int long - unsigned int - size_t Py_ssize_t - float - double - DecodeFSDefault + size_t + unsigned int + unsigned long -None of these take parameters. For the first three, return -1 to indicate -error. For ``DecodeFSDefault``, the return type is ``const char *``; return a ``NULL`` -pointer to indicate an error. +None of these take parameters. +For all of these, return ``-1`` to indicate error. To see all the return converters Argument Clinic supports, along with their parameters (if any), From 5245cb64d9a898804edf683427ef4d4fd20ccaec Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 5 May 2023 13:40:18 +0200 Subject: [PATCH 057/204] gh-68395: Avoid naming conflicts by mangling variable names in Argument Clinic (#104065) Add all internally used variable names to CLINIC_PREFIXED_ARGS. --- Lib/test/clinic.test | 169 +++++++++++++++++++++++++++++++++++++++++ Tools/clinic/clinic.py | 13 +++- 2 files changed, 181 insertions(+), 1 deletion(-) diff --git a/Lib/test/clinic.test b/Lib/test/clinic.test index 53e5df5ba872ed..564205274edd73 100644 --- a/Lib/test/clinic.test +++ b/Lib/test/clinic.test @@ -4102,3 +4102,172 @@ exit: static PyObject * test_paramname_module_impl(PyObject *module, PyObject *mod) /*[clinic end generated code: output=4a2a849ecbcc8b53 input=afefe259667f13ba]*/ + +/*[clinic input] +mangle1 + + args: object + kwnames: object + return_value: object + _keywords: object + _parser: object + argsbuf: object + fastargs: object + nargs: object + noptargs: object + +[clinic start generated code]*/ + +PyDoc_STRVAR(mangle1__doc__, +"mangle1($module, /, args, kwnames, return_value, _keywords, _parser,\n" +" argsbuf, fastargs, nargs, noptargs)\n" +"--\n" +"\n"); + +#define MANGLE1_METHODDEF \ + {"mangle1", _PyCFunction_CAST(mangle1), METH_FASTCALL|METH_KEYWORDS, mangle1__doc__}, + +static PyObject * +mangle1_impl(PyObject *module, PyObject *args, PyObject *kwnames, + PyObject *return_value, PyObject *_keywords, PyObject *_parser, + PyObject *argsbuf, PyObject *fastargs, PyObject *nargs, + PyObject *noptargs); + +static PyObject * +mangle1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 9 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(args), &_Py_ID(kwnames), &_Py_ID(return_value), &_Py_ID(_keywords), &_Py_ID(_parser), &_Py_ID(argsbuf), &_Py_ID(fastargs), &_Py_ID(nargs), &_Py_ID(noptargs), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"args", "kwnames", "return_value", "_keywords", "_parser", "argsbuf", "fastargs", "nargs", "noptargs", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "mangle1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[9]; + PyObject *__clinic_args; + PyObject *__clinic_kwnames; + PyObject *__clinic_return_value; + PyObject *__clinic__keywords; + PyObject *__clinic__parser; + PyObject *__clinic_argsbuf; + PyObject *__clinic_fastargs; + PyObject *__clinic_nargs; + PyObject *__clinic_noptargs; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 9, 9, 0, argsbuf); + if (!args) { + goto exit; + } + __clinic_args = args[0]; + __clinic_kwnames = args[1]; + __clinic_return_value = args[2]; + __clinic__keywords = args[3]; + __clinic__parser = args[4]; + __clinic_argsbuf = args[5]; + __clinic_fastargs = args[6]; + __clinic_nargs = args[7]; + __clinic_noptargs = args[8]; + return_value = mangle1_impl(module, __clinic_args, __clinic_kwnames, __clinic_return_value, __clinic__keywords, __clinic__parser, __clinic_argsbuf, __clinic_fastargs, __clinic_nargs, __clinic_noptargs); + +exit: + return return_value; +} + +static PyObject * +mangle1_impl(PyObject *module, PyObject *args, PyObject *kwnames, + PyObject *return_value, PyObject *_keywords, PyObject *_parser, + PyObject *argsbuf, PyObject *fastargs, PyObject *nargs, + PyObject *noptargs) +/*[clinic end generated code: output=083e5076be9987c3 input=a3ed51bdedf8a3c7]*/ + +/*[clinic input] +mangle2 + + args: object + kwargs: object + return_value: object + +[clinic start generated code]*/ + +PyDoc_STRVAR(mangle2__doc__, +"mangle2($module, /, args, kwargs, return_value)\n" +"--\n" +"\n"); + +#define MANGLE2_METHODDEF \ + {"mangle2", _PyCFunction_CAST(mangle2), METH_FASTCALL|METH_KEYWORDS, mangle2__doc__}, + +static PyObject * +mangle2_impl(PyObject *module, PyObject *args, PyObject *kwargs, + PyObject *return_value); + +static PyObject * +mangle2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(args), &_Py_ID(kwargs), &_Py_ID(return_value), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"args", "kwargs", "return_value", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "mangle2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject *__clinic_args; + PyObject *__clinic_kwargs; + PyObject *__clinic_return_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + if (!args) { + goto exit; + } + __clinic_args = args[0]; + __clinic_kwargs = args[1]; + __clinic_return_value = args[2]; + return_value = mangle2_impl(module, __clinic_args, __clinic_kwargs, __clinic_return_value); + +exit: + return return_value; +} + +static PyObject * +mangle2_impl(PyObject *module, PyObject *args, PyObject *kwargs, + PyObject *return_value) +/*[clinic end generated code: output=2ebb62aaefe7590a input=391766fee51bad7a]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index d3e120c0e7a8ee..2746b24c333e66 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -43,7 +43,18 @@ NO_VARARG = "PY_SSIZE_T_MAX" CLINIC_PREFIX = "__clinic_" -CLINIC_PREFIXED_ARGS = {"args"} +CLINIC_PREFIXED_ARGS = { + "_keywords", + "_parser", + "args", + "argsbuf", + "fastargs", + "kwargs", + "kwnames", + "nargs", + "noptargs", + "return_value", +} class Unspecified: def __repr__(self): From d0b4abedfb8b0322df835065f85551d097cfecb8 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 5 May 2023 14:08:24 +0200 Subject: [PATCH 058/204] gh-64631: Test exception messages in cloned Argument Clinic funcs (#104167) --- Lib/test/test_clinic.py | 13 ++ Modules/_testclinic.c | 81 ++++++++++ Modules/clinic/_testclinic.c.h | 260 ++++++++++++++++++++++++++++++++- 3 files changed, 353 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 4abf739cf52ca3..660f7a1e296d8f 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1284,6 +1284,19 @@ def test_gh_99240_double_free(self): with self.assertRaisesRegex(TypeError, expected_error): ac_tester.gh_99240_double_free('a', '\0b') + def test_cloned_func_exception_message(self): + incorrect_arg = -1 # f1() and f2() accept a single str + with self.assertRaisesRegex(TypeError, "clone_f1"): + ac_tester.clone_f1(incorrect_arg) + with self.assertRaisesRegex(TypeError, "clone_f2"): + ac_tester.clone_f2(incorrect_arg) + + def test_cloned_func_with_converter_exception_message(self): + for name in "clone_with_conv_f1", "clone_with_conv_f2": + with self.subTest(name=name): + func = getattr(ac_tester, name) + self.assertEqual(func(), name) + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 91fdee24d328d9..6ff55a2755cf5a 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -9,6 +9,19 @@ #include "Python.h" + +// Used for clone_with_conv_f1 and clone_with_conv_v2 +typedef struct { + const char *name; +} custom_t; + +static int +custom_converter(PyObject *obj, custom_t *val) +{ + return 1; +} + + #include "clinic/_testclinic.c.h" @@ -1117,6 +1130,70 @@ gh_99240_double_free_impl(PyObject *module, char *a, char *b) } +/*[clinic input] +_testclinic.clone_f1 as clone_f1 + path: str +[clinic start generated code]*/ + +static PyObject * +clone_f1_impl(PyObject *module, const char *path) +/*[clinic end generated code: output=8c30b5620ba86715 input=9c614b7f025ebf70]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +_testclinic.clone_f2 as clone_f2 = _testclinic.clone_f1 +[clinic start generated code]*/ + +static PyObject * +clone_f2_impl(PyObject *module, const char *path) +/*[clinic end generated code: output=6aa1c39bec3f5d9b input=1aaaf47d6ed2324a]*/ +{ + Py_RETURN_NONE; +} + + +/*[python input] +class custom_t_converter(CConverter): + type = 'custom_t' + converter = 'custom_converter' + + def pre_render(self): + self.c_default = f'''{{ + .name = "{self.function.name}", + }}''' + +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=b2fb801e99a06bf6]*/ + + +/*[clinic input] +_testclinic.clone_with_conv_f1 as clone_with_conv_f1 + path: custom_t = None +[clinic start generated code]*/ + +static PyObject * +clone_with_conv_f1_impl(PyObject *module, custom_t path) +/*[clinic end generated code: output=f7e030ffd5439cb0 input=bc77bc80dec3f46d]*/ +{ + return PyUnicode_FromString(path.name); +} + + +/*[clinic input] +_testclinic.clone_with_conv_f2 as clone_with_conv_f2 = _testclinic.clone_with_conv_f1 +[clinic start generated code]*/ + +static PyObject * +clone_with_conv_f2_impl(PyObject *module, custom_t path) +/*[clinic end generated code: output=9d7fdd6a75eecee4 input=cff459a205fa83bb]*/ +{ + return PyUnicode_FromString(path.name); +} + + static PyMethodDef tester_methods[] = { TEST_EMPTY_FUNCTION_METHODDEF OBJECTS_CONVERTER_METHODDEF @@ -1168,6 +1245,10 @@ static PyMethodDef tester_methods[] = { GH_32092_KW_PASS_METHODDEF GH_99233_REFCOUNT_METHODDEF GH_99240_DOUBLE_FREE_METHODDEF + CLONE_F1_METHODDEF + CLONE_F2_METHODDEF + CLONE_WITH_CONV_F1_METHODDEF + CLONE_WITH_CONV_F2_METHODDEF {NULL, NULL} }; diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 831f58ca650aab..cc69f5c3d2fe9f 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -2817,4 +2817,262 @@ gh_99240_double_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=e8211606b03d733a input=a9049054013a1b77]*/ + +PyDoc_STRVAR(clone_f1__doc__, +"clone_f1($module, /, path)\n" +"--\n" +"\n"); + +#define CLONE_F1_METHODDEF \ + {"clone_f1", _PyCFunction_CAST(clone_f1), METH_FASTCALL|METH_KEYWORDS, clone_f1__doc__}, + +static PyObject * +clone_f1_impl(PyObject *module, const char *path); + +static PyObject * +clone_f1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "clone_f1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + const char *path; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("clone_f1", "argument 'path'", "str", args[0]); + goto exit; + } + Py_ssize_t path_length; + path = PyUnicode_AsUTF8AndSize(args[0], &path_length); + if (path == NULL) { + goto exit; + } + if (strlen(path) != (size_t)path_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + return_value = clone_f1_impl(module, path); + +exit: + return return_value; +} + +PyDoc_STRVAR(clone_f2__doc__, +"clone_f2($module, /, path)\n" +"--\n" +"\n"); + +#define CLONE_F2_METHODDEF \ + {"clone_f2", _PyCFunction_CAST(clone_f2), METH_FASTCALL|METH_KEYWORDS, clone_f2__doc__}, + +static PyObject * +clone_f2_impl(PyObject *module, const char *path); + +static PyObject * +clone_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "clone_f2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + const char *path; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("clone_f2", "argument 'path'", "str", args[0]); + goto exit; + } + Py_ssize_t path_length; + path = PyUnicode_AsUTF8AndSize(args[0], &path_length); + if (path == NULL) { + goto exit; + } + if (strlen(path) != (size_t)path_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + return_value = clone_f2_impl(module, path); + +exit: + return return_value; +} + +PyDoc_STRVAR(clone_with_conv_f1__doc__, +"clone_with_conv_f1($module, /, path=None)\n" +"--\n" +"\n"); + +#define CLONE_WITH_CONV_F1_METHODDEF \ + {"clone_with_conv_f1", _PyCFunction_CAST(clone_with_conv_f1), METH_FASTCALL|METH_KEYWORDS, clone_with_conv_f1__doc__}, + +static PyObject * +clone_with_conv_f1_impl(PyObject *module, custom_t path); + +static PyObject * +clone_with_conv_f1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "clone_with_conv_f1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + custom_t path = { + .name = "clone_with_conv_f1", + }; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (!custom_converter(args[0], &path)) { + goto exit; + } +skip_optional_pos: + return_value = clone_with_conv_f1_impl(module, path); + +exit: + return return_value; +} + +PyDoc_STRVAR(clone_with_conv_f2__doc__, +"clone_with_conv_f2($module, /, path=None)\n" +"--\n" +"\n"); + +#define CLONE_WITH_CONV_F2_METHODDEF \ + {"clone_with_conv_f2", _PyCFunction_CAST(clone_with_conv_f2), METH_FASTCALL|METH_KEYWORDS, clone_with_conv_f2__doc__}, + +static PyObject * +clone_with_conv_f2_impl(PyObject *module, custom_t path); + +static PyObject * +clone_with_conv_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "clone_with_conv_f2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + custom_t path = { + .name = "clone_with_conv_f2", + }; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (!custom_converter(args[0], &path)) { + goto exit; + } +skip_optional_pos: + return_value = clone_with_conv_f2_impl(module, path); + +exit: + return return_value; +} +/*[clinic end generated code: output=f58202a6e5df2d16 input=a9049054013a1b77]*/ From 45a9e3834a6ed20ee250e2e5a8583dffcef0eb73 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 5 May 2023 14:38:38 +0200 Subject: [PATCH 059/204] gh-64595: Argument Clinic: Touch source file if any output file changed (#104152) --- Lib/test/test_clinic.py | 7 ++++--- Tools/clinic/clinic.py | 19 +++++++++++++------ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 660f7a1e296d8f..6aaf4d1ed8d560 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -99,8 +99,9 @@ def test_eol(self): # the last line of the block got corrupted. c = clinic.Clinic(clinic.CLanguage(None), filename="file") raw = "/*[clinic]\nfoo\n[clinic]*/" - cooked = c.parse(raw).splitlines() - end_line = cooked[2].rstrip() + cooked, _ = c.parse(raw) + lines = cooked.splitlines() + end_line = lines[2].rstrip() # this test is redundant, it's just here explicitly to catch # the regression test so we don't forget what it looked like self.assertNotEqual(end_line, "[clinic]*/[clinic]*/") @@ -259,7 +260,7 @@ def _test_clinic(self, input, output): c = clinic.Clinic(language, filename="file") c.parsers['inert'] = InertParser(c) c.parsers['copy'] = CopyParser(c) - computed = c.parse(input) + computed, _ = c.parse(input) self.assertEqual(output, computed) def test_clinic_1(self): diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 2746b24c333e66..5f6e67e7d65a1d 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -1954,12 +1954,12 @@ def dump(self): return_converters = {} -def write_file(filename, new_contents): +def write_file(filename, new_contents, force=False): try: with open(filename, 'r', encoding="utf-8") as fp: old_contents = fp.read() - if old_contents == new_contents: + if old_contents == new_contents and not force: # no change: avoid modifying the file modification time return except FileNotFoundError: @@ -2123,6 +2123,8 @@ def parse(self, input): traceback.format_exc().rstrip()) printer.print_block(block) + clinic_out = [] + # these are destinations not buffers for name, destination in self.destinations.items(): if destination.type == 'suppress': @@ -2162,10 +2164,11 @@ def parse(self, input): block.input = 'preserve\n' printer_2 = BlockPrinter(self.language) printer_2.print_block(block, core_includes=True) - write_file(destination.filename, printer_2.f.getvalue()) + pair = destination.filename, printer_2.f.getvalue() + clinic_out.append(pair) continue - return printer.f.getvalue() + return printer.f.getvalue(), clinic_out def _module_and_class(self, fields): @@ -2221,9 +2224,13 @@ def parse_file(filename, *, verify=True, output=None): return clinic = Clinic(language, verify=verify, filename=filename) - cooked = clinic.parse(raw) + src_out, clinic_out = clinic.parse(raw) - write_file(output, cooked) + # If clinic output changed, force updating the source file as well. + force = bool(clinic_out) + write_file(output, src_out, force=force) + for fn, data in clinic_out: + write_file(fn, data) def compute_checksum(input, length=None): From a0df9ee8fc77443510ab7e9ba8fd830f255a8fec Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 5 May 2023 17:53:07 +0100 Subject: [PATCH 060/204] GH-96803: Add three C-API functions to make _PyInterpreterFrame less opaque for users of PEP 523. (GH-96849) --- Include/cpython/frameobject.h | 17 +++++++++++++++++ Include/internal/pycore_frame.h | 2 -- ...22-09-15-15-21-34.gh-issue-96803.ynBKIS.rst | 6 ++++++ Modules/_tracemalloc.c | 3 ++- Objects/frameobject.c | 2 +- Objects/genobject.c | 3 ++- Python/ceval.c | 3 ++- Python/frame.c | 18 +++++++++++++++++- Python/traceback.c | 2 +- 9 files changed, 48 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2022-09-15-15-21-34.gh-issue-96803.ynBKIS.rst diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index 4e19535c656f2c..6f3efe36ede5d8 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -4,6 +4,8 @@ # error "this header file must not be included directly" #endif +struct _PyInterpreterFrame; + /* Standard object interface */ PyAPI_FUNC(PyFrameObject *) PyFrame_New(PyThreadState *, PyCodeObject *, @@ -27,3 +29,18 @@ PyAPI_FUNC(int) _PyFrame_IsEntryFrame(PyFrameObject *frame); PyAPI_FUNC(int) PyFrame_FastToLocalsWithError(PyFrameObject *f); PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); + +/* The following functions are for use by debuggers and other tools + * implementing custom frame evaluators with PEP 523. */ + +/* Returns the code object of the frame (strong reference). + * Does not raise an exception. */ +PyAPI_FUNC(PyCodeObject *) PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame); + +/* Returns a byte ofsset into the last executed instruction. + * Does not raise an exception. */ +PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame); + +/* Returns the currently executing line number, or -1 if there is no line number. + * Does not raise an exception. */ +PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame); diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index d8d7fe9ef2ebde..3d3cbbff7aae81 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -265,8 +265,6 @@ _PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func, int null_l return new_frame; } -int _PyInterpreterFrame_GetLine(_PyInterpreterFrame *frame); - static inline PyGenObject *_PyFrame_GetGenerator(_PyInterpreterFrame *frame) { diff --git a/Misc/NEWS.d/next/C API/2022-09-15-15-21-34.gh-issue-96803.ynBKIS.rst b/Misc/NEWS.d/next/C API/2022-09-15-15-21-34.gh-issue-96803.ynBKIS.rst new file mode 100644 index 00000000000000..6fc56d2249f581 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-09-15-15-21-34.gh-issue-96803.ynBKIS.rst @@ -0,0 +1,6 @@ +Add unstable C-API functions to get the code object, lasti and line number from +the internal ``_PyInterpreterFrame`` in the limited API. The functions are: + +* ``PyCodeObject * PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame)`` +* ``int PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame)`` +* ``int PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame)`` diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index d69c5636486da9..c5714d5e7d5a0f 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -7,6 +7,7 @@ #include "pycore_runtime.h" // _Py_ID() #include "pycore_traceback.h" #include +#include "frameobject.h" // _PyInterpreterFrame_GetLine #include // malloc() @@ -257,7 +258,7 @@ static void tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame) { frame->filename = &_Py_STR(anon_unknown); - int lineno = _PyInterpreterFrame_GetLine(pyframe); + int lineno = PyUnstable_InterpreterFrame_GetLine(pyframe); if (lineno < 0) { lineno = 0; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index ef0070199ab2c0..d0eca447c012ea 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -38,7 +38,7 @@ PyFrame_GetLineNumber(PyFrameObject *f) return f->f_lineno; } else { - return _PyInterpreterFrame_GetLine(f->f_frame); + return PyUnstable_InterpreterFrame_GetLine(f->f_frame); } } diff --git a/Objects/genobject.c b/Objects/genobject.c index 6316fa9865fe65..937d497753e970 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -12,6 +12,7 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "structmember.h" // PyMemberDef #include "opcode.h" // SEND +#include "frameobject.h" // _PyInterpreterFrame_GetLine #include "pystats.h" static PyObject *gen_close(PyGenObject *, PyObject *); @@ -1322,7 +1323,7 @@ compute_cr_origin(int origin_depth, _PyInterpreterFrame *current_frame) frame = current_frame; for (int i = 0; i < frame_count; ++i) { PyCodeObject *code = frame->f_code; - int line = _PyInterpreterFrame_GetLine(frame); + int line = PyUnstable_InterpreterFrame_GetLine(frame); PyObject *frameinfo = Py_BuildValue("OiO", code->co_filename, line, code->co_name); if (!frameinfo) { diff --git a/Python/ceval.c b/Python/ceval.c index 958689debc87f8..56a3b123f46331 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -27,6 +27,7 @@ #include "pycore_dict.h" #include "dictobject.h" #include "pycore_frame.h" +#include "frameobject.h" // _PyInterpreterFrame_GetLine #include "opcode.h" #include "pydtrace.h" #include "setobject.h" @@ -785,7 +786,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", frame->f_code->co_filename, - _PyInterpreterFrame_GetLine(frame), + PyUnstable_InterpreterFrame_GetLine(frame), opcode); goto error; diff --git a/Python/frame.c b/Python/frame.c index c2c0be30113912..d792b92fa57560 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -144,8 +144,24 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) Py_DECREF(frame->f_funcobj); } +/* Unstable API functions */ + +PyCodeObject * +PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame) +{ + PyCodeObject *code = frame->f_code; + Py_INCREF(code); + return code; +} + +int +PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame) +{ + return _PyInterpreterFrame_LASTI(frame) * sizeof(_Py_CODEUNIT); +} + int -_PyInterpreterFrame_GetLine(_PyInterpreterFrame *frame) +PyUnstable_InterpreterFrame_GetLine(_PyInterpreterFrame *frame) { int addr = _PyInterpreterFrame_LASTI(frame) * sizeof(_Py_CODEUNIT); return PyCode_Addr2Line(frame->f_code, addr); diff --git a/Python/traceback.c b/Python/traceback.c index 097f69c76abfe1..b2479542047308 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -1180,7 +1180,7 @@ dump_frame(int fd, _PyInterpreterFrame *frame) PUTS(fd, "???"); } - int lineno = _PyInterpreterFrame_GetLine(frame); + int lineno = PyUnstable_InterpreterFrame_GetLine(frame); PUTS(fd, ", line "); if (lineno >= 0) { _Py_DumpDecimal(fd, (size_t)lineno); From b9797417315cc2d1700cb2d427685016d3380711 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 5 May 2023 10:38:47 -0700 Subject: [PATCH 061/204] gh-103533: Use PEP 669 APIs for cprofile (GH-103534) --- Lib/test/test_cprofile.py | 19 +- ...-04-14-06-32-54.gh-issue-103533.n_AfcS.rst | 1 + Modules/_lsprof.c | 252 +++++++++++++----- Tools/c-analyzer/cpython/ignored.tsv | 1 + 4 files changed, 200 insertions(+), 73 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-14-06-32-54.gh-issue-103533.n_AfcS.rst diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index 98648528bc81f2..484b8f8e3a365c 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -25,7 +25,6 @@ def test_bad_counter_during_dealloc(self): with support.catch_unraisable_exception() as cm: obj = _lsprof.Profiler(lambda: int) obj.enable() - obj = _lsprof.Profiler(1) obj.disable() obj.clear() @@ -37,10 +36,11 @@ def test_profile_enable_disable(self): self.addCleanup(prof.disable) prof.enable() - self.assertIs(sys.getprofile(), prof) + self.assertEqual( + sys.monitoring.get_tool(sys.monitoring.PROFILER_ID), "cProfile") prof.disable() - self.assertIs(sys.getprofile(), None) + self.assertIs(sys.monitoring.get_tool(sys.monitoring.PROFILER_ID), None) def test_profile_as_context_manager(self): prof = self.profilerclass() @@ -53,10 +53,19 @@ def test_profile_as_context_manager(self): # profile should be set as the global profiler inside the # with-block - self.assertIs(sys.getprofile(), prof) + self.assertEqual( + sys.monitoring.get_tool(sys.monitoring.PROFILER_ID), "cProfile") # profile shouldn't be set once we leave the with-block. - self.assertIs(sys.getprofile(), None) + self.assertIs(sys.monitoring.get_tool(sys.monitoring.PROFILER_ID), None) + + def test_second_profiler(self): + pr = self.profilerclass() + pr2 = self.profilerclass() + pr.enable() + self.assertRaises(ValueError, pr2.enable) + pr.disable() + class TestCommandLine(unittest.TestCase): def test_sort(self): diff --git a/Misc/NEWS.d/next/Library/2023-04-14-06-32-54.gh-issue-103533.n_AfcS.rst b/Misc/NEWS.d/next/Library/2023-04-14-06-32-54.gh-issue-103533.n_AfcS.rst new file mode 100644 index 00000000000000..1008ea076c71a0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-14-06-32-54.gh-issue-103533.n_AfcS.rst @@ -0,0 +1 @@ +Update :mod:`cProfile` to use PEP 669 API diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 83d034ae7eed78..a7ce328cb5307a 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -49,6 +49,8 @@ typedef struct { int flags; PyObject *externalTimer; double externalTimerUnit; + int tool_id; + PyObject* missing; } ProfilerObject; #define POF_ENABLED 0x001 @@ -399,64 +401,6 @@ ptrace_leave_call(PyObject *self, void *key) pObj->freelistProfilerContext = pContext; } -static int -profiler_callback(PyObject *self, PyFrameObject *frame, int what, - PyObject *arg) -{ - switch (what) { - - /* the 'frame' of a called function is about to start its execution */ - case PyTrace_CALL: - { - PyCodeObject *code = PyFrame_GetCode(frame); - ptrace_enter_call(self, (void *)code, (PyObject *)code); - Py_DECREF(code); - break; - } - - /* the 'frame' of a called function is about to finish - (either normally or with an exception) */ - case PyTrace_RETURN: - { - PyCodeObject *code = PyFrame_GetCode(frame); - ptrace_leave_call(self, (void *)code); - Py_DECREF(code); - break; - } - - /* case PyTrace_EXCEPTION: - If the exception results in the function exiting, a - PyTrace_RETURN event will be generated, so we don't need to - handle it. */ - - /* the Python function 'frame' is issuing a call to the built-in - function 'arg' */ - case PyTrace_C_CALL: - if ((((ProfilerObject *)self)->flags & POF_BUILTINS) - && PyCFunction_Check(arg)) { - ptrace_enter_call(self, - ((PyCFunctionObject *)arg)->m_ml, - arg); - } - break; - - /* the call to the built-in function 'arg' is returning into its - caller 'frame' */ - case PyTrace_C_RETURN: /* ...normally */ - case PyTrace_C_EXCEPTION: /* ...with an exception set */ - if ((((ProfilerObject *)self)->flags & POF_BUILTINS) - && PyCFunction_Check(arg)) { - ptrace_leave_call(self, - ((PyCFunctionObject *)arg)->m_ml); - } - break; - - default: - break; - } - return 0; -} - static int pending_exception(ProfilerObject *pObj) { @@ -650,6 +594,99 @@ setBuiltins(ProfilerObject *pObj, int nvalue) return 0; } +PyObject* pystart_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +{ + PyObject* code = args[0]; + ptrace_enter_call((PyObject*)self, (void *)code, (PyObject *)code); + + Py_RETURN_NONE; +} + +PyObject* pyreturn_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +{ + PyObject* code = args[0]; + ptrace_leave_call((PyObject*)self, (void *)code); + + Py_RETURN_NONE; +} + +PyObject* get_cfunc_from_callable(PyObject* callable, PyObject* self_arg, PyObject* missing) +{ + // return a new reference + if (PyCFunction_Check(callable)) { + Py_INCREF(callable); + return (PyObject*)((PyCFunctionObject *)callable); + } + if (Py_TYPE(callable) == &PyMethodDescr_Type) { + /* For backwards compatibility need to + * convert to builtin method */ + + /* If no arg, skip */ + if (self_arg == missing) { + return NULL; + } + PyObject *meth = Py_TYPE(callable)->tp_descr_get( + callable, self_arg, (PyObject*)Py_TYPE(self_arg)); + if (meth == NULL) { + return NULL; + } + if (PyCFunction_Check(meth)) { + return (PyObject*)((PyCFunctionObject *)meth); + } + } + return NULL; +} + +PyObject* ccall_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +{ + if (self->flags & POF_BUILTINS) { + PyObject* callable = args[2]; + PyObject* self_arg = args[3]; + + PyObject* cfunc = get_cfunc_from_callable(callable, self_arg, self->missing); + + if (cfunc) { + ptrace_enter_call((PyObject*)self, + ((PyCFunctionObject *)cfunc)->m_ml, + cfunc); + Py_DECREF(cfunc); + } + } + Py_RETURN_NONE; +} + +PyObject* creturn_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +{ + if (self->flags & POF_BUILTINS) { + PyObject* callable = args[2]; + PyObject* self_arg = args[3]; + + PyObject* cfunc = get_cfunc_from_callable(callable, self_arg, self->missing); + + if (cfunc) { + ptrace_leave_call((PyObject*)self, + ((PyCFunctionObject *)cfunc)->m_ml); + Py_DECREF(cfunc); + } + } + Py_RETURN_NONE; +} + +static const struct { + int event; + const char* callback_method; +} callback_table[] = { + {PY_MONITORING_EVENT_PY_START, "_pystart_callback"}, + {PY_MONITORING_EVENT_PY_RESUME, "_pystart_callback"}, + {PY_MONITORING_EVENT_PY_RETURN, "_pyreturn_callback"}, + {PY_MONITORING_EVENT_PY_YIELD, "_pyreturn_callback"}, + {PY_MONITORING_EVENT_PY_UNWIND, "_pyreturn_callback"}, + {PY_MONITORING_EVENT_CALL, "_ccall_callback"}, + {PY_MONITORING_EVENT_C_RETURN, "_creturn_callback"}, + {PY_MONITORING_EVENT_C_RAISE, "_creturn_callback"}, + {0, NULL} +}; + PyDoc_STRVAR(enable_doc, "\ enable(subcalls=True, builtins=True)\n\ \n\ @@ -666,6 +703,8 @@ profiler_enable(ProfilerObject *self, PyObject *args, PyObject *kwds) int subcalls = -1; int builtins = -1; static char *kwlist[] = {"subcalls", "builtins", 0}; + int all_events = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|pp:enable", kwlist, &subcalls, &builtins)) return NULL; @@ -673,11 +712,37 @@ profiler_enable(ProfilerObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyThreadState *tstate = _PyThreadState_GET(); - if (_PyEval_SetProfile(tstate, profiler_callback, (PyObject*)self) < 0) { + PyObject* monitoring = _PyImport_GetModuleAttrString("sys", "monitoring"); + if (!monitoring) { + return NULL; + } + + if (PyObject_CallMethod(monitoring, "use_tool_id", "is", self->tool_id, "cProfile") == NULL) { + PyErr_Format(PyExc_ValueError, "Another profiling tool is already active"); + Py_DECREF(monitoring); + return NULL; + } + + for (int i = 0; callback_table[i].callback_method; i++) { + PyObject* callback = PyObject_GetAttrString((PyObject*)self, callback_table[i].callback_method); + if (!callback) { + Py_DECREF(monitoring); + return NULL; + } + Py_XDECREF(PyObject_CallMethod(monitoring, "register_callback", "iiO", self->tool_id, + (1 << callback_table[i].event), + callback)); + Py_DECREF(callback); + all_events |= (1 << callback_table[i].event); + } + + if (!PyObject_CallMethod(monitoring, "set_events", "ii", self->tool_id, all_events)) { + Py_DECREF(monitoring); return NULL; } + Py_DECREF(monitoring); + self->flags |= POF_ENABLED; Py_RETURN_NONE; } @@ -707,13 +772,44 @@ Stop collecting profiling information.\n\ static PyObject* profiler_disable(ProfilerObject *self, PyObject* noarg) { - PyThreadState *tstate = _PyThreadState_GET(); - if (_PyEval_SetProfile(tstate, NULL, NULL) < 0) { - return NULL; + if (self->flags & POF_ENABLED) { + PyObject* result = NULL; + PyObject* monitoring = _PyImport_GetModuleAttrString("sys", "monitoring"); + + if (!monitoring) { + return NULL; + } + + for (int i = 0; callback_table[i].callback_method; i++) { + result = PyObject_CallMethod(monitoring, "register_callback", "iiO", self->tool_id, + (1 << callback_table[i].event), Py_None); + if (!result) { + Py_DECREF(monitoring); + return NULL; + } + Py_DECREF(result); + } + + result = PyObject_CallMethod(monitoring, "set_events", "ii", self->tool_id, 0); + if (!result) { + Py_DECREF(monitoring); + return NULL; + } + Py_DECREF(result); + + result = PyObject_CallMethod(monitoring, "free_tool_id", "i", self->tool_id); + if (!result) { + Py_DECREF(monitoring); + return NULL; + } + Py_DECREF(result); + + Py_DECREF(monitoring); + + self->flags &= ~POF_ENABLED; + flush_unmatched(self); } - self->flags &= ~POF_ENABLED; - flush_unmatched(self); if (pending_exception(self)) { return NULL; } @@ -778,17 +874,37 @@ profiler_init(ProfilerObject *pObj, PyObject *args, PyObject *kw) return -1; pObj->externalTimerUnit = timeunit; Py_XSETREF(pObj->externalTimer, Py_XNewRef(timer)); + pObj->tool_id = PY_MONITORING_PROFILER_ID; + + PyObject* monitoring = _PyImport_GetModuleAttrString("sys", "monitoring"); + if (!monitoring) { + return -1; + } + pObj->missing = PyObject_GetAttrString(monitoring, "MISSING"); + if (!pObj->missing) { + Py_DECREF(monitoring); + return -1; + } + Py_DECREF(monitoring); return 0; } static PyMethodDef profiler_methods[] = { _LSPROF_PROFILER_GETSTATS_METHODDEF - {"enable", _PyCFunction_CAST(profiler_enable), + {"enable", _PyCFunction_CAST(profiler_enable), METH_VARARGS | METH_KEYWORDS, enable_doc}, - {"disable", (PyCFunction)profiler_disable, + {"disable", (PyCFunction)profiler_disable, METH_NOARGS, disable_doc}, - {"clear", (PyCFunction)profiler_clear, + {"clear", (PyCFunction)profiler_clear, METH_NOARGS, clear_doc}, + {"_pystart_callback", _PyCFunction_CAST(pystart_callback), + METH_FASTCALL, NULL}, + {"_pyreturn_callback", _PyCFunction_CAST(pyreturn_callback), + METH_FASTCALL, NULL}, + {"_ccall_callback", _PyCFunction_CAST(ccall_callback), + METH_FASTCALL, NULL}, + {"_creturn_callback", _PyCFunction_CAST(creturn_callback), + METH_FASTCALL, NULL}, {NULL, NULL} }; diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index fee493ff7f1666..7ba116dcb171cf 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -216,6 +216,7 @@ Modules/_io/_iomodule.c - static_types - Modules/_io/textio.c - encodefuncs - Modules/_io/winconsoleio.c - _PyWindowsConsoleIO_Type - Modules/_localemodule.c - langinfo_constants - +Modules/_lsprof.c - callback_table - Modules/_pickle.c - READ_WHOLE_LINE - Modules/_sqlite/module.c - error_codes - Modules/_sre/sre.c pattern_repr flag_names - From 1afe0e0320c6f19418d44d682ad95ba0c689c595 Mon Sep 17 00:00:00 2001 From: Alexey Namyotkin <62434915+nametkin@users.noreply.github.com> Date: Fri, 5 May 2023 21:52:24 +0300 Subject: [PATCH 062/204] gh-69152: Add _proxy_response_headers attribute to HTTPConnection (#26152) Add _proxy_response_headers attribute to HTTPConnection (#26152) --------- Co-authored-by: Senthil Kumaran --- Lib/http/client.py | 18 +++++++----------- Lib/test/test_httplib.py | 14 ++++++++++++++ .../2021-05-16-14-28-30.bpo-24964.Oa5Ie_.rst | 3 +++ 3 files changed, 24 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2021-05-16-14-28-30.bpo-24964.Oa5Ie_.rst diff --git a/Lib/http/client.py b/Lib/http/client.py index 74f7bcb68fb6bc..50f2b4680769c8 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -858,6 +858,7 @@ def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, self._tunnel_host = None self._tunnel_port = None self._tunnel_headers = {} + self._proxy_response_headers = None (self.host, self.port) = self._get_hostport(host, port) @@ -944,21 +945,16 @@ def _tunnel(self): try: (version, code, message) = response._read_status() + self._proxy_response_headers = parse_headers(response.fp) + + if self.debuglevel > 0: + for hdr, val in self._proxy_response_headers.items(): + print("header:", hdr + ":", val) + if code != http.HTTPStatus.OK: self.close() raise OSError(f"Tunnel connection failed: {code} {message.strip()}") - while True: - line = response.fp.readline(_MAXLINE + 1) - if len(line) > _MAXLINE: - raise LineTooLong("header line") - if not line: - # for sites which EOF without sending a trailer - break - if line in (b'\r\n', b'\n', b''): - break - if self.debuglevel > 0: - print('header:', line.decode()) finally: response.close() diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 37f77fe0a320c7..4b1d355f550b49 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -2390,6 +2390,20 @@ def test_tunnel_debuglog(self): lines = output.getvalue().splitlines() self.assertIn('header: {}'.format(expected_header), lines) + def test_proxy_response_headers(self): + expected_header = ('X-Dummy', '1') + response_text = ( + 'HTTP/1.0 200 OK\r\n' + '{0}\r\n\r\n'.format(':'.join(expected_header)) + ) + + self.conn._create_connection = self._create_connection(response_text) + self.conn.set_tunnel('destination.com') + + self.conn.request('PUT', '/', '') + headers = self.conn._proxy_response_headers + self.assertIn(expected_header, headers.items()) + def test_tunnel_leak(self): sock = None diff --git a/Misc/NEWS.d/next/Library/2021-05-16-14-28-30.bpo-24964.Oa5Ie_.rst b/Misc/NEWS.d/next/Library/2021-05-16-14-28-30.bpo-24964.Oa5Ie_.rst new file mode 100644 index 00000000000000..ba113673b7fbe5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-05-16-14-28-30.bpo-24964.Oa5Ie_.rst @@ -0,0 +1,3 @@ +Added attribute '_proxy_response_headers' to HTTPConnection class. This +attribute contains the headers of the proxy server response to the CONNECT +request. From d00d94214971621e6a3541425ee8c8072023ca1a Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 5 May 2023 20:04:53 +0100 Subject: [PATCH 063/204] GH-100479: Add `pathlib.PurePath.with_segments()` (GH-103975) Add `pathlib.PurePath.with_segments()`, which creates a path object from arguments. This method is called whenever a derivative path is created, such as from `pathlib.PurePath.parent`. Subclasses may override this method to share information between path objects. Co-authored-by: Alex Waygood --- Doc/library/pathlib.rst | 28 +++++++- Doc/whatsnew/3.12.rst | 5 ++ Lib/pathlib.py | 66 ++++++++++--------- Lib/test/test_pathlib.py | 52 ++++++++++----- ...-04-03-22-02-35.gh-issue-100479.kNBjQm.rst | 4 ++ 5 files changed, 108 insertions(+), 47 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-03-22-02-35.gh-issue-100479.kNBjQm.rst diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 14118127835bbe..5ffa33d4e61f19 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -530,10 +530,10 @@ Pure paths provide the following methods and properties: unintended effects. -.. method:: PurePath.joinpath(*other) +.. method:: PurePath.joinpath(*pathsegments) Calling this method is equivalent to combining the path with each of - the *other* arguments in turn:: + the given *pathsegments* in turn:: >>> PurePosixPath('/etc').joinpath('passwd') PurePosixPath('/etc/passwd') @@ -680,6 +680,30 @@ Pure paths provide the following methods and properties: PureWindowsPath('README') +.. method:: PurePath.with_segments(*pathsegments) + + Create a new path object of the same type by combining the given + *pathsegments*. This method is called whenever a derivative path is created, + such as from :attr:`parent` and :meth:`relative_to`. Subclasses may + override this method to pass information to derivative paths, for example:: + + from pathlib import PurePosixPath + + class MyPath(PurePosixPath): + def __init__(self, *pathsegments, session_id): + super().__init__(*pathsegments) + self.session_id = session_id + + def with_segments(self, *pathsegments): + return type(self)(*pathsegments, session_id=self.session_id) + + etc = MyPath('/etc', session_id=42) + hosts = etc / 'hosts' + print(hosts.session_id) # 42 + + .. versionadded:: 3.12 + + .. _concrete-paths: diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 4f952e2a37ef2f..ccddc8bd832f29 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -348,6 +348,11 @@ inspect pathlib ------- +* Add support for subclassing :class:`pathlib.PurePath` and + :class:`~pathlib.Path`, plus their Posix- and Windows-specific variants. + Subclasses may override the :meth:`~pathlib.PurePath.with_segments` method + to pass information between path instances. + * Add :meth:`~pathlib.Path.walk` for walking the directory trees and generating all file or directory names within them, similar to :func:`os.walk`. (Contributed by Stanislav Zmiev in :gh:`90385`.) diff --git a/Lib/pathlib.py b/Lib/pathlib.py index f32e1e2d822834..9aa3c1e52447d1 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -204,11 +204,10 @@ def _select_from(self, parent_path, scandir): class _PathParents(Sequence): """This object provides sequence-like access to the logical ancestors of a path. Don't try to construct it yourself.""" - __slots__ = ('_pathcls', '_drv', '_root', '_tail') + __slots__ = ('_path', '_drv', '_root', '_tail') def __init__(self, path): - # We don't store the instance to avoid reference cycles - self._pathcls = type(path) + self._path = path self._drv = path.drive self._root = path.root self._tail = path._tail @@ -224,11 +223,11 @@ def __getitem__(self, idx): raise IndexError(idx) if idx < 0: idx += len(self) - return self._pathcls._from_parsed_parts(self._drv, self._root, - self._tail[:-idx - 1]) + return self._path._from_parsed_parts(self._drv, self._root, + self._tail[:-idx - 1]) def __repr__(self): - return "<{}.parents>".format(self._pathcls.__name__) + return "<{}.parents>".format(type(self._path).__name__) class PurePath(object): @@ -316,6 +315,13 @@ def __init__(self, *args): else: self._raw_path = self._flavour.join(*paths) + def with_segments(self, *pathsegments): + """Construct a new path object from any number of path-like objects. + Subclasses may override this method to customize how new path objects + are created from methods like `iterdir()`. + """ + return type(self)(*pathsegments) + @classmethod def _parse_path(cls, path): if not path: @@ -342,15 +348,14 @@ def _load_parts(self): self._root = root self._tail_cached = tail - @classmethod - def _from_parsed_parts(cls, drv, root, tail): - path = cls._format_parsed_parts(drv, root, tail) - self = cls(path) - self._str = path or '.' - self._drv = drv - self._root = root - self._tail_cached = tail - return self + def _from_parsed_parts(self, drv, root, tail): + path_str = self._format_parsed_parts(drv, root, tail) + path = self.with_segments(path_str) + path._str = path_str or '.' + path._drv = drv + path._root = root + path._tail_cached = tail + return path @classmethod def _format_parsed_parts(cls, drv, root, tail): @@ -584,8 +589,7 @@ def relative_to(self, other, /, *_deprecated, walk_up=False): "scheduled for removal in Python {remove}") warnings._deprecated("pathlib.PurePath.relative_to(*args)", msg, remove=(3, 14)) - path_cls = type(self) - other = path_cls(other, *_deprecated) + other = self.with_segments(other, *_deprecated) for step, path in enumerate([other] + list(other.parents)): if self.is_relative_to(path): break @@ -594,7 +598,7 @@ def relative_to(self, other, /, *_deprecated, walk_up=False): if step and not walk_up: raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") parts = ['..'] * step + self._tail[len(path._tail):] - return path_cls(*parts) + return self.with_segments(*parts) def is_relative_to(self, other, /, *_deprecated): """Return True if the path is relative to another path or False. @@ -605,7 +609,7 @@ def is_relative_to(self, other, /, *_deprecated): "scheduled for removal in Python {remove}") warnings._deprecated("pathlib.PurePath.is_relative_to(*args)", msg, remove=(3, 14)) - other = type(self)(other, *_deprecated) + other = self.with_segments(other, *_deprecated) return other == self or other in self.parents @property @@ -617,13 +621,13 @@ def parts(self): else: return tuple(self._tail) - def joinpath(self, *args): + def joinpath(self, *pathsegments): """Combine this path with one or several arguments, and return a new path representing either a subpath (if all arguments are relative paths) or a totally different path (if one of the arguments is anchored). """ - return self.__class__(self, *args) + return self.with_segments(self, *pathsegments) def __truediv__(self, key): try: @@ -633,7 +637,7 @@ def __truediv__(self, key): def __rtruediv__(self, key): try: - return type(self)(key, self) + return self.with_segments(key, self) except TypeError: return NotImplemented @@ -650,6 +654,8 @@ def parent(self): @property def parents(self): """A sequence of this path's logical parents.""" + # The value of this property should not be cached on the path object, + # as doing so would introduce a reference cycle. return _PathParents(self) def is_absolute(self): @@ -680,7 +686,7 @@ def match(self, path_pattern): """ Return True if this path matches the given pattern. """ - pat = type(self)(path_pattern) + pat = self.with_segments(path_pattern) if not pat.parts: raise ValueError("empty pattern") pat_parts = pat._parts_normcase @@ -755,7 +761,7 @@ def _make_child_relpath(self, name): path_str = f'{path_str}{name}' else: path_str = name - path = type(self)(path_str) + path = self.with_segments(path_str) path._str = path_str path._drv = self.drive path._root = self.root @@ -805,7 +811,7 @@ def samefile(self, other_path): try: other_st = other_path.stat() except AttributeError: - other_st = self.__class__(other_path).stat() + other_st = self.with_segments(other_path).stat() return self._flavour.samestat(st, other_st) def iterdir(self): @@ -867,7 +873,7 @@ def absolute(self): cwd = self._flavour.abspath(self.drive) else: cwd = os.getcwd() - return type(self)(cwd, self) + return self.with_segments(cwd, self) def resolve(self, strict=False): """ @@ -885,7 +891,7 @@ def check_eloop(e): except OSError as e: check_eloop(e) raise - p = type(self)(s) + p = self.with_segments(s) # In non-strict mode, realpath() doesn't raise on symlink loops. # Ensure we get an exception by calling stat() @@ -975,7 +981,7 @@ def readlink(self): """ if not hasattr(os, "readlink"): raise NotImplementedError("os.readlink() not available on this system") - return type(self)(os.readlink(self)) + return self.with_segments(os.readlink(self)) def touch(self, mode=0o666, exist_ok=True): """ @@ -1064,7 +1070,7 @@ def rename(self, target): Returns the new Path instance pointing to the target path. """ os.rename(self, target) - return self.__class__(target) + return self.with_segments(target) def replace(self, target): """ @@ -1077,7 +1083,7 @@ def replace(self, target): Returns the new Path instance pointing to the target path. """ os.replace(self, target) - return self.__class__(target) + return self.with_segments(target) def symlink_to(self, target, target_is_directory=False): """ diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index a932e03df4236d..7586610833b063 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -29,11 +29,12 @@ # class _BasePurePathSubclass(object): - init_called = False + def __init__(self, *pathsegments, session_id): + super().__init__(*pathsegments) + self.session_id = session_id - def __init__(self, *args): - super().__init__(*args) - self.init_called = True + def with_segments(self, *pathsegments): + return type(self)(*pathsegments, session_id=self.session_id) class _BasePurePathTest(object): @@ -121,20 +122,21 @@ def test_str_subclass_common(self): self._check_str_subclass('a/b.txt') self._check_str_subclass('/a/b.txt') - def test_init_called_common(self): + def test_with_segments_common(self): class P(_BasePurePathSubclass, self.cls): pass - p = P('foo', 'bar') - self.assertTrue((p / 'foo').init_called) - self.assertTrue(('foo' / p).init_called) - self.assertTrue(p.joinpath('foo').init_called) - self.assertTrue(p.with_name('foo').init_called) - self.assertTrue(p.with_stem('foo').init_called) - self.assertTrue(p.with_suffix('.foo').init_called) - self.assertTrue(p.relative_to('foo').init_called) - self.assertTrue(p.parent.init_called) + p = P('foo', 'bar', session_id=42) + self.assertEqual(42, (p / 'foo').session_id) + self.assertEqual(42, ('foo' / p).session_id) + self.assertEqual(42, p.joinpath('foo').session_id) + self.assertEqual(42, p.with_name('foo').session_id) + self.assertEqual(42, p.with_stem('foo').session_id) + self.assertEqual(42, p.with_suffix('.foo').session_id) + self.assertEqual(42, p.with_segments('foo').session_id) + self.assertEqual(42, p.relative_to('foo').session_id) + self.assertEqual(42, p.parent.session_id) for parent in p.parents: - self.assertTrue(parent.init_called) + self.assertEqual(42, parent.session_id) def _get_drive_root_parts(self, parts): path = self.cls(*parts) @@ -1647,6 +1649,26 @@ def test_home(self): env['HOME'] = os.path.join(BASE, 'home') self._test_home(self.cls.home()) + def test_with_segments(self): + class P(_BasePurePathSubclass, self.cls): + pass + p = P(BASE, session_id=42) + self.assertEqual(42, p.absolute().session_id) + self.assertEqual(42, p.resolve().session_id) + self.assertEqual(42, p.with_segments('~').expanduser().session_id) + self.assertEqual(42, (p / 'fileA').rename(p / 'fileB').session_id) + self.assertEqual(42, (p / 'fileB').replace(p / 'fileA').session_id) + if os_helper.can_symlink(): + self.assertEqual(42, (p / 'linkA').readlink().session_id) + for path in p.iterdir(): + self.assertEqual(42, path.session_id) + for path in p.glob('*'): + self.assertEqual(42, path.session_id) + for path in p.rglob('*'): + self.assertEqual(42, path.session_id) + for dirpath, dirnames, filenames in p.walk(): + self.assertEqual(42, dirpath.session_id) + def test_samefile(self): fileA_path = os.path.join(BASE, 'fileA') fileB_path = os.path.join(BASE, 'dirB', 'fileB') diff --git a/Misc/NEWS.d/next/Library/2023-04-03-22-02-35.gh-issue-100479.kNBjQm.rst b/Misc/NEWS.d/next/Library/2023-04-03-22-02-35.gh-issue-100479.kNBjQm.rst new file mode 100644 index 00000000000000..58db90480d2ff0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-03-22-02-35.gh-issue-100479.kNBjQm.rst @@ -0,0 +1,4 @@ +Add :meth:`pathlib.PurePath.with_segments`, which creates a path object from +arguments. This method is called whenever a derivative path is created, such +as from :attr:`pathlib.PurePath.parent`. Subclasses may override this method +to share information between path objects. From 55671fe04700ccb4e73c8db3dd1e9c031dafe700 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 5 May 2023 13:23:00 -0600 Subject: [PATCH 064/204] gh-99113: Share the GIL via PyInterpreterState.ceval.gil (gh-104203) In preparation for a per-interpreter GIL, we add PyInterpreterState.ceval.gil, set it to the shared GIL for each interpreter, and use that rather than using _PyRuntime.ceval.gil directly. Note that _PyRuntime.ceval.gil is still the actual GIL. --- Include/internal/pycore_ceval.h | 2 +- Include/internal/pycore_ceval_state.h | 3 + Modules/_xxsubinterpretersmodule.c | 1 + Python/ceval_gil.c | 97 ++++++++++++++++----------- Python/pystate.c | 2 +- 5 files changed, 65 insertions(+), 40 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index deda070a6dea79..0bbc9efdda3ba7 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -96,7 +96,7 @@ _PyEval_Vector(PyThreadState *tstate, PyObject* const* args, size_t argcount, PyObject *kwnames); -extern int _PyEval_ThreadsInitialized(struct pyruntimestate *runtime); +extern int _PyEval_ThreadsInitialized(void); extern PyStatus _PyEval_InitGIL(PyThreadState *tstate); extern void _PyEval_FiniGIL(PyInterpreterState *interp); diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 9ba42eb03b2676..1a00ec80270e0c 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -49,6 +49,8 @@ struct _ceval_runtime_state { the main thread of the main interpreter can handle signals: see _Py_ThreadCanHandleSignals(). */ _Py_atomic_int signals_pending; + + /* This is (only) used indirectly through PyInterpreterState.ceval.gil. */ struct _gil_runtime_state gil; }; @@ -83,6 +85,7 @@ struct _pending_calls { struct _ceval_state { int recursion_limit; + struct _gil_runtime_state *gil; /* This single variable consolidates all requests to break out of the fast path in the eval loop. */ _Py_atomic_int eval_breaker; diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 95273ab278d996..433fecfda655de 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -513,6 +513,7 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) // Create and initialize the new interpreter. PyThreadState *save_tstate = _PyThreadState_GET(); + assert(save_tstate != NULL); const PyInterpreterConfig config = isolated ? (PyInterpreterConfig)_PyInterpreterConfig_INIT : (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 4d501c89661743..ad33a58dedd820 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -229,6 +229,9 @@ static void _gil_initialize(struct _gil_runtime_state *gil) static int gil_created(struct _gil_runtime_state *gil) { + if (gil == NULL) { + return 0; + } return (_Py_atomic_load_explicit(&gil->locked, _Py_memory_order_acquire) >= 0); } @@ -273,10 +276,9 @@ static void recreate_gil(struct _gil_runtime_state *gil) #endif static void -drop_gil(struct _ceval_runtime_state *ceval, struct _ceval_state *ceval2, - PyThreadState *tstate) +drop_gil(struct _ceval_state *ceval, PyThreadState *tstate) { - struct _gil_runtime_state *gil = &ceval->gil; + struct _gil_runtime_state *gil = ceval->gil; if (!_Py_atomic_load_relaxed(&gil->locked)) { Py_FatalError("drop_gil: GIL is not locked"); } @@ -296,7 +298,7 @@ drop_gil(struct _ceval_runtime_state *ceval, struct _ceval_state *ceval2, MUTEX_UNLOCK(gil->mutex); #ifdef FORCE_SWITCHING - if (_Py_atomic_load_relaxed(&ceval2->gil_drop_request) && tstate != NULL) { + if (_Py_atomic_load_relaxed(&ceval->gil_drop_request) && tstate != NULL) { MUTEX_LOCK(gil->switch_mutex); /* Not switched yet => wait */ if (((PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) == tstate) @@ -358,9 +360,8 @@ take_gil(PyThreadState *tstate) assert(is_tstate_valid(tstate)); PyInterpreterState *interp = tstate->interp; - struct _ceval_runtime_state *ceval = &interp->runtime->ceval; - struct _ceval_state *ceval2 = &interp->ceval; - struct _gil_runtime_state *gil = &ceval->gil; + struct _ceval_state *ceval = &interp->ceval; + struct _gil_runtime_state *gil = ceval->gil; /* Check that _PyEval_InitThreads() was called to create the lock */ assert(gil_created(gil)); @@ -434,12 +435,12 @@ take_gil(PyThreadState *tstate) in take_gil() while the main thread called wait_for_thread_shutdown() from Py_Finalize(). */ MUTEX_UNLOCK(gil->mutex); - drop_gil(ceval, ceval2, tstate); + drop_gil(ceval, tstate); PyThread_exit_thread(); } assert(is_tstate_valid(tstate)); - if (_Py_atomic_load_relaxed(&ceval2->gil_drop_request)) { + if (_Py_atomic_load_relaxed(&ceval->gil_drop_request)) { RESET_GIL_DROP_REQUEST(interp); } else { @@ -448,7 +449,7 @@ take_gil(PyThreadState *tstate) handle signals. Note: RESET_GIL_DROP_REQUEST() calls COMPUTE_EVAL_BREAKER(). */ - COMPUTE_EVAL_BREAKER(interp, ceval, ceval2); + COMPUTE_EVAL_BREAKER(interp, &_PyRuntime.ceval, ceval); } /* Don't access tstate if the thread must exit */ @@ -463,63 +464,86 @@ take_gil(PyThreadState *tstate) void _PyEval_SetSwitchInterval(unsigned long microseconds) { - struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil; + /* XXX per-interpreter GIL */ + PyInterpreterState *interp = _PyInterpreterState_Main(); + struct _gil_runtime_state *gil = interp->ceval.gil; + assert(gil != NULL); gil->interval = microseconds; } unsigned long _PyEval_GetSwitchInterval(void) { - struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil; + /* XXX per-interpreter GIL */ + PyInterpreterState *interp = _PyInterpreterState_Main(); + struct _gil_runtime_state *gil = interp->ceval.gil; + assert(gil != NULL); return gil->interval; } int -_PyEval_ThreadsInitialized(_PyRuntimeState *runtime) +_PyEval_ThreadsInitialized(void) { - return gil_created(&runtime->ceval.gil); + /* XXX per-interpreter GIL */ + PyInterpreterState *interp = _PyInterpreterState_Main(); + if (interp == NULL) { + return 0; + } + struct _gil_runtime_state *gil = interp->ceval.gil; + return gil_created(gil); } int PyEval_ThreadsInitialized(void) { - _PyRuntimeState *runtime = &_PyRuntime; - return _PyEval_ThreadsInitialized(runtime); + return _PyEval_ThreadsInitialized(); } PyStatus _PyEval_InitGIL(PyThreadState *tstate) { + assert(tstate->interp->ceval.gil == NULL); + + /* XXX per-interpreter GIL */ + struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil; if (!_Py_IsMainInterpreter(tstate->interp)) { /* Currently, the GIL is shared by all interpreters, and only the main interpreter is responsible to create and destroy it. */ + assert(gil_created(gil)); + tstate->interp->ceval.gil = gil; return _PyStatus_OK(); } - struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil; assert(!gil_created(gil)); PyThread_init_thread(); create_gil(gil); - - take_gil(tstate); - assert(gil_created(gil)); + tstate->interp->ceval.gil = gil; + take_gil(tstate); return _PyStatus_OK(); } void _PyEval_FiniGIL(PyInterpreterState *interp) { + if (interp->ceval.gil == NULL) { + /* It was already finalized (or hasn't been initialized yet). */ + return; + } + + /* XXX per-interpreter GIL */ + struct _gil_runtime_state *gil = &interp->runtime->ceval.gil; if (!_Py_IsMainInterpreter(interp)) { /* Currently, the GIL is shared by all interpreters, and only the main interpreter is responsible to create and destroy it. */ + assert(interp->ceval.gil == gil); + interp->ceval.gil = NULL; return; } - struct _gil_runtime_state *gil = &interp->runtime->ceval.gil; if (!gil_created(gil)) { /* First Py_InitializeFromConfig() call: the GIL doesn't exist yet: do nothing. */ @@ -528,6 +552,7 @@ _PyEval_FiniGIL(PyInterpreterState *interp) destroy_gil(gil); assert(!gil_created(gil)); + interp->ceval.gil = NULL; } void @@ -555,22 +580,19 @@ PyEval_AcquireLock(void) void PyEval_ReleaseLock(void) { - _PyRuntimeState *runtime = &_PyRuntime; PyThreadState *tstate = _PyThreadState_GET(); /* This function must succeed when the current thread state is NULL. We therefore avoid PyThreadState_Get() which dumps a fatal error in debug mode. */ - struct _ceval_runtime_state *ceval = &runtime->ceval; - struct _ceval_state *ceval2 = &tstate->interp->ceval; - drop_gil(ceval, ceval2, tstate); + struct _ceval_state *ceval = &tstate->interp->ceval; + drop_gil(ceval, tstate); } void _PyEval_ReleaseLock(PyThreadState *tstate) { - struct _ceval_runtime_state *ceval = &tstate->interp->runtime->ceval; - struct _ceval_state *ceval2 = &tstate->interp->ceval; - drop_gil(ceval, ceval2, tstate); + struct _ceval_state *ceval = &tstate->interp->ceval; + drop_gil(ceval, tstate); } void @@ -595,9 +617,8 @@ PyEval_ReleaseThread(PyThreadState *tstate) if (new_tstate != tstate) { Py_FatalError("wrong thread state"); } - struct _ceval_runtime_state *ceval = &runtime->ceval; - struct _ceval_state *ceval2 = &tstate->interp->ceval; - drop_gil(ceval, ceval2, tstate); + struct _ceval_state *ceval = &tstate->interp->ceval; + drop_gil(ceval, tstate); } #ifdef HAVE_FORK @@ -607,9 +628,9 @@ PyEval_ReleaseThread(PyThreadState *tstate) PyStatus _PyEval_ReInitThreads(PyThreadState *tstate) { - _PyRuntimeState *runtime = tstate->interp->runtime; + assert(tstate->interp == _PyInterpreterState_Main()); - struct _gil_runtime_state *gil = &runtime->ceval.gil; + struct _gil_runtime_state *gil = tstate->interp->ceval.gil; if (!gil_created(gil)) { return _PyStatus_OK(); } @@ -644,10 +665,9 @@ PyEval_SaveThread(void) PyThreadState *tstate = _PyThreadState_Swap(runtime, NULL); _Py_EnsureTstateNotNULL(tstate); - struct _ceval_runtime_state *ceval = &runtime->ceval; - struct _ceval_state *ceval2 = &tstate->interp->ceval; - assert(gil_created(&ceval->gil)); - drop_gil(ceval, ceval2, tstate); + struct _ceval_state *ceval = &tstate->interp->ceval; + assert(gil_created(ceval->gil)); + drop_gil(ceval, tstate); return tstate; } @@ -906,6 +926,7 @@ Py_MakePendingCalls(void) void _PyEval_InitRuntimeState(struct _ceval_runtime_state *ceval) { + /* XXX per-interpreter GIL */ _gil_initialize(&ceval->gil); } @@ -964,7 +985,7 @@ _Py_HandlePending(PyThreadState *tstate) if (_PyThreadState_Swap(runtime, NULL) != tstate) { Py_FatalError("tstate mix-up"); } - drop_gil(ceval, interp_ceval_state, tstate); + drop_gil(interp_ceval_state, tstate); /* Other threads may run now */ diff --git a/Python/pystate.c b/Python/pystate.c index f09d37657f9c27..75bd9f41e301a3 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2186,7 +2186,7 @@ PyGILState_Ensure(void) /* Ensure that _PyEval_InitThreads() and _PyGILState_Init() have been called by Py_Initialize() */ - assert(_PyEval_ThreadsInitialized(runtime)); + assert(_PyEval_ThreadsInitialized()); assert(gilstate_tss_initialized(runtime)); assert(runtime->gilstate.autoInterpreterState != NULL); From 1c420e138fd828895b6bd3c44ef99156e8796095 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 5 May 2023 14:04:55 -0600 Subject: [PATCH 065/204] gh-104108: Add the Py_mod_multiple_interpreters Module Def Slot (gh-104148) I'll be adding a value to indicate support for per-interpreter GIL in gh-99114. --- Include/moduleobject.h | 7 +- Lib/test/test_import/__init__.py | 82 ++++++++++++++----- ...-05-03-17-46-47.gh-issue-104108.GOxAYt.rst | 6 ++ Modules/_testmultiphase.c | 19 +++++ Objects/moduleobject.c | 30 +++++++ 5 files changed, 122 insertions(+), 22 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-03-17-46-47.gh-issue-104108.GOxAYt.rst diff --git a/Include/moduleobject.h b/Include/moduleobject.h index 555564ec73b4a2..7ac6f6e8a4a24e 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -78,11 +78,16 @@ struct PyModuleDef_Slot { #define Py_mod_create 1 #define Py_mod_exec 2 +#define Py_mod_multiple_interpreters 3 #ifndef Py_LIMITED_API -#define _Py_mod_LAST_SLOT 2 +#define _Py_mod_LAST_SLOT 3 #endif +/* for Py_mod_multiple_interpreters: */ +#define Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED ((void *)0) +#define Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED ((void *)1) + #endif /* New in 3.5 */ struct PyModuleDef { diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 41dfdaabe24664..9211639b016e7a 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1652,26 +1652,44 @@ def pipe(self): os.set_blocking(r, False) return (r, w) - def import_script(self, name, fd, check_override=None): + def import_script(self, name, fd, filename=None, check_override=None): override_text = '' if check_override is not None: override_text = f''' - import _imp - _imp._override_multi_interp_extensions_check({check_override}) - ''' - return textwrap.dedent(f''' - import os, sys - {override_text} - try: - import {name} - except ImportError as exc: - text = 'ImportError: ' + str(exc) - else: - text = 'okay' - os.write({fd}, text.encode('utf-8')) - ''') + import _imp + _imp._override_multi_interp_extensions_check({check_override}) + ''' + if filename: + return textwrap.dedent(f''' + from importlib.util import spec_from_loader, module_from_spec + from importlib.machinery import ExtensionFileLoader + import os, sys + {override_text} + loader = ExtensionFileLoader({name!r}, {filename!r}) + spec = spec_from_loader({name!r}, loader) + try: + module = module_from_spec(spec) + loader.exec_module(module) + except ImportError as exc: + text = 'ImportError: ' + str(exc) + else: + text = 'okay' + os.write({fd}, text.encode('utf-8')) + ''') + else: + return textwrap.dedent(f''' + import os, sys + {override_text} + try: + import {name} + except ImportError as exc: + text = 'ImportError: ' + str(exc) + else: + text = 'okay' + os.write({fd}, text.encode('utf-8')) + ''') - def run_here(self, name, *, + def run_here(self, name, filename=None, *, check_singlephase_setting=False, check_singlephase_override=None, isolated=False, @@ -1700,26 +1718,30 @@ def run_here(self, name, *, ) r, w = self.pipe() - script = self.import_script(name, w, check_singlephase_override) + script = self.import_script(name, w, filename, + check_singlephase_override) ret = run_in_subinterp_with_config(script, **kwargs) self.assertEqual(ret, 0) return os.read(r, 100) - def check_compatible_here(self, name, *, strict=False, isolated=False): + def check_compatible_here(self, name, filename=None, *, + strict=False, + isolated=False, + ): # Verify that the named module may be imported in a subinterpreter. # (See run_here() for more info.) - out = self.run_here(name, + out = self.run_here(name, filename, check_singlephase_setting=strict, isolated=isolated, ) self.assertEqual(out, b'okay') - def check_incompatible_here(self, name, *, isolated=False): + def check_incompatible_here(self, name, filename=None, *, isolated=False): # Differences from check_compatible_here(): # * verify that import fails # * "strict" is always True - out = self.run_here(name, + out = self.run_here(name, filename, check_singlephase_setting=True, isolated=isolated, ) @@ -1820,6 +1842,24 @@ def test_multi_init_extension_compat(self): with self.subTest(f'{module}: strict, fresh'): self.check_compatible_fresh(module, strict=True) + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_multi_init_extension_non_isolated_compat(self): + modname = '_test_non_isolated' + filename = _testmultiphase.__file__ + loader = ExtensionFileLoader(modname, filename) + spec = importlib.util.spec_from_loader(modname, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + sys.modules[modname] = module + + require_extension(module) + with self.subTest(f'{modname}: isolated'): + self.check_incompatible_here(modname, filename, isolated=True) + with self.subTest(f'{modname}: not isolated'): + self.check_incompatible_here(modname, filename, isolated=False) + with self.subTest(f'{modname}: not strict'): + self.check_compatible_here(modname, filename, strict=False) + def test_python_compat(self): module = 'threading' require_pure_python(module) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-03-17-46-47.gh-issue-104108.GOxAYt.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-03-17-46-47.gh-issue-104108.GOxAYt.rst new file mode 100644 index 00000000000000..dad843636493ae --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-03-17-46-47.gh-issue-104108.GOxAYt.rst @@ -0,0 +1,6 @@ +Multi-phase init extension modules may now indicate whether or not they +actually support multiple interpreters. By default such modules are +expected to support use in multiple interpreters. In the uncommon case that +one does not, it may use the new ``Py_mod_multiple_interpreters`` module def +slot. A value of ``0`` means the module does not support them. ``1`` means +it does. The default is ``1``. diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index cf8990a2df0a9b..bc7d8b64a94322 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -884,3 +884,22 @@ PyInit__test_module_state_shared(void) } return module; } + + +/* multiple interpreters supports */ + +static PyModuleDef_Slot non_isolated_slots[] = { + {Py_mod_exec, execfunc}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {0, NULL}, +}; + +static PyModuleDef non_isolated_def = TEST_MODULE_DEF("_test_non_isolated", + non_isolated_slots, + testexport_methods); + +PyMODINIT_FUNC +PyInit__test_non_isolated(void) +{ + return PyModuleDef_Init(&non_isolated_def); +} diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index ffcd90ed122e77..63f4226dd9e94a 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -245,6 +245,8 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio PyObject *(*create)(PyObject *, PyModuleDef*) = NULL; PyObject *nameobj; PyObject *m = NULL; + int has_multiple_interpreters_slot = 0; + void *multiple_interpreters = (void *)0; int has_execution_slots = 0; const char *name; int ret; @@ -287,6 +289,17 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio case Py_mod_exec: has_execution_slots = 1; break; + case Py_mod_multiple_interpreters: + if (has_multiple_interpreters_slot) { + PyErr_Format( + PyExc_SystemError, + "module %s has more than one 'multiple interpreters' slots", + name); + goto error; + } + multiple_interpreters = cur_slot->value; + has_multiple_interpreters_slot = 1; + break; default: assert(cur_slot->slot < 0 || cur_slot->slot > _Py_mod_LAST_SLOT); PyErr_Format( @@ -297,6 +310,20 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio } } + /* By default, multi-phase init modules are expected + to work under multiple interpreters. */ + if (!has_multiple_interpreters_slot) { + multiple_interpreters = Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED; + } + if (multiple_interpreters == Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (!_Py_IsMainInterpreter(interp) + && _PyImport_CheckSubinterpIncompatibleExtensionAllowed(name) < 0) + { + goto error; + } + } + if (create) { m = create(spec, def); if (m == NULL) { @@ -421,6 +448,9 @@ PyModule_ExecDef(PyObject *module, PyModuleDef *def) return -1; } break; + case Py_mod_multiple_interpreters: + /* handled in PyModule_FromDefAndSpec2 */ + break; default: PyErr_Format( PyExc_SystemError, From a9c6e0618f26270e2591b3d99ffeef55eea02a33 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 5 May 2023 15:11:27 -0600 Subject: [PATCH 066/204] gh-99113: Add Py_MOD_PER_INTERPRETER_GIL_SUPPORTED (gh-104205) Here we are doing no more than adding the value for Py_mod_multiple_interpreters and using it for stdlib modules. We will start checking for it in gh-104206 (once PyInterpreterState.ceval.own_gil is added in gh-104204). --- Include/moduleobject.h | 1 + Modules/_abc.c | 1 + Modules/_asynciomodule.c | 1 + Modules/_bisectmodule.c | 1 + Modules/_blake2/blake2module.c | 3 ++- Modules/_bz2module.c | 1 + Modules/_codecsmodule.c | 1 + Modules/_collectionsmodule.c | 1 + Modules/_contextvarsmodule.c | 1 + Modules/_cryptmodule.c | 1 + Modules/_csv.c | 1 + Modules/_ctypes/_ctypes_test.c | 1 + Modules/_curses_panel.c | 3 +++ Modules/_dbmmodule.c | 1 + Modules/_elementtree.c | 3 +++ Modules/_functoolsmodule.c | 1 + Modules/_gdbmmodule.c | 1 + Modules/_hashopenssl.c | 1 + Modules/_heapqmodule.c | 1 + Modules/_json.c | 1 + Modules/_localemodule.c | 1 + Modules/_lsprof.c | 3 +++ Modules/_lzmamodule.c | 1 + Modules/_multiprocessing/multiprocessing.c | 1 + Modules/_multiprocessing/posixshmem.c | 7 +++++++ Modules/_opcode.c | 8 +++++++- Modules/_operator.c | 1 + Modules/_pickle.c | 1 + Modules/_posixsubprocess.c | 1 + Modules/_queuemodule.c | 1 + Modules/_randommodule.c | 1 + Modules/_scproxy.c | 1 + Modules/_sha3/sha3module.c | 1 + Modules/_sqlite/module.c | 1 + Modules/_sre/sre.c | 1 + Modules/_ssl.c | 3 +++ Modules/_stat.c | 1 + Modules/_statisticsmodule.c | 1 + Modules/_struct.c | 1 + Modules/_testinternalcapi.c | 1 + Modules/_testmultiphase.c | 6 ++++++ Modules/_threadmodule.c | 1 + Modules/_typingmodule.c | 1 + Modules/_uuidmodule.c | 1 + Modules/_weakref.c | 1 + Modules/_winapi.c | 1 + Modules/_xxinterpchannelsmodule.c | 1 + Modules/_xxsubinterpretersmodule.c | 1 + Modules/_zoneinfo.c | 5 ++++- Modules/arraymodule.c | 1 + Modules/atexitmodule.c | 6 ++++++ Modules/audioop.c | 1 + Modules/binascii.c | 1 + Modules/cjkcodecs/cjkcodecs.h | 1 + Modules/cjkcodecs/multibytecodec.c | 1 + Modules/cmathmodule.c | 1 + Modules/errnomodule.c | 1 + Modules/faulthandler.c | 2 ++ Modules/fcntlmodule.c | 1 + Modules/gcmodule.c | 1 + Modules/grpmodule.c | 1 + Modules/itertoolsmodule.c | 1 + Modules/mathmodule.c | 1 + Modules/md5module.c | 1 + Modules/mmapmodule.c | 1 + Modules/nismodule.c | 3 +++ Modules/overlapped.c | 1 + Modules/posixmodule.c | 1 + Modules/pwdmodule.c | 1 + Modules/pyexpat.c | 3 +++ Modules/resource.c | 1 + Modules/selectmodule.c | 1 + Modules/sha1module.c | 1 + Modules/sha2module.c | 1 + Modules/signalmodule.c | 1 + Modules/socketmodule.c | 1 + Modules/spwdmodule.c | 1 + Modules/symtablemodule.c | 1 + Modules/syslogmodule.c | 1 + Modules/termios.c | 1 + Modules/timemodule.c | 1 + Modules/unicodedata.c | 1 + Modules/xxlimited.c | 1 + Modules/xxlimited_35.c | 1 + Modules/xxmodule.c | 1 + Modules/xxsubtype.c | 1 + Modules/zlibmodule.c | 1 + Objects/moduleobject.c | 3 ++- Objects/unicodeobject.c | 6 ++++++ PC/_testconsole.c | 1 + PC/msvcrtmodule.c | 1 + PC/winreg.c | 1 + PC/winsound.c | 1 + Parser/asdl_c.py | 1 + Python/Python-ast.c | 1 + Python/Python-tokenize.c | 1 + Python/_warnings.c | 1 + Python/import.c | 1 + Python/marshal.c | 1 + 99 files changed, 144 insertions(+), 4 deletions(-) diff --git a/Include/moduleobject.h b/Include/moduleobject.h index 7ac6f6e8a4a24e..b8bdfe29d80406 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -87,6 +87,7 @@ struct PyModuleDef_Slot { /* for Py_mod_multiple_interpreters: */ #define Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED ((void *)0) #define Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED ((void *)1) +#define Py_MOD_PER_INTERPRETER_GIL_SUPPORTED ((void *)2) #endif /* New in 3.5 */ diff --git a/Modules/_abc.c b/Modules/_abc.c index 9694331339aa78..d3e405dadb664a 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -944,6 +944,7 @@ _abcmodule_free(void *module) static PyModuleDef_Slot _abcmodule_slots[] = { {Py_mod_exec, _abcmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index fef34d65523870..822d5f2a41de33 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3803,6 +3803,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 30801c2f87eee7..0773bbd191931d 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -457,6 +457,7 @@ bisect_modexec(PyObject *m) static PyModuleDef_Slot bisect_slots[] = { {Py_mod_exec, bisect_modexec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_blake2/blake2module.c b/Modules/_blake2/blake2module.c index 44d783b40d0453..0d1d88c6603684 100644 --- a/Modules/_blake2/blake2module.c +++ b/Modules/_blake2/blake2module.c @@ -127,6 +127,7 @@ blake2_exec(PyObject *m) static PyModuleDef_Slot _blake2_slots[] = { {Py_mod_exec, blake2_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; @@ -146,4 +147,4 @@ PyMODINIT_FUNC PyInit__blake2(void) { return PyModuleDef_Init(&blake2_module); -} \ No newline at end of file +} diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index 8e7b8e8078af4e..97bd44b4ac9694 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -799,6 +799,7 @@ _bz2_free(void *module) static struct PyModuleDef_Slot _bz2_slots[] = { {Py_mod_exec, _bz2_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index d5035d20600ae2..777c753bd7c2a9 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -1049,6 +1049,7 @@ static PyMethodDef _codecs_functions[] = { }; static PyModuleDef_Slot _codecs_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index a9b1425177c3d7..9a81531bdffb16 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -2571,6 +2571,7 @@ collections_exec(PyObject *module) { static struct PyModuleDef_Slot collections_slots[] = { {Py_mod_exec, collections_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_contextvarsmodule.c b/Modules/_contextvarsmodule.c index d13b5962c13c44..f621c1de6d42d6 100644 --- a/Modules/_contextvarsmodule.c +++ b/Modules/_contextvarsmodule.c @@ -44,6 +44,7 @@ _contextvars_exec(PyObject *m) static struct PyModuleDef_Slot _contextvars_slots[] = { {Py_mod_exec, _contextvars_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_cryptmodule.c b/Modules/_cryptmodule.c index 72a4f44600d92c..75035084c9cd29 100644 --- a/Modules/_cryptmodule.c +++ b/Modules/_cryptmodule.c @@ -58,6 +58,7 @@ static PyMethodDef crypt_methods[] = { }; static PyModuleDef_Slot _crypt_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_csv.c b/Modules/_csv.c index 2217cc2ca7a775..0cde5c5a8bdc68 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -1798,6 +1798,7 @@ csv_exec(PyObject *module) { static PyModuleDef_Slot csv_slots[] = { {Py_mod_exec, csv_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index a8811d03cc91a2..ce652b362d5bb3 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -1054,6 +1054,7 @@ _testfunc_pylist_append(PyObject *list, PyObject *item) } static struct PyModuleDef_Slot _ctypes_test_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index 2144345de01ba3..a3124ff80551e0 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -690,6 +690,9 @@ _curses_panel_exec(PyObject *mod) static PyModuleDef_Slot _curses_slots[] = { {Py_mod_exec, _curses_panel_exec}, + // XXX gh-103092: fix isolation. + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index 54376022dcb182..9908174c94c450 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -583,6 +583,7 @@ _dbm_module_free(void *module) static PyModuleDef_Slot _dbmmodule_slots[] = { {Py_mod_exec, _dbm_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 97be89a167104f..42de3c675c2e5a 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -4419,6 +4419,9 @@ module_exec(PyObject *m) static struct PyModuleDef_Slot elementtree_slots[] = { {Py_mod_exec, module_exec}, + // XXX gh-103092: fix isolation. + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 4032ba79374fa4..a8001d71223fdc 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -1520,6 +1520,7 @@ _functools_free(void *module) static struct PyModuleDef_Slot _functools_slots[] = { {Py_mod_exec, _functools_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index 4e8acdefc722b2..4dbb5741b2ede8 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -793,6 +793,7 @@ _gdbm_module_free(void *module) static PyModuleDef_Slot _gdbm_module_slots[] = { {Py_mod_exec, _gdbm_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 7476e5dc7dd61e..99d0b72819137e 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -2260,6 +2260,7 @@ static PyModuleDef_Slot hashlib_slots[] = { {Py_mod_exec, hashlib_md_meth_names}, {Py_mod_exec, hashlib_init_constructors}, {Py_mod_exec, hashlib_exception}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index 07ddc7b0851241..00285ae01f8574 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -682,6 +682,7 @@ heapq_exec(PyObject *m) static struct PyModuleDef_Slot heapq_slots[] = { {Py_mod_exec, heapq_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_json.c b/Modules/_json.c index fa8e2a936d2c33..c90de05b046b00 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1801,6 +1801,7 @@ _json_exec(PyObject *module) static PyModuleDef_Slot _json_slots[] = { {Py_mod_exec, _json_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 96675cdfb661ad..1ada7305117bb7 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -874,6 +874,7 @@ _locale_exec(PyObject *module) static struct PyModuleDef_Slot _locale_slots[] = { {Py_mod_exec, _locale_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index a7ce328cb5307a..1c84f66ee6f579 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -1001,6 +1001,9 @@ _lsprof_exec(PyObject *module) static PyModuleDef_Slot _lsprofslots[] = { {Py_mod_exec, _lsprof_exec}, + // XXX gh-103092: fix isolation. + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index bccab8639159e7..e34fbad230d51a 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -1611,6 +1611,7 @@ static PyMethodDef lzma_methods[] = { static PyModuleDef_Slot lzma_slots[] = { {Py_mod_exec, lzma_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 2463e1e1a8bf7e..8f9daa5c3de0cc 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -276,6 +276,7 @@ multiprocessing_exec(PyObject *module) static PyModuleDef_Slot multiprocessing_slots[] = { {Py_mod_exec, multiprocessing_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index d64ded4168228f..88c93fe313785c 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -110,12 +110,19 @@ static PyMethodDef module_methods[ ] = { }; +static PyModuleDef_Slot module_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL} +}; + + static struct PyModuleDef _posixshmemmodule = { PyModuleDef_HEAD_INIT, .m_name = "_posixshmem", .m_doc = "POSIX shared memory module", .m_size = 0, .m_methods = module_methods, + .m_slots = module_slots, }; /* Module init function */ diff --git a/Modules/_opcode.c b/Modules/_opcode.c index 99be977417743e..b70d426fa29bc0 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -94,12 +94,18 @@ opcode_functions[] = { {NULL, NULL, 0, NULL} }; +static PyModuleDef_Slot module_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL} +}; + static struct PyModuleDef opcodemodule = { PyModuleDef_HEAD_INIT, .m_name = "_opcode", .m_doc = "Opcode support module.", .m_size = 0, - .m_methods = opcode_functions + .m_methods = opcode_functions, + .m_slots = module_slots, }; PyMODINIT_FUNC diff --git a/Modules/_operator.c b/Modules/_operator.c index 38335b6995016c..68ccc90562d38d 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1828,6 +1828,7 @@ operator_exec(PyObject *module) static struct PyModuleDef_Slot operator_slots[] = { {Py_mod_exec, operator_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 360c7910f67187..bf7ecae0cc0e50 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -7912,6 +7912,7 @@ _pickle_exec(PyObject *m) static PyModuleDef_Slot pickle_slots[] = { {Py_mod_exec, _pickle_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index f5bce8cd7628ad..2bf83db0e228fb 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -1140,6 +1140,7 @@ static PyMethodDef module_methods[] = { }; static PyModuleDef_Slot _posixsubprocess_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index af19dd6c198b67..d36a911a57c02c 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -431,6 +431,7 @@ queuemodule_exec(PyObject *module) static PyModuleDef_Slot queuemodule_slots[] = { {Py_mod_exec, queuemodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 6e22053239305a..fda5ef267fb470 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -624,6 +624,7 @@ _random_exec(PyObject *module) static PyModuleDef_Slot _random_slots[] = { {Py_mod_exec, _random_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_scproxy.c b/Modules/_scproxy.c index 344b66f9aad522..e66918016b8da6 100644 --- a/Modules/_scproxy.c +++ b/Modules/_scproxy.c @@ -232,6 +232,7 @@ static PyMethodDef mod_methods[] = { }; static PyModuleDef_Slot _scproxy_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_sha3/sha3module.c b/Modules/_sha3/sha3module.c index 633a0c0ea08d2a..93abc3b2710e55 100644 --- a/Modules/_sha3/sha3module.c +++ b/Modules/_sha3/sha3module.c @@ -641,6 +641,7 @@ _sha3_exec(PyObject *m) static PyModuleDef_Slot _sha3_slots[] = { {Py_mod_exec, _sha3_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 9c42faa232c70d..27bd42f4595e1c 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -785,6 +785,7 @@ module_exec(PyObject *module) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 4b6290a5967932..f8a1a05a318889 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -3221,6 +3221,7 @@ sre_exec(PyObject *m) static PyModuleDef_Slot sre_slots[] = { {Py_mod_exec, sre_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index c9e2f24d66cc00..016a5a5cbca548 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -6161,6 +6161,9 @@ static PyModuleDef_Slot sslmodule_slots[] = { {Py_mod_exec, sslmodule_init_constants}, {Py_mod_exec, sslmodule_init_versioninfo}, {Py_mod_exec, sslmodule_init_strings}, + // XXX gh-103092: fix isolation. + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_stat.c b/Modules/_stat.c index 546e6a5f94ca15..4218799103b59d 100644 --- a/Modules/_stat.c +++ b/Modules/_stat.c @@ -612,6 +612,7 @@ stat_exec(PyObject *module) static PyModuleDef_Slot stat_slots[] = { {Py_mod_exec, stat_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_statisticsmodule.c b/Modules/_statisticsmodule.c index b9d1e4f1616036..1d5465fbe6d04e 100644 --- a/Modules/_statisticsmodule.c +++ b/Modules/_statisticsmodule.c @@ -129,6 +129,7 @@ PyDoc_STRVAR(statistics_doc, "Accelerators for the statistics module.\n"); static struct PyModuleDef_Slot _statisticsmodule_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_struct.c b/Modules/_struct.c index 3db7b991acd0a1..26434f714de5cc 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -2572,6 +2572,7 @@ _structmodule_exec(PyObject *m) static PyModuleDef_Slot _structmodule_slots[] = { {Py_mod_exec, _structmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 1e38f1aa63499b..24152412107c09 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -789,6 +789,7 @@ module_exec(PyObject *module) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index bc7d8b64a94322..58b064bb17cd87 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -441,6 +441,7 @@ static int execfunc(PyObject *m) static PyModuleDef_Slot main_slots[] = { {Py_mod_exec, execfunc}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; @@ -745,6 +746,7 @@ PyInit__testmultiphase_create_unreported_exception(void) static PyModuleDef_Slot slots_nonmodule_with_exec_slots[] = { {Py_mod_create, createfunc_nonmodule}, {Py_mod_exec, execfunc}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; @@ -765,6 +767,7 @@ execfunc_err(PyObject *mod) static PyModuleDef_Slot slots_exec_err[] = { {Py_mod_exec, execfunc_err}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; @@ -786,6 +789,7 @@ execfunc_raise(PyObject *spec) static PyModuleDef_Slot slots_exec_raise[] = { {Py_mod_exec, execfunc_raise}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; @@ -807,6 +811,7 @@ execfunc_unreported_exception(PyObject *mod) static PyModuleDef_Slot slots_exec_unreported_exception[] = { {Py_mod_exec, execfunc_unreported_exception}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; @@ -845,6 +850,7 @@ meth_state_access_exec(PyObject *m) static PyModuleDef_Slot meth_state_access_slots[] = { {Py_mod_exec, meth_state_access_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index fd2fd9ab25f113..5d753b4a0ebc5e 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1710,6 +1710,7 @@ The 'threading' module provides a more convenient interface."); static PyModuleDef_Slot thread_module_slots[] = { {Py_mod_exec, thread_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_typingmodule.c b/Modules/_typingmodule.c index 262dddb63fd5fe..64286375636aff 100644 --- a/Modules/_typingmodule.c +++ b/Modules/_typingmodule.c @@ -36,6 +36,7 @@ PyDoc_STRVAR(typing_doc, "Accelerators for the typing module.\n"); static struct PyModuleDef_Slot _typingmodule_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_uuidmodule.c b/Modules/_uuidmodule.c index eae38f5c98cc7f..ed3b2fedfd4d88 100644 --- a/Modules/_uuidmodule.c +++ b/Modules/_uuidmodule.c @@ -106,6 +106,7 @@ static PyMethodDef uuid_methods[] = { static PyModuleDef_Slot uuid_slots[] = { {Py_mod_exec, uuid_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_weakref.c b/Modules/_weakref.c index 157a852ae9a378..387b8fa9d0a6f1 100644 --- a/Modules/_weakref.c +++ b/Modules/_weakref.c @@ -174,6 +174,7 @@ weakref_exec(PyObject *module) static struct PyModuleDef_Slot weakref_slots[] = { {Py_mod_exec, weakref_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_winapi.c b/Modules/_winapi.c index fa380b8b798405..473bcb4736e925 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -2259,6 +2259,7 @@ static int winapi_exec(PyObject *m) static PyModuleDef_Slot winapi_slots[] = { {Py_mod_exec, winapi_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index 13b005eaef9866..616dd577688116 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -2418,6 +2418,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 433fecfda655de..d7daae254638ec 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -822,6 +822,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index c215a75b804fdb..3b2d282d65cab9 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -2822,7 +2822,10 @@ zoneinfomodule_exec(PyObject *m) } static PyModuleDef_Slot zoneinfomodule_slots[] = { - {Py_mod_exec, zoneinfomodule_exec}, {0, NULL}}; + {Py_mod_exec, zoneinfomodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL}, +}; static struct PyModuleDef zoneinfomodule = { .m_base = PyModuleDef_HEAD_INIT, diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 798a7629257966..f94bbec8e0bb3c 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -3111,6 +3111,7 @@ array_modexec(PyObject *m) static PyModuleDef_Slot arrayslots[] = { {Py_mod_exec, array_modexec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/atexitmodule.c b/Modules/atexitmodule.c index 47afd7f0751039..5882d405636400 100644 --- a/Modules/atexitmodule.c +++ b/Modules/atexitmodule.c @@ -314,12 +314,18 @@ upon normal program termination.\n\ Two public functions, register and unregister, are defined.\n\ "); +static PyModuleDef_Slot atexitmodule_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL} +}; + static struct PyModuleDef atexitmodule = { PyModuleDef_HEAD_INIT, .m_name = "atexit", .m_doc = atexit__doc__, .m_size = 0, .m_methods = atexit_methods, + .m_slots = atexitmodule_slots, }; PyMODINIT_FUNC diff --git a/Modules/audioop.c b/Modules/audioop.c index 9325f82f9a17e0..604306d449265c 100644 --- a/Modules/audioop.c +++ b/Modules/audioop.c @@ -1975,6 +1975,7 @@ audioop_exec(PyObject* module) static PyModuleDef_Slot audioop_slots[] = { {Py_mod_exec, audioop_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/binascii.c b/Modules/binascii.c index 95ddb26988d6c9..4ecff4793be9a0 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -1291,6 +1291,7 @@ binascii_exec(PyObject *module) { static PyModuleDef_Slot binascii_slots[] = { {Py_mod_exec, binascii_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/cjkcodecs/cjkcodecs.h b/Modules/cjkcodecs/cjkcodecs.h index e553ff3e17b898..36bc7024df9acc 100644 --- a/Modules/cjkcodecs/cjkcodecs.h +++ b/Modules/cjkcodecs/cjkcodecs.h @@ -502,6 +502,7 @@ static struct PyMethodDef _cjk_methods[] = { static PyModuleDef_Slot _cjk_slots[] = { {Py_mod_exec, _cjk_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 233fc3020fd6a8..b501e4fb923232 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -2062,6 +2062,7 @@ static struct PyMethodDef _multibytecodec_methods[] = { static PyModuleDef_Slot _multibytecodec_slots[] = { {Py_mod_exec, _multibytecodec_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index b4f7e5424b4ccf..914a697f8e173b 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -1411,6 +1411,7 @@ cmath_exec(PyObject *mod) static PyModuleDef_Slot cmath_slots[] = { {Py_mod_exec, cmath_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index df4e494ba8a973..fddde960a5fe9a 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -940,6 +940,7 @@ errno_exec(PyObject *module) static PyModuleDef_Slot errno_slots[] = { {Py_mod_exec, errno_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 9b4e4199cdc20a..428b090193f093 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -1274,6 +1274,8 @@ PyExec_faulthandler(PyObject *module) { static PyModuleDef_Slot faulthandler_slots[] = { {Py_mod_exec, PyExec_faulthandler}, + // XXX gh-103092: fix isolation. + //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index 9a8ec8dc9858d7..6ca0b62bc5dca8 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -686,6 +686,7 @@ fcntl_exec(PyObject *module) static PyModuleDef_Slot fcntl_slots[] = { {Py_mod_exec, fcntl_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index f4d5186ff1552e..26ddcdd538a4d4 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -2044,6 +2044,7 @@ gcmodule_exec(PyObject *module) static PyModuleDef_Slot gcmodule_slots[] = { {Py_mod_exec, gcmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/grpmodule.c b/Modules/grpmodule.c index f6298ca0ee84c1..57cdde6064c24e 100644 --- a/Modules/grpmodule.c +++ b/Modules/grpmodule.c @@ -327,6 +327,7 @@ grpmodule_exec(PyObject *module) static PyModuleDef_Slot grpmodule_slots[] = { {Py_mod_exec, grpmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index c986e02867ca82..555eab09935e9e 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -4693,6 +4693,7 @@ itertoolsmodule_exec(PyObject *mod) static struct PyModuleDef_Slot itertoolsmodule_slots[] = { {Py_mod_exec, itertoolsmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 4a2381d9611776..3737a9654575ab 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -4064,6 +4064,7 @@ static PyMethodDef math_methods[] = { static PyModuleDef_Slot math_slots[] = { {Py_mod_exec, math_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/md5module.c b/Modules/md5module.c index 4f7bc77a8836a3..86605771d9643f 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -340,6 +340,7 @@ md5_exec(PyObject *m) static PyModuleDef_Slot _md5_slots[] = { {Py_mod_exec, md5_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index fe76ca6eafaa88..a470dd3c2f3bba 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1713,6 +1713,7 @@ mmap_exec(PyObject *module) static PyModuleDef_Slot mmap_slots[] = { {Py_mod_exec, mmap_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/nismodule.c b/Modules/nismodule.c index ec7f6d8031e84b..6d094490cea731 100644 --- a/Modules/nismodule.c +++ b/Modules/nismodule.c @@ -502,6 +502,9 @@ nis_exec(PyObject *module) static PyModuleDef_Slot nis_slots[] = { {Py_mod_exec, nis_exec}, + // XXX gh-103092: fix isolation. + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/overlapped.c b/Modules/overlapped.c index 02c0f401be4c9e..ac637316583d2d 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -2050,6 +2050,7 @@ overlapped_exec(PyObject *module) static PyModuleDef_Slot overlapped_slots[] = { {Py_mod_exec, overlapped_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index b395c265c72d0c..5022fdeb03703a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -16793,6 +16793,7 @@ posixmodule_exec(PyObject *m) static PyModuleDef_Slot posixmodile_slots[] = { {Py_mod_exec, posixmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/pwdmodule.c b/Modules/pwdmodule.c index a757380bd09f70..cc2e2a43893971 100644 --- a/Modules/pwdmodule.c +++ b/Modules/pwdmodule.c @@ -336,6 +336,7 @@ pwdmodule_exec(PyObject *module) static PyModuleDef_Slot pwdmodule_slots[] = { {Py_mod_exec, pwdmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 0a744998b6c514..c0fbd4d39f0096 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -2056,6 +2056,9 @@ pyexpat_free(void *module) static PyModuleDef_Slot pyexpat_slots[] = { {Py_mod_exec, pyexpat_exec}, + // XXX gh-103092: fix isolation. + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/resource.c b/Modules/resource.c index a97fb870062b82..2a8158c9be5359 100644 --- a/Modules/resource.c +++ b/Modules/resource.c @@ -514,6 +514,7 @@ resource_exec(PyObject *module) static struct PyModuleDef_Slot resource_slots[] = { {Py_mod_exec, resource_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 5a1e40d0b4a482..79bd5b59ab68f9 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -2651,6 +2651,7 @@ _select_exec(PyObject *m) static PyModuleDef_Slot _select_slots[] = { {Py_mod_exec, _select_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/sha1module.c b/Modules/sha1module.c index f8d4056fd34b65..bdb76c56f1a6e8 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -344,6 +344,7 @@ _sha1_exec(PyObject *module) static PyModuleDef_Slot _sha1_slots[] = { {Py_mod_exec, _sha1_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/sha2module.c b/Modules/sha2module.c index 72de20b44762d7..37d9b5c538fd0b 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -785,6 +785,7 @@ static int sha2_exec(PyObject *module) static PyModuleDef_Slot _sha2_slots[] = { {Py_mod_exec, sha2_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index fdd1450050fa1b..2350236ad46b25 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -1695,6 +1695,7 @@ _signal_module_free(void *module) static PyModuleDef_Slot signal_slots[] = { {Py_mod_exec, signal_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index e5478382e11f89..60219593be61e2 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -8870,6 +8870,7 @@ socket_exec(PyObject *m) static struct PyModuleDef_Slot socket_slots[] = { {Py_mod_exec, socket_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/spwdmodule.c b/Modules/spwdmodule.c index 42123c93b59365..13f1115feefa86 100644 --- a/Modules/spwdmodule.c +++ b/Modules/spwdmodule.c @@ -224,6 +224,7 @@ spwdmodule_exec(PyObject *module) static PyModuleDef_Slot spwdmodule_slots[] = { {Py_mod_exec, spwdmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index 91538b4fb15cbd..1cc319cc3410d8 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -100,6 +100,7 @@ symtable_init_constants(PyObject *m) static PyModuleDef_Slot symtable_slots[] = { {Py_mod_exec, symtable_init_constants}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index f45aa5227f1cbf..6db8de9c491dd9 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -406,6 +406,7 @@ syslog_exec(PyObject *module) static PyModuleDef_Slot syslog_slots[] = { {Py_mod_exec, syslog_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/termios.c b/Modules/termios.c index fcc8f042679870..169a36fc6477d8 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -1253,6 +1253,7 @@ termios_exec(PyObject *mod) static PyModuleDef_Slot termios_slots[] = { {Py_mod_exec, termios_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/timemodule.c b/Modules/timemodule.c index c50e689bb6986c..3607855dbd8f27 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -2107,6 +2107,7 @@ time_module_free(void *module) static struct PyModuleDef_Slot time_slots[] = { {Py_mod_exec, time_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index c108f14871f946..41dcd5f8f883f2 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -1516,6 +1516,7 @@ unicodedata_exec(PyObject *module) static PyModuleDef_Slot unicodedata_slots[] = { {Py_mod_exec, unicodedata_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 5f5297ba6337af..3935c00fc26530 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -390,6 +390,7 @@ xx_modexec(PyObject *m) static PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_modexec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/xxlimited_35.c b/Modules/xxlimited_35.c index 361c7e76d77f50..1ff3ef1cb6f296 100644 --- a/Modules/xxlimited_35.c +++ b/Modules/xxlimited_35.c @@ -293,6 +293,7 @@ xx_modexec(PyObject *m) static PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_modexec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/xxmodule.c b/Modules/xxmodule.c index a676fdb4ec773a..1e4e0ea3743ce3 100644 --- a/Modules/xxmodule.c +++ b/Modules/xxmodule.c @@ -383,6 +383,7 @@ xx_exec(PyObject *m) static struct PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/xxsubtype.c b/Modules/xxsubtype.c index 8512baf7cd0a2d..744ba7bf5d28b6 100644 --- a/Modules/xxsubtype.c +++ b/Modules/xxsubtype.c @@ -286,6 +286,7 @@ xxsubtype_exec(PyObject* m) static struct PyModuleDef_Slot xxsubtype_slots[] = { {Py_mod_exec, xxsubtype_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index e2f7dbaca87a9f..b67844a67c315c 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -2109,6 +2109,7 @@ zlib_exec(PyObject *mod) static PyModuleDef_Slot zlib_slots[] = { {Py_mod_exec, zlib_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 63f4226dd9e94a..c100d018d3f0df 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -250,6 +250,7 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio int has_execution_slots = 0; const char *name; int ret; + PyInterpreterState *interp = _PyInterpreterState_GET(); PyModuleDef_Init(def); @@ -316,13 +317,13 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio multiple_interpreters = Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED; } if (multiple_interpreters == Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED) { - PyInterpreterState *interp = _PyInterpreterState_GET(); if (!_Py_IsMainInterpreter(interp) && _PyImport_CheckSubinterpIncompatibleExtensionAllowed(name) < 0) { goto error; } } + // XXX Do a similar check once we have PyInterpreterState.ceval.own_gil. if (create) { m = create(spec, def); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 6ae68cc20f7dee..1585a582f00aad 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -15192,12 +15192,18 @@ static PyMethodDef _string_methods[] = { {NULL, NULL} }; +static PyModuleDef_Slot module_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL} +}; + static struct PyModuleDef _string_module = { PyModuleDef_HEAD_INIT, .m_name = "_string", .m_doc = PyDoc_STR("string helper module"), .m_size = 0, .m_methods = _string_methods, + .m_slots = module_slots, }; PyMODINIT_FUNC diff --git a/PC/_testconsole.c b/PC/_testconsole.c index f14a2d45b1be26..3221b985d01ba0 100644 --- a/PC/_testconsole.c +++ b/PC/_testconsole.c @@ -31,6 +31,7 @@ static int execfunc(PyObject *m) PyModuleDef_Slot testconsole_slots[] = { {Py_mod_exec, execfunc}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index 090254befc934d..53ef26b732f615 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -661,6 +661,7 @@ exec_module(PyObject* m) static PyModuleDef_Slot msvcrt_slots[] = { {Py_mod_exec, exec_module}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/PC/winreg.c b/PC/winreg.c index 4884125c3609ad..e2d5322f458c2a 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -2184,6 +2184,7 @@ exec_module(PyObject *m) static PyModuleDef_Slot winreg_slots[] = { {Py_mod_exec, exec_module}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/PC/winsound.c b/PC/winsound.c index 17ce2ef423b1f9..68a917810f884d 100644 --- a/PC/winsound.c +++ b/PC/winsound.c @@ -235,6 +235,7 @@ exec_module(PyObject *module) static PyModuleDef_Slot sound_slots[] = { {Py_mod_exec, exec_module}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index b44e303ac2594b..5d5a05a70ca7ec 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1206,6 +1206,7 @@ def visitModule(self, mod): self.emit(""" static PyModuleDef_Slot astmodule_slots[] = { {Py_mod_exec, astmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 6c878474afb192..81ab71c0fc3b29 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -12193,6 +12193,7 @@ astmodule_exec(PyObject *m) static PyModuleDef_Slot astmodule_slots[] = { {Py_mod_exec, astmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 416dc5971bca3d..3394a5108cb535 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -151,6 +151,7 @@ static PyMethodDef tokenize_methods[] = { static PyModuleDef_Slot tokenizemodule_slots[] = { {Py_mod_exec, tokenizemodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Python/_warnings.c b/Python/_warnings.c index d510381c365b66..5644db9a3770cb 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1449,6 +1449,7 @@ warnings_module_exec(PyObject *module) static PyModuleDef_Slot warnings_slots[] = { {Py_mod_exec, warnings_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Python/import.c b/Python/import.c index 0bf107b28d3990..9e1857d5f3e4e6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3840,6 +3840,7 @@ imp_module_exec(PyObject *module) static PyModuleDef_Slot imp_slots[] = { {Py_mod_exec, imp_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Python/marshal.c b/Python/marshal.c index 2966139cec9ae9..208996b05fc484 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -1870,6 +1870,7 @@ marshal_module_exec(PyObject *mod) static PyModuleDef_Slot marshalmodule_slots[] = { {Py_mod_exec, marshal_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; From 66558d2a16ee42afc0e2c02e6a90bfd62dcb67f6 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 5 May 2023 23:35:24 +0200 Subject: [PATCH 067/204] gh-104146: Remove unused var 'parser_body_declarations' from clinic.py (#104214) --- Tools/clinic/clinic.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 5f6e67e7d65a1d..704325670bc994 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -814,13 +814,11 @@ def output_templates(self, f): # parser_body_fields remembers the fields passed in to the # previous call to parser_body. this is used for an awful hack. parser_body_fields = () - parser_body_declarations = '' def parser_body(prototype, *fields, declarations=''): - nonlocal parser_body_fields, parser_body_declarations + nonlocal parser_body_fields add, output = text_accumulator() add(prototype) parser_body_fields = fields - parser_body_declarations = declarations fields = list(fields) fields.insert(0, normalize_snippet(""" From f3e7eb48f86057919c347f56dabf417acfd55845 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 5 May 2023 15:59:20 -0600 Subject: [PATCH 068/204] gh-99113: Add PyInterpreterConfig.own_gil (gh-104204) We also add PyInterpreterState.ceval.own_gil to record if the interpreter actually has its own GIL. Note that for now we don't actually respect own_gil; all interpreters still share the one GIL. However, PyInterpreterState.ceval.own_gil does reflect PyInterpreterConfig.own_gil. That lie is a temporary one that we will fix when the GIL really becomes per-interpreter. --- Include/cpython/initconfig.h | 3 +++ Include/internal/pycore_ceval.h | 2 +- Include/internal/pycore_ceval_state.h | 1 + Lib/test/test_capi/test_misc.py | 36 +++++++++++++++++++-------- Lib/test/test_embed.py | 1 + Lib/test/test_import/__init__.py | 1 + Lib/test/test_threading.py | 1 + Modules/_testcapimodule.c | 12 +++++++-- Modules/_testinternalcapi.c | 7 ++++++ Python/ceval_gil.c | 23 ++++++++++++++++- Python/pylifecycle.c | 12 +++++---- 11 files changed, 80 insertions(+), 19 deletions(-) diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 9c1783d272f1cd..efae2409b50069 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -252,6 +252,7 @@ typedef struct { int allow_threads; int allow_daemon_threads; int check_multi_interp_extensions; + int own_gil; } PyInterpreterConfig; #define _PyInterpreterConfig_INIT \ @@ -262,6 +263,7 @@ typedef struct { .allow_threads = 1, \ .allow_daemon_threads = 0, \ .check_multi_interp_extensions = 1, \ + .own_gil = 1, \ } #define _PyInterpreterConfig_LEGACY_INIT \ @@ -272,6 +274,7 @@ typedef struct { .allow_threads = 1, \ .allow_daemon_threads = 1, \ .check_multi_interp_extensions = 0, \ + .own_gil = 0, \ } /* --- Helper functions --------------------------------------- */ diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 0bbc9efdda3ba7..b7a9bf40425bc7 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -97,7 +97,7 @@ _PyEval_Vector(PyThreadState *tstate, PyObject *kwnames); extern int _PyEval_ThreadsInitialized(void); -extern PyStatus _PyEval_InitGIL(PyThreadState *tstate); +extern PyStatus _PyEval_InitGIL(PyThreadState *tstate, int own_gil); extern void _PyEval_FiniGIL(PyInterpreterState *interp); extern void _PyEval_ReleaseLock(PyThreadState *tstate); diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 1a00ec80270e0c..4781dd5735dcf6 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -86,6 +86,7 @@ struct _pending_calls { struct _ceval_state { int recursion_limit; struct _gil_runtime_state *gil; + int own_gil; /* This single variable consolidates all requests to break out of the fast path in the eval loop. */ _Py_atomic_int eval_breaker; diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 22be3c0814278e..3fc2c07f933061 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -1401,23 +1401,37 @@ def test_configured_settings(self): DAEMON_THREADS = 1<<11 FORK = 1<<15 EXEC = 1<<16 - - features = ['obmalloc', 'fork', 'exec', 'threads', 'daemon_threads', - 'extensions'] + ALL_FLAGS = (OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS + | EXTENSIONS); + + features = [ + 'obmalloc', + 'fork', + 'exec', + 'threads', + 'daemon_threads', + 'extensions', + 'own_gil', + ] kwlist = [f'allow_{n}' for n in features] kwlist[0] = 'use_main_obmalloc' - kwlist[-1] = 'check_multi_interp_extensions' + kwlist[-2] = 'check_multi_interp_extensions' + kwlist[-1] = 'own_gil' # expected to work for config, expected in { - (True, True, True, True, True, True): - OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS | EXTENSIONS, - (True, False, False, False, False, False): OBMALLOC, - (False, False, False, True, False, True): THREADS | EXTENSIONS, + (True, True, True, True, True, True, True): + (ALL_FLAGS, True), + (True, False, False, False, False, False, False): + (OBMALLOC, False), + (False, False, False, True, False, True, False): + (THREADS | EXTENSIONS, False), }.items(): kwargs = dict(zip(kwlist, config)) + exp_flags, exp_gil = expected expected = { - 'feature_flags': expected, + 'feature_flags': exp_flags, + 'own_gil': exp_gil, } with self.subTest(config): r, w = os.pipe() @@ -1437,7 +1451,7 @@ def test_configured_settings(self): # expected to fail for config in [ - (False, False, False, False, False, False), + (False, False, False, False, False, False, False), ]: kwargs = dict(zip(kwlist, config)) with self.subTest(config): @@ -1473,6 +1487,7 @@ def test_overridden_setting_extensions_subinterp_check(self): 'allow_exec': True, 'allow_threads': True, 'allow_daemon_threads': True, + 'own_gil': False, } def check(enabled, override): @@ -1483,6 +1498,7 @@ def check(enabled, override): flags = BASE_FLAGS | EXTENSIONS if enabled else BASE_FLAGS settings = { 'feature_flags': flags, + 'own_gil': False, } expected = { diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index c9691bbf304915..582392ecddcb91 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1666,6 +1666,7 @@ def test_init_main_interpreter_settings(self): # All optional features should be enabled. 'feature_flags': OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS, + 'own_gil': True, } out, err = self.run_embedded_interpreter( 'test_init_main_interpreter_settings', diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 9211639b016e7a..773b7094c6b8ce 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1640,6 +1640,7 @@ class SubinterpImportTests(unittest.TestCase): ) ISOLATED = dict( use_main_obmalloc=False, + own_gil=True, ) NOT_ISOLATED = {k: not v for k, v in ISOLATED.items()} diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index fdd74c37e26235..97165264b34bbe 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1349,6 +1349,7 @@ def func(): allow_threads={allowed}, allow_daemon_threads={daemon_allowed}, check_multi_interp_extensions=False, + own_gil=False, ) """) with test.support.SuppressCrashReport(): diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 376f04f23e324a..79ab7d3f5555c2 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1488,6 +1488,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) int allow_threads = -1; int allow_daemon_threads = -1; int check_multi_interp_extensions = -1; + int own_gil = -1; int r; PyThreadState *substate, *mainstate; /* only initialise 'cflags.cf_flags' to test backwards compatibility */ @@ -1500,13 +1501,15 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) "allow_threads", "allow_daemon_threads", "check_multi_interp_extensions", + "own_gil", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "s$pppppp:run_in_subinterp_with_config", kwlist, + "s$ppppppp:run_in_subinterp_with_config", kwlist, &code, &use_main_obmalloc, &allow_fork, &allow_exec, &allow_threads, &allow_daemon_threads, - &check_multi_interp_extensions)) { + &check_multi_interp_extensions, + &own_gil)) { return NULL; } if (use_main_obmalloc < 0) { @@ -1525,6 +1528,10 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) PyErr_SetString(PyExc_ValueError, "missing allow_threads"); return NULL; } + if (own_gil < 0) { + PyErr_SetString(PyExc_ValueError, "missing own_gil"); + return NULL; + } if (allow_daemon_threads < 0) { PyErr_SetString(PyExc_ValueError, "missing allow_daemon_threads"); return NULL; @@ -1545,6 +1552,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) .allow_threads = allow_threads, .allow_daemon_threads = allow_daemon_threads, .check_multi_interp_extensions = check_multi_interp_extensions, + .own_gil = own_gil, }; PyStatus status = Py_NewInterpreterFromConfig(&substate, &config); if (PyStatus_Exception(status)) { diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 24152412107c09..f35e3b48df9321 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -729,6 +729,13 @@ get_interp_settings(PyObject *self, PyObject *args) return NULL; } + /* "own GIL" */ + PyObject *own_gil = interp->ceval.own_gil ? Py_True : Py_False; + if (PyDict_SetItemString(settings, "own_gil", own_gil) != 0) { + Py_DECREF(settings); + return NULL; + } + return settings; } diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index ad33a58dedd820..a390bec80d556c 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -500,9 +500,18 @@ PyEval_ThreadsInitialized(void) } PyStatus -_PyEval_InitGIL(PyThreadState *tstate) +_PyEval_InitGIL(PyThreadState *tstate, int own_gil) { assert(tstate->interp->ceval.gil == NULL); + if (!own_gil) { + PyInterpreterState *main_interp = _PyInterpreterState_Main(); + assert(tstate->interp != main_interp); + struct _gil_runtime_state *gil = main_interp->ceval.gil; + assert(gil_created(gil)); + tstate->interp->ceval.gil = gil; + tstate->interp->ceval.own_gil = 0; + return _PyStatus_OK(); + } /* XXX per-interpreter GIL */ struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil; @@ -512,8 +521,11 @@ _PyEval_InitGIL(PyThreadState *tstate) and destroy it. */ assert(gil_created(gil)); tstate->interp->ceval.gil = gil; + // XXX For now we lie. + tstate->interp->ceval.own_gil = 1; return _PyStatus_OK(); } + assert(own_gil); assert(!gil_created(gil)); @@ -521,6 +533,7 @@ _PyEval_InitGIL(PyThreadState *tstate) create_gil(gil); assert(gil_created(gil)); tstate->interp->ceval.gil = gil; + tstate->interp->ceval.own_gil = 1; take_gil(tstate); return _PyStatus_OK(); } @@ -530,6 +543,14 @@ _PyEval_FiniGIL(PyInterpreterState *interp) { if (interp->ceval.gil == NULL) { /* It was already finalized (or hasn't been initialized yet). */ + assert(!interp->ceval.own_gil); + return; + } + else if (!interp->ceval.own_gil) { + PyInterpreterState *main_interp = _PyInterpreterState_Main(); + assert(interp != main_interp); + assert(interp->ceval.gil == main_interp->ceval.gil); + interp->ceval.gil = NULL; return; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 97957d3f17e6db..705708698c6a8b 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -585,7 +585,7 @@ init_interp_settings(PyInterpreterState *interp, static PyStatus -init_interp_create_gil(PyThreadState *tstate) +init_interp_create_gil(PyThreadState *tstate, int own_gil) { PyStatus status; @@ -600,7 +600,7 @@ init_interp_create_gil(PyThreadState *tstate) } /* Create the GIL and take it */ - status = _PyEval_InitGIL(tstate); + status = _PyEval_InitGIL(tstate, own_gil); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -632,7 +632,9 @@ pycore_create_interpreter(_PyRuntimeState *runtime, return status; } - const PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; + PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; + // The main interpreter always has its own GIL. + config.own_gil = 1; status = init_interp_settings(interp, &config); if (_PyStatus_EXCEPTION(status)) { return status; @@ -645,7 +647,7 @@ pycore_create_interpreter(_PyRuntimeState *runtime, _PyThreadState_Bind(tstate); (void) PyThreadState_Swap(tstate); - status = init_interp_create_gil(tstate); + status = init_interp_create_gil(tstate, config.own_gil); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -2047,7 +2049,7 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) goto error; } - status = init_interp_create_gil(tstate); + status = init_interp_create_gil(tstate, config->own_gil); if (_PyStatus_EXCEPTION(status)) { goto error; } From 52d8f36e8c9f6048367d7bdfede3698e3f5f70d0 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Fri, 5 May 2023 16:44:03 -0700 Subject: [PATCH 069/204] gh-104144: Skip scheduling a done callback if a TaskGroup task completes eagerly (#104140) Co-authored-by: Carl Meyer --- Lib/asyncio/taskgroups.py | 10 ++++++++-- .../2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 0fdea3697ece3d..06b2e0db86a1fe 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -164,8 +164,14 @@ def create_task(self, coro, *, name=None, context=None): else: task = self._loop.create_task(coro, context=context) tasks._set_task_name(task, name) - task.add_done_callback(self._on_task_done) - self._tasks.add(task) + # optimization: Immediately call the done callback if the task is + # already done (e.g. if the coro was able to complete eagerly), + # and skip scheduling a done callback + if task.done(): + self._on_task_done(task) + else: + self._tasks.add(task) + task.add_done_callback(self._on_task_done) return task # Since Python 3.8 Tasks propagate all exceptions correctly, diff --git a/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst b/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst new file mode 100644 index 00000000000000..59870de3e02edd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst @@ -0,0 +1 @@ +Optimize :class:`asyncio.TaskGroup` when using :func:`asyncio.eager_task_factory`. Skip scheduling done callbacks when all tasks finish without blocking. From 8b7f37dd4c297138e9f4a256ff6750cf1402b421 Mon Sep 17 00:00:00 2001 From: Jacob Bower <1978924+jbower-fb@users.noreply.github.com> Date: Fri, 5 May 2023 16:50:06 -0700 Subject: [PATCH 070/204] gh-97696: Remove redundant #include (#104216) Remove "#include cpython/context.h"` from `_asynciomodule.c`. It's already included in `Python.h`. --- Modules/_asynciomodule.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 822d5f2a41de33..39c33fed74e221 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -8,7 +8,6 @@ #include "pycore_runtime_init.h" // _Py_ID() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "structmember.h" // PyMemberDef -#include "cpython/context.h" #include // offsetof() From c84029179c3287f9c357ccac231fe78469c6f068 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 6 May 2023 01:53:55 +0200 Subject: [PATCH 071/204] gh-101819: Prepare to modernize the _io extension (#104178) * Add references to static types to _PyIO_State: * PyBufferedIOBase_Type * PyBytesIOBuffer_Type * PyIncrementalNewlineDecoder_Type * PyRawIOBase_Type * PyTextIOBase_Type * Add the defining class to methods: * _io.BytesIO.getbuffer() * _io.FileIO.close() * Add get_io_state_by_cls() function. * Add state parameter to _textiowrapper_decode() * _io_TextIOWrapper___init__() now sets self->state before calling _textiowrapper_set_decoder(). Co-authored-by: Erlend E. Aasland --- Modules/_io/_iomodule.c | 35 ++++++++++++++++++++++++++-------- Modules/_io/_iomodule.h | 14 ++++++++++++++ Modules/_io/bufferedio.c | 2 +- Modules/_io/bytesio.c | 10 +++++++--- Modules/_io/clinic/bytesio.c.h | 14 +++++++++----- Modules/_io/clinic/fileio.c.h | 14 +++++++++----- Modules/_io/fileio.c | 13 +++++++++---- Modules/_io/stringio.c | 5 +++-- Modules/_io/textio.c | 22 ++++++++++++--------- 9 files changed, 92 insertions(+), 37 deletions(-) diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 8ec3a6081c98d9..403968af1b996c 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -583,13 +583,18 @@ iomodule_traverse(PyObject *mod, visitproc visit, void *arg) { Py_VISIT(state->locale_module); Py_VISIT(state->unsupported_operation); + Py_VISIT(state->PyIncrementalNewlineDecoder_Type); + Py_VISIT(state->PyRawIOBase_Type); + Py_VISIT(state->PyBufferedIOBase_Type); Py_VISIT(state->PyBufferedRWPair_Type); Py_VISIT(state->PyBufferedRandom_Type); Py_VISIT(state->PyBufferedReader_Type); Py_VISIT(state->PyBufferedWriter_Type); + Py_VISIT(state->PyBytesIOBuffer_Type); Py_VISIT(state->PyBytesIO_Type); Py_VISIT(state->PyFileIO_Type); Py_VISIT(state->PyStringIO_Type); + Py_VISIT(state->PyTextIOBase_Type); Py_VISIT(state->PyTextIOWrapper_Type); return 0; } @@ -604,13 +609,18 @@ iomodule_clear(PyObject *mod) { Py_CLEAR(state->locale_module); Py_CLEAR(state->unsupported_operation); + Py_CLEAR(state->PyIncrementalNewlineDecoder_Type); + Py_CLEAR(state->PyRawIOBase_Type); + Py_CLEAR(state->PyBufferedIOBase_Type); Py_CLEAR(state->PyBufferedRWPair_Type); Py_CLEAR(state->PyBufferedRandom_Type); Py_CLEAR(state->PyBufferedReader_Type); Py_CLEAR(state->PyBufferedWriter_Type); + Py_CLEAR(state->PyBytesIOBuffer_Type); Py_CLEAR(state->PyBytesIO_Type); Py_CLEAR(state->PyFileIO_Type); Py_CLEAR(state->PyStringIO_Type); + Py_CLEAR(state->PyTextIOBase_Type); Py_CLEAR(state->PyTextIOWrapper_Type); return 0; } @@ -749,24 +759,33 @@ PyInit__io(void) } } + // Base classes + state->PyIncrementalNewlineDecoder_Type = (PyTypeObject *)Py_NewRef(&PyIncrementalNewlineDecoder_Type); + + // PyIOBase_Type subclasses + state->PyRawIOBase_Type = (PyTypeObject *)Py_NewRef(&PyRawIOBase_Type); + state->PyBufferedIOBase_Type = (PyTypeObject *)Py_NewRef(&PyBufferedIOBase_Type); + state->PyTextIOBase_Type = (PyTypeObject *)Py_NewRef(&PyTextIOBase_Type); + // PyBufferedIOBase_Type(PyIOBase_Type) subclasses - ADD_TYPE(m, state->PyBytesIO_Type, &bytesio_spec, &PyBufferedIOBase_Type); + ADD_TYPE(m, state->PyBytesIO_Type, &bytesio_spec, state->PyBufferedIOBase_Type); ADD_TYPE(m, state->PyBufferedWriter_Type, &bufferedwriter_spec, - &PyBufferedIOBase_Type); + state->PyBufferedIOBase_Type); ADD_TYPE(m, state->PyBufferedReader_Type, &bufferedreader_spec, - &PyBufferedIOBase_Type); + state->PyBufferedIOBase_Type); ADD_TYPE(m, state->PyBufferedRWPair_Type, &bufferedrwpair_spec, - &PyBufferedIOBase_Type); + state->PyBufferedIOBase_Type); ADD_TYPE(m, state->PyBufferedRandom_Type, &bufferedrandom_spec, - &PyBufferedIOBase_Type); + state->PyBufferedIOBase_Type); // PyRawIOBase_Type(PyIOBase_Type) subclasses - ADD_TYPE(m, state->PyFileIO_Type, &fileio_spec, &PyRawIOBase_Type); + state->PyBytesIOBuffer_Type = (PyTypeObject *)Py_NewRef(&_PyBytesIOBuffer_Type); + ADD_TYPE(m, state->PyFileIO_Type, &fileio_spec, state->PyRawIOBase_Type); // PyTextIOBase_Type(PyIOBase_Type) subclasses - ADD_TYPE(m, state->PyStringIO_Type, &stringio_spec, &PyTextIOBase_Type); + ADD_TYPE(m, state->PyStringIO_Type, &stringio_spec, state->PyTextIOBase_Type); ADD_TYPE(m, state->PyTextIOWrapper_Type, &textiowrapper_spec, - &PyTextIOBase_Type); + state->PyTextIOBase_Type); state->initialized = 1; diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index d7224e56f9a722..8a788fbb8185c5 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -5,6 +5,7 @@ #include "exports.h" #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_typeobject.h" // _PyType_GetModuleState() #include "structmember.h" /* ABCs */ @@ -147,13 +148,18 @@ typedef struct { PyObject *unsupported_operation; /* Types */ + PyTypeObject *PyIncrementalNewlineDecoder_Type; + PyTypeObject *PyRawIOBase_Type; + PyTypeObject *PyBufferedIOBase_Type; PyTypeObject *PyBufferedRWPair_Type; PyTypeObject *PyBufferedRandom_Type; PyTypeObject *PyBufferedReader_Type; PyTypeObject *PyBufferedWriter_Type; + PyTypeObject *PyBytesIOBuffer_Type; PyTypeObject *PyBytesIO_Type; PyTypeObject *PyFileIO_Type; PyTypeObject *PyStringIO_Type; + PyTypeObject *PyTextIOBase_Type; PyTypeObject *PyTextIOWrapper_Type; } _PyIO_State; @@ -168,6 +174,14 @@ get_io_state(PyObject *module) return (_PyIO_State *)state; } +static inline _PyIO_State * +get_io_state_by_cls(PyTypeObject *cls) +{ + void *state = _PyType_GetModuleState(cls); + assert(state != NULL); + return (_PyIO_State *)state; +} + static inline _PyIO_State * find_io_state_by_def(PyTypeObject *type) { diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 2c71768be97870..723d16b47fef9f 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -2231,7 +2231,7 @@ bufferedrwpair_close(rwpair *self, PyObject *Py_UNUSED(ignored)) } else { Py_DECREF(ret); - } + } ret = _forward_call(self->reader, &_Py_ID(close), NULL); if (exc != NULL) { _PyErr_ChainExceptions1(exc); diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 7e9d28b3b9655c..9c7a28357987ba 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -308,14 +308,18 @@ _io_BytesIO_flush_impl(bytesio *self) /*[clinic input] _io.BytesIO.getbuffer + cls: defining_class + / + Get a read-write view over the contents of the BytesIO object. [clinic start generated code]*/ static PyObject * -_io_BytesIO_getbuffer_impl(bytesio *self) -/*[clinic end generated code: output=72cd7c6e13aa09ed input=8f738ef615865176]*/ +_io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls) +/*[clinic end generated code: output=045091d7ce87fe4e input=0668fbb48f95dffa]*/ { - PyTypeObject *type = &_PyBytesIOBuffer_Type; + _PyIO_State *state = get_io_state_by_cls(cls); + PyTypeObject *type = state->PyBytesIOBuffer_Type; bytesiobuf *buf; PyObject *view; diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index 84b58db6c7a702..9550c8728c251e 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -87,15 +87,19 @@ PyDoc_STRVAR(_io_BytesIO_getbuffer__doc__, "Get a read-write view over the contents of the BytesIO object."); #define _IO_BYTESIO_GETBUFFER_METHODDEF \ - {"getbuffer", (PyCFunction)_io_BytesIO_getbuffer, METH_NOARGS, _io_BytesIO_getbuffer__doc__}, + {"getbuffer", _PyCFunction_CAST(_io_BytesIO_getbuffer), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_BytesIO_getbuffer__doc__}, static PyObject * -_io_BytesIO_getbuffer_impl(bytesio *self); +_io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls); static PyObject * -_io_BytesIO_getbuffer(bytesio *self, PyObject *Py_UNUSED(ignored)) +_io_BytesIO_getbuffer(bytesio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _io_BytesIO_getbuffer_impl(self); + if (nargs) { + PyErr_SetString(PyExc_TypeError, "getbuffer() takes no arguments"); + return NULL; + } + return _io_BytesIO_getbuffer_impl(self, cls); } PyDoc_STRVAR(_io_BytesIO_getvalue__doc__, @@ -534,4 +538,4 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=a44770efbaeb80dd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=098584d485420b65 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index b6e9bd5b65a029..dfad8a58c4723e 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -18,15 +18,19 @@ PyDoc_STRVAR(_io_FileIO_close__doc__, "called more than once without error."); #define _IO_FILEIO_CLOSE_METHODDEF \ - {"close", (PyCFunction)_io_FileIO_close, METH_NOARGS, _io_FileIO_close__doc__}, + {"close", _PyCFunction_CAST(_io_FileIO_close), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_close__doc__}, static PyObject * -_io_FileIO_close_impl(fileio *self); +_io_FileIO_close_impl(fileio *self, PyTypeObject *cls); static PyObject * -_io_FileIO_close(fileio *self, PyObject *Py_UNUSED(ignored)) +_io_FileIO_close(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _io_FileIO_close_impl(self); + if (nargs) { + PyErr_SetString(PyExc_TypeError, "close() takes no arguments"); + return NULL; + } + return _io_FileIO_close_impl(self, cls); } PyDoc_STRVAR(_io_FileIO___init____doc__, @@ -466,4 +470,4 @@ _io_FileIO_isatty(fileio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=27f883807a6c29ae input=a9049054013a1b77]*/ +/*[clinic end generated code: output=29ed2ae6c451c139 input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 1118d86e6c9a10..cc0e7307b9da77 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -130,6 +130,9 @@ internal_close(fileio *self) /*[clinic input] _io.FileIO.close + cls: defining_class + / + Close the file. A closed file cannot be used for further I/O operations. close() may be @@ -137,18 +140,20 @@ called more than once without error. [clinic start generated code]*/ static PyObject * -_io_FileIO_close_impl(fileio *self) -/*[clinic end generated code: output=7737a319ef3bad0b input=f35231760d54a522]*/ +_io_FileIO_close_impl(fileio *self, PyTypeObject *cls) +/*[clinic end generated code: output=c30cbe9d1f23ca58 input=70da49e63db7c64d]*/ { PyObject *res; - PyObject *exc; int rc; - res = PyObject_CallMethodOneArg((PyObject*)&PyRawIOBase_Type, + _PyIO_State *state = get_io_state_by_cls(cls); + res = PyObject_CallMethodOneArg((PyObject*)state->PyRawIOBase_Type, &_Py_ID(close), (PyObject *)self); if (!self->closefd) { self->fd = -1; return res; } + + PyObject *exc; if (res == NULL) { exc = PyErr_GetRaisedException(); } diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 54c050f0be4688..13d3b870b39a81 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -716,9 +716,10 @@ _io_StringIO___init___impl(stringio *self, PyObject *value, self->writenl = Py_NewRef(self->readnl); } + _PyIO_State *module_state = find_io_state_by_def(Py_TYPE(self)); if (self->readuniversal) { self->decoder = PyObject_CallFunctionObjArgs( - (PyObject *)&PyIncrementalNewlineDecoder_Type, + (PyObject *)module_state->PyIncrementalNewlineDecoder_Type, Py_None, self->readtranslate ? Py_True : Py_False, NULL); if (self->decoder == NULL) return -1; @@ -750,7 +751,7 @@ _io_StringIO___init___impl(stringio *self, PyObject *value, self->state = STATE_ACCUMULATING; } self->pos = 0; - self->module_state = find_io_state_by_def(Py_TYPE(self)); + self->module_state = module_state; self->closed = 0; self->ok = 1; return 0; diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 3ff84cb623af74..2dba382f4f8fb0 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -18,10 +18,10 @@ /*[clinic input] module _io -class _io.IncrementalNewlineDecoder "nldecoder_object *" "&PyIncrementalNewlineDecoder_Type" +class _io.IncrementalNewlineDecoder "nldecoder_object *" "clinic_state()->PyIncrementalNewlineDecoder_Type" class _io.TextIOWrapper "textio *" "clinic_state()->TextIOWrapper_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=d3f032e90f74c8f2]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=81f67cf54eaa6001]*/ /* TextIOBase */ @@ -872,8 +872,9 @@ _textiowrapper_set_decoder(textio *self, PyObject *codec_info, return -1; if (self->readuniversal) { + _PyIO_State *state = self->state; PyObject *incrementalDecoder = PyObject_CallFunctionObjArgs( - (PyObject *)&PyIncrementalNewlineDecoder_Type, + (PyObject *)state->PyIncrementalNewlineDecoder_Type, self->decoder, self->readtranslate ? Py_True : Py_False, NULL); if (incrementalDecoder == NULL) return -1; @@ -884,11 +885,12 @@ _textiowrapper_set_decoder(textio *self, PyObject *codec_info, } static PyObject* -_textiowrapper_decode(PyObject *decoder, PyObject *bytes, int eof) +_textiowrapper_decode(_PyIO_State *state, PyObject *decoder, PyObject *bytes, + int eof) { PyObject *chars; - if (Py_IS_TYPE(decoder, &PyIncrementalNewlineDecoder_Type)) + if (Py_IS_TYPE(decoder, state->PyIncrementalNewlineDecoder_Type)) chars = _PyIncrementalNewlineDecoder_decode(decoder, bytes, eof); else chars = PyObject_CallMethodObjArgs(decoder, &_Py_ID(decode), bytes, @@ -1167,6 +1169,8 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, self->buffer = Py_NewRef(buffer); /* Build the decoder object */ + _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); + self->state = state; if (_textiowrapper_set_decoder(self, codec_info, PyUnicode_AsUTF8(errors)) != 0) goto error; @@ -1177,7 +1181,6 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, /* Finished sorting out the codec details */ Py_CLEAR(codec_info); - _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); if (Py_IS_TYPE(buffer, state->PyBufferedReader_Type) || Py_IS_TYPE(buffer, state->PyBufferedWriter_Type) || Py_IS_TYPE(buffer, state->PyBufferedRandom_Type)) @@ -1214,7 +1217,6 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, goto error; } - self->state = state; self->ok = 1; return 0; @@ -1843,7 +1845,8 @@ textiowrapper_read_chunk(textio *self, Py_ssize_t size_hint) nbytes = input_chunk_buf.len; eof = (nbytes == 0); - decoded_chars = _textiowrapper_decode(self->decoder, input_chunk, eof); + decoded_chars = _textiowrapper_decode(self->state, self->decoder, + input_chunk, eof); PyBuffer_Release(&input_chunk_buf); if (decoded_chars == NULL) goto fail; @@ -1913,7 +1916,8 @@ _io_TextIOWrapper_read_impl(textio *self, Py_ssize_t n) if (bytes == NULL) goto fail; - if (Py_IS_TYPE(self->decoder, &PyIncrementalNewlineDecoder_Type)) + _PyIO_State *state = self->state; + if (Py_IS_TYPE(self->decoder, state->PyIncrementalNewlineDecoder_Type)) decoded = _PyIncrementalNewlineDecoder_decode(self->decoder, bytes, 1); else From 4cd95dce0b8d7bb8a16468ec8b5b3429555417f1 Mon Sep 17 00:00:00 2001 From: Sam Morris Date: Sat, 6 May 2023 03:40:19 +0100 Subject: [PATCH 072/204] gh-102215: importlib documentation cleanups --- Doc/library/importlib.metadata.rst | 14 +++++++------- Doc/library/importlib.resources.rst | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 3097bcf47b627f..d2cc769e2c8400 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -1,11 +1,11 @@ .. _using: -================================= - Using :mod:`!importlib.metadata` -================================= +======================================================== +:mod:`!importlib.metadata` -- Accessing package metadata +======================================================== .. module:: importlib.metadata - :synopsis: The implementation of the importlib metadata. + :synopsis: Accessing package metadata .. versionadded:: 3.8 .. versionchanged:: 3.10 @@ -13,7 +13,7 @@ **Source code:** :source:`Lib/importlib/metadata/__init__.py` -``importlib_metadata`` is a library that provides access to +``importlib.metadata`` is a library that provides access to the metadata of an installed `Distribution Package `_, such as its entry points or its top-level names (`Import Package `_\s, modules, if any). @@ -24,7 +24,7 @@ API`_ and `metadata API`_ of ``pkg_resources``. Along with this package can eliminate the need to use the older and less efficient ``pkg_resources`` package. -``importlib_metadata`` operates on third-party *distribution packages* +``importlib.metadata`` operates on third-party *distribution packages* installed into Python's ``site-packages`` directory via tools such as `pip `_. Specifically, it works with distributions with discoverable @@ -368,7 +368,7 @@ system :ref:`finders `. To find a distribution package's m ``importlib.metadata`` queries the list of :term:`meta path finders ` on :data:`sys.meta_path`. -By default ``importlib_metadata`` installs a finder for distribution packages +By default ``importlib.metadata`` installs a finder for distribution packages found on the file system. This finder doesn't actually find any *distributions*, but it can find their metadata. diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index 4c6aa59bf9f58f..755693840fecd8 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -1,5 +1,5 @@ -:mod:`importlib.resources` -- Resources ---------------------------------------- +:mod:`importlib.resources` -- Package resource reading, opening and access +-------------------------------------------------------------------------- .. module:: importlib.resources :synopsis: Package resource reading, opening, and access @@ -97,7 +97,7 @@ for example, a package and its resources can be imported from a zip file using Deprecated functions --------------------- +^^^^^^^^^^^^^^^^^^^^ An older, deprecated set of functions is still available, but is scheduled for removal in a future version of Python. From f5088006ca8e9654fbc3de119462f0ab764e408b Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 6 May 2023 04:54:08 +0100 Subject: [PATCH 073/204] GH-97950: Use new-style index directive ('builtin') (#104164) * Uncomment builtin removal in pairindextypes * Use new-style index directive ('builtin') - C API * Use new-style index directive ('builtin') - Extending * Use new-style index directive ('builtin') - Library * Use new-style index directive ('builtin') - Reference * Use new-style index directive ('builtin') - Tutorial --- Doc/c-api/dict.rst | 2 +- Doc/c-api/import.rst | 4 +-- Doc/c-api/list.rst | 4 +-- Doc/c-api/mapping.rst | 2 +- Doc/c-api/number.rst | 12 +++---- Doc/c-api/object.rst | 12 +++---- Doc/c-api/sequence.rst | 4 +-- Doc/c-api/set.rst | 2 +- Doc/c-api/structures.rst | 4 +-- Doc/c-api/typeobj.rst | 4 +-- Doc/extending/newtypes.rst | 2 +- Doc/library/dis.rst | 2 +- Doc/library/functions.rst | 2 +- Doc/library/pprint.rst | 4 +-- Doc/library/stdtypes.rst | 24 ++++++------- Doc/library/types.rst | 2 +- Doc/reference/compound_stmts.rst | 2 +- Doc/reference/datamodel.rst | 50 +++++++++++++-------------- Doc/reference/simple_stmts.rst | 8 ++--- Doc/reference/toplevel_components.rst | 2 +- Doc/tools/extensions/pyspecific.py | 2 +- Doc/tutorial/inputoutput.rst | 2 +- Doc/tutorial/stdlib.rst | 2 +- 23 files changed, 77 insertions(+), 77 deletions(-) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index f02abb01f0220b..0ca8ad624b2034 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -154,7 +154,7 @@ Dictionary Objects .. c:function:: Py_ssize_t PyDict_Size(PyObject *p) - .. index:: builtin: len + .. index:: pair: built-in function; len Return the number of items in the dictionary. This is equivalent to ``len(p)`` on a dictionary. diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 8e5af32b65e02c..79843ba521ab93 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -41,7 +41,7 @@ Importing Modules .. c:function:: PyObject* PyImport_ImportModuleEx(const char *name, PyObject *globals, PyObject *locals, PyObject *fromlist) - .. index:: builtin: __import__ + .. index:: pair: built-in function; __import__ Import a module. This is best described by referring to the built-in Python function :func:`__import__`. @@ -120,7 +120,7 @@ Importing Modules .. c:function:: PyObject* PyImport_ExecCodeModule(const char *name, PyObject *co) - .. index:: builtin: compile + .. index:: pair: built-in function; compile Given a module name (possibly of the form ``package.module``) and a code object read from a Python bytecode file or obtained from the built-in function diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index 317421f0db8453..dbf35611eccd3e 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -45,7 +45,7 @@ List Objects .. c:function:: Py_ssize_t PyList_Size(PyObject *list) - .. index:: builtin: len + .. index:: pair: built-in function; len Return the length of the list object in *list*; this is equivalent to ``len(list)`` on a list object. @@ -138,7 +138,7 @@ List Objects .. c:function:: PyObject* PyList_AsTuple(PyObject *list) - .. index:: builtin: tuple + .. index:: pair: built-in function; tuple Return a new tuple object containing the contents of *list*; equivalent to ``tuple(list)``. diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst index 3c9d282c6d0ab0..cffb0ed50fb77d 100644 --- a/Doc/c-api/mapping.rst +++ b/Doc/c-api/mapping.rst @@ -20,7 +20,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. c:function:: Py_ssize_t PyMapping_Size(PyObject *o) Py_ssize_t PyMapping_Length(PyObject *o) - .. index:: builtin: len + .. index:: pair: built-in function; len Returns the number of keys in object *o* on success, and ``-1`` on failure. This is equivalent to the Python expression ``len(o)``. diff --git a/Doc/c-api/number.rst b/Doc/c-api/number.rst index 70b91f8c2d0ca1..13d3c5af956905 100644 --- a/Doc/c-api/number.rst +++ b/Doc/c-api/number.rst @@ -64,7 +64,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_Divmod(PyObject *o1, PyObject *o2) - .. index:: builtin: divmod + .. index:: pair: built-in function; divmod See the built-in function :func:`divmod`. Returns ``NULL`` on failure. This is the equivalent of the Python expression ``divmod(o1, o2)``. @@ -72,7 +72,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_Power(PyObject *o1, PyObject *o2, PyObject *o3) - .. index:: builtin: pow + .. index:: pair: built-in function; pow See the built-in function :func:`pow`. Returns ``NULL`` on failure. This is the equivalent of the Python expression ``pow(o1, o2, o3)``, where *o3* is optional. @@ -94,7 +94,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_Absolute(PyObject *o) - .. index:: builtin: abs + .. index:: pair: built-in function; abs Returns the absolute value of *o*, or ``NULL`` on failure. This is the equivalent of the Python expression ``abs(o)``. @@ -192,7 +192,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_InPlacePower(PyObject *o1, PyObject *o2, PyObject *o3) - .. index:: builtin: pow + .. index:: pair: built-in function; pow See the built-in function :func:`pow`. Returns ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python @@ -238,7 +238,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_Long(PyObject *o) - .. index:: builtin: int + .. index:: pair: built-in function; int Returns the *o* converted to an integer object on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``int(o)``. @@ -246,7 +246,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_Float(PyObject *o) - .. index:: builtin: float + .. index:: pair: built-in function; float Returns the *o* converted to a float object on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``float(o)``. diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index a0c3194ab0fb90..a25ff244c9f07c 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -190,7 +190,7 @@ Object Protocol .. c:function:: PyObject* PyObject_Repr(PyObject *o) - .. index:: builtin: repr + .. index:: pair: built-in function; repr Compute a string representation of object *o*. Returns the string representation on success, ``NULL`` on failure. This is the equivalent of the @@ -202,7 +202,7 @@ Object Protocol .. c:function:: PyObject* PyObject_ASCII(PyObject *o) - .. index:: builtin: ascii + .. index:: pair: built-in function; ascii As :c:func:`PyObject_Repr`, compute a string representation of object *o*, but escape the non-ASCII characters in the string returned by @@ -227,7 +227,7 @@ Object Protocol .. c:function:: PyObject* PyObject_Bytes(PyObject *o) - .. index:: builtin: bytes + .. index:: pair: built-in function; bytes Compute a bytes representation of object *o*. ``NULL`` is returned on failure and a bytes object on success. This is equivalent to the Python @@ -278,7 +278,7 @@ Object Protocol .. c:function:: Py_hash_t PyObject_Hash(PyObject *o) - .. index:: builtin: hash + .. index:: pair: built-in function; hash Compute and return the hash value of an object *o*. On failure, return ``-1``. This is the equivalent of the Python expression ``hash(o)``. @@ -312,7 +312,7 @@ Object Protocol .. c:function:: PyObject* PyObject_Type(PyObject *o) - .. index:: builtin: type + .. index:: pair: built-in function; type When *o* is non-``NULL``, returns a type object corresponding to the object type of object *o*. On failure, raises :exc:`SystemError` and returns ``NULL``. This @@ -332,7 +332,7 @@ Object Protocol .. c:function:: Py_ssize_t PyObject_Size(PyObject *o) Py_ssize_t PyObject_Length(PyObject *o) - .. index:: builtin: len + .. index:: pair: built-in function; len Return the length of object *o*. If the object *o* provides either the sequence and mapping protocols, the sequence length is returned. On error, ``-1`` is diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index c78d273f9f149f..402a3e5e09ff56 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -18,7 +18,7 @@ Sequence Protocol .. c:function:: Py_ssize_t PySequence_Size(PyObject *o) Py_ssize_t PySequence_Length(PyObject *o) - .. index:: builtin: len + .. index:: pair: built-in function; len Returns the number of objects in sequence *o* on success, and ``-1`` on failure. This is equivalent to the Python expression ``len(o)``. @@ -120,7 +120,7 @@ Sequence Protocol .. c:function:: PyObject* PySequence_Tuple(PyObject *o) - .. index:: builtin: tuple + .. index:: pair: built-in function; tuple Return a tuple object with the same contents as the sequence or iterable *o*, or ``NULL`` on failure. If *o* is a tuple, a new reference will be returned, diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index 8e8af60252250e..d642a5f1902e2e 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -107,7 +107,7 @@ or :class:`frozenset` or instances of their subtypes. .. c:function:: Py_ssize_t PySet_Size(PyObject *anyset) - .. index:: builtin: len + .. index:: pair: built-in function; len Return the length of a :class:`set` or :class:`frozenset` object. Equivalent to ``len(anyset)``. Raises a :exc:`PyExc_SystemError` if *anyset* is not a diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 338db6378d240e..aae1b951804491 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -347,7 +347,7 @@ method. .. data:: METH_CLASS - .. index:: builtin: classmethod + .. index:: pair: built-in function; classmethod The method will be passed the type object as the first parameter rather than an instance of the type. This is used to create *class methods*, @@ -357,7 +357,7 @@ method. .. data:: METH_STATIC - .. index:: builtin: staticmethod + .. index:: pair: built-in function; staticmethod The method will be passed ``NULL`` as the first parameter rather than an instance of the type. This is used to create *static methods*, similar to diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index e13db3fb2211e2..0584989233de3f 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -805,7 +805,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. c:member:: reprfunc PyTypeObject.tp_repr - .. index:: builtin: repr + .. index:: pair: built-in function; repr An optional pointer to a function that implements the built-in function :func:`repr`. @@ -870,7 +870,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. c:member:: hashfunc PyTypeObject.tp_hash - .. index:: builtin: hash + .. index:: pair: built-in function; hash An optional pointer to a function that implements the built-in function :func:`hash`. diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 56b40acdb69fed..6852a385f0c63c 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -149,7 +149,7 @@ done. This can be done using the :c:func:`PyErr_Fetch` and .. index:: single: string; object representation - builtin: repr + pair: built-in function; repr Object Presentation ------------------- diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 6c3f436ddb1494..296d8a9c66faa4 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1367,7 +1367,7 @@ iterations of the loop. .. opcode:: BUILD_SLICE (argc) - .. index:: builtin: slice + .. index:: pair: built-in function; slice Pushes a slice object on the stack. *argc* must be 2 or 3. If it is 2, implements:: diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 80b7b8b4f4ed07..48a832db60e919 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -562,7 +562,7 @@ are always available. They are listed here in alphabetical order. Raises an :ref:`auditing event ` ``exec`` with the code object as the argument. Code compilation events may also be raised. -.. index:: builtin: exec +.. index:: pair: built-in function; exec .. function:: exec(object, globals=None, locals=None, /, *, closure=None) diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index 4e29192311fc21..d8269ef48cb36a 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -159,7 +159,7 @@ The :mod:`pprint` module defines one class: .. function:: isreadable(object) - .. index:: builtin: eval + .. index:: pair: built-in function; eval Determine if the formatted representation of *object* is "readable", or can be used to reconstruct the value using :func:`eval`. This always returns ``False`` @@ -218,7 +218,7 @@ created. .. method:: PrettyPrinter.isreadable(object) - .. index:: builtin: eval + .. index:: pair: built-in function; eval Determine if the formatted representation of the object is "readable," or can be used to reconstruct the value using :func:`eval`. Note that this returns diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index c739485c5564da..9203afbf6a4e8a 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -244,9 +244,9 @@ and imaginary parts. .. index:: single: arithmetic - builtin: int - builtin: float - builtin: complex + pair: built-in function; int + pair: built-in function; float + pair: built-in function; complex single: operator; + (plus) single: + (plus); unary operator single: + (plus); binary operator @@ -945,9 +945,9 @@ operations have the same priority as the corresponding numeric operations. [3]_ .. index:: triple: operations on; sequence; types - builtin: len - builtin: min - builtin: max + pair: built-in function; len + pair: built-in function; min + pair: built-in function; max pair: concatenation; operation pair: repetition; operation pair: subscript; operation @@ -1113,7 +1113,7 @@ Immutable Sequence Types .. index:: triple: immutable; sequence; types pair: object; tuple - builtin: hash + pair: built-in function; hash The only operation that immutable sequence types generally implement that is not also implemented by mutable sequence types is support for the :func:`hash` @@ -4419,7 +4419,7 @@ Mapping Types --- :class:`dict` triple: operations on; mapping; types triple: operations on; dictionary; type pair: statement; del - builtin: len + pair: built-in function; len A :term:`mapping` object maps :term:`hashable` values to arbitrary objects. Mappings are mutable objects. There is currently only one standard mapping @@ -5348,7 +5348,7 @@ Code Objects ------------ .. index:: - builtin: compile + pair: built-in function; compile single: __code__ (function object attribute) Code objects are used by the implementation to represent "pseudo-compiled" @@ -5362,8 +5362,8 @@ Accessing ``__code__`` raises an :ref:`auditing event ` ``object.__getattr__`` with arguments ``obj`` and ``"__code__"``. .. index:: - builtin: exec - builtin: eval + pair: built-in function; exec + pair: built-in function; eval A code object can be executed or evaluated by passing it (instead of a source string) to the :func:`exec` or :func:`eval` built-in functions. @@ -5377,7 +5377,7 @@ Type Objects ------------ .. index:: - builtin: type + pair: built-in function; type pair: module; types Type objects represent the various object types. An object's type is accessed diff --git a/Doc/library/types.rst b/Doc/library/types.rst index a15fb5cfa49473..8cbe17df16f107 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -186,7 +186,7 @@ Standard names are defined for the following types: .. class:: CodeType(**kwargs) - .. index:: builtin: compile + .. index:: pair: built-in function; compile The type for code objects such as returned by :func:`compile`. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index e9c1c493ae4278..9d1e5b6c596d9f 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -188,7 +188,7 @@ those made in the suite of the for-loop:: .. index:: - builtin: range + pair: built-in function; range Names in the target list are not deleted when the loop is finished, but if the sequence is empty, they will not have been assigned to at all by the loop. Hint: diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 0a9cabc158b9e4..c0734e49f29192 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -21,8 +21,8 @@ conformance to Von Neumann's model of a "stored program computer", code is also represented by objects.) .. index:: - builtin: id - builtin: type + pair: built-in function; id + pair: built-in function; type single: identity of an object single: value of an object single: type of an object @@ -267,7 +267,7 @@ Ellipsis Sequences .. index:: - builtin: len + pair: built-in function; len pair: object; sequence single: index operation single: item selection @@ -308,8 +308,8 @@ Sequences Strings .. index:: - builtin: chr - builtin: ord + pair: built-in function; chr + pair: built-in function; ord single: character single: integer single: Unicode @@ -384,7 +384,7 @@ Sequences Set types .. index:: - builtin: len + pair: built-in function; len pair: object; set type These represent unordered, finite sets of unique, immutable objects. As such, @@ -418,7 +418,7 @@ Set types Mappings .. index:: - builtin: len + pair: built-in function; len single: subscription pair: object; mapping @@ -908,7 +908,7 @@ Class instances I/O objects (also known as file objects) .. index:: - builtin: open + pair: built-in function; open pair: module; io single: popen() (in module os) single: makefile() (socket method) @@ -1177,7 +1177,7 @@ Internal types and the ``tb_next`` attribute of existing instances can be updated. Slice objects - .. index:: builtin: slice + .. index:: pair: built-in function; slice Slice objects are used to represent slices for :meth:`~object.__getitem__` @@ -1411,7 +1411,7 @@ Basic customization .. method:: object.__bytes__(self) - .. index:: builtin: bytes + .. index:: pair: built-in function; bytes Called by :ref:`bytes ` to compute a byte-string representation of an object. This should return a :class:`bytes` object. @@ -1419,7 +1419,7 @@ Basic customization .. index:: single: string; __format__() (object method) pair: string; conversion - builtin: print + pair: built-in function; print .. method:: object.__format__(self, format_spec) @@ -1499,7 +1499,7 @@ Basic customization .. index:: pair: object; dictionary - builtin: hash + pair: built-in function; hash Called by built-in function :func:`hash` and for operations on members of hashed collections including :class:`set`, :class:`frozenset`, and @@ -2050,7 +2050,7 @@ Metaclasses .. index:: single: metaclass - builtin: type + pair: built-in function; type single: = (equals); class definition By default, classes are constructed using :func:`type`. The class body is @@ -2477,7 +2477,7 @@ through the object's keys; for sequences, it should iterate through the values. .. method:: object.__len__(self) .. index:: - builtin: len + pair: built-in function; len single: __bool__() (object method) Called to implement the built-in function :func:`len`. Should return the length @@ -2635,9 +2635,9 @@ left undefined. object.__or__(self, other) .. index:: - builtin: divmod - builtin: pow - builtin: pow + pair: built-in function; divmod + pair: built-in function; pow + pair: built-in function; pow These methods are called to implement the binary arithmetic operations (``+``, ``-``, ``*``, ``@``, ``/``, ``//``, ``%``, :func:`divmod`, @@ -2670,8 +2670,8 @@ left undefined. object.__ror__(self, other) .. index:: - builtin: divmod - builtin: pow + pair: built-in function; divmod + pair: built-in function; pow These methods are called to implement the binary arithmetic operations (``+``, ``-``, ``*``, ``@``, ``/``, ``//``, ``%``, :func:`divmod`, @@ -2683,7 +2683,7 @@ left undefined. ``type(y).__rsub__(y, x)`` is called if ``type(x).__sub__(x, y)`` returns *NotImplemented*. - .. index:: builtin: pow + .. index:: pair: built-in function; pow Note that ternary :func:`pow` will not try calling :meth:`__rpow__` (the coercion rules would become too complicated). @@ -2730,7 +2730,7 @@ left undefined. object.__abs__(self) object.__invert__(self) - .. index:: builtin: abs + .. index:: pair: built-in function; abs Called to implement the unary arithmetic operations (``-``, ``+``, :func:`abs` and ``~``). @@ -2741,9 +2741,9 @@ left undefined. object.__float__(self) .. index:: - builtin: complex - builtin: int - builtin: float + pair: built-in function; complex + pair: built-in function; int + pair: built-in function; float Called to implement the built-in functions :func:`complex`, :func:`int` and :func:`float`. Should return a value @@ -2768,7 +2768,7 @@ left undefined. object.__floor__(self) object.__ceil__(self) - .. index:: builtin: round + .. index:: pair: built-in function; round Called to implement the built-in function :func:`round` and :mod:`math` functions :func:`~math.trunc`, :func:`~math.floor` and :func:`~math.ceil`. diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index ca1aa744389bda..f7a8b44d195417 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -53,7 +53,7 @@ An expression statement evaluates the expression list (which may be a single expression). .. index:: - builtin: repr + pair: built-in function; repr pair: object; None pair: string; conversion single: output @@ -970,9 +970,9 @@ annotation. them or silently change the meaning of the program. .. index:: - builtin: exec - builtin: eval - builtin: compile + pair: built-in function; exec + pair: built-in function; eval + pair: built-in function; compile **Programmer's note:** :keyword:`global` is a directive to the parser. It applies only to code parsed at the same time as the :keyword:`!global` statement. diff --git a/Doc/reference/toplevel_components.rst b/Doc/reference/toplevel_components.rst index ee472ace6e2129..dd3d3d6878e289 100644 --- a/Doc/reference/toplevel_components.rst +++ b/Doc/reference/toplevel_components.rst @@ -98,7 +98,7 @@ Expression input ================ .. index:: single: input -.. index:: builtin: eval +.. index:: pair: built-in function; eval :func:`eval` is used for expression input. It ignores leading whitespace. The string argument to :func:`eval` must have the following form: diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 8fc44cab44455a..4fe54e30b82b25 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -695,7 +695,7 @@ def patch_pairindextypes(app) -> None: pairindextypes.pop('object', None) pairindextypes.pop('exception', None) pairindextypes.pop('statement', None) - # pairindextypes.pop('builtin', None) + pairindextypes.pop('builtin', None) def setup(app): diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index 3fcf4e3f43a300..f5cdd84cbadefe 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -285,7 +285,7 @@ Reading and Writing Files ========================= .. index:: - builtin: open + pair: built-in function; open pair: object; file :func:`open` returns a :term:`file object`, and is most commonly used with diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 4f5ada90eb57bc..6bae279c5e9cde 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -24,7 +24,7 @@ Be sure to use the ``import os`` style instead of ``from os import *``. This will keep :func:`os.open` from shadowing the built-in :func:`open` function which operates much differently. -.. index:: builtin: help +.. index:: pair: built-in function; help The built-in :func:`dir` and :func:`help` functions are useful as interactive aids for working with large modules like :mod:`os`:: From 6616710731b9ad1a4e6b73696e8bd5c40cf8762d Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sat, 6 May 2023 17:58:32 +0300 Subject: [PATCH 074/204] gh-104233: Fix "unused variable" warning in `ceval_gil.c` (#104234) --- Python/ceval_gil.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index a390bec80d556c..1ac0dbcf2ecfec 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -547,9 +547,11 @@ _PyEval_FiniGIL(PyInterpreterState *interp) return; } else if (!interp->ceval.own_gil) { +#ifdef Py_DEBUG PyInterpreterState *main_interp = _PyInterpreterState_Main(); assert(interp != main_interp); assert(interp->ceval.gil == main_interp->ceval.gil); +#endif interp->ceval.gil = NULL; return; } From e407661e7a70ad49a69df3058634bc4fccebbcd6 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 6 May 2023 11:04:41 -0400 Subject: [PATCH 075/204] gh-65772: Clean-up turtle module (#104218) * Remove the unused, private, and undocumented name `_ver` and the commented-out `print` call. * Don't add math functions to `__all__`. Beginners should learn to `import math` to access them. * Gregor Lindel, who wrote this version of turtle, dropped plans to implement turtle on another toolkit at least a decade ago. Drop `_dot` code preparing for this, but add a hint comment. * `_Screen` is meant to be a singleton class. To enforce that, it needs either a `__new__` that returns the singleton or `else...raise` in `__iter__`. Merely removing the `if` clauses as suggested might break something if a user were to call `_Screen` directly. Leave the code alone until a problem is evident. * Turtledemo injects into _Screen both _root and _canvas, configured as it needs them to be. Making _canvas an `__init__` option would require skipping some but not all of the lines under 'if _Screen._canvas is None:`. Leave working code alone. --- Lib/turtle.py | 61 +++++++------------ ...3-05-05-18-52-22.gh-issue-65772.w5P5Wv.rst | 1 + 2 files changed, 22 insertions(+), 40 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-05-18-52-22.gh-issue-65772.w5P5Wv.rst diff --git a/Lib/turtle.py b/Lib/turtle.py index 2de406e0f517af..cf111158b7c149 100644 --- a/Lib/turtle.py +++ b/Lib/turtle.py @@ -21,7 +21,6 @@ # misrepresented as being the original software. # 3. This notice may not be removed or altered from any source distribution. - """ Turtle graphics is a popular way for introducing programming to kids. It was part of the original Logo programming language developed @@ -97,13 +96,8 @@ Behind the scenes there are some features included with possible extensions in mind. These will be commented and documented elsewhere. - """ -_ver = "turtle 1.1b- - for Python 3.1 - 4. 5. 2009" - -# print(_ver) - import tkinter as TK import types import math @@ -141,7 +135,7 @@ _tg_utilities = ['write_docstringdict', 'done'] __all__ = (_tg_classes + _tg_screen_functions + _tg_turtle_functions + - _tg_utilities + ['Terminator']) # + _math_functions) + _tg_utilities + ['Terminator']) _alias_list = ['addshape', 'backward', 'bk', 'fd', 'ht', 'lt', 'pd', 'pos', 'pu', 'rt', 'seth', 'setpos', 'setposition', 'st', @@ -598,9 +592,6 @@ def _write(self, pos, txt, align, font, pencolor): x0, y0, x1, y1 = self.cv.bbox(item) return item, x1-1 -## def _dot(self, pos, size, color): -## """may be implemented for some other graphics toolkit""" - def _onclick(self, item, fun, num=1, add=None): """Bind fun to mouse-click event on turtle. fun must be a function with two arguments, the coordinates @@ -2726,7 +2717,7 @@ def _cc(self, args): if not ((0 <= r <= 255) and (0 <= g <= 255) and (0 <= b <= 255)): raise TurtleGraphicsError("bad color sequence: %s" % str(args)) return "#%02x%02x%02x" % (r, g, b) - + def teleport(self, x=None, y=None, *, fill_gap: bool = False) -> None: """Instantly move turtle to an absolute position. @@ -2738,14 +2729,14 @@ def teleport(self, x=None, y=None, *, fill_gap: bool = False) -> None: call: teleport(x, y) # two coordinates --or: teleport(x) # teleport to x position, keeping y as is --or: teleport(y=y) # teleport to y position, keeping x as is - --or: teleport(x, y, fill_gap=True) + --or: teleport(x, y, fill_gap=True) # teleport but fill the gap in between Move turtle to an absolute position. Unlike goto(x, y), a line will not be drawn. The turtle's orientation does not change. If currently filling, the polygon(s) teleported from will be filled after leaving, and filling will begin again after teleporting. This can be disabled - with fill_gap=True, which makes the imaginary line traveled during + with fill_gap=True, which makes the imaginary line traveled during teleporting act as a fill barrier like in goto(x, y). Example (for a Turtle instance named turtle): @@ -2773,7 +2764,7 @@ def teleport(self, x=None, y=None, *, fill_gap: bool = False) -> None: self._position = Vec2D(new_x, new_y) self.pen(pendown=pendown) if was_filling and not fill_gap: - self.begin_fill() + self.begin_fill() def clone(self): """Create and return a clone of the turtle. @@ -3455,27 +3446,22 @@ def dot(self, size=None, *color): if size is None: size = self._pensize + max(self._pensize, 4) color = self._colorstr(color) - if hasattr(self.screen, "_dot"): - item = self.screen._dot(self._position, size, color) - self.items.append(item) - if self.undobuffer: - self.undobuffer.push(("dot", item)) - else: - pen = self.pen() - if self.undobuffer: - self.undobuffer.push(["seq"]) - self.undobuffer.cumulate = True - try: - if self.resizemode() == 'auto': - self.ht() - self.pendown() - self.pensize(size) - self.pencolor(color) - self.forward(0) - finally: - self.pen(pen) - if self.undobuffer: - self.undobuffer.cumulate = False + # If screen were to gain a dot function, see GH #104218. + pen = self.pen() + if self.undobuffer: + self.undobuffer.push(["seq"]) + self.undobuffer.cumulate = True + try: + if self.resizemode() == 'auto': + self.ht() + self.pendown() + self.pensize(size) + self.pencolor(color) + self.forward(0) + finally: + self.pen(pen) + if self.undobuffer: + self.undobuffer.cumulate = False def _write(self, txt, align, font): """Performs the writing for write() @@ -3751,11 +3737,6 @@ class _Screen(TurtleScreen): _title = _CFG["title"] def __init__(self): - # XXX there is no need for this code to be conditional, - # as there will be only a single _Screen instance, anyway - # XXX actually, the turtle demo is injecting root window, - # so perhaps the conditional creation of a root should be - # preserved (perhaps by passing it as an optional parameter) if _Screen._root is None: _Screen._root = self._root = _Root() self._root.title(_Screen._title) diff --git a/Misc/NEWS.d/next/Library/2023-05-05-18-52-22.gh-issue-65772.w5P5Wv.rst b/Misc/NEWS.d/next/Library/2023-05-05-18-52-22.gh-issue-65772.w5P5Wv.rst new file mode 100644 index 00000000000000..54b0190192863c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-05-18-52-22.gh-issue-65772.w5P5Wv.rst @@ -0,0 +1 @@ +Remove unneeded comments and code in turtle.py. From 96f95df48e41ccf984de1ee1312c81809fd9e876 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 6 May 2023 11:09:08 -0400 Subject: [PATCH 076/204] Rewrite the turtledemo makeGraphFrame method (#104224) Replace `self._canvas` and `self.scanvas`, both bound to `canvas`, with `self.canvas, which is accessed in other methods. Replace `_s_` with `screen` and `_s_._canvas` with `canvas`. Add a comment explaining the unorthodox use of function turtle.Screen and singleton class turtle._Screen. --- Lib/turtledemo/__main__.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py index caea022da4a688..f6c9d6aa6f9a32 100755 --- a/Lib/turtledemo/__main__.py +++ b/Lib/turtledemo/__main__.py @@ -203,10 +203,10 @@ def __init__(self, filename=None): def onResize(self, event): - cwidth = self._canvas.winfo_width() - cheight = self._canvas.winfo_height() - self._canvas.xview_moveto(0.5*(self.canvwidth-cwidth)/self.canvwidth) - self._canvas.yview_moveto(0.5*(self.canvheight-cheight)/self.canvheight) + cwidth = self.canvas.winfo_width() + cheight = self.canvas.winfo_height() + self.canvas.xview_moveto(0.5*(self.canvwidth-cwidth)/self.canvwidth) + self.canvas.yview_moveto(0.5*(self.canvheight-cheight)/self.canvheight) def makeTextFrame(self, root): self.text_frame = text_frame = Frame(root) @@ -237,19 +237,23 @@ def makeTextFrame(self, root): return text_frame def makeGraphFrame(self, root): + # t._Screen is a singleton class instantiated or retrieved + # by calling Screen. Since tdemo canvas needs a different + # configuration, we manually set class attributes before + # calling Screen and manually call superclass init after. turtle._Screen._root = root + self.canvwidth = 1000 self.canvheight = 800 - turtle._Screen._canvas = self._canvas = canvas = turtle.ScrolledCanvas( + turtle._Screen._canvas = self.canvas = canvas = turtle.ScrolledCanvas( root, 800, 600, self.canvwidth, self.canvheight) canvas.adjustScrolls() canvas._rootwindow.bind('', self.onResize) canvas._canvas['borderwidth'] = 0 - self.screen = _s_ = turtle.Screen() - turtle.TurtleScreen.__init__(_s_, _s_._canvas) - self.scanvas = _s_._canvas - turtle.RawTurtle.screens = [_s_] + self.screen = screen = turtle.Screen() + turtle.TurtleScreen.__init__(screen, canvas) + turtle.RawTurtle.screens = [screen] return canvas def set_txtsize(self, size): @@ -373,7 +377,7 @@ def startDemo(self): def clearCanvas(self): self.refreshCanvas() self.screen._delete("all") - self.scanvas.config(cursor="") + self.canvas.config(cursor="") self.configGUI(NORMAL, DISABLED, DISABLED) def stopIt(self): From 263abd333d18b8825cf6d68a5051818826dbffce Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Sat, 6 May 2023 08:15:27 -0700 Subject: [PATCH 077/204] gh-104144: Optimize gather to finish eagerly when all futures complete eagerly (#104138) --- Lib/asyncio/tasks.py | 13 ++++++++++++- .../2023-05-03-16-50-24.gh-issue-104144.yNkjL8.rst | 3 +++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-03-16-50-24.gh-issue-104144.yNkjL8.rst diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index aa5269ade19a7f..7eb647bd129819 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -813,6 +813,7 @@ def _done_callback(fut): children = [] nfuts = 0 nfinished = 0 + done_futs = [] loop = None outer = None # bpo-46672 for arg in coros_or_futures: @@ -829,7 +830,10 @@ def _done_callback(fut): nfuts += 1 arg_to_fut[arg] = fut - fut.add_done_callback(_done_callback) + if fut.done(): + done_futs.append(fut) + else: + fut.add_done_callback(_done_callback) else: # There's a duplicate Future object in coros_or_futures. @@ -838,6 +842,13 @@ def _done_callback(fut): children.append(fut) outer = _GatheringFuture(children, loop=loop) + # Run done callbacks after GatheringFuture created so any post-processing + # can be performed at this point + # optimization: in the special case that *all* futures finished eagerly, + # this will effectively complete the gather eagerly, with the last + # callback setting the result (or exception) on outer before returning it + for fut in done_futs: + _done_callback(fut) return outer diff --git a/Misc/NEWS.d/next/Library/2023-05-03-16-50-24.gh-issue-104144.yNkjL8.rst b/Misc/NEWS.d/next/Library/2023-05-03-16-50-24.gh-issue-104144.yNkjL8.rst new file mode 100644 index 00000000000000..b975d48ed3385c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-03-16-50-24.gh-issue-104144.yNkjL8.rst @@ -0,0 +1,3 @@ +Optimize :func:`asyncio.gather` when using :func:`asyncio.eager_task_factory` +to complete eagerly if all fututres completed eagerly. +Avoid scheduling done callbacks for futures that complete eagerly. From 376137f6ec73e0800e49cec6100e401f6154b693 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sat, 6 May 2023 17:48:07 +0100 Subject: [PATCH 078/204] gh-90953: Emit deprecation warnings for `ast` features deprecated in Python 3.8 (#104199) `ast.Num`, `ast.Str`, `ast.Bytes`, `ast.Ellipsis` and `ast.NameConstant` now all emit deprecation warnings on import, access, instantation or `isinstance()` checks. Co-authored-by: Serhiy Storchaka --- Doc/whatsnew/3.12.rst | 13 + Lib/ast.py | 82 ++- Lib/test/test_ast.py | 508 +++++++++++++----- .../2022-02-19-14-19-34.bpo-46797.6BXZX4.rst | 4 + 4 files changed, 472 insertions(+), 135 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-02-19-14-19-34.bpo-46797.6BXZX4.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index ccddc8bd832f29..ec04178238b6b0 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -844,6 +844,19 @@ Pending Removal in Python 3.14 use :func:`importlib.util.find_spec` instead. (Contributed by Nikita Sobolev in :gh:`97850`.) +* The following :mod:`ast` features have been deprecated in documentation since + Python 3.8, now cause a :exc:`DeprecationWarning` to be emitted at runtime + when they are accessed or used, and will be removed in Python 3.14: + + * :class:`!ast.Num` + * :class:`!ast.Str` + * :class:`!ast.Bytes` + * :class:`!ast.NameConstant` + * :class:`!ast.Ellipsis` + + Use :class:`ast.Constant` instead. + (Contributed by Serhiy Storchaka in :gh:`90953`.) + Pending Removal in Future Versions ---------------------------------- diff --git a/Lib/ast.py b/Lib/ast.py index d9733a79d3a78f..65152047a22370 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -294,9 +294,7 @@ def get_docstring(node, clean=True): if not(node.body and isinstance(node.body[0], Expr)): return None node = node.body[0].value - if isinstance(node, Str): - text = node.s - elif isinstance(node, Constant) and isinstance(node.value, str): + if isinstance(node, Constant) and isinstance(node.value, str): text = node.value else: return None @@ -499,20 +497,52 @@ def generic_visit(self, node): return node +_DEPRECATED_VALUE_ALIAS_MESSAGE = ( + "{name} is deprecated and will be removed in Python {remove}; use value instead" +) +_DEPRECATED_CLASS_MESSAGE = ( + "{name} is deprecated and will be removed in Python {remove}; " + "use ast.Constant instead" +) + + # If the ast module is loaded more than once, only add deprecated methods once if not hasattr(Constant, 'n'): # The following code is for backward compatibility. # It will be removed in future. - def _getter(self): + def _n_getter(self): + """Deprecated. Use value instead.""" + import warnings + warnings._deprecated( + "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) + ) + return self.value + + def _n_setter(self, value): + import warnings + warnings._deprecated( + "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) + ) + self.value = value + + def _s_getter(self): """Deprecated. Use value instead.""" + import warnings + warnings._deprecated( + "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) + ) return self.value - def _setter(self, value): + def _s_setter(self, value): + import warnings + warnings._deprecated( + "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) + ) self.value = value - Constant.n = property(_getter, _setter) - Constant.s = property(_getter, _setter) + Constant.n = property(_n_getter, _n_setter) + Constant.s = property(_s_getter, _s_setter) class _ABC(type): @@ -520,6 +550,13 @@ def __init__(cls, *args): cls.__doc__ = """Deprecated AST node class. Use ast.Constant instead""" def __instancecheck__(cls, inst): + if cls in _const_types: + import warnings + warnings._deprecated( + f"ast.{cls.__qualname__}", + message=_DEPRECATED_CLASS_MESSAGE, + remove=(3, 14) + ) if not isinstance(inst, Constant): return False if cls in _const_types: @@ -543,6 +580,10 @@ def _new(cls, *args, **kwargs): if pos < len(args): raise TypeError(f"{cls.__name__} got multiple values for argument {key!r}") if cls in _const_types: + import warnings + warnings._deprecated( + f"ast.{cls.__qualname__}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) + ) return Constant(*args, **kwargs) return Constant.__new__(cls, *args, **kwargs) @@ -565,10 +606,19 @@ class Ellipsis(Constant, metaclass=_ABC): _fields = () def __new__(cls, *args, **kwargs): - if cls is Ellipsis: + if cls is _ast_Ellipsis: + import warnings + warnings._deprecated( + "ast.Ellipsis", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) + ) return Constant(..., *args, **kwargs) return Constant.__new__(cls, *args, **kwargs) +# Keep another reference to Ellipsis in the global namespace +# so it can be referenced in Ellipsis.__new__ +# (The original "Ellipsis" name is removed from the global namespace later on) +_ast_Ellipsis = Ellipsis + _const_types = { Num: (int, float, complex), Str: (str,), @@ -1699,6 +1749,22 @@ def unparse(ast_obj): return unparser.visit(ast_obj) +_deprecated_globals = { + name: globals().pop(name) + for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis') +} + +def __getattr__(name): + if name in _deprecated_globals: + globals()[name] = value = _deprecated_globals[name] + import warnings + warnings._deprecated( + f"ast.{name}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) + ) + return value + raise AttributeError(f"module 'ast' has no attribute '{name}'") + + def main(): import argparse diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 8eef7baec70118..fdd21aca06ffdd 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -8,9 +8,11 @@ import unittest import warnings import weakref +from functools import partial from textwrap import dedent from test import support +from test.support.import_helper import import_fresh_module from test.support import os_helper, script_helper from test.support.ast_helper import ASTTestMixin @@ -267,6 +269,7 @@ def to_tuple(t): # excepthandler, arguments, keywords, alias class AST_Tests(unittest.TestCase): + maxDiff = None def _is_ast_node(self, name, node): if not isinstance(node, type): @@ -435,16 +438,42 @@ def test_base_classes(self): self.assertTrue(issubclass(ast.comprehension, ast.AST)) self.assertTrue(issubclass(ast.Gt, ast.AST)) + def test_import_deprecated(self): + ast = import_fresh_module('ast') + depr_regex = ( + r'ast\.{} is deprecated and will be removed in Python 3.14; ' + r'use ast\.Constant instead' + ) + for name in 'Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis': + with self.assertWarnsRegex(DeprecationWarning, depr_regex.format(name)): + getattr(ast, name) + + def test_field_attr_existence_deprecated(self): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', '', DeprecationWarning) + from ast import Num, Str, Bytes, NameConstant, Ellipsis + + for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis'): + item = getattr(ast, name) + if self._is_ast_node(name, item): + with self.subTest(item): + with self.assertWarns(DeprecationWarning): + x = item() + if isinstance(x, ast.AST): + self.assertIs(type(x._fields), tuple) + def test_field_attr_existence(self): for name, item in ast.__dict__.items(): + # These emit DeprecationWarnings + if name in {'Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis'}: + continue + # constructor has a different signature + if name == 'Index': + continue if self._is_ast_node(name, item): - if name == 'Index': - # Index(value) just returns value now. - # The argument is required. - continue x = item() if isinstance(x, ast.AST): - self.assertEqual(type(x._fields), tuple) + self.assertIs(type(x._fields), tuple) def test_arguments(self): x = ast.arguments() @@ -459,25 +488,108 @@ def test_arguments(self): self.assertEqual(x.args, 2) self.assertEqual(x.vararg, 3) + def test_field_attr_writable_deprecated(self): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', '', DeprecationWarning) + x = ast.Num() + # We can assign to _fields + x._fields = 666 + self.assertEqual(x._fields, 666) + def test_field_attr_writable(self): - x = ast.Num() + x = ast.Constant() # We can assign to _fields x._fields = 666 self.assertEqual(x._fields, 666) + def test_classattrs_deprecated(self): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', '', DeprecationWarning) + from ast import Num, Str, Bytes, NameConstant, Ellipsis + + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('always', '', DeprecationWarning) + x = ast.Num() + self.assertEqual(x._fields, ('value', 'kind')) + + with self.assertRaises(AttributeError): + x.value + + with self.assertRaises(AttributeError): + x.n + + x = ast.Num(42) + self.assertEqual(x.value, 42) + self.assertEqual(x.n, 42) + + with self.assertRaises(AttributeError): + x.lineno + + with self.assertRaises(AttributeError): + x.foobar + + x = ast.Num(lineno=2) + self.assertEqual(x.lineno, 2) + + x = ast.Num(42, lineno=0) + self.assertEqual(x.lineno, 0) + self.assertEqual(x._fields, ('value', 'kind')) + self.assertEqual(x.value, 42) + self.assertEqual(x.n, 42) + + self.assertRaises(TypeError, ast.Num, 1, None, 2) + self.assertRaises(TypeError, ast.Num, 1, None, 2, lineno=0) + + # Arbitrary keyword arguments are supported + self.assertEqual(ast.Num(1, foo='bar').foo, 'bar') + + with self.assertRaisesRegex(TypeError, "Num got multiple values for argument 'n'"): + ast.Num(1, n=2) + + self.assertEqual(ast.Num(42).n, 42) + self.assertEqual(ast.Num(4.25).n, 4.25) + self.assertEqual(ast.Num(4.25j).n, 4.25j) + self.assertEqual(ast.Str('42').s, '42') + self.assertEqual(ast.Bytes(b'42').s, b'42') + self.assertIs(ast.NameConstant(True).value, True) + self.assertIs(ast.NameConstant(False).value, False) + self.assertIs(ast.NameConstant(None).value, None) + + self.assertEqual([str(w.message) for w in wlog], [ + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', + 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', + ]) + def test_classattrs(self): - x = ast.Num() + x = ast.Constant() self.assertEqual(x._fields, ('value', 'kind')) with self.assertRaises(AttributeError): x.value - with self.assertRaises(AttributeError): - x.n - - x = ast.Num(42) + x = ast.Constant(42) self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) with self.assertRaises(AttributeError): x.lineno @@ -485,36 +597,23 @@ def test_classattrs(self): with self.assertRaises(AttributeError): x.foobar - x = ast.Num(lineno=2) + x = ast.Constant(lineno=2) self.assertEqual(x.lineno, 2) - x = ast.Num(42, lineno=0) + x = ast.Constant(42, lineno=0) self.assertEqual(x.lineno, 0) self.assertEqual(x._fields, ('value', 'kind')) self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - self.assertRaises(TypeError, ast.Num, 1, None, 2) - self.assertRaises(TypeError, ast.Num, 1, None, 2, lineno=0) + self.assertRaises(TypeError, ast.Constant, 1, None, 2) + self.assertRaises(TypeError, ast.Constant, 1, None, 2, lineno=0) # Arbitrary keyword arguments are supported self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar') - self.assertEqual(ast.Num(1, foo='bar').foo, 'bar') - with self.assertRaisesRegex(TypeError, "Num got multiple values for argument 'n'"): - ast.Num(1, n=2) with self.assertRaisesRegex(TypeError, "Constant got multiple values for argument 'value'"): ast.Constant(1, value=2) - self.assertEqual(ast.Num(42).n, 42) - self.assertEqual(ast.Num(4.25).n, 4.25) - self.assertEqual(ast.Num(4.25j).n, 4.25j) - self.assertEqual(ast.Str('42').s, '42') - self.assertEqual(ast.Bytes(b'42').s, b'42') - self.assertIs(ast.NameConstant(True).value, True) - self.assertIs(ast.NameConstant(False).value, False) - self.assertIs(ast.NameConstant(None).value, None) - self.assertEqual(ast.Constant(42).value, 42) self.assertEqual(ast.Constant(4.25).value, 4.25) self.assertEqual(ast.Constant(4.25j).value, 4.25j) @@ -526,85 +625,211 @@ def test_classattrs(self): self.assertIs(ast.Constant(...).value, ...) def test_realtype(self): - self.assertEqual(type(ast.Num(42)), ast.Constant) - self.assertEqual(type(ast.Num(4.25)), ast.Constant) - self.assertEqual(type(ast.Num(4.25j)), ast.Constant) - self.assertEqual(type(ast.Str('42')), ast.Constant) - self.assertEqual(type(ast.Bytes(b'42')), ast.Constant) - self.assertEqual(type(ast.NameConstant(True)), ast.Constant) - self.assertEqual(type(ast.NameConstant(False)), ast.Constant) - self.assertEqual(type(ast.NameConstant(None)), ast.Constant) - self.assertEqual(type(ast.Ellipsis()), ast.Constant) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', '', DeprecationWarning) + from ast import Num, Str, Bytes, NameConstant, Ellipsis + + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('always', '', DeprecationWarning) + self.assertIs(type(ast.Num(42)), ast.Constant) + self.assertIs(type(ast.Num(4.25)), ast.Constant) + self.assertIs(type(ast.Num(4.25j)), ast.Constant) + self.assertIs(type(ast.Str('42')), ast.Constant) + self.assertIs(type(ast.Bytes(b'42')), ast.Constant) + self.assertIs(type(ast.NameConstant(True)), ast.Constant) + self.assertIs(type(ast.NameConstant(False)), ast.Constant) + self.assertIs(type(ast.NameConstant(None)), ast.Constant) + self.assertIs(type(ast.Ellipsis()), ast.Constant) + + self.assertEqual([str(w.message) for w in wlog], [ + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Ellipsis is deprecated and will be removed in Python 3.14; use ast.Constant instead', + ]) def test_isinstance(self): - self.assertTrue(isinstance(ast.Num(42), ast.Num)) - self.assertTrue(isinstance(ast.Num(4.2), ast.Num)) - self.assertTrue(isinstance(ast.Num(4.2j), ast.Num)) - self.assertTrue(isinstance(ast.Str('42'), ast.Str)) - self.assertTrue(isinstance(ast.Bytes(b'42'), ast.Bytes)) - self.assertTrue(isinstance(ast.NameConstant(True), ast.NameConstant)) - self.assertTrue(isinstance(ast.NameConstant(False), ast.NameConstant)) - self.assertTrue(isinstance(ast.NameConstant(None), ast.NameConstant)) - self.assertTrue(isinstance(ast.Ellipsis(), ast.Ellipsis)) - - self.assertTrue(isinstance(ast.Constant(42), ast.Num)) - self.assertTrue(isinstance(ast.Constant(4.2), ast.Num)) - self.assertTrue(isinstance(ast.Constant(4.2j), ast.Num)) - self.assertTrue(isinstance(ast.Constant('42'), ast.Str)) - self.assertTrue(isinstance(ast.Constant(b'42'), ast.Bytes)) - self.assertTrue(isinstance(ast.Constant(True), ast.NameConstant)) - self.assertTrue(isinstance(ast.Constant(False), ast.NameConstant)) - self.assertTrue(isinstance(ast.Constant(None), ast.NameConstant)) - self.assertTrue(isinstance(ast.Constant(...), ast.Ellipsis)) - - self.assertFalse(isinstance(ast.Str('42'), ast.Num)) - self.assertFalse(isinstance(ast.Num(42), ast.Str)) - self.assertFalse(isinstance(ast.Str('42'), ast.Bytes)) - self.assertFalse(isinstance(ast.Num(42), ast.NameConstant)) - self.assertFalse(isinstance(ast.Num(42), ast.Ellipsis)) - self.assertFalse(isinstance(ast.NameConstant(True), ast.Num)) - self.assertFalse(isinstance(ast.NameConstant(False), ast.Num)) - - self.assertFalse(isinstance(ast.Constant('42'), ast.Num)) - self.assertFalse(isinstance(ast.Constant(42), ast.Str)) - self.assertFalse(isinstance(ast.Constant('42'), ast.Bytes)) - self.assertFalse(isinstance(ast.Constant(42), ast.NameConstant)) - self.assertFalse(isinstance(ast.Constant(42), ast.Ellipsis)) - self.assertFalse(isinstance(ast.Constant(True), ast.Num)) - self.assertFalse(isinstance(ast.Constant(False), ast.Num)) - - self.assertFalse(isinstance(ast.Constant(), ast.Num)) - self.assertFalse(isinstance(ast.Constant(), ast.Str)) - self.assertFalse(isinstance(ast.Constant(), ast.Bytes)) - self.assertFalse(isinstance(ast.Constant(), ast.NameConstant)) - self.assertFalse(isinstance(ast.Constant(), ast.Ellipsis)) + from ast import Constant + + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', '', DeprecationWarning) + from ast import Num, Str, Bytes, NameConstant, Ellipsis + + cls_depr_msg = ( + 'ast.{} is deprecated and will be removed in Python 3.14; ' + 'use ast.Constant instead' + ) + + assertNumDeprecated = partial( + self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Num") + ) + assertStrDeprecated = partial( + self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Str") + ) + assertBytesDeprecated = partial( + self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Bytes") + ) + assertNameConstantDeprecated = partial( + self.assertWarnsRegex, + DeprecationWarning, + cls_depr_msg.format("NameConstant") + ) + assertEllipsisDeprecated = partial( + self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Ellipsis") + ) + + for arg in 42, 4.2, 4.2j: + with self.subTest(arg=arg): + with assertNumDeprecated(): + n = Num(arg) + with assertNumDeprecated(): + self.assertIsInstance(n, Num) + + with assertStrDeprecated(): + s = Str('42') + with assertStrDeprecated(): + self.assertIsInstance(s, Str) + + with assertBytesDeprecated(): + b = Bytes(b'42') + with assertBytesDeprecated(): + self.assertIsInstance(b, Bytes) + + for arg in True, False, None: + with self.subTest(arg=arg): + with assertNameConstantDeprecated(): + n = NameConstant(arg) + with assertNameConstantDeprecated(): + self.assertIsInstance(n, NameConstant) + + with assertEllipsisDeprecated(): + e = Ellipsis() + with assertEllipsisDeprecated(): + self.assertIsInstance(e, Ellipsis) + + for arg in 42, 4.2, 4.2j: + with self.subTest(arg=arg): + with assertNumDeprecated(): + self.assertIsInstance(Constant(arg), Num) + + with assertStrDeprecated(): + self.assertIsInstance(Constant('42'), Str) + + with assertBytesDeprecated(): + self.assertIsInstance(Constant(b'42'), Bytes) + + for arg in True, False, None: + with self.subTest(arg=arg): + with assertNameConstantDeprecated(): + self.assertIsInstance(Constant(arg), NameConstant) + + with assertEllipsisDeprecated(): + self.assertIsInstance(Constant(...), Ellipsis) + + with assertStrDeprecated(): + s = Str('42') + assertNumDeprecated(self.assertNotIsInstance, s, Num) + assertBytesDeprecated(self.assertNotIsInstance, s, Bytes) + + with assertNumDeprecated(): + n = Num(42) + assertStrDeprecated(self.assertNotIsInstance, n, Str) + assertNameConstantDeprecated(self.assertNotIsInstance, n, NameConstant) + assertEllipsisDeprecated(self.assertNotIsInstance, n, Ellipsis) + + with assertNameConstantDeprecated(): + n = NameConstant(True) + with assertNumDeprecated(): + self.assertNotIsInstance(n, Num) + + with assertNameConstantDeprecated(): + n = NameConstant(False) + with assertNumDeprecated(): + self.assertNotIsInstance(n, Num) + + for arg in '42', True, False: + with self.subTest(arg=arg): + with assertNumDeprecated(): + self.assertNotIsInstance(Constant(arg), Num) + + assertStrDeprecated(self.assertNotIsInstance, Constant(42), Str) + assertBytesDeprecated(self.assertNotIsInstance, Constant('42'), Bytes) + assertNameConstantDeprecated(self.assertNotIsInstance, Constant(42), NameConstant) + assertEllipsisDeprecated(self.assertNotIsInstance, Constant(42), Ellipsis) + assertNumDeprecated(self.assertNotIsInstance, Constant(), Num) + assertStrDeprecated(self.assertNotIsInstance, Constant(), Str) + assertBytesDeprecated(self.assertNotIsInstance, Constant(), Bytes) + assertNameConstantDeprecated(self.assertNotIsInstance, Constant(), NameConstant) + assertEllipsisDeprecated(self.assertNotIsInstance, Constant(), Ellipsis) class S(str): pass - self.assertTrue(isinstance(ast.Constant(S('42')), ast.Str)) - self.assertFalse(isinstance(ast.Constant(S('42')), ast.Num)) + with assertStrDeprecated(): + self.assertIsInstance(Constant(S('42')), Str) + with assertNumDeprecated(): + self.assertNotIsInstance(Constant(S('42')), Num) + + def test_constant_subclasses_deprecated(self): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', '', DeprecationWarning) + from ast import Num - def test_subclasses(self): - class N(ast.Num): + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('always', '', DeprecationWarning) + class N(ast.Num): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.z = 'spam' + class N2(ast.Num): + pass + + n = N(42) + self.assertEqual(n.n, 42) + self.assertEqual(n.z, 'spam') + self.assertIs(type(n), N) + self.assertIsInstance(n, N) + self.assertIsInstance(n, ast.Num) + self.assertNotIsInstance(n, N2) + self.assertNotIsInstance(ast.Num(42), N) + n = N(n=42) + self.assertEqual(n.n, 42) + self.assertIs(type(n), N) + + self.assertEqual([str(w.message) for w in wlog], [ + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + ]) + + def test_constant_subclasses(self): + class N(ast.Constant): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.z = 'spam' - class N2(ast.Num): + class N2(ast.Constant): pass n = N(42) - self.assertEqual(n.n, 42) + self.assertEqual(n.value, 42) self.assertEqual(n.z, 'spam') self.assertEqual(type(n), N) self.assertTrue(isinstance(n, N)) - self.assertTrue(isinstance(n, ast.Num)) + self.assertTrue(isinstance(n, ast.Constant)) self.assertFalse(isinstance(n, N2)) - self.assertFalse(isinstance(ast.Num(42), N)) - n = N(n=42) - self.assertEqual(n.n, 42) + self.assertFalse(isinstance(ast.Constant(42), N)) + n = N(value=42) + self.assertEqual(n.value, 42) self.assertEqual(type(n), N) def test_module(self): - body = [ast.Num(42)] + body = [ast.Constant(42)] x = ast.Module(body, []) self.assertEqual(x.body, body) @@ -617,8 +842,8 @@ def test_nodeclasses(self): x.foobarbaz = 5 self.assertEqual(x.foobarbaz, 5) - n1 = ast.Num(1) - n3 = ast.Num(3) + n1 = ast.Constant(1) + n3 = ast.Constant(3) addop = ast.Add() x = ast.BinOp(n1, addop, n3) self.assertEqual(x.left, n1) @@ -987,7 +1212,7 @@ def test_dump_incomplete(self): def test_copy_location(self): src = ast.parse('1 + 1', mode='eval') - src.body.right = ast.copy_location(ast.Num(2), src.body.right) + src.body.right = ast.copy_location(ast.Constant(2), src.body.right) self.assertEqual(ast.dump(src, include_attributes=True), 'Expression(body=BinOp(left=Constant(value=1, lineno=1, col_offset=0, ' 'end_lineno=1, end_col_offset=1), op=Add(), right=Constant(value=2, ' @@ -1004,7 +1229,7 @@ def test_copy_location(self): def test_fix_missing_locations(self): src = ast.parse('write("spam")') src.body.append(ast.Expr(ast.Call(ast.Name('spam', ast.Load()), - [ast.Str('eggs')], []))) + [ast.Constant('eggs')], []))) self.assertEqual(src, ast.fix_missing_locations(src)) self.maxDiff = None self.assertEqual(ast.dump(src, include_attributes=True), @@ -1317,9 +1542,9 @@ def arguments(args=None, posonlyargs=None, vararg=None, check(arguments(args=args), "must have Load context") check(arguments(posonlyargs=args), "must have Load context") check(arguments(kwonlyargs=args), "must have Load context") - check(arguments(defaults=[ast.Num(3)]), + check(arguments(defaults=[ast.Constant(3)]), "more positional defaults than args") - check(arguments(kw_defaults=[ast.Num(4)]), + check(arguments(kw_defaults=[ast.Constant(4)]), "length of kwonlyargs is not the same as kw_defaults") args = [ast.arg("x", ast.Name("x", ast.Load()))] check(arguments(args=args, defaults=[ast.Name("x", ast.Store())]), @@ -1372,9 +1597,9 @@ def test_delete(self): "must have Del context") def test_assign(self): - self.stmt(ast.Assign([], ast.Num(3)), "empty targets on Assign") - self.stmt(ast.Assign([None], ast.Num(3)), "None disallowed") - self.stmt(ast.Assign([ast.Name("x", ast.Load())], ast.Num(3)), + self.stmt(ast.Assign([], ast.Constant(3)), "empty targets on Assign") + self.stmt(ast.Assign([None], ast.Constant(3)), "None disallowed") + self.stmt(ast.Assign([ast.Name("x", ast.Load())], ast.Constant(3)), "must have Store context") self.stmt(ast.Assign([ast.Name("x", ast.Store())], ast.Name("y", ast.Store())), @@ -1402,39 +1627,39 @@ def test_for(self): self.stmt(ast.For(x, y, [p], [e]), "must have Load context") def test_while(self): - self.stmt(ast.While(ast.Num(3), [], []), "empty body on While") + self.stmt(ast.While(ast.Constant(3), [], []), "empty body on While") self.stmt(ast.While(ast.Name("x", ast.Store()), [ast.Pass()], []), "must have Load context") - self.stmt(ast.While(ast.Num(3), [ast.Pass()], + self.stmt(ast.While(ast.Constant(3), [ast.Pass()], [ast.Expr(ast.Name("x", ast.Store()))]), "must have Load context") def test_if(self): - self.stmt(ast.If(ast.Num(3), [], []), "empty body on If") + self.stmt(ast.If(ast.Constant(3), [], []), "empty body on If") i = ast.If(ast.Name("x", ast.Store()), [ast.Pass()], []) self.stmt(i, "must have Load context") - i = ast.If(ast.Num(3), [ast.Expr(ast.Name("x", ast.Store()))], []) + i = ast.If(ast.Constant(3), [ast.Expr(ast.Name("x", ast.Store()))], []) self.stmt(i, "must have Load context") - i = ast.If(ast.Num(3), [ast.Pass()], + i = ast.If(ast.Constant(3), [ast.Pass()], [ast.Expr(ast.Name("x", ast.Store()))]) self.stmt(i, "must have Load context") def test_with(self): p = ast.Pass() self.stmt(ast.With([], [p]), "empty items on With") - i = ast.withitem(ast.Num(3), None) + i = ast.withitem(ast.Constant(3), None) self.stmt(ast.With([i], []), "empty body on With") i = ast.withitem(ast.Name("x", ast.Store()), None) self.stmt(ast.With([i], [p]), "must have Load context") - i = ast.withitem(ast.Num(3), ast.Name("x", ast.Load())) + i = ast.withitem(ast.Constant(3), ast.Name("x", ast.Load())) self.stmt(ast.With([i], [p]), "must have Store context") def test_raise(self): - r = ast.Raise(None, ast.Num(3)) + r = ast.Raise(None, ast.Constant(3)) self.stmt(r, "Raise with cause but no exception") r = ast.Raise(ast.Name("x", ast.Store()), None) self.stmt(r, "must have Load context") - r = ast.Raise(ast.Num(4), ast.Name("x", ast.Store())) + r = ast.Raise(ast.Constant(4), ast.Name("x", ast.Store())) self.stmt(r, "must have Load context") def test_try(self): @@ -1505,11 +1730,11 @@ def test_expr(self): def test_boolop(self): b = ast.BoolOp(ast.And(), []) self.expr(b, "less than 2 values") - b = ast.BoolOp(ast.And(), [ast.Num(3)]) + b = ast.BoolOp(ast.And(), [ast.Constant(3)]) self.expr(b, "less than 2 values") - b = ast.BoolOp(ast.And(), [ast.Num(4), None]) + b = ast.BoolOp(ast.And(), [ast.Constant(4), None]) self.expr(b, "None disallowed") - b = ast.BoolOp(ast.And(), [ast.Num(4), ast.Name("x", ast.Store())]) + b = ast.BoolOp(ast.And(), [ast.Constant(4), ast.Name("x", ast.Store())]) self.expr(b, "must have Load context") def test_unaryop(self): @@ -1597,11 +1822,11 @@ def test_compare(self): left = ast.Name("x", ast.Load()) comp = ast.Compare(left, [ast.In()], []) self.expr(comp, "no comparators") - comp = ast.Compare(left, [ast.In()], [ast.Num(4), ast.Num(5)]) + comp = ast.Compare(left, [ast.In()], [ast.Constant(4), ast.Constant(5)]) self.expr(comp, "different number of comparators and operands") - comp = ast.Compare(ast.Num("blah"), [ast.In()], [left]) + comp = ast.Compare(ast.Constant("blah"), [ast.In()], [left]) self.expr(comp) - comp = ast.Compare(left, [ast.In()], [ast.Num("blah")]) + comp = ast.Compare(left, [ast.In()], [ast.Constant("blah")]) self.expr(comp) def test_call(self): @@ -1617,23 +1842,37 @@ def test_call(self): self.expr(call, "must have Load context") def test_num(self): - class subint(int): - pass - class subfloat(float): - pass - class subcomplex(complex): - pass - for obj in "0", "hello": - self.expr(ast.Num(obj)) - for obj in subint(), subfloat(), subcomplex(): - self.expr(ast.Num(obj), "invalid type", exc=TypeError) + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('ignore', '', DeprecationWarning) + from ast import Num + + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('always', '', DeprecationWarning) + class subint(int): + pass + class subfloat(float): + pass + class subcomplex(complex): + pass + for obj in "0", "hello": + self.expr(ast.Num(obj)) + for obj in subint(), subfloat(), subcomplex(): + self.expr(ast.Num(obj), "invalid type", exc=TypeError) + + self.assertEqual([str(w.message) for w in wlog], [ + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + ]) def test_attribute(self): attr = ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load()) self.expr(attr, "must have Load context") def test_subscript(self): - sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Num(3), + sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Constant(3), ast.Load()) self.expr(sub, "must have Load context") x = ast.Name("x", ast.Load()) @@ -1653,7 +1892,7 @@ def test_subscript(self): def test_starred(self): left = ast.List([ast.Starred(ast.Name("x", ast.Load()), ast.Store())], ast.Store()) - assign = ast.Assign([left], ast.Num(4)) + assign = ast.Assign([left], ast.Constant(4)) self.stmt(assign, "must have Store context") def _sequence(self, fac): @@ -1668,7 +1907,17 @@ def test_tuple(self): self._sequence(ast.Tuple) def test_nameconstant(self): - self.expr(ast.NameConstant(4)) + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('ignore', '', DeprecationWarning) + from ast import NameConstant + + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('always', '', DeprecationWarning) + self.expr(ast.NameConstant(4)) + + self.assertEqual([str(w.message) for w in wlog], [ + 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', + ]) def test_stdlib_validates(self): stdlib = os.path.dirname(ast.__file__) @@ -2357,10 +2606,15 @@ def visit_Ellipsis(self, node): ]) self.assertEqual([str(w.message) for w in wlog], [ 'visit_Num is deprecated; add visit_Constant', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', 'visit_Num is deprecated; add visit_Constant', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', 'visit_Num is deprecated; add visit_Constant', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', 'visit_Str is deprecated; add visit_Constant', + 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', 'visit_Bytes is deprecated; add visit_Constant', + 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', 'visit_NameConstant is deprecated; add visit_Constant', 'visit_NameConstant is deprecated; add visit_Constant', 'visit_Ellipsis is deprecated; add visit_Constant', diff --git a/Misc/NEWS.d/next/Library/2022-02-19-14-19-34.bpo-46797.6BXZX4.rst b/Misc/NEWS.d/next/Library/2022-02-19-14-19-34.bpo-46797.6BXZX4.rst new file mode 100644 index 00000000000000..6539efbc9d0eb0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-02-19-14-19-34.bpo-46797.6BXZX4.rst @@ -0,0 +1,4 @@ +Deprecation warnings are now emitted for :class:`!ast.Num`, +:class:`!ast.Bytes`, :class:`!ast.Str`, :class:`!ast.NameConstant` and +:class:`!ast.Ellipsis`. These have been documented as deprecated since Python +3.8, and will be removed in Python 3.14. From de7f694e3c92797fe65f04cd2c6941ed0446bb24 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sat, 6 May 2023 19:03:07 +0100 Subject: [PATCH 079/204] GH-103548: Improve performance of `pathlib.Path.[is_]absolute()` (GH-103549) Improve performance of `pathlib.Path.absolute()` and `cwd()` by joining paths only when necessary. Also improve performance of `PurePath.is_absolute()` on Posix by skipping path parsing and normalization. --- Lib/pathlib.py | 11 ++++++++++- .../2023-04-14-21-16-05.gh-issue-103548.lagdpp.rst | 4 ++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-14-21-16-05.gh-issue-103548.lagdpp.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 9aa3c1e52447d1..480c354ce8b656 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -664,7 +664,7 @@ def is_absolute(self): # ntpath.isabs() is defective - see GH-44626 . if self._flavour is ntpath: return bool(self.drive and self.root) - return self._flavour.isabs(self) + return self._flavour.isabs(self._raw_path) def is_reserved(self): """Return True if the path contains one of the special names reserved @@ -873,6 +873,15 @@ def absolute(self): cwd = self._flavour.abspath(self.drive) else: cwd = os.getcwd() + # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). + # We pass only one argument to with_segments() to avoid the cost + # of joining, and we exploit the fact that getcwd() returns a + # fully-normalized string by storing it in _str. This is used to + # implement Path.cwd(). + if not self.root and not self._tail: + result = self.with_segments(cwd) + result._str = cwd + return result return self.with_segments(cwd, self) def resolve(self, strict=False): diff --git a/Misc/NEWS.d/next/Library/2023-04-14-21-16-05.gh-issue-103548.lagdpp.rst b/Misc/NEWS.d/next/Library/2023-04-14-21-16-05.gh-issue-103548.lagdpp.rst new file mode 100644 index 00000000000000..238f2868867472 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-14-21-16-05.gh-issue-103548.lagdpp.rst @@ -0,0 +1,4 @@ +Improve performance of :meth:`pathlib.Path.absolute` and +:meth:`~pathlib.Path.cwd` by joining paths only when necessary. Also improve +performance of :meth:`pathlib.PurePath.is_absolute` on Posix by skipping path +parsing and normalization. From 3b14b51d11ae23a915299a8f9bf650ca2ae90566 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 6 May 2023 22:26:06 +0200 Subject: [PATCH 080/204] gh-101819: Remove unused 'locale_module' from _io state (#104246) The locale module reference was introduced by 932ff8368 in 2013, and rendered unused by 710e82630 (gh-23050) in 2020. --- Modules/_io/_iomodule.c | 3 --- Modules/_io/_iomodule.h | 2 -- 2 files changed, 5 deletions(-) diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 403968af1b996c..99b8a8e09ecf0a 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -580,7 +580,6 @@ iomodule_traverse(PyObject *mod, visitproc visit, void *arg) { _PyIO_State *state = get_io_state(mod); if (!state->initialized) return 0; - Py_VISIT(state->locale_module); Py_VISIT(state->unsupported_operation); Py_VISIT(state->PyIncrementalNewlineDecoder_Type); @@ -605,8 +604,6 @@ iomodule_clear(PyObject *mod) { _PyIO_State *state = get_io_state(mod); if (!state->initialized) return 0; - if (state->locale_module != NULL) - Py_CLEAR(state->locale_module); Py_CLEAR(state->unsupported_operation); Py_CLEAR(state->PyIncrementalNewlineDecoder_Type); diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index 8a788fbb8185c5..c971c987cf5fee 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -143,8 +143,6 @@ extern PyModuleDef _PyIO_Module; typedef struct { int initialized; - PyObject *locale_module; - PyObject *unsupported_operation; /* Types */ From fff193bbfebe7b00229856b1e8105ab3de36437f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 6 May 2023 15:57:35 -0600 Subject: [PATCH 081/204] gh-99113: Add a check for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED (gh-104206) Py_MOD_PER_INTERPRETER_GIL_SUPPORTED is a new supported value for Py_mod_multiple_interpreters, added in gh-104205. --- Lib/test/test_import/__init__.py | 20 +++++++ .../test_importlib/extension/test_loader.py | 2 + ...3-05-05-13-18-56.gh-issue-99113.hT1ajK.rst | 11 ++++ Modules/_testmultiphase.c | 60 ++++++++++++++++++- Objects/moduleobject.c | 8 ++- 5 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-05-13-18-56.gh-issue-99113.hT1ajK.rst diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 773b7094c6b8ce..e2384a08ecaa90 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1861,6 +1861,26 @@ def test_multi_init_extension_non_isolated_compat(self): with self.subTest(f'{modname}: not strict'): self.check_compatible_here(modname, filename, strict=False) + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_multi_init_extension_per_interpreter_gil_compat(self): + modname = '_test_shared_gil_only' + filename = _testmultiphase.__file__ + loader = ExtensionFileLoader(modname, filename) + spec = importlib.util.spec_from_loader(modname, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + sys.modules[modname] = module + + require_extension(module) + with self.subTest(f'{modname}: isolated, strict'): + self.check_incompatible_here(modname, filename, isolated=True) + with self.subTest(f'{modname}: not isolated, strict'): + self.check_compatible_here(modname, filename, + strict=True, isolated=False) + with self.subTest(f'{modname}: not isolated, not strict'): + self.check_compatible_here(modname, filename, + strict=False, isolated=False) + def test_python_compat(self): module = 'threading' require_pure_python(module) diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index 3bf2bbdcdcc4e6..3a74b821eaee49 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -348,6 +348,8 @@ def test_bad_modules(self): 'exec_err', 'exec_raise', 'exec_unreported_exception', + 'multiple_create_slots', + 'multiple_multiple_interpreters_slots', ]: with self.subTest(name_base): name = self.name + '_' + name_base diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-05-13-18-56.gh-issue-99113.hT1ajK.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-05-13-18-56.gh-issue-99113.hT1ajK.rst new file mode 100644 index 00000000000000..afd26750846167 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-05-13-18-56.gh-issue-99113.hT1ajK.rst @@ -0,0 +1,11 @@ +Multi-phase init extension modules may now indicate that they support +running in subinterpreters that have their own GIL. This is done by using +``Py_MOD_PER_INTERPRETER_GIL_SUPPORTED`` as the value for the +``Py_mod_multiple_interpreters`` module def slot. Otherwise the module, by +default, cannot be imported in such subinterpreters. (This does not affect +the main interpreter or subinterpreters that do not have their own GIL.) In +addition to the isolation that multi-phase init already normally requires, +support for per-interpreter GIL involves one additional constraint: +thread-safety. If the module has external (linked) dependencies and those +libraries have any state that isn't thread-safe then the module must do the +additional work to add thread-safety. This should be an uncommon case. diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index 58b064bb17cd87..ca71b6156b005d 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -681,6 +681,27 @@ PyInit__testmultiphase_export_unreported_exception(void) return PyModuleDef_Init(&main_def); } +static PyObject* +createfunc_noop(PyObject *spec, PyModuleDef *def) +{ + return PyModule_New("spam"); +} + +static PyModuleDef_Slot slots_multiple_create_slots[] = { + {Py_mod_create, createfunc_noop}, + {Py_mod_create, createfunc_noop}, + {0, NULL}, +}; + +static PyModuleDef def_multiple_create_slots = TEST_MODULE_DEF( + "_testmultiphase_multiple_create_slots", slots_multiple_create_slots, NULL); + +PyMODINIT_FUNC +PyInit__testmultiphase_multiple_create_slots(void) +{ + return PyModuleDef_Init(&def_multiple_create_slots); +} + static PyObject* createfunc_null(PyObject *spec, PyModuleDef *def) { @@ -892,7 +913,24 @@ PyInit__test_module_state_shared(void) } -/* multiple interpreters supports */ +/* multiple interpreters support */ + +static PyModuleDef_Slot slots_multiple_multiple_interpreters_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL}, +}; + +static PyModuleDef def_multiple_multiple_interpreters_slots = TEST_MODULE_DEF( + "_testmultiphase_multiple_multiple_interpreters_slots", + slots_multiple_multiple_interpreters_slots, + NULL); + +PyMODINIT_FUNC +PyInit__testmultiphase_multiple_multiple_interpreters_slots(void) +{ + return PyModuleDef_Init(&def_multiple_multiple_interpreters_slots); +} static PyModuleDef_Slot non_isolated_slots[] = { {Py_mod_exec, execfunc}, @@ -909,3 +947,23 @@ PyInit__test_non_isolated(void) { return PyModuleDef_Init(&non_isolated_def); } + + +static PyModuleDef_Slot shared_gil_only_slots[] = { + {Py_mod_exec, execfunc}, + /* Note that Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED is the default. + We put it here explicitly to draw attention to the contrast + with Py_MOD_PER_INTERPRETER_GIL_SUPPORTED. */ + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED}, + {0, NULL}, +}; + +static PyModuleDef shared_gil_only_def = TEST_MODULE_DEF("_test_shared_gil_only", + shared_gil_only_slots, + testexport_methods); + +PyMODINIT_FUNC +PyInit__test_shared_gil_only(void) +{ + return PyModuleDef_Init(&shared_gil_only_def); +} diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index c100d018d3f0df..985be58d02c784 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -323,7 +323,13 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio goto error; } } - // XXX Do a similar check once we have PyInterpreterState.ceval.own_gil. + else if (multiple_interpreters != Py_MOD_PER_INTERPRETER_GIL_SUPPORTED + && interp->ceval.own_gil + && !_Py_IsMainInterpreter(interp) + && _PyImport_CheckSubinterpIncompatibleExtensionAllowed(name) < 0) + { + goto error; + } if (create) { m = create(spec, def); From 92d8bfffbf377e91d8b92666525cb8700bb1d5e8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 6 May 2023 15:59:30 -0600 Subject: [PATCH 082/204] gh-99113: Make Sure the GIL is Acquired at the Right Places (gh-104208) This is a pre-requisite for a per-interpreter GIL. Without it this change isn't strictly necessary. However, there is no real downside otherwise. --- Include/internal/pycore_ceval.h | 2 + Python/ceval_gil.c | 84 ++++++++++++++++++++++----------- Python/pylifecycle.c | 25 ++++++++-- Python/pystate.c | 42 +++++++++++++---- 4 files changed, 113 insertions(+), 40 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index b7a9bf40425bc7..9fd8571cbc87f4 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -100,7 +100,9 @@ extern int _PyEval_ThreadsInitialized(void); extern PyStatus _PyEval_InitGIL(PyThreadState *tstate, int own_gil); extern void _PyEval_FiniGIL(PyInterpreterState *interp); +extern void _PyEval_AcquireLock(PyThreadState *tstate); extern void _PyEval_ReleaseLock(PyThreadState *tstate); +extern PyThreadState * _PyThreadState_SwapNoGIL(PyThreadState *); extern void _PyEval_DeactivateOpCache(void); diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 1ac0dbcf2ecfec..9958856bae8019 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -499,42 +499,66 @@ PyEval_ThreadsInitialized(void) return _PyEval_ThreadsInitialized(); } +static inline int +current_thread_holds_gil(struct _gil_runtime_state *gil, PyThreadState *tstate) +{ + if (((PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) != tstate) { + return 0; + } + return _Py_atomic_load_relaxed(&gil->locked); +} + +static void +init_shared_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil) +{ + assert(gil_created(gil)); + interp->ceval.gil = gil; + interp->ceval.own_gil = 0; +} + +static void +init_own_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil) +{ + assert(!gil_created(gil)); + create_gil(gil); + assert(gil_created(gil)); + interp->ceval.gil = gil; + interp->ceval.own_gil = 1; +} + PyStatus _PyEval_InitGIL(PyThreadState *tstate, int own_gil) { assert(tstate->interp->ceval.gil == NULL); + int locked; if (!own_gil) { PyInterpreterState *main_interp = _PyInterpreterState_Main(); assert(tstate->interp != main_interp); struct _gil_runtime_state *gil = main_interp->ceval.gil; - assert(gil_created(gil)); - tstate->interp->ceval.gil = gil; - tstate->interp->ceval.own_gil = 0; - return _PyStatus_OK(); + init_shared_gil(tstate->interp, gil); + locked = current_thread_holds_gil(gil, tstate); } - /* XXX per-interpreter GIL */ - struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil; - if (!_Py_IsMainInterpreter(tstate->interp)) { + else if (!_Py_IsMainInterpreter(tstate->interp)) { /* Currently, the GIL is shared by all interpreters, and only the main interpreter is responsible to create and destroy it. */ - assert(gil_created(gil)); - tstate->interp->ceval.gil = gil; + struct _gil_runtime_state *main_gil = _PyInterpreterState_Main()->ceval.gil; + init_shared_gil(tstate->interp, main_gil); // XXX For now we lie. tstate->interp->ceval.own_gil = 1; - return _PyStatus_OK(); + locked = current_thread_holds_gil(main_gil, tstate); + } + else { + PyThread_init_thread(); + // XXX per-interpreter GIL: switch to interp->_gil. + init_own_gil(tstate->interp, &tstate->interp->runtime->ceval.gil); + locked = 0; + } + if (!locked) { + take_gil(tstate); } - assert(own_gil); - - assert(!gil_created(gil)); - PyThread_init_thread(); - create_gil(gil); - assert(gil_created(gil)); - tstate->interp->ceval.gil = gil; - tstate->interp->ceval.own_gil = 1; - take_gil(tstate); return _PyStatus_OK(); } @@ -611,9 +635,17 @@ PyEval_ReleaseLock(void) drop_gil(ceval, tstate); } +void +_PyEval_AcquireLock(PyThreadState *tstate) +{ + _Py_EnsureTstateNotNULL(tstate); + take_gil(tstate); +} + void _PyEval_ReleaseLock(PyThreadState *tstate) { + _Py_EnsureTstateNotNULL(tstate); struct _ceval_state *ceval = &tstate->interp->ceval; drop_gil(ceval, tstate); } @@ -625,7 +657,7 @@ PyEval_AcquireThread(PyThreadState *tstate) take_gil(tstate); - if (_PyThreadState_Swap(tstate->interp->runtime, tstate) != NULL) { + if (_PyThreadState_SwapNoGIL(tstate) != NULL) { Py_FatalError("non-NULL old thread state"); } } @@ -635,8 +667,7 @@ PyEval_ReleaseThread(PyThreadState *tstate) { assert(is_tstate_valid(tstate)); - _PyRuntimeState *runtime = tstate->interp->runtime; - PyThreadState *new_tstate = _PyThreadState_Swap(runtime, NULL); + PyThreadState *new_tstate = _PyThreadState_SwapNoGIL(NULL); if (new_tstate != tstate) { Py_FatalError("wrong thread state"); } @@ -684,8 +715,7 @@ _PyEval_SignalAsyncExc(PyInterpreterState *interp) PyThreadState * PyEval_SaveThread(void) { - _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = _PyThreadState_Swap(runtime, NULL); + PyThreadState *tstate = _PyThreadState_SwapNoGIL(NULL); _Py_EnsureTstateNotNULL(tstate); struct _ceval_state *ceval = &tstate->interp->ceval; @@ -701,7 +731,7 @@ PyEval_RestoreThread(PyThreadState *tstate) take_gil(tstate); - _PyThreadState_Swap(tstate->interp->runtime, tstate); + _PyThreadState_SwapNoGIL(tstate); } @@ -1005,7 +1035,7 @@ _Py_HandlePending(PyThreadState *tstate) /* GIL drop request */ if (_Py_atomic_load_relaxed_int32(&interp_ceval_state->gil_drop_request)) { /* Give another thread a chance */ - if (_PyThreadState_Swap(runtime, NULL) != tstate) { + if (_PyThreadState_SwapNoGIL(NULL) != tstate) { Py_FatalError("tstate mix-up"); } drop_gil(interp_ceval_state, tstate); @@ -1014,7 +1044,7 @@ _Py_HandlePending(PyThreadState *tstate) take_gil(tstate); - if (_PyThreadState_Swap(runtime, tstate) != NULL) { + if (_PyThreadState_SwapNoGIL(tstate) != NULL) { Py_FatalError("orphan tstate"); } } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 705708698c6a8b..61f87c5eba60ed 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -591,6 +591,7 @@ init_interp_create_gil(PyThreadState *tstate, int own_gil) /* finalize_interp_delete() comment explains why _PyEval_FiniGIL() is only called here. */ + // XXX This is broken with a per-interpreter GIL. _PyEval_FiniGIL(tstate->interp); /* Auto-thread-state API */ @@ -645,7 +646,8 @@ pycore_create_interpreter(_PyRuntimeState *runtime, return _PyStatus_ERR("can't make first thread"); } _PyThreadState_Bind(tstate); - (void) PyThreadState_Swap(tstate); + // XXX For now we do this before the GIL is created. + (void) _PyThreadState_SwapNoGIL(tstate); status = init_interp_create_gil(tstate, config.own_gil); if (_PyStatus_EXCEPTION(status)) { @@ -2025,11 +2027,20 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) } _PyThreadState_Bind(tstate); - PyThreadState *save_tstate = PyThreadState_Swap(tstate); + // XXX For now we do this before the GIL is created. + PyThreadState *save_tstate = _PyThreadState_SwapNoGIL(tstate); + int has_gil = 0; + + /* From this point until the init_interp_create_gil() call, + we must not do anything that requires that the GIL be held + (or otherwise exist). That applies whether or not the new + interpreter has its own GIL (e.g. the main interpreter). */ /* Copy the current interpreter config into the new interpreter */ const PyConfig *src_config; if (save_tstate != NULL) { + // XXX Might new_interpreter() have been called without the GIL held? + _PyEval_ReleaseLock(save_tstate); src_config = _PyInterpreterState_GetConfig(save_tstate->interp); } else @@ -2039,11 +2050,13 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) src_config = _PyInterpreterState_GetConfig(main_interp); } + /* This does not require that the GIL be held. */ status = _PyConfig_Copy(&interp->config, src_config); if (_PyStatus_EXCEPTION(status)) { goto error; } + /* This does not require that the GIL be held. */ status = init_interp_settings(interp, config); if (_PyStatus_EXCEPTION(status)) { goto error; @@ -2053,6 +2066,7 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) if (_PyStatus_EXCEPTION(status)) { goto error; } + has_gil = 1; status = pycore_interp_init(tstate); if (_PyStatus_EXCEPTION(status)) { @@ -2072,7 +2086,12 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) /* Oops, it didn't work. Undo it all. */ PyErr_PrintEx(0); - PyThreadState_Swap(save_tstate); + if (has_gil) { + PyThreadState_Swap(save_tstate); + } + else { + _PyThreadState_SwapNoGIL(save_tstate); + } PyThreadState_Clear(tstate); PyThreadState_Delete(tstate); PyInterpreterState_Delete(interp); diff --git a/Python/pystate.c b/Python/pystate.c index 75bd9f41e301a3..f14934361dab78 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1863,17 +1863,11 @@ PyThreadState_Get(void) } -PyThreadState * -_PyThreadState_Swap(_PyRuntimeState *runtime, PyThreadState *newts) +static void +_swap_thread_states(_PyRuntimeState *runtime, + PyThreadState *oldts, PyThreadState *newts) { -#if defined(Py_DEBUG) - /* This can be called from PyEval_RestoreThread(). Similar - to it, we need to ensure errno doesn't change. - */ - int err = errno; -#endif - PyThreadState *oldts = current_fast_get(runtime); - + // XXX Do this only if oldts != NULL? current_fast_clear(runtime); if (oldts != NULL) { @@ -1887,6 +1881,20 @@ _PyThreadState_Swap(_PyRuntimeState *runtime, PyThreadState *newts) current_fast_set(runtime, newts); tstate_activate(newts); } +} + +PyThreadState * +_PyThreadState_SwapNoGIL(PyThreadState *newts) +{ +#if defined(Py_DEBUG) + /* This can be called from PyEval_RestoreThread(). Similar + to it, we need to ensure errno doesn't change. + */ + int err = errno; +#endif + + PyThreadState *oldts = current_fast_get(&_PyRuntime); + _swap_thread_states(&_PyRuntime, oldts, newts); #if defined(Py_DEBUG) errno = err; @@ -1894,6 +1902,20 @@ _PyThreadState_Swap(_PyRuntimeState *runtime, PyThreadState *newts) return oldts; } +PyThreadState * +_PyThreadState_Swap(_PyRuntimeState *runtime, PyThreadState *newts) +{ + PyThreadState *oldts = current_fast_get(runtime); + if (oldts != NULL) { + _PyEval_ReleaseLock(oldts); + } + _swap_thread_states(runtime, oldts, newts); + if (newts != NULL) { + _PyEval_AcquireLock(newts); + } + return oldts; +} + PyThreadState * PyThreadState_Swap(PyThreadState *newts) { From 42f54d1f9244784fec99e0610aa05a5051e594bb Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Sun, 7 May 2023 02:53:48 +0400 Subject: [PATCH 083/204] gh-101640: Make argparse _print_message catch any write error (#101802) * In particular, don't exit when trying to print to stderr = None. * Add tests Co-authored-by: Terry Jan Reedy --- Lib/argparse.py | 8 +++-- Lib/test/test_argparse.py | 31 +++++++++++++++++++ ...-02-09-22-24-34.gh-issue-101640.oFuEpB.rst | 1 + 3 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-09-22-24-34.gh-issue-101640.oFuEpB.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index a819d2650e85f0..68089a5c1e80b0 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -2605,9 +2605,11 @@ def print_help(self, file=None): def _print_message(self, message, file=None): if message: - if file is None: - file = _sys.stderr - file.write(message) + file = file or _sys.stderr + try: + file.write(message) + except (AttributeError, OSError): + pass # =============== # Exiting methods diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 861da2326d1214..0659d244d35686 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -1,5 +1,7 @@ # Author: Steven J. Bethard . +import contextlib +import functools import inspect import io import operator @@ -35,6 +37,35 @@ def getvalue(self): return self.buffer.raw.getvalue().decode('utf-8') +class StdStreamTest(unittest.TestCase): + + def test_skip_invalid_stderr(self): + parser = argparse.ArgumentParser() + with ( + contextlib.redirect_stderr(None), + mock.patch('argparse._sys.exit') + ): + parser.exit(status=0, message='foo') + + def test_skip_invalid_stdout(self): + parser = argparse.ArgumentParser() + for func in ( + parser.print_usage, + parser.print_help, + functools.partial(parser.parse_args, ['-h']) + ): + with ( + self.subTest(func=func), + contextlib.redirect_stdout(None), + # argparse uses stderr as a fallback + StdIOBuffer() as mocked_stderr, + contextlib.redirect_stderr(mocked_stderr), + mock.patch('argparse._sys.exit'), + ): + func() + self.assertRegex(mocked_stderr.getvalue(), r'usage:') + + class TestCase(unittest.TestCase): def setUp(self): diff --git a/Misc/NEWS.d/next/Library/2023-02-09-22-24-34.gh-issue-101640.oFuEpB.rst b/Misc/NEWS.d/next/Library/2023-02-09-22-24-34.gh-issue-101640.oFuEpB.rst new file mode 100644 index 00000000000000..917cf0f97b9e06 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-09-22-24-34.gh-issue-101640.oFuEpB.rst @@ -0,0 +1 @@ +:class:`argparse.ArgumentParser` now catches errors when writing messages, such as when :data:`sys.stderr` is ``None``. Patch by Oleg Iarygin. From 4ee2068c34bd45eddba7f6a8ee83f62d5b6932fc Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Sat, 6 May 2023 18:31:53 -0700 Subject: [PATCH 084/204] gh-104254: Document the optional keyword-only "context" argument to Task constructor (#104251) (This was added in 3.11. It was already documented for `create_task()`, but not for `Task()`.) --- Doc/library/asyncio-task.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index f8727b98066990..a46ebc1c3d25a9 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -1014,7 +1014,7 @@ Introspection Task Object =========== -.. class:: Task(coro, *, loop=None, name=None) +.. class:: Task(coro, *, loop=None, name=None, context=None) A :class:`Future-like ` object that runs a Python :ref:`coroutine `. Not thread-safe. @@ -1049,9 +1049,10 @@ Task Object APIs except :meth:`Future.set_result` and :meth:`Future.set_exception`. - Tasks support the :mod:`contextvars` module. When a Task - is created it copies the current context and later runs its - coroutine in the copied context. + An optional keyword-only *context* argument allows specifying a + custom :class:`contextvars.Context` for the *coro* to run in. + If no *context* is provided, the Task copies the current context + and later runs its coroutine in the copied context. .. versionchanged:: 3.7 Added support for the :mod:`contextvars` module. @@ -1063,6 +1064,9 @@ Task Object Deprecation warning is emitted if *loop* is not specified and there is no running event loop. + .. versionchanged:: 3.11 + Added the *context* parameter. + .. method:: done() Return ``True`` if the Task is *done*. From b35711d17a90251bdd57d255090e07daafe89f6c Mon Sep 17 00:00:00 2001 From: Tomas R Date: Sun, 7 May 2023 04:05:34 +0200 Subject: [PATCH 085/204] gh-103886: Improve `builtins.__doc__` (#104179) Co-authored-by: Jelle Zijlstra --- Python/bltinmodule.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 8840bbabe4b584..ddddc03ca316e0 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -3014,9 +3014,16 @@ static PyMethodDef builtin_methods[] = { }; PyDoc_STRVAR(builtin_doc, -"Built-in functions, exceptions, and other objects.\n\ +"Built-in functions, types, exceptions, and other objects.\n\ \n\ -Noteworthy: None is the `nil' object; Ellipsis represents `...' in slices."); +This module provides direct access to all 'built-in'\n\ +identifiers of Python; for example, builtins.len is\n\ +the full name for the built-in function len().\n\ +\n\ +This module is not normally accessed explicitly by most\n\ +applications, but can be useful in modules that provide\n\ +objects with the same name as a built-in value, but in\n\ +which the built-in of that name is also needed."); static struct PyModuleDef builtinsmodule = { PyModuleDef_HEAD_INIT, From c53547c907371be53c8016145d73ba4ea0a22756 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Sat, 6 May 2023 21:25:45 -0700 Subject: [PATCH 086/204] gh-97696: Use `PyObject_CallMethodNoArgs` and inline is_loop_running check in `_asyncio` (#104255) --- Modules/_asynciomodule.c | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 39c33fed74e221..3830245abe87b3 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -2063,21 +2063,6 @@ swap_current_task(asyncio_state *state, PyObject *loop, PyObject *task) return prev_task; } -static int -is_loop_running(PyObject *loop) -{ - PyObject *func = PyObject_GetAttr(loop, &_Py_ID(is_running)); - if (func == NULL) { - PyErr_Format(PyExc_TypeError, "Loop missing is_running()"); - return -1; - } - PyObject *res = PyObject_CallNoArgs(func); - int retval = Py_IsTrue(res); - Py_DECREF(func); - Py_DECREF(res); - return !!retval; -} - /* ----- Task */ /*[clinic input] @@ -2148,11 +2133,13 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, } if (eager_start) { - int loop_running = is_loop_running(self->task_loop); - if (loop_running == -1) { + PyObject *res = PyObject_CallMethodNoArgs(loop, &_Py_ID(is_running)); + if (res == NULL) { return -1; } - if (loop_running) { + int is_loop_running = Py_IsTrue(res); + Py_DECREF(res); + if (is_loop_running) { if (task_eager_start(state, self)) { return -1; } From 69621d1b09c996e43a1e13d2fa4c317d3dd4d738 Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Sun, 7 May 2023 13:41:42 +0900 Subject: [PATCH 087/204] gh-104018: remove unused format "z" handling in string formatfloat() (#104107) This is a cleanup overlooked in PR #104033. --- Include/internal/pycore_format.h | 2 -- Objects/bytesobject.c | 3 --- Objects/unicodeobject.c | 2 -- Python/ast_opt.c | 1 - 4 files changed, 8 deletions(-) diff --git a/Include/internal/pycore_format.h b/Include/internal/pycore_format.h index 1899609e77ef20..1b8d57539ca505 100644 --- a/Include/internal/pycore_format.h +++ b/Include/internal/pycore_format.h @@ -14,14 +14,12 @@ extern "C" { * F_BLANK ' ' * F_ALT '#' * F_ZERO '0' - * F_NO_NEG_0 'z' */ #define F_LJUST (1<<0) #define F_SIGN (1<<1) #define F_BLANK (1<<2) #define F_ALT (1<<3) #define F_ZERO (1<<4) -#define F_NO_NEG_0 (1<<5) #ifdef __cplusplus } diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index e7e85cc19cda75..abbf3eeb16c35c 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -423,9 +423,6 @@ formatfloat(PyObject *v, int flags, int prec, int type, if (flags & F_ALT) { dtoa_flags |= Py_DTSF_ALT; } - if (flags & F_NO_NEG_0) { - dtoa_flags |= Py_DTSF_NO_NEG_0; - } p = PyOS_double_to_string(x, type, prec, dtoa_flags, NULL); if (p == NULL) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 1585a582f00aad..7726f2fb17afde 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -13452,8 +13452,6 @@ formatfloat(PyObject *v, struct unicode_format_arg_t *arg, if (arg->flags & F_ALT) dtoa_flags |= Py_DTSF_ALT; - if (arg->flags & F_NO_NEG_0) - dtoa_flags |= Py_DTSF_NO_NEG_0; p = PyOS_double_to_string(x, arg->ch, prec, dtoa_flags, NULL); if (p == NULL) return -1; diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 8270fa8e372d93..3883ec9e21c765 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -317,7 +317,6 @@ simple_format_arg_parse(PyObject *fmt, Py_ssize_t *ppos, case ' ': *flags |= F_BLANK; continue; case '#': *flags |= F_ALT; continue; case '0': *flags |= F_ZERO; continue; - case 'z': *flags |= F_NO_NEG_0; continue; } break; } From 472938316a85c706c06ad1b3727a205d5bffcb1f Mon Sep 17 00:00:00 2001 From: ymki4360 <132453923+ymki4360@users.noreply.github.com> Date: Sun, 7 May 2023 13:44:46 +0900 Subject: [PATCH 088/204] Re-enable commented-out test in test_generators.py (#104130) --- Lib/test/test_generators.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index cc782ea1ee5dff..31680b5a92e0f3 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -2141,11 +2141,10 @@ def printsolution(self, x): ... SyntaxError: 'yield' outside function -# Pegen does not produce this error message yet -# >>> def f(): x = yield = y -# Traceback (most recent call last): -# ... -# SyntaxError: assignment to yield expression not possible +>>> def f(): x = yield = y +Traceback (most recent call last): + ... +SyntaxError: assignment to yield expression not possible >>> def f(): (yield bar) = y Traceback (most recent call last): From 39523796554c41f16e74961f7a90dfc30b0eed64 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 7 May 2023 11:20:34 +0200 Subject: [PATCH 089/204] gh-101819: Port _io.PyIncrementalNewlineDecoder_Type to heap type (#104249) --- Modules/_io/_iomodule.c | 3 +- Modules/_io/_iomodule.h | 4 +- Modules/_io/textio.c | 82 ++++++++++----------- Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 - 4 files changed, 42 insertions(+), 48 deletions(-) diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 99b8a8e09ecf0a..ce9fcca971f99d 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -660,7 +660,6 @@ struct PyModuleDef _PyIO_Module = { static PyTypeObject* static_types[] = { // Base classes &PyIOBase_Type, - &PyIncrementalNewlineDecoder_Type, // PyIOBase_Type subclasses &PyBufferedIOBase_Type, @@ -757,7 +756,7 @@ PyInit__io(void) } // Base classes - state->PyIncrementalNewlineDecoder_Type = (PyTypeObject *)Py_NewRef(&PyIncrementalNewlineDecoder_Type); + ADD_TYPE(m, state->PyIncrementalNewlineDecoder_Type, &nldecoder_spec, NULL); // PyIOBase_Type subclasses state->PyRawIOBase_Type = (PyTypeObject *)Py_NewRef(&PyRawIOBase_Type); diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index c971c987cf5fee..f191cea7fcc47b 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -14,9 +14,6 @@ extern PyTypeObject PyRawIOBase_Type; extern PyTypeObject PyBufferedIOBase_Type; extern PyTypeObject PyTextIOBase_Type; -/* Concrete classes */ -extern PyTypeObject PyIncrementalNewlineDecoder_Type; - /* Type specs */ extern PyType_Spec bufferedrandom_spec; extern PyType_Spec bufferedreader_spec; @@ -24,6 +21,7 @@ extern PyType_Spec bufferedrwpair_spec; extern PyType_Spec bufferedwriter_spec; extern PyType_Spec bytesio_spec; extern PyType_Spec fileio_spec; +extern PyType_Spec nldecoder_spec; extern PyType_Spec stringio_spec; extern PyType_Spec textiowrapper_spec; diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 2dba382f4f8fb0..070687a83d1bc2 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -248,12 +248,32 @@ _io_IncrementalNewlineDecoder___init___impl(nldecoder_object *self, return 0; } -static void -incrementalnewlinedecoder_dealloc(nldecoder_object *self) +static int +incrementalnewlinedecoder_traverse(nldecoder_object *self, visitproc visit, + void *arg) +{ + Py_VISIT(Py_TYPE(self)); + Py_VISIT(self->decoder); + Py_VISIT(self->errors); + return 0; +} + +static int +incrementalnewlinedecoder_clear(nldecoder_object *self) { Py_CLEAR(self->decoder); Py_CLEAR(self->errors); - Py_TYPE(self)->tp_free((PyObject *)self); + return 0; +} + +static void +incrementalnewlinedecoder_dealloc(nldecoder_object *self) +{ + PyTypeObject *tp = Py_TYPE(self); + _PyObject_GC_UNTRACK(self); + (void)incrementalnewlinedecoder_clear(self); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); } static int @@ -3176,45 +3196,23 @@ static PyGetSetDef incrementalnewlinedecoder_getset[] = { {NULL} }; -PyTypeObject PyIncrementalNewlineDecoder_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io.IncrementalNewlineDecoder", /*tp_name*/ - sizeof(nldecoder_object), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)incrementalnewlinedecoder_dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ - _io_IncrementalNewlineDecoder___init____doc__, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /*tp_weaklistoffset*/ - 0, /* tp_iter */ - 0, /* tp_iternext */ - incrementalnewlinedecoder_methods, /* tp_methods */ - 0, /* tp_members */ - incrementalnewlinedecoder_getset, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - _io_IncrementalNewlineDecoder___init__, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ +static PyType_Slot nldecoder_slots[] = { + {Py_tp_dealloc, incrementalnewlinedecoder_dealloc}, + {Py_tp_doc, (void *)_io_IncrementalNewlineDecoder___init____doc__}, + {Py_tp_methods, incrementalnewlinedecoder_methods}, + {Py_tp_getset, incrementalnewlinedecoder_getset}, + {Py_tp_traverse, incrementalnewlinedecoder_traverse}, + {Py_tp_clear, incrementalnewlinedecoder_clear}, + {Py_tp_init, _io_IncrementalNewlineDecoder___init__}, + {0, NULL}, +}; + +PyType_Spec nldecoder_spec = { + .name = "_io.IncrementalNewlineDecoder", + .basicsize = sizeof(nldecoder_object), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = nldecoder_slots, }; diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 165bd74587d7c7..b195dab9ccc83c 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -321,7 +321,6 @@ Modules/_io/bufferedio.c - PyBufferedIOBase_Type - Modules/_io/bytesio.c - _PyBytesIOBuffer_Type - Modules/_io/iobase.c - PyIOBase_Type - Modules/_io/iobase.c - PyRawIOBase_Type - -Modules/_io/textio.c - PyIncrementalNewlineDecoder_Type - Modules/_io/textio.c - PyTextIOBase_Type - Modules/_io/winconsoleio.c - PyWindowsConsoleIO_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorBase_Type - From cab1298a6022ddf12ddcdadd74bb8741650d8e9f Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 7 May 2023 11:23:11 +0200 Subject: [PATCH 090/204] gh-101819: Adapt _io.PyWindowsConsoleIO_Type to heap type (#104197) --- Modules/_io/_iomodule.c | 22 ++--- Modules/_io/_iomodule.h | 7 +- Modules/_io/winconsoleio.c | 91 ++++++++------------- Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 - 4 files changed, 51 insertions(+), 70 deletions(-) diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index ce9fcca971f99d..b72b847c663411 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -321,7 +321,7 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode, #ifdef HAVE_WINDOWS_CONSOLE_IO const PyConfig *config = _Py_GetConfig(); if (!config->legacy_windows_stdio && _PyIO_get_console_type(path_or_fd) != '\0') { - RawIO_class = (PyObject *)&PyWindowsConsoleIO_Type; + RawIO_class = (PyObject *)state->PyWindowsConsoleIO_Type; encoding = "utf-8"; } #endif @@ -595,6 +595,9 @@ iomodule_traverse(PyObject *mod, visitproc visit, void *arg) { Py_VISIT(state->PyStringIO_Type); Py_VISIT(state->PyTextIOBase_Type); Py_VISIT(state->PyTextIOWrapper_Type); +#ifdef HAVE_WINDOWS_CONSOLE_IO + Py_VISIT(state->PyWindowsConsoleIO_Type); +#endif return 0; } @@ -619,6 +622,9 @@ iomodule_clear(PyObject *mod) { Py_CLEAR(state->PyStringIO_Type); Py_CLEAR(state->PyTextIOBase_Type); Py_CLEAR(state->PyTextIOWrapper_Type); +#ifdef HAVE_WINDOWS_CONSOLE_IO + Py_CLEAR(state->PyWindowsConsoleIO_Type); +#endif return 0; } @@ -668,22 +674,12 @@ static PyTypeObject* static_types[] = { // PyRawIOBase_Type(PyIOBase_Type) subclasses &_PyBytesIOBuffer_Type, -#ifdef HAVE_WINDOWS_CONSOLE_IO - &PyWindowsConsoleIO_Type, -#endif }; PyStatus _PyIO_InitTypes(PyInterpreterState *interp) { -#ifdef HAVE_WINDOWS_CONSOLE_IO - if (_Py_IsMainInterpreter(interp)) { - // Set type base classes - PyWindowsConsoleIO_Type.tp_base = &PyRawIOBase_Type; - } -#endif - for (size_t i=0; i < Py_ARRAY_LENGTH(static_types); i++) { PyTypeObject *type = static_types[i]; if (_PyStaticType_InitBuiltin(interp, type) < 0) { @@ -777,6 +773,10 @@ PyInit__io(void) // PyRawIOBase_Type(PyIOBase_Type) subclasses state->PyBytesIOBuffer_Type = (PyTypeObject *)Py_NewRef(&_PyBytesIOBuffer_Type); ADD_TYPE(m, state->PyFileIO_Type, &fileio_spec, state->PyRawIOBase_Type); +#ifdef MS_WINDOWS + ADD_TYPE(m, state->PyWindowsConsoleIO_Type, &winconsoleio_spec, + state->PyRawIOBase_Type); +#endif // PyTextIOBase_Type(PyIOBase_Type) subclasses ADD_TYPE(m, state->PyStringIO_Type, &stringio_spec, state->PyTextIOBase_Type); diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index f191cea7fcc47b..00e6a19db2b85f 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -26,8 +26,8 @@ extern PyType_Spec stringio_spec; extern PyType_Spec textiowrapper_spec; #ifdef HAVE_WINDOWS_CONSOLE_IO -extern PyTypeObject PyWindowsConsoleIO_Type; -#endif /* HAVE_WINDOWS_CONSOLE_IO */ +extern PyType_Spec winconsoleio_spec; +#endif /* These functions are used as METH_NOARGS methods, are normally called * with args=NULL, and return a new reference. @@ -157,6 +157,9 @@ typedef struct { PyTypeObject *PyStringIO_Type; PyTypeObject *PyTextIOBase_Type; PyTypeObject *PyTextIOWrapper_Type; +#ifdef MS_WINDOWS + PyTypeObject *PyWindowsConsoleIO_Type; +#endif } _PyIO_State; #define IO_MOD_STATE(mod) ((_PyIO_State *)PyModule_GetState(mod)) diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index f836e230243020..fdb57cff7c04d6 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -137,9 +137,9 @@ char _PyIO_get_console_type(PyObject *path_or_fd) { /*[clinic input] module _io -class _io._WindowsConsoleIO "winconsoleio *" "&PyWindowsConsoleIO_Type" +class _io._WindowsConsoleIO "winconsoleio *" "clinic_state()->PyWindowsConsoleIO_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e897fdc1fba4e131]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=05526e723011ab36]*/ typedef struct { PyObject_HEAD @@ -156,8 +156,6 @@ typedef struct { wchar_t wbuf; } winconsoleio; -PyTypeObject PyWindowsConsoleIO_Type; - int _PyWindowsConsoleIO_closed(PyObject *self) { @@ -265,7 +263,10 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, int fd_is_own = 0; HANDLE handle = NULL; - assert(PyObject_TypeCheck(self, (PyTypeObject *)&PyWindowsConsoleIO_Type)); +#ifdef Py_DEBUG + _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); + assert(PyObject_TypeCheck(self, state->PyWindowsConsoleIO_Type)); +#endif if (self->fd >= 0) { if (self->closefd) { /* Have to close the existing file first. */ @@ -417,6 +418,7 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, static int winconsoleio_traverse(winconsoleio *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); Py_VISIT(self->dict); return 0; } @@ -431,6 +433,7 @@ winconsoleio_clear(winconsoleio *self) static void winconsoleio_dealloc(winconsoleio *self) { + PyTypeObject *tp = Py_TYPE(self); self->finalizing = 1; if (_PyIOBase_finalize((PyObject *) self) < 0) return; @@ -438,7 +441,8 @@ winconsoleio_dealloc(winconsoleio *self) if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) self); Py_CLEAR(self->dict); - Py_TYPE(self)->tp_free((PyObject *)self); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); } static PyObject * @@ -1078,7 +1082,9 @@ _io__WindowsConsoleIO_isatty_impl(winconsoleio *self) Py_RETURN_TRUE; } +#define clinic_state() (IO_STATE()) #include "clinic/winconsoleio.c.h" +#undef clinic_state static PyMethodDef winconsoleio_methods[] = { _IO__WINDOWSCONSOLEIO_READ_METHODDEF @@ -1124,59 +1130,32 @@ static PyGetSetDef winconsoleio_getsetlist[] = { static PyMemberDef winconsoleio_members[] = { {"_blksize", T_UINT, offsetof(winconsoleio, blksize), 0}, {"_finalizing", T_BOOL, offsetof(winconsoleio, finalizing), 0}, + {"__weaklistoffset__", T_PYSSIZET, offsetof(winconsoleio, weakreflist), READONLY}, + {"__dictoffset__", T_PYSSIZET, offsetof(winconsoleio, dict), READONLY}, {NULL} }; -PyTypeObject PyWindowsConsoleIO_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io._WindowsConsoleIO", - sizeof(winconsoleio), - 0, - (destructor)winconsoleio_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)winconsoleio_repr, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - _io__WindowsConsoleIO___init____doc__, /* tp_doc */ - (traverseproc)winconsoleio_traverse, /* tp_traverse */ - (inquiry)winconsoleio_clear, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(winconsoleio, weakreflist), /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - winconsoleio_methods, /* tp_methods */ - winconsoleio_members, /* tp_members */ - winconsoleio_getsetlist, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - offsetof(winconsoleio, dict), /* tp_dictoffset */ - _io__WindowsConsoleIO___init__, /* tp_init */ - PyType_GenericAlloc, /* tp_alloc */ - winconsoleio_new, /* tp_new */ - PyObject_GC_Del, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0, /* tp_finalize */ +static PyType_Slot winconsoleio_slots[] = { + {Py_tp_dealloc, winconsoleio_dealloc}, + {Py_tp_repr, winconsoleio_repr}, + {Py_tp_getattro, PyObject_GenericGetAttr}, + {Py_tp_doc, (void *)_io__WindowsConsoleIO___init____doc__}, + {Py_tp_traverse, winconsoleio_traverse}, + {Py_tp_clear, winconsoleio_clear}, + {Py_tp_methods, winconsoleio_methods}, + {Py_tp_members, winconsoleio_members}, + {Py_tp_getset, winconsoleio_getsetlist}, + {Py_tp_init, _io__WindowsConsoleIO___init__}, + {Py_tp_new, winconsoleio_new}, + {0, NULL}, +}; + +PyType_Spec winconsoleio_spec = { + .name = "_io._WindowsConsoleIO", + .basicsize = sizeof(winconsoleio), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = winconsoleio_slots, }; #endif /* HAVE_WINDOWS_CONSOLE_IO */ diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index b195dab9ccc83c..ffe15152448af7 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -322,7 +322,6 @@ Modules/_io/bytesio.c - _PyBytesIOBuffer_Type - Modules/_io/iobase.c - PyIOBase_Type - Modules/_io/iobase.c - PyRawIOBase_Type - Modules/_io/textio.c - PyTextIOBase_Type - -Modules/_io/winconsoleio.c - PyWindowsConsoleIO_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorBase_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorDerived_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorNopGet_Type - From a05bad3254e2ae5fdf558dfdb65899a2298d8ded Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 7 May 2023 12:55:31 +0200 Subject: [PATCH 091/204] gh-100370: fix OverflowError in sqlite3.Connection.blobopen for 32-bit builds (#103902) --- Lib/test/test_sqlite3/test_dbapi.py | 8 ++++++ ...-04-27-00-45-41.gh-issue-100370.MgZ3KY.rst | 2 ++ Modules/_sqlite/clinic/connection.c.h | 9 +++---- Modules/_sqlite/connection.c | 26 ++++++++++++++++--- 4 files changed, 36 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-27-00-45-41.gh-issue-100370.MgZ3KY.rst diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 1bb0e13e356e78..328b0467e7fa3d 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -1495,6 +1495,14 @@ def test_blob_closed_db_read(self): "Cannot operate on a closed database", blob.read) + def test_blob_32bit_rowid(self): + # gh-100370: we should not get an OverflowError for 32-bit rowids + with memory_database() as cx: + rowid = 2**32 + cx.execute("create table t(t blob)") + cx.execute("insert into t(rowid, t) values (?, zeroblob(1))", (rowid,)) + cx.blobopen('t', 't', rowid) + @threading_helper.requires_working_threading() class ThreadTests(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2023-04-27-00-45-41.gh-issue-100370.MgZ3KY.rst b/Misc/NEWS.d/next/Library/2023-04-27-00-45-41.gh-issue-100370.MgZ3KY.rst new file mode 100644 index 00000000000000..9022d55c48cb11 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-27-00-45-41.gh-issue-100370.MgZ3KY.rst @@ -0,0 +1,2 @@ +Fix potential :exc:`OverflowError` in :meth:`sqlite3.Connection.blobopen` +for 32-bit builds. Patch by Erlend E. Aasland. diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index 182754cca36d61..417abcc4626170 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -228,7 +228,7 @@ PyDoc_STRVAR(blobopen__doc__, static PyObject * blobopen_impl(pysqlite_Connection *self, const char *table, const char *col, - int row, int readonly, const char *name); + sqlite3_int64 row, int readonly, const char *name); static PyObject * blobopen(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -263,7 +263,7 @@ blobopen(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyO Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; const char *table; const char *col; - int row; + sqlite3_int64 row; int readonly = 0; const char *name = "main"; @@ -297,8 +297,7 @@ blobopen(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyO PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - row = _PyLong_AsInt(args[2]); - if (row == -1 && PyErr_Occurred()) { + if (!sqlite3_int64_converter(args[2], &row)) { goto exit; } if (!noptargs) { @@ -1666,4 +1665,4 @@ getconfig(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=8b03149c115ee6da input=a9049054013a1b77]*/ +/*[clinic end generated code: output=834a99827555bf1a input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index aec3aa8bbf4ed8..7bbb462ed54dfa 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -118,6 +118,20 @@ autocommit_converter(PyObject *val, enum autocommit_mode *result) return 0; } +static int +sqlite3_int64_converter(PyObject *obj, sqlite3_int64 *result) +{ + if (!PyLong_Check(obj)) { + PyErr_SetString(PyExc_TypeError, "expected 'int'"); + return 0; + } + *result = _pysqlite_long_as_int64(obj); + if (PyErr_Occurred()) { + return 0; + } + return 1; +} + #define clinic_state() (pysqlite_get_state_by_type(Py_TYPE(self))) #include "clinic/connection.c.h" #undef clinic_state @@ -188,8 +202,12 @@ class Autocommit_converter(CConverter): type = "enum autocommit_mode" converter = "autocommit_converter" +class sqlite3_int64_converter(CConverter): + type = "sqlite3_int64" + converter = "sqlite3_int64_converter" + [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=bc2aa6c7ba0c5f8f]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=dff8760fb1eba6a1]*/ // NB: This needs to be in sync with the sqlite3.connect docstring /*[clinic input] @@ -483,7 +501,7 @@ _sqlite3.Connection.blobopen as blobopen Table name. column as col: str Column name. - row: int + row: sqlite3_int64 Row index. / * @@ -497,8 +515,8 @@ Open and return a BLOB object. static PyObject * blobopen_impl(pysqlite_Connection *self, const char *table, const char *col, - int row, int readonly, const char *name) -/*[clinic end generated code: output=0c8e2e58516d0b5c input=fa73c83aa7a7ddee]*/ + sqlite3_int64 row, int readonly, const char *name) +/*[clinic end generated code: output=6a02d43efb885d1c input=23576bd1108d8774]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; From 7a7eaff95c7a400449822bbabd94524b8f87299c Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 7 May 2023 16:01:27 +0200 Subject: [PATCH 092/204] gh-101819: Port _io.PyBytesIOBuffer_Type to heap type (#104264) --- Modules/_io/_iomodule.c | 5 +- Modules/_io/_iomodule.h | 3 +- Modules/_io/bytesio.c | 71 ++++++++------------- Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 - 4 files changed, 29 insertions(+), 51 deletions(-) diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index b72b847c663411..c05407b5d61815 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -671,9 +671,6 @@ static PyTypeObject* static_types[] = { &PyBufferedIOBase_Type, &PyRawIOBase_Type, &PyTextIOBase_Type, - - // PyRawIOBase_Type(PyIOBase_Type) subclasses - &_PyBytesIOBuffer_Type, }; @@ -753,6 +750,7 @@ PyInit__io(void) // Base classes ADD_TYPE(m, state->PyIncrementalNewlineDecoder_Type, &nldecoder_spec, NULL); + ADD_TYPE(m, state->PyBytesIOBuffer_Type, &bytesiobuf_spec, NULL); // PyIOBase_Type subclasses state->PyRawIOBase_Type = (PyTypeObject *)Py_NewRef(&PyRawIOBase_Type); @@ -771,7 +769,6 @@ PyInit__io(void) state->PyBufferedIOBase_Type); // PyRawIOBase_Type(PyIOBase_Type) subclasses - state->PyBytesIOBuffer_Type = (PyTypeObject *)Py_NewRef(&_PyBytesIOBuffer_Type); ADD_TYPE(m, state->PyFileIO_Type, &fileio_spec, state->PyRawIOBase_Type); #ifdef MS_WINDOWS ADD_TYPE(m, state->PyWindowsConsoleIO_Type, &winconsoleio_spec, diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index 00e6a19db2b85f..1bf301c9cf0a94 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -20,6 +20,7 @@ extern PyType_Spec bufferedreader_spec; extern PyType_Spec bufferedrwpair_spec; extern PyType_Spec bufferedwriter_spec; extern PyType_Spec bytesio_spec; +extern PyType_Spec bytesiobuf_spec; extern PyType_Spec fileio_spec; extern PyType_Spec nldecoder_spec; extern PyType_Spec stringio_spec; @@ -194,5 +195,3 @@ extern _PyIO_State *_PyIO_get_module_state(void); #ifdef HAVE_WINDOWS_CONSOLE_IO extern char _PyIO_get_console_type(PyObject *); #endif - -extern Py_EXPORTED_SYMBOL PyTypeObject _PyBytesIOBuffer_Type; diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 9c7a28357987ba..3fddfc2ed0bc9c 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -1090,9 +1090,17 @@ bytesiobuf_releasebuffer(bytesiobuf *obj, Py_buffer *view) b->exports--; } +static int +bytesiobuf_clear(bytesiobuf *self) +{ + Py_CLEAR(self->source); + return 0; +} + static int bytesiobuf_traverse(bytesiobuf *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); Py_VISIT(self->source); return 0; } @@ -1100,54 +1108,29 @@ bytesiobuf_traverse(bytesiobuf *self, visitproc visit, void *arg) static void bytesiobuf_dealloc(bytesiobuf *self) { + PyTypeObject *tp = Py_TYPE(self); /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(self); - Py_CLEAR(self->source); - Py_TYPE(self)->tp_free(self); + (void)bytesiobuf_clear(self); + tp->tp_free(self); + Py_DECREF(tp); } -static PyBufferProcs bytesiobuf_as_buffer = { - (getbufferproc) bytesiobuf_getbuffer, - (releasebufferproc) bytesiobuf_releasebuffer, +static PyType_Slot bytesiobuf_slots[] = { + {Py_tp_dealloc, bytesiobuf_dealloc}, + {Py_tp_traverse, bytesiobuf_traverse}, + {Py_tp_clear, bytesiobuf_clear}, + + // Buffer protocol + {Py_bf_getbuffer, bytesiobuf_getbuffer}, + {Py_bf_releasebuffer, bytesiobuf_releasebuffer}, + {0, NULL}, }; -Py_EXPORTED_SYMBOL PyTypeObject _PyBytesIOBuffer_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io._BytesIOBuffer", /*tp_name*/ - sizeof(bytesiobuf), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)bytesiobuf_dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - &bytesiobuf_as_buffer, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - 0, /*tp_doc*/ - (traverseproc)bytesiobuf_traverse, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - 0, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - 0, /*tp_new*/ +PyType_Spec bytesiobuf_spec = { + .name = "_io._BytesIOBuffer", + .basicsize = sizeof(bytesiobuf), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION), + .slots = bytesiobuf_slots, }; diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index ffe15152448af7..453f63ec3f1cfb 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -318,7 +318,6 @@ Python/instrumentation.c - _PyInstrumentation_MISSING - ## static types Modules/_io/bufferedio.c - PyBufferedIOBase_Type - -Modules/_io/bytesio.c - _PyBytesIOBuffer_Type - Modules/_io/iobase.c - PyIOBase_Type - Modules/_io/iobase.c - PyRawIOBase_Type - Modules/_io/textio.c - PyTextIOBase_Type - From 60f588478f0a3d88e86b97acecbcb569142f4636 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 7 May 2023 17:54:40 +0100 Subject: [PATCH 093/204] GH-100479: Fix pathlib test failure on WASI (#104215) --- Lib/test/test_pathlib.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 7586610833b063..e25c77f2ba8af6 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1655,7 +1655,8 @@ class P(_BasePurePathSubclass, self.cls): p = P(BASE, session_id=42) self.assertEqual(42, p.absolute().session_id) self.assertEqual(42, p.resolve().session_id) - self.assertEqual(42, p.with_segments('~').expanduser().session_id) + if not is_wasi: # WASI has no user accounts. + self.assertEqual(42, p.with_segments('~').expanduser().session_id) self.assertEqual(42, (p / 'fileA').rename(p / 'fileB').session_id) self.assertEqual(42, (p / 'fileB').replace(p / 'fileA').session_id) if os_helper.can_symlink(): From 1b19bd1a88e6c410fc9cd08db48e0d35cfa8bb5a Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sun, 7 May 2023 18:45:09 +0100 Subject: [PATCH 094/204] gh-103193: cache calls to `inspect._shadowed_dict` in `inspect.getattr_static` (#104267) Co-authored-by: Carl Meyer --- Doc/whatsnew/3.12.rst | 7 ++++--- Lib/inspect.py | 8 ++++++-- Lib/test/test_inspect.py | 22 ++++++++++++++++++++++ 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index ec04178238b6b0..65b3e9ffb8072d 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -342,8 +342,9 @@ inspect (Contributed by Thomas Krennwallner in :issue:`35759`.) * The performance of :func:`inspect.getattr_static` has been considerably - improved. Most calls to the function should be around 2x faster than they - were in Python 3.11. (Contributed by Alex Waygood in :gh:`103193`.) + improved. Most calls to the function should be at least 2x faster than they + were in Python 3.11, and some may be 6x faster or more. (Contributed by Alex + Waygood in :gh:`103193`.) pathlib ------- @@ -597,7 +598,7 @@ typing :func:`runtime-checkable protocols ` has changed significantly. Most ``isinstance()`` checks against protocols with only a few members should be at least 2x faster than in 3.11, and some may be 20x - faster or more. However, ``isinstance()`` checks against protocols with seven + faster or more. However, ``isinstance()`` checks against protocols with fourteen or more members may be slower than in Python 3.11. (Contributed by Alex Waygood in :gh:`74690` and :gh:`103193`.) diff --git a/Lib/inspect.py b/Lib/inspect.py index 95da7fb71a3997..a64e85e4fd67a4 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1794,8 +1794,9 @@ def _check_class(klass, attr): return entry.__dict__[attr] return _sentinel -def _shadowed_dict(klass): - for entry in _static_getmro(klass): +@functools.lru_cache() +def _shadowed_dict_from_mro_tuple(mro): + for entry in mro: dunder_dict = _get_dunder_dict_of_class(entry) if '__dict__' in dunder_dict: class_dict = dunder_dict['__dict__'] @@ -1805,6 +1806,9 @@ def _shadowed_dict(klass): return class_dict return _sentinel +def _shadowed_dict(klass): + return _shadowed_dict_from_mro_tuple(_static_getmro(klass)) + def getattr_static(obj, attr, default=_sentinel): """Retrieve attributes without triggering dynamic lookup via the descriptor protocol, __getattr__ or __getattribute__. diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 42e3d709bd683f..dd0325a43e0f58 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -2111,6 +2111,28 @@ def __dict__(self): self.assertEqual(inspect.getattr_static(foo, 'a'), 3) self.assertFalse(test.called) + def test_mutated_mro(self): + test = self + test.called = False + + class Foo(dict): + a = 3 + @property + def __dict__(self): + test.called = True + return {} + + class Bar(dict): + a = 4 + + class Baz(Bar): pass + + baz = Baz() + self.assertEqual(inspect.getattr_static(baz, 'a'), 4) + Baz.__bases__ = (Foo,) + self.assertEqual(inspect.getattr_static(baz, 'a'), 3) + self.assertFalse(test.called) + def test_custom_object_dict(self): test = self test.called = False From 2c2dc61e8d01f44e5b7e63dd99196460a80905f1 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Sun, 7 May 2023 18:47:28 +0100 Subject: [PATCH 095/204] gh-104240: make _PyCompile_CodeGen support different compilation modes (#104241) --- Include/internal/pycore_compile.h | 3 +- .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 3 ++ Lib/test/test_compiler_codegen.py | 4 +++ Modules/_testinternalcapi.c | 8 +++-- Modules/clinic/_testinternalcapi.c.h | 29 +++++++++++++------ Python/compile.c | 8 +++-- 9 files changed, 43 insertions(+), 15 deletions(-) diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 4bd4ef57238f98..d2b12c91fe7a00 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -97,7 +97,8 @@ PyAPI_FUNC(PyObject*) _PyCompile_CodeGen( PyObject *ast, PyObject *filename, PyCompilerFlags *flags, - int optimize); + int optimize, + int compile_mode); PyAPI_FUNC(PyObject*) _PyCompile_OptimizeCfg( PyObject *instructions, diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 9377fd8526e3a2..7e495817981f06 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -847,6 +847,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(command)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(comment_factory)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(compile_mode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(consts)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(context)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cookie)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index ed9b2bb44ddffc..8ebfee85c87c23 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -335,6 +335,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(code) STRUCT_FOR_ID(command) STRUCT_FOR_ID(comment_factory) + STRUCT_FOR_ID(compile_mode) STRUCT_FOR_ID(consts) STRUCT_FOR_ID(context) STRUCT_FOR_ID(cookie) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 6ade8fb6eade03..7b9c73dd1edf3b 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -841,6 +841,7 @@ extern "C" { INIT_ID(code), \ INIT_ID(command), \ INIT_ID(comment_factory), \ + INIT_ID(compile_mode), \ INIT_ID(consts), \ INIT_ID(context), \ INIT_ID(cookie), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 0b33ea187e60ff..8e086edbdf8193 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -858,6 +858,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(comment_factory); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(compile_mode); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(consts); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index 022753e0c99483..ea57df9cd2400b 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -25,6 +25,8 @@ def test_if_expression(self): ('LOAD_CONST', 2, 1), exit_lbl, ('POP_TOP', None), + ('LOAD_CONST', 3), + ('RETURN_VALUE', None), ] self.codegen_test(snippet, expected) @@ -46,5 +48,7 @@ def test_for_loop(self): ('JUMP', loop_lbl), exit_lbl, ('END_FOR', None), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), ] self.codegen_test(snippet, expected) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index f35e3b48df9321..40ad6f88868daf 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -593,17 +593,19 @@ _testinternalcapi.compiler_codegen -> object ast: object filename: object optimize: int + compile_mode: int = 0 Apply compiler code generation to an AST. [clinic start generated code]*/ static PyObject * _testinternalcapi_compiler_codegen_impl(PyObject *module, PyObject *ast, - PyObject *filename, int optimize) -/*[clinic end generated code: output=fbbbbfb34700c804 input=e9fbe6562f7f75e4]*/ + PyObject *filename, int optimize, + int compile_mode) +/*[clinic end generated code: output=40a68f6e13951cc8 input=a0e00784f1517cd7]*/ { PyCompilerFlags *flags = NULL; - return _PyCompile_CodeGen(ast, filename, flags, optimize); + return _PyCompile_CodeGen(ast, filename, flags, optimize, compile_mode); } diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index 89573222572594..41dd50437956c4 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -9,7 +9,7 @@ preserve PyDoc_STRVAR(_testinternalcapi_compiler_codegen__doc__, -"compiler_codegen($module, /, ast, filename, optimize)\n" +"compiler_codegen($module, /, ast, filename, optimize, compile_mode=0)\n" "--\n" "\n" "Apply compiler code generation to an AST."); @@ -19,7 +19,8 @@ PyDoc_STRVAR(_testinternalcapi_compiler_codegen__doc__, static PyObject * _testinternalcapi_compiler_codegen_impl(PyObject *module, PyObject *ast, - PyObject *filename, int optimize); + PyObject *filename, int optimize, + int compile_mode); static PyObject * _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -27,14 +28,14 @@ _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_s PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 3 + #define NUM_KEYWORDS 4 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(ast), &_Py_ID(filename), &_Py_ID(optimize), }, + .ob_item = { &_Py_ID(ast), &_Py_ID(filename), &_Py_ID(optimize), &_Py_ID(compile_mode), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -43,19 +44,21 @@ _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_s # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"ast", "filename", "optimize", NULL}; + static const char * const _keywords[] = {"ast", "filename", "optimize", "compile_mode", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "compiler_codegen", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; PyObject *ast; PyObject *filename; int optimize; + int compile_mode = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 4, 0, argsbuf); if (!args) { goto exit; } @@ -65,7 +68,15 @@ _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_s if (optimize == -1 && PyErr_Occurred()) { goto exit; } - return_value = _testinternalcapi_compiler_codegen_impl(module, ast, filename, optimize); + if (!noptargs) { + goto skip_optional_pos; + } + compile_mode = _PyLong_AsInt(args[3]); + if (compile_mode == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_pos: + return_value = _testinternalcapi_compiler_codegen_impl(module, ast, filename, optimize, compile_mode); exit: return return_value; @@ -190,4 +201,4 @@ _testinternalcapi_assemble_code_object(PyObject *module, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=d5e08c9d67f9721f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ab661d56a14b1a1c input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index cbe5403aafbc87..f875e4e17e0a9f 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -7258,7 +7258,7 @@ cfg_to_instructions(cfg_builder *g) PyObject * _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, - int optimize) + int optimize, int compile_mode) { PyObject *res = NULL; @@ -7272,7 +7272,7 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, return NULL; } - mod_ty mod = PyAST_obj2mod(ast, arena, 0 /* exec */); + mod_ty mod = PyAST_obj2mod(ast, arena, compile_mode); if (mod == NULL || !_PyAST_Validate(mod)) { _PyArena_Free(arena); return NULL; @@ -7287,6 +7287,10 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, if (compiler_codegen(c, mod) < 0) { goto finally; } + int addNone = mod->kind != Expression_kind; + if (add_return_at_end(c, addNone) < 0) { + return NULL; + } res = instr_sequence_to_instructions(INSTR_SEQUENCE(c)); From e8d77b03e08a4c7e7dde0830c5a12a0b41ff7c33 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 7 May 2023 20:07:07 +0100 Subject: [PATCH 096/204] GH-89812: Churn `pathlib.Path` methods (GH-104243) Re-arrange `pathlib.Path` methods in source code. No other changes. The methods are arranged as follows: 1. `stat()` and dependants (`exists()`, `is_dir()`, etc) 2. `open()` and dependants (`read_text()`, `write_bytes()`, etc) 3. `iterdir()` and dependants (`glob()`, `walk()`, etc) 4. All other `Path` methods This patch prepares the ground for a new `_AbstractPath` class, which will support the methods in groups 1, 2 and 3 above. By churning the methods here, subsequent patches will be easier to review and less likely to break things. --- Lib/pathlib.py | 606 ++++++++++++++++++++++++------------------------- 1 file changed, 303 insertions(+), 303 deletions(-) diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 480c354ce8b656..68255aa3e511ec 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -740,68 +740,164 @@ class Path(PurePath): """ __slots__ = () - def __init__(self, *args, **kwargs): - if kwargs: - msg = ("support for supplying keyword arguments to pathlib.PurePath " - "is deprecated and scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) - super().__init__(*args) + def stat(self, *, follow_symlinks=True): + """ + Return the result of the stat() system call on this path, like + os.stat() does. + """ + return os.stat(self, follow_symlinks=follow_symlinks) - def __new__(cls, *args, **kwargs): - if cls is Path: - cls = WindowsPath if os.name == 'nt' else PosixPath - return object.__new__(cls) + def lstat(self): + """ + Like stat(), except if the path points to a symlink, the symlink's + status information is returned, rather than its target's. + """ + return self.stat(follow_symlinks=False) - def _make_child_relpath(self, name): - path_str = str(self) - tail = self._tail - if tail: - path_str = f'{path_str}{self._flavour.sep}{name}' - elif path_str != '.': - path_str = f'{path_str}{name}' - else: - path_str = name - path = self.with_segments(path_str) - path._str = path_str - path._drv = self.drive - path._root = self.root - path._tail_cached = tail + [name] - return path - def __enter__(self): - # In previous versions of pathlib, __exit__() marked this path as - # closed; subsequent attempts to perform I/O would raise an IOError. - # This functionality was never documented, and had the effect of - # making Path objects mutable, contrary to PEP 428. - # In Python 3.9 __exit__() was made a no-op. - # In Python 3.11 __enter__() began emitting DeprecationWarning. - # In Python 3.13 __enter__() and __exit__() should be removed. - warnings.warn("pathlib.Path.__enter__() is deprecated and scheduled " - "for removal in Python 3.13; Path objects as a context " - "manager is a no-op", - DeprecationWarning, stacklevel=2) - return self + # Convenience functions for querying the stat results - def __exit__(self, t, v, tb): - pass + def exists(self, *, follow_symlinks=True): + """ + Whether this path exists. - # Public API + This method normally follows symlinks; to check whether a symlink exists, + add the argument follow_symlinks=False. + """ + try: + self.stat(follow_symlinks=follow_symlinks) + except OSError as e: + if not _ignore_error(e): + raise + return False + except ValueError: + # Non-encodable path + return False + return True - @classmethod - def cwd(cls): - """Return a new path pointing to the current working directory.""" - # We call 'absolute()' rather than using 'os.getcwd()' directly to - # enable users to replace the implementation of 'absolute()' in a - # subclass and benefit from the new behaviour here. This works because - # os.path.abspath('.') == os.getcwd(). - return cls().absolute() + def is_dir(self): + """ + Whether this path is a directory. + """ + try: + return S_ISDIR(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False - @classmethod - def home(cls): - """Return a new path pointing to the user's home directory (as - returned by os.path.expanduser('~')). + def is_file(self): """ - return cls("~").expanduser() + Whether this path is a regular file (also True for symlinks pointing + to regular files). + """ + try: + return S_ISREG(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_mount(self): + """ + Check if this path is a mount point + """ + return self._flavour.ismount(self) + + def is_symlink(self): + """ + Whether this path is a symbolic link. + """ + try: + return S_ISLNK(self.lstat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist + return False + except ValueError: + # Non-encodable path + return False + + def is_junction(self): + """ + Whether this path is a junction. + """ + return self._flavour.isjunction(self) + + def is_block_device(self): + """ + Whether this path is a block device. + """ + try: + return S_ISBLK(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_char_device(self): + """ + Whether this path is a character device. + """ + try: + return S_ISCHR(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_fifo(self): + """ + Whether this path is a FIFO. + """ + try: + return S_ISFIFO(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_socket(self): + """ + Whether this path is a socket. + """ + try: + return S_ISSOCK(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False def samefile(self, other_path): """Return whether other_path is the same or not as this file @@ -814,6 +910,51 @@ def samefile(self, other_path): other_st = self.with_segments(other_path).stat() return self._flavour.samestat(st, other_st) + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + """ + Open the file pointed by this path and return a file object, as + the built-in open() function does. + """ + if "b" not in mode: + encoding = io.text_encoding(encoding) + return io.open(self, mode, buffering, encoding, errors, newline) + + def read_bytes(self): + """ + Open the file in bytes mode, read it, and close the file. + """ + with self.open(mode='rb') as f: + return f.read() + + def read_text(self, encoding=None, errors=None): + """ + Open the file in text mode, read it, and close the file. + """ + encoding = io.text_encoding(encoding) + with self.open(mode='r', encoding=encoding, errors=errors) as f: + return f.read() + + def write_bytes(self, data): + """ + Open the file in bytes mode, write to it, and close the file. + """ + # type-check for the buffer interface before truncating the file + view = memoryview(data) + with self.open(mode='wb') as f: + return f.write(view) + + def write_text(self, data, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, write to it, and close the file. + """ + if not isinstance(data, str): + raise TypeError('data must be str, not %s' % + data.__class__.__name__) + encoding = io.text_encoding(encoding) + with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: + return f.write(data) + def iterdir(self): """Yield path objects of the directory contents. @@ -829,6 +970,22 @@ def _scandir(self): # includes scandir(), which is used to implement glob(). return os.scandir(self) + def _make_child_relpath(self, name): + path_str = str(self) + tail = self._tail + if tail: + path_str = f'{path_str}{self._flavour.sep}{name}' + elif path_str != '.': + path_str = f'{path_str}{name}' + else: + path_str = name + path = self.with_segments(path_str) + path._str = path_str + path._drv = self.drive + path._root = self.root + path._tail_cached = tail + [name] + return path + def glob(self, pattern, *, case_sensitive=None): """Iterate over this subtree and yield all existing files (of any kind, including directories) matching the given relative pattern. @@ -860,6 +1017,98 @@ def rglob(self, pattern, *, case_sensitive=None): for p in selector.select_from(self): yield p + def walk(self, top_down=True, on_error=None, follow_symlinks=False): + """Walk the directory tree from this directory, similar to os.walk().""" + sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) + paths = [self] + + while paths: + path = paths.pop() + if isinstance(path, tuple): + yield path + continue + + # We may not have read permission for self, in which case we can't + # get a list of the files the directory contains. os.walk() + # always suppressed the exception in that instance, rather than + # blow up for a minor reason when (say) a thousand readable + # directories are still left to visit. That logic is copied here. + try: + scandir_it = path._scandir() + except OSError as error: + if on_error is not None: + on_error(error) + continue + + with scandir_it: + dirnames = [] + filenames = [] + for entry in scandir_it: + try: + is_dir = entry.is_dir(follow_symlinks=follow_symlinks) + except OSError: + # Carried over from os.path.isdir(). + is_dir = False + + if is_dir: + dirnames.append(entry.name) + else: + filenames.append(entry.name) + + if top_down: + yield path, dirnames, filenames + else: + paths.append((path, dirnames, filenames)) + + paths += [path._make_child_relpath(d) for d in reversed(dirnames)] + + def __init__(self, *args, **kwargs): + if kwargs: + msg = ("support for supplying keyword arguments to pathlib.PurePath " + "is deprecated and scheduled for removal in Python {remove}") + warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) + super().__init__(*args) + + def __new__(cls, *args, **kwargs): + if cls is Path: + cls = WindowsPath if os.name == 'nt' else PosixPath + return object.__new__(cls) + + def __enter__(self): + # In previous versions of pathlib, __exit__() marked this path as + # closed; subsequent attempts to perform I/O would raise an IOError. + # This functionality was never documented, and had the effect of + # making Path objects mutable, contrary to PEP 428. + # In Python 3.9 __exit__() was made a no-op. + # In Python 3.11 __enter__() began emitting DeprecationWarning. + # In Python 3.13 __enter__() and __exit__() should be removed. + warnings.warn("pathlib.Path.__enter__() is deprecated and scheduled " + "for removal in Python 3.13; Path objects as a context " + "manager is a no-op", + DeprecationWarning, stacklevel=2) + return self + + def __exit__(self, t, v, tb): + pass + + # Public API + + @classmethod + def cwd(cls): + """Return a new path pointing to the current working directory.""" + # We call 'absolute()' rather than using 'os.getcwd()' directly to + # enable users to replace the implementation of 'absolute()' in a + # subclass and benefit from the new behaviour here. This works because + # os.path.abspath('.') == os.getcwd(). + return cls().absolute() + + @classmethod + def home(cls): + """Return a new path pointing to the user's home directory (as + returned by os.path.expanduser('~')). + """ + return cls("~").expanduser() + def absolute(self): """Return an absolute version of this path by prepending the current working directory. No normalization or symlink resolution is performed. @@ -911,13 +1160,6 @@ def check_eloop(e): check_eloop(e) return p - def stat(self, *, follow_symlinks=True): - """ - Return the result of the stat() system call on this path, like - os.stat() does. - """ - return os.stat(self, follow_symlinks=follow_symlinks) - def owner(self): """ Return the login name of the file owner. @@ -939,51 +1181,6 @@ def group(self): except ImportError: raise NotImplementedError("Path.group() is unsupported on this system") - def open(self, mode='r', buffering=-1, encoding=None, - errors=None, newline=None): - """ - Open the file pointed by this path and return a file object, as - the built-in open() function does. - """ - if "b" not in mode: - encoding = io.text_encoding(encoding) - return io.open(self, mode, buffering, encoding, errors, newline) - - def read_bytes(self): - """ - Open the file in bytes mode, read it, and close the file. - """ - with self.open(mode='rb') as f: - return f.read() - - def read_text(self, encoding=None, errors=None): - """ - Open the file in text mode, read it, and close the file. - """ - encoding = io.text_encoding(encoding) - with self.open(mode='r', encoding=encoding, errors=errors) as f: - return f.read() - - def write_bytes(self, data): - """ - Open the file in bytes mode, write to it, and close the file. - """ - # type-check for the buffer interface before truncating the file - view = memoryview(data) - with self.open(mode='wb') as f: - return f.write(view) - - def write_text(self, data, encoding=None, errors=None, newline=None): - """ - Open the file in text mode, write to it, and close the file. - """ - if not isinstance(data, str): - raise TypeError('data must be str, not %s' % - data.__class__.__name__) - encoding = io.text_encoding(encoding) - with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: - return f.write(data) - def readlink(self): """ Return the path to which the symbolic link points. @@ -1061,13 +1258,6 @@ def rmdir(self): """ os.rmdir(self) - def lstat(self): - """ - Like stat(), except if the path points to a symlink, the symlink's - status information is returned, rather than its target's. - """ - return self.stat(follow_symlinks=False) - def rename(self, target): """ Rename this path to the target path. @@ -1113,151 +1303,6 @@ def hardlink_to(self, target): raise NotImplementedError("os.link() not available on this system") os.link(target, self) - - # Convenience functions for querying the stat results - - def exists(self, *, follow_symlinks=True): - """ - Whether this path exists. - - This method normally follows symlinks; to check whether a symlink exists, - add the argument follow_symlinks=False. - """ - try: - self.stat(follow_symlinks=follow_symlinks) - except OSError as e: - if not _ignore_error(e): - raise - return False - except ValueError: - # Non-encodable path - return False - return True - - def is_dir(self): - """ - Whether this path is a directory. - """ - try: - return S_ISDIR(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_file(self): - """ - Whether this path is a regular file (also True for symlinks pointing - to regular files). - """ - try: - return S_ISREG(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_mount(self): - """ - Check if this path is a mount point - """ - return self._flavour.ismount(self) - - def is_symlink(self): - """ - Whether this path is a symbolic link. - """ - try: - return S_ISLNK(self.lstat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist - return False - except ValueError: - # Non-encodable path - return False - - def is_junction(self): - """ - Whether this path is a junction. - """ - return self._flavour.isjunction(self) - - def is_block_device(self): - """ - Whether this path is a block device. - """ - try: - return S_ISBLK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_char_device(self): - """ - Whether this path is a character device. - """ - try: - return S_ISCHR(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_fifo(self): - """ - Whether this path is a FIFO. - """ - try: - return S_ISFIFO(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_socket(self): - """ - Whether this path is a socket. - """ - try: - return S_ISSOCK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - def expanduser(self): """ Return a new path with expanded ~ and ~user constructs (as returned by os.path.expanduser) @@ -1272,51 +1317,6 @@ def expanduser(self): return self - def walk(self, top_down=True, on_error=None, follow_symlinks=False): - """Walk the directory tree from this directory, similar to os.walk().""" - sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) - paths = [self] - - while paths: - path = paths.pop() - if isinstance(path, tuple): - yield path - continue - - # We may not have read permission for self, in which case we can't - # get a list of the files the directory contains. os.walk() - # always suppressed the exception in that instance, rather than - # blow up for a minor reason when (say) a thousand readable - # directories are still left to visit. That logic is copied here. - try: - scandir_it = path._scandir() - except OSError as error: - if on_error is not None: - on_error(error) - continue - - with scandir_it: - dirnames = [] - filenames = [] - for entry in scandir_it: - try: - is_dir = entry.is_dir(follow_symlinks=follow_symlinks) - except OSError: - # Carried over from os.path.isdir(). - is_dir = False - - if is_dir: - dirnames.append(entry.name) - else: - filenames.append(entry.name) - - if top_down: - yield path, dirnames, filenames - else: - paths.append((path, dirnames, filenames)) - - paths += [path._make_child_relpath(d) for d in reversed(dirnames)] - class PosixPath(Path, PurePosixPath): """Path subclass for non-Windows systems. From 8d95012c95988dc517db6e09348aab996868699c Mon Sep 17 00:00:00 2001 From: Arthur Pastel Date: Sun, 7 May 2023 21:42:26 +0200 Subject: [PATCH 097/204] gh-103650: Fix perf maps address format (#103651) --- Lib/test/test_perf_profiler.py | 12 +++++++++--- .../2023-04-20-16-17-51.gh-issue-103650.K1MFXR.rst | 1 + Python/perf_trampoline.c | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-20-16-17-51.gh-issue-103650.K1MFXR.rst diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index 2b977d78d39324..5418f9f35485f8 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -1,4 +1,5 @@ import unittest +import string import subprocess import sys import sysconfig @@ -70,9 +71,14 @@ def baz(): perf_file = pathlib.Path(f"/tmp/perf-{process.pid}.map") self.assertTrue(perf_file.exists()) perf_file_contents = perf_file.read_text() - self.assertIn(f"py::foo:{script}", perf_file_contents) - self.assertIn(f"py::bar:{script}", perf_file_contents) - self.assertIn(f"py::baz:{script}", perf_file_contents) + perf_lines = perf_file_contents.splitlines(); + expected_symbols = [f"py::foo:{script}", f"py::bar:{script}", f"py::baz:{script}"] + for expected_symbol in expected_symbols: + perf_line = next((line for line in perf_lines if expected_symbol in line), None) + self.assertIsNotNone(perf_line, f"Could not find {expected_symbol} in perf file") + perf_addr = perf_line.split(" ")[0] + self.assertFalse(perf_addr.startswith("0x"), "Address should not be prefixed with 0x") + self.assertTrue(set(perf_addr).issubset(string.hexdigits), "Address should contain only hex characters") def test_trampoline_works_with_forks(self): code = """if 1: diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-20-16-17-51.gh-issue-103650.K1MFXR.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-20-16-17-51.gh-issue-103650.K1MFXR.rst new file mode 100644 index 00000000000000..5434660e9d6ffb --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-04-20-16-17-51.gh-issue-103650.K1MFXR.rst @@ -0,0 +1 @@ +Change the perf map format to remove the '0x' prefix from the addresses diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 1957ab82c33951..3b183280e1f24c 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -253,7 +253,7 @@ perf_map_write_entry(void *state, const void *code_addr, NULL); return; } - fprintf(method_file, "%p %x py::%s:%s\n", code_addr, code_size, entry, + fprintf(method_file, "%" PRIxPTR " %x py::%s:%s\n", (uintptr_t) code_addr, code_size, entry, filename); fflush(method_file); } From c0ece3dc9791694e960952ba74070efaaa79a676 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 7 May 2023 22:12:50 +0100 Subject: [PATCH 098/204] GH-102613: Improve performance of `pathlib.Path.rglob()` (GH-104244) Stop de-duplicating results in `_RecursiveWildcardSelector`. A new `_DoubleRecursiveWildcardSelector` class is introduced which performs de-duplication, but this is used _only_ for patterns with multiple non-adjacent `**` segments, such as `path.glob('**/foo/**')`. By avoiding the use of a set, `PurePath.__hash__()` is not called, and so paths do not need to be stringified and case-normalised. Also merge adjacent '**' segments in patterns. --- Lib/pathlib.py | 54 +++++++++++++------ Lib/test/test_pathlib.py | 6 ++- ...-05-06-20-37-46.gh-issue-102613.QZG9iX.rst | 3 ++ 3 files changed, 45 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-06-20-37-46.gh-issue-102613.QZG9iX.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 68255aa3e511ec..20ec1ce9d80374 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -64,17 +64,25 @@ def _is_case_sensitive(flavour): @functools.lru_cache() def _make_selector(pattern_parts, flavour, case_sensitive): pat = pattern_parts[0] - child_parts = pattern_parts[1:] if not pat: return _TerminatingSelector() if pat == '**': - cls = _RecursiveWildcardSelector - elif pat == '..': - cls = _ParentSelector - elif '**' in pat: - raise ValueError("Invalid pattern: '**' can only be an entire path component") + child_parts_idx = 1 + while child_parts_idx < len(pattern_parts) and pattern_parts[child_parts_idx] == '**': + child_parts_idx += 1 + child_parts = pattern_parts[child_parts_idx:] + if '**' in child_parts: + cls = _DoubleRecursiveWildcardSelector + else: + cls = _RecursiveWildcardSelector else: - cls = _WildcardSelector + child_parts = pattern_parts[1:] + if pat == '..': + cls = _ParentSelector + elif '**' in pat: + raise ValueError("Invalid pattern: '**' can only be an entire path component") + else: + cls = _WildcardSelector return cls(pat, child_parts, flavour, case_sensitive) @@ -183,20 +191,32 @@ def _iterate_directories(self, parent_path, scandir): def _select_from(self, parent_path, scandir): try: - yielded = set() - try: - successor_select = self.successor._select_from - for starting_point in self._iterate_directories(parent_path, scandir): - for p in successor_select(starting_point, scandir): - if p not in yielded: - yield p - yielded.add(p) - finally: - yielded.clear() + successor_select = self.successor._select_from + for starting_point in self._iterate_directories(parent_path, scandir): + for p in successor_select(starting_point, scandir): + yield p except PermissionError: return +class _DoubleRecursiveWildcardSelector(_RecursiveWildcardSelector): + """ + Like _RecursiveWildcardSelector, but also de-duplicates results from + successive selectors. This is necessary if the pattern contains + multiple non-adjacent '**' segments. + """ + + def _select_from(self, parent_path, scandir): + yielded = set() + try: + for p in super()._select_from(parent_path, scandir): + if p not in yielded: + yield p + yielded.add(p) + finally: + yielded.clear() + + # # Public API # diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index e25c77f2ba8af6..ee0ef9a34c385c 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1853,13 +1853,14 @@ def _check(path, pattern, case_sensitive, expected): def test_rglob_common(self): def _check(glob, expected): - self.assertEqual(set(glob), { P(BASE, q) for q in expected }) + self.assertEqual(sorted(glob), sorted(P(BASE, q) for q in expected)) P = self.cls p = P(BASE) it = p.rglob("fileA") self.assertIsInstance(it, collections.abc.Iterator) _check(it, ["fileA"]) _check(p.rglob("fileB"), ["dirB/fileB"]) + _check(p.rglob("**/fileB"), ["dirB/fileB"]) _check(p.rglob("*/fileA"), []) if not os_helper.can_symlink(): _check(p.rglob("*/fileB"), ["dirB/fileB"]) @@ -1883,9 +1884,12 @@ def _check(glob, expected): _check(p.rglob("*"), ["dirC/fileC", "dirC/novel.txt", "dirC/dirD", "dirC/dirD/fileD"]) _check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p.rglob("**/file*"), ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p.rglob("dir*/**"), ["dirC/dirD"]) _check(p.rglob("*/*"), ["dirC/dirD/fileD"]) _check(p.rglob("*/"), ["dirC/dirD"]) _check(p.rglob(""), ["dirC", "dirC/dirD"]) + _check(p.rglob("**"), ["dirC", "dirC/dirD"]) # gh-91616, a re module regression _check(p.rglob("*.txt"), ["dirC/novel.txt"]) _check(p.rglob("*.*"), ["dirC/novel.txt"]) diff --git a/Misc/NEWS.d/next/Library/2023-05-06-20-37-46.gh-issue-102613.QZG9iX.rst b/Misc/NEWS.d/next/Library/2023-05-06-20-37-46.gh-issue-102613.QZG9iX.rst new file mode 100644 index 00000000000000..01f8b948d2cb65 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-06-20-37-46.gh-issue-102613.QZG9iX.rst @@ -0,0 +1,3 @@ +Improve performance of :meth:`pathlib.Path.glob` when expanding recursive +wildcards ("``**``") by merging adjacent wildcards and de-duplicating +results only when necessary. From 06c2a4858b8806abc700a0471434067910db54ec Mon Sep 17 00:00:00 2001 From: chgnrdv <52372310+chgnrdv@users.noreply.github.com> Date: Mon, 8 May 2023 00:15:44 +0300 Subject: [PATCH 099/204] gh-104265 Disallow instantiation of `_csv.Reader` and `_csv.Writer` (#104266) --- Lib/test/test_csv.py | 9 ++++++++- .../2023-05-07-19-56-45.gh-issue-104265.fVblry.rst | 4 ++++ Modules/_csv.c | 4 ++-- 3 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-07-19-56-45.gh-issue-104265.fVblry.rst diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 8fb97bc0c1a1a7..de7ac97d72cb8e 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -10,7 +10,7 @@ import gc import pickle from test import support -from test.support import warnings_helper +from test.support import warnings_helper, import_helper, check_disallow_instantiation from itertools import permutations from textwrap import dedent from collections import OrderedDict @@ -1430,5 +1430,12 @@ def test_subclassable(self): # issue 44089 class Foo(csv.Error): ... + @support.cpython_only + def test_disallow_instantiation(self): + _csv = import_helper.import_module("_csv") + for tp in _csv.Reader, _csv.Writer: + with self.subTest(tp=tp): + check_disallow_instantiation(self, tp) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2023-05-07-19-56-45.gh-issue-104265.fVblry.rst b/Misc/NEWS.d/next/Library/2023-05-07-19-56-45.gh-issue-104265.fVblry.rst new file mode 100644 index 00000000000000..9c582844bf909b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-07-19-56-45.gh-issue-104265.fVblry.rst @@ -0,0 +1,4 @@ +Prevent possible crash by disallowing instantiation of the +:class:`!_csv.Reader` and :class:`!_csv.Writer` types. +The regression was introduced in 3.10.0a4 with PR 23224 (:issue:`14935`). +Patch by Radislav Chugunov. diff --git a/Modules/_csv.c b/Modules/_csv.c index 0cde5c5a8bdc68..9ab2ad266c2739 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -1000,7 +1000,7 @@ PyType_Spec Reader_Type_spec = { .name = "_csv.reader", .basicsize = sizeof(ReaderObj), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION), .slots = Reader_Type_slots }; @@ -1431,7 +1431,7 @@ PyType_Spec Writer_Type_spec = { .name = "_csv.writer", .basicsize = sizeof(WriterObj), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION), .slots = Writer_Type_slots, }; From ac020624b32820e8e6e272122b94883f8e75ac61 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 7 May 2023 23:55:37 +0200 Subject: [PATCH 100/204] gh-64660: Don't hardcode Argument Clinic return converter result variable name (#104200) --- Tools/clinic/clinic.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 704325670bc994..a6f330d1502dad 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -356,6 +356,7 @@ def __init__(self): # you should check the _return_value for errors, and # "goto exit" if there are any. self.return_conversion = [] + self.converter_retval = "_return_value" # The C statements required to do some operations # after the end of parsing but before cleaning up. @@ -3893,15 +3894,15 @@ def __init__(self, *, py_default=None, **kwargs): def return_converter_init(self): pass - def declare(self, data, name="_return_value"): + def declare(self, data): line = [] add = line.append add(self.type) if not self.type.endswith('*'): add(' ') - add(name + ';') + add(data.converter_retval + ';') data.declarations.append(''.join(line)) - data.return_value = name + data.return_value = data.converter_retval def err_occurred_if(self, expr, data): data.return_conversion.append('if (({}) && PyErr_Occurred()) {{\n goto exit;\n}}\n'.format(expr)) @@ -3923,8 +3924,10 @@ class bool_return_converter(CReturnConverter): def render(self, function, data): self.declare(data) - self.err_occurred_if("_return_value == -1", data) - data.return_conversion.append('return_value = PyBool_FromLong((long)_return_value);\n') + self.err_occurred_if(f"{data.converter_retval} == -1", data) + data.return_conversion.append( + f'return_value = PyBool_FromLong((long){data.converter_retval});\n' + ) class long_return_converter(CReturnConverter): type = 'long' @@ -3934,9 +3937,10 @@ class long_return_converter(CReturnConverter): def render(self, function, data): self.declare(data) - self.err_occurred_if("_return_value == {}-1".format(self.unsigned_cast), data) + self.err_occurred_if(f"{data.converter_retval} == {self.unsigned_cast}-1", data) data.return_conversion.append( - ''.join(('return_value = ', self.conversion_fn, '(', self.cast, '_return_value);\n'))) + f'return_value = {self.conversion_fn}({self.cast}{data.converter_retval});\n' + ) class int_return_converter(long_return_converter): type = 'int' @@ -3978,9 +3982,10 @@ class double_return_converter(CReturnConverter): def render(self, function, data): self.declare(data) - self.err_occurred_if("_return_value == -1.0", data) + self.err_occurred_if(f"{data.converter_retval} == -1.0", data) data.return_conversion.append( - 'return_value = PyFloat_FromDouble(' + self.cast + '_return_value);\n') + f'return_value = PyFloat_FromDouble({self.cast}{data.converter_retval});\n' + ) class float_return_converter(double_return_converter): type = 'float' From 01cc9c1ff79bf18fe34c05c6cd573e79ff9487c3 Mon Sep 17 00:00:00 2001 From: Burak Saler <59198732+buraksaler@users.noreply.github.com> Date: Mon, 8 May 2023 02:43:50 +0300 Subject: [PATCH 101/204] gh-104273: Remove redundant len() calls in argparse function (#104274) --- Lib/argparse.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/argparse.py b/Lib/argparse.py index 68089a5c1e80b0..f5f44ff02c0d38 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -345,21 +345,22 @@ def _format_usage(self, usage, actions, groups, prefix): def get_lines(parts, indent, prefix=None): lines = [] line = [] + indent_length = len(indent) if prefix is not None: line_len = len(prefix) - 1 else: - line_len = len(indent) - 1 + line_len = indent_length - 1 for part in parts: if line_len + 1 + len(part) > text_width and line: lines.append(indent + ' '.join(line)) line = [] - line_len = len(indent) - 1 + line_len = indent_length - 1 line.append(part) line_len += len(part) + 1 if line: lines.append(indent + ' '.join(line)) if prefix is not None: - lines[0] = lines[0][len(indent):] + lines[0] = lines[0][indent_length:] return lines # if prog is short, follow it with optionals or positionals From 15665d896bae9c3d8b60bd7210ac1b7dc533b093 Mon Sep 17 00:00:00 2001 From: Jonathan Protzenko Date: Sun, 7 May 2023 20:50:04 -0700 Subject: [PATCH 102/204] gh-99108: Replace SHA3 implementation HACL* version (#103597) Replaces our built-in SHA3 implementation with a verified one from the HACL* project. This implementation is used when OpenSSL does not provide SHA3 or is not present. 3.11 shiped with a very slow tiny sha3 implementation to get off of the <=3.10 reference implementation that wound up having serious bugs. This brings us back to a reasonably performing built-in implementation consistent with what we've just replaced our other guaranteed available standard hash algorithms with: code from the HACL* project. --------- Co-authored-by: Gregory P. Smith --- Makefile.pre.in | 2 +- ...3-04-17-14-38-12.gh-issue-99108.720lG8.rst | 2 + Modules/Setup | 2 +- Modules/Setup.stdlib.in | 2 +- Modules/_hacl/Hacl_Hash_SHA3.c | 826 ++++++++++++++++++ Modules/_hacl/Hacl_Hash_SHA3.h | 136 +++ Modules/_hacl/Hacl_Streaming_Types.h | 17 + Modules/_hacl/include/krml/internal/target.h | 44 + .../_hacl/include/krml/lowstar_endianness.h | 5 +- Modules/_hacl/internal/Hacl_Hash_SHA3.h | 65 ++ Modules/_hacl/python_hacl_namespaces.h | 23 + Modules/_hacl/refresh.sh | 11 +- Modules/_sha3/LICENSE | 22 - Modules/_sha3/README.txt | 8 - Modules/_sha3/sha3.c | 193 ---- Modules/_sha3/sha3.h | 49 -- Modules/{_sha3 => }/clinic/sha3module.c.h | 4 +- Modules/{_sha3 => }/sha3module.c | 191 ++-- PCbuild/pythoncore.vcxproj | 9 +- PCbuild/pythoncore.vcxproj.filters | 19 +- Tools/c-analyzer/cpython/_parser.py | 8 +- 21 files changed, 1204 insertions(+), 434 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2023-04-17-14-38-12.gh-issue-99108.720lG8.rst create mode 100644 Modules/_hacl/Hacl_Hash_SHA3.c create mode 100644 Modules/_hacl/Hacl_Hash_SHA3.h create mode 100644 Modules/_hacl/internal/Hacl_Hash_SHA3.h delete mode 100644 Modules/_sha3/LICENSE delete mode 100644 Modules/_sha3/README.txt delete mode 100644 Modules/_sha3/sha3.c delete mode 100644 Modules/_sha3/sha3.h rename Modules/{_sha3 => }/clinic/sha3module.c.h (98%) rename Modules/{_sha3 => }/sha3module.c (75%) diff --git a/Makefile.pre.in b/Makefile.pre.in index 736a520d0e8fb6..329466580b9cb0 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2698,7 +2698,7 @@ MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_MD5.h Modules/_hacl/Hacl_Hash_MD5.c MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_SHA1.h Modules/_hacl/Hacl_Hash_SHA1.c MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_A) -MODULE__SHA3_DEPS=$(srcdir)/Modules/_sha3/sha3.c $(srcdir)/Modules/_sha3/sha3.h $(srcdir)/Modules/hashlib.h +MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_SHA3.h Modules/_hacl/Hacl_Hash_SHA3.c MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data.h $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/testcapi_long.h $(srcdir)/Modules/_testcapi/parts.h diff --git a/Misc/NEWS.d/next/Security/2023-04-17-14-38-12.gh-issue-99108.720lG8.rst b/Misc/NEWS.d/next/Security/2023-04-17-14-38-12.gh-issue-99108.720lG8.rst new file mode 100644 index 00000000000000..f259acf753831c --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-04-17-14-38-12.gh-issue-99108.720lG8.rst @@ -0,0 +1,2 @@ +Upgrade built-in :mod:`hashlib` SHA3 implementation to a verified implementation +from the ``HACL*`` project. Used when OpenSSL is not present or lacks SHA3. diff --git a/Modules/Setup b/Modules/Setup index 1c6f2f7ea5182d..e5bc078af62c41 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -166,7 +166,7 @@ PYTHONPATH=$(COREPYTHONPATH) #_md5 md5module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_MD5.c -D_BSD_SOURCE -D_DEFAULT_SOURCE #_sha1 sha1module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA1.c -D_BSD_SOURCE -D_DEFAULT_SOURCE #_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a -#_sha3 _sha3/sha3module.c +#_sha3 sha3module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_sha3.c -D_BSD_SOURCE -D_DEFAULT_SOURCE # text encodings and unicode #_codecs_cn cjkcodecs/_codecs_cn.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index a7803cf7c00e68..8e66576b5c5f00 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -80,7 +80,7 @@ @MODULE__MD5_TRUE@_md5 md5module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_MD5.c -D_BSD_SOURCE -D_DEFAULT_SOURCE @MODULE__SHA1_TRUE@_sha1 sha1module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA1.c -D_BSD_SOURCE -D_DEFAULT_SOURCE @MODULE__SHA2_TRUE@_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a -@MODULE__SHA3_TRUE@_sha3 _sha3/sha3module.c +@MODULE__SHA3_TRUE@_sha3 sha3module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA3.c -D_BSD_SOURCE -D_DEFAULT_SOURCE @MODULE__BLAKE2_TRUE@_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c ############################################################################ diff --git a/Modules/_hacl/Hacl_Hash_SHA3.c b/Modules/_hacl/Hacl_Hash_SHA3.c new file mode 100644 index 00000000000000..100afe7c2c6d1f --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_SHA3.c @@ -0,0 +1,826 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#include "internal/Hacl_Hash_SHA3.h" + +static uint32_t block_len(Spec_Hash_Definitions_hash_alg a) +{ + switch (a) + { + case Spec_Hash_Definitions_SHA3_224: + { + return (uint32_t)144U; + } + case Spec_Hash_Definitions_SHA3_256: + { + return (uint32_t)136U; + } + case Spec_Hash_Definitions_SHA3_384: + { + return (uint32_t)104U; + } + case Spec_Hash_Definitions_SHA3_512: + { + return (uint32_t)72U; + } + case Spec_Hash_Definitions_Shake128: + { + return (uint32_t)168U; + } + case Spec_Hash_Definitions_Shake256: + { + return (uint32_t)136U; + } + default: + { + KRML_HOST_EPRINTF("KaRaMeL incomplete match at %s:%d\n", __FILE__, __LINE__); + KRML_HOST_EXIT(253U); + } + } +} + +static uint32_t hash_len(Spec_Hash_Definitions_hash_alg a) +{ + switch (a) + { + case Spec_Hash_Definitions_SHA3_224: + { + return (uint32_t)28U; + } + case Spec_Hash_Definitions_SHA3_256: + { + return (uint32_t)32U; + } + case Spec_Hash_Definitions_SHA3_384: + { + return (uint32_t)48U; + } + case Spec_Hash_Definitions_SHA3_512: + { + return (uint32_t)64U; + } + default: + { + KRML_HOST_EPRINTF("KaRaMeL incomplete match at %s:%d\n", __FILE__, __LINE__); + KRML_HOST_EXIT(253U); + } + } +} + +void +Hacl_Hash_SHA3_update_multi_sha3( + Spec_Hash_Definitions_hash_alg a, + uint64_t *s, + uint8_t *blocks, + uint32_t n_blocks +) +{ + for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) + { + uint8_t *block = blocks + i * block_len(a); + Hacl_Impl_SHA3_absorb_inner(block_len(a), block, s); + } +} + +void +Hacl_Hash_SHA3_update_last_sha3( + Spec_Hash_Definitions_hash_alg a, + uint64_t *s, + uint8_t *input, + uint32_t input_len +) +{ + uint8_t suffix; + if (a == Spec_Hash_Definitions_Shake128 || a == Spec_Hash_Definitions_Shake256) + { + suffix = (uint8_t)0x1fU; + } + else + { + suffix = (uint8_t)0x06U; + } + uint32_t len = block_len(a); + if (input_len == len) + { + Hacl_Impl_SHA3_absorb_inner(len, input, s); + uint8_t *uu____0 = input + input_len; + uint8_t lastBlock_[200U] = { 0U }; + uint8_t *lastBlock = lastBlock_; + memcpy(lastBlock, uu____0, (uint32_t)0U * sizeof (uint8_t)); + lastBlock[0U] = suffix; + Hacl_Impl_SHA3_loadState(len, lastBlock, s); + if (!((suffix & (uint8_t)0x80U) == (uint8_t)0U) && (uint32_t)0U == len - (uint32_t)1U) + { + Hacl_Impl_SHA3_state_permute(s); + } + uint8_t nextBlock_[200U] = { 0U }; + uint8_t *nextBlock = nextBlock_; + nextBlock[len - (uint32_t)1U] = (uint8_t)0x80U; + Hacl_Impl_SHA3_loadState(len, nextBlock, s); + Hacl_Impl_SHA3_state_permute(s); + return; + } + uint8_t lastBlock_[200U] = { 0U }; + uint8_t *lastBlock = lastBlock_; + memcpy(lastBlock, input, input_len * sizeof (uint8_t)); + lastBlock[input_len] = suffix; + Hacl_Impl_SHA3_loadState(len, lastBlock, s); + if (!((suffix & (uint8_t)0x80U) == (uint8_t)0U) && input_len == len - (uint32_t)1U) + { + Hacl_Impl_SHA3_state_permute(s); + } + uint8_t nextBlock_[200U] = { 0U }; + uint8_t *nextBlock = nextBlock_; + nextBlock[len - (uint32_t)1U] = (uint8_t)0x80U; + Hacl_Impl_SHA3_loadState(len, nextBlock, s); + Hacl_Impl_SHA3_state_permute(s); +} + +typedef struct hash_buf2_s +{ + Hacl_Streaming_Keccak_hash_buf fst; + Hacl_Streaming_Keccak_hash_buf snd; +} +hash_buf2; + +Spec_Hash_Definitions_hash_alg Hacl_Streaming_Keccak_get_alg(Hacl_Streaming_Keccak_state *s) +{ + Hacl_Streaming_Keccak_state scrut = *s; + Hacl_Streaming_Keccak_hash_buf block_state = scrut.block_state; + return block_state.fst; +} + +Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_hash_alg a) +{ + KRML_CHECK_SIZE(sizeof (uint8_t), block_len(a)); + uint8_t *buf0 = (uint8_t *)KRML_HOST_CALLOC(block_len(a), sizeof (uint8_t)); + uint64_t *buf = (uint64_t *)KRML_HOST_CALLOC((uint32_t)25U, sizeof (uint64_t)); + Hacl_Streaming_Keccak_hash_buf block_state = { .fst = a, .snd = buf }; + Hacl_Streaming_Keccak_state + s = { .block_state = block_state, .buf = buf0, .total_len = (uint64_t)(uint32_t)0U }; + Hacl_Streaming_Keccak_state + *p = (Hacl_Streaming_Keccak_state *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_Keccak_state)); + p[0U] = s; + uint64_t *s1 = block_state.snd; + for (uint32_t _i = 0U; _i < (uint32_t)25U; ++_i) + ((void **)s1)[_i] = (void *)(uint64_t)0U; + return p; +} + +void Hacl_Streaming_Keccak_free(Hacl_Streaming_Keccak_state *s) +{ + Hacl_Streaming_Keccak_state scrut = *s; + uint8_t *buf = scrut.buf; + Hacl_Streaming_Keccak_hash_buf block_state = scrut.block_state; + uint64_t *s1 = block_state.snd; + KRML_HOST_FREE(s1); + KRML_HOST_FREE(buf); + KRML_HOST_FREE(s); +} + +Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_copy(Hacl_Streaming_Keccak_state *s0) +{ + Hacl_Streaming_Keccak_state scrut0 = *s0; + Hacl_Streaming_Keccak_hash_buf block_state0 = scrut0.block_state; + uint8_t *buf0 = scrut0.buf; + uint64_t total_len0 = scrut0.total_len; + Spec_Hash_Definitions_hash_alg i = block_state0.fst; + KRML_CHECK_SIZE(sizeof (uint8_t), block_len(i)); + uint8_t *buf1 = (uint8_t *)KRML_HOST_CALLOC(block_len(i), sizeof (uint8_t)); + memcpy(buf1, buf0, block_len(i) * sizeof (uint8_t)); + uint64_t *buf = (uint64_t *)KRML_HOST_CALLOC((uint32_t)25U, sizeof (uint64_t)); + Hacl_Streaming_Keccak_hash_buf block_state = { .fst = i, .snd = buf }; + hash_buf2 scrut = { .fst = block_state0, .snd = block_state }; + uint64_t *s_dst = scrut.snd.snd; + uint64_t *s_src = scrut.fst.snd; + memcpy(s_dst, s_src, (uint32_t)25U * sizeof (uint64_t)); + Hacl_Streaming_Keccak_state + s = { .block_state = block_state, .buf = buf1, .total_len = total_len0 }; + Hacl_Streaming_Keccak_state + *p = (Hacl_Streaming_Keccak_state *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_Keccak_state)); + p[0U] = s; + return p; +} + +void Hacl_Streaming_Keccak_reset(Hacl_Streaming_Keccak_state *s) +{ + Hacl_Streaming_Keccak_state scrut = *s; + uint8_t *buf = scrut.buf; + Hacl_Streaming_Keccak_hash_buf block_state = scrut.block_state; + uint64_t *s1 = block_state.snd; + for (uint32_t _i = 0U; _i < (uint32_t)25U; ++_i) + ((void **)s1)[_i] = (void *)(uint64_t)0U; + Hacl_Streaming_Keccak_state + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s[0U] = tmp; +} + +uint32_t +Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint32_t len) +{ + Hacl_Streaming_Keccak_state s = *p; + Hacl_Streaming_Keccak_hash_buf block_state = s.block_state; + uint64_t total_len = s.total_len; + Spec_Hash_Definitions_hash_alg i = block_state.fst; + if ((uint64_t)len > (uint64_t)0xffffffffU - total_len) + { + return (uint32_t)1U; + } + uint32_t sz; + if (total_len % (uint64_t)block_len(i) == (uint64_t)0U && total_len > (uint64_t)0U) + { + sz = block_len(i); + } + else + { + sz = (uint32_t)(total_len % (uint64_t)block_len(i)); + } + if (len <= block_len(i) - sz) + { + Hacl_Streaming_Keccak_state s1 = *p; + Hacl_Streaming_Keccak_hash_buf block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)block_len(i) == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = block_len(i); + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)block_len(i)); + } + uint8_t *buf2 = buf + sz1; + memcpy(buf2, data, len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)len; + *p + = + ( + (Hacl_Streaming_Keccak_state){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len2 + } + ); + } + else if (sz == (uint32_t)0U) + { + Hacl_Streaming_Keccak_state s1 = *p; + Hacl_Streaming_Keccak_hash_buf block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)block_len(i) == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = block_len(i); + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)block_len(i)); + } + if (!(sz1 == (uint32_t)0U)) + { + Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; + uint64_t *s2 = block_state1.snd; + Hacl_Hash_SHA3_update_multi_sha3(a1, s2, buf, block_len(i) / block_len(a1)); + } + uint32_t ite; + if ((uint64_t)len % (uint64_t)block_len(i) == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + { + ite = block_len(i); + } + else + { + ite = (uint32_t)((uint64_t)len % (uint64_t)block_len(i)); + } + uint32_t n_blocks = (len - ite) / block_len(i); + uint32_t data1_len = n_blocks * block_len(i); + uint32_t data2_len = len - data1_len; + uint8_t *data1 = data; + uint8_t *data2 = data + data1_len; + Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; + uint64_t *s2 = block_state1.snd; + Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data1, data1_len / block_len(a1)); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *p + = + ( + (Hacl_Streaming_Keccak_state){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)len + } + ); + } + else + { + uint32_t diff = block_len(i) - sz; + uint8_t *data1 = data; + uint8_t *data2 = data + diff; + Hacl_Streaming_Keccak_state s1 = *p; + Hacl_Streaming_Keccak_hash_buf block_state10 = s1.block_state; + uint8_t *buf0 = s1.buf; + uint64_t total_len10 = s1.total_len; + uint32_t sz10; + if (total_len10 % (uint64_t)block_len(i) == (uint64_t)0U && total_len10 > (uint64_t)0U) + { + sz10 = block_len(i); + } + else + { + sz10 = (uint32_t)(total_len10 % (uint64_t)block_len(i)); + } + uint8_t *buf2 = buf0 + sz10; + memcpy(buf2, data1, diff * sizeof (uint8_t)); + uint64_t total_len2 = total_len10 + (uint64_t)diff; + *p + = + ( + (Hacl_Streaming_Keccak_state){ + .block_state = block_state10, + .buf = buf0, + .total_len = total_len2 + } + ); + Hacl_Streaming_Keccak_state s10 = *p; + Hacl_Streaming_Keccak_hash_buf block_state1 = s10.block_state; + uint8_t *buf = s10.buf; + uint64_t total_len1 = s10.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)block_len(i) == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = block_len(i); + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)block_len(i)); + } + if (!(sz1 == (uint32_t)0U)) + { + Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; + uint64_t *s2 = block_state1.snd; + Hacl_Hash_SHA3_update_multi_sha3(a1, s2, buf, block_len(i) / block_len(a1)); + } + uint32_t ite; + if + ( + (uint64_t)(len - diff) + % (uint64_t)block_len(i) + == (uint64_t)0U + && (uint64_t)(len - diff) > (uint64_t)0U + ) + { + ite = block_len(i); + } + else + { + ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)block_len(i)); + } + uint32_t n_blocks = (len - diff - ite) / block_len(i); + uint32_t data1_len = n_blocks * block_len(i); + uint32_t data2_len = len - diff - data1_len; + uint8_t *data11 = data2; + uint8_t *data21 = data2 + data1_len; + Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; + uint64_t *s2 = block_state1.snd; + Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data11, data1_len / block_len(a1)); + uint8_t *dst = buf; + memcpy(dst, data21, data2_len * sizeof (uint8_t)); + *p + = + ( + (Hacl_Streaming_Keccak_state){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)(len - diff) + } + ); + } + return (uint32_t)0U; +} + +static void +finish_( + Spec_Hash_Definitions_hash_alg a, + Hacl_Streaming_Keccak_state *p, + uint8_t *dst, + uint32_t l +) +{ + Hacl_Streaming_Keccak_state scrut0 = *p; + Hacl_Streaming_Keccak_hash_buf block_state = scrut0.block_state; + uint8_t *buf_ = scrut0.buf; + uint64_t total_len = scrut0.total_len; + uint32_t r; + if (total_len % (uint64_t)block_len(a) == (uint64_t)0U && total_len > (uint64_t)0U) + { + r = block_len(a); + } + else + { + r = (uint32_t)(total_len % (uint64_t)block_len(a)); + } + uint8_t *buf_1 = buf_; + uint64_t buf[25U] = { 0U }; + Hacl_Streaming_Keccak_hash_buf tmp_block_state = { .fst = a, .snd = buf }; + hash_buf2 scrut = { .fst = block_state, .snd = tmp_block_state }; + uint64_t *s_dst = scrut.snd.snd; + uint64_t *s_src = scrut.fst.snd; + memcpy(s_dst, s_src, (uint32_t)25U * sizeof (uint64_t)); + uint32_t ite0; + if (r % block_len(a) == (uint32_t)0U && r > (uint32_t)0U) + { + ite0 = block_len(a); + } + else + { + ite0 = r % block_len(a); + } + uint8_t *buf_last = buf_1 + r - ite0; + uint8_t *buf_multi = buf_1; + Spec_Hash_Definitions_hash_alg a1 = tmp_block_state.fst; + uint64_t *s0 = tmp_block_state.snd; + Hacl_Hash_SHA3_update_multi_sha3(a1, s0, buf_multi, (uint32_t)0U / block_len(a1)); + Spec_Hash_Definitions_hash_alg a10 = tmp_block_state.fst; + uint64_t *s1 = tmp_block_state.snd; + Hacl_Hash_SHA3_update_last_sha3(a10, s1, buf_last, r); + Spec_Hash_Definitions_hash_alg a11 = tmp_block_state.fst; + uint64_t *s = tmp_block_state.snd; + if (a11 == Spec_Hash_Definitions_Shake128 || a11 == Spec_Hash_Definitions_Shake256) + { + uint32_t ite; + if (a11 == Spec_Hash_Definitions_Shake128 || a11 == Spec_Hash_Definitions_Shake256) + { + ite = l; + } + else + { + ite = hash_len(a11); + } + Hacl_Impl_SHA3_squeeze(s, block_len(a11), ite, dst); + return; + } + Hacl_Impl_SHA3_squeeze(s, block_len(a11), hash_len(a11), dst); +} + +Hacl_Streaming_Keccak_error_code +Hacl_Streaming_Keccak_finish(Hacl_Streaming_Keccak_state *s, uint8_t *dst) +{ + Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + if (a1 == Spec_Hash_Definitions_Shake128 || a1 == Spec_Hash_Definitions_Shake256) + { + return Hacl_Streaming_Keccak_InvalidAlgorithm; + } + finish_(a1, s, dst, hash_len(a1)); + return Hacl_Streaming_Keccak_Success; +} + +Hacl_Streaming_Keccak_error_code +Hacl_Streaming_Keccak_squeeze(Hacl_Streaming_Keccak_state *s, uint8_t *dst, uint32_t l) +{ + Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + if (!(a1 == Spec_Hash_Definitions_Shake128 || a1 == Spec_Hash_Definitions_Shake256)) + { + return Hacl_Streaming_Keccak_InvalidAlgorithm; + } + if (l == (uint32_t)0U) + { + return Hacl_Streaming_Keccak_InvalidLength; + } + finish_(a1, s, dst, l); + return Hacl_Streaming_Keccak_Success; +} + +uint32_t Hacl_Streaming_Keccak_block_len(Hacl_Streaming_Keccak_state *s) +{ + Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + return block_len(a1); +} + +uint32_t Hacl_Streaming_Keccak_hash_len(Hacl_Streaming_Keccak_state *s) +{ + Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + return hash_len(a1); +} + +bool Hacl_Streaming_Keccak_is_shake(Hacl_Streaming_Keccak_state *s) +{ + Spec_Hash_Definitions_hash_alg uu____0 = Hacl_Streaming_Keccak_get_alg(s); + return uu____0 == Spec_Hash_Definitions_Shake128 || uu____0 == Spec_Hash_Definitions_Shake256; +} + +void +Hacl_SHA3_shake128_hacl( + uint32_t inputByteLen, + uint8_t *input, + uint32_t outputByteLen, + uint8_t *output +) +{ + Hacl_Impl_SHA3_keccak((uint32_t)1344U, + (uint32_t)256U, + inputByteLen, + input, + (uint8_t)0x1FU, + outputByteLen, + output); +} + +void +Hacl_SHA3_shake256_hacl( + uint32_t inputByteLen, + uint8_t *input, + uint32_t outputByteLen, + uint8_t *output +) +{ + Hacl_Impl_SHA3_keccak((uint32_t)1088U, + (uint32_t)512U, + inputByteLen, + input, + (uint8_t)0x1FU, + outputByteLen, + output); +} + +void Hacl_SHA3_sha3_224(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +{ + Hacl_Impl_SHA3_keccak((uint32_t)1152U, + (uint32_t)448U, + inputByteLen, + input, + (uint8_t)0x06U, + (uint32_t)28U, + output); +} + +void Hacl_SHA3_sha3_256(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +{ + Hacl_Impl_SHA3_keccak((uint32_t)1088U, + (uint32_t)512U, + inputByteLen, + input, + (uint8_t)0x06U, + (uint32_t)32U, + output); +} + +void Hacl_SHA3_sha3_384(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +{ + Hacl_Impl_SHA3_keccak((uint32_t)832U, + (uint32_t)768U, + inputByteLen, + input, + (uint8_t)0x06U, + (uint32_t)48U, + output); +} + +void Hacl_SHA3_sha3_512(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +{ + Hacl_Impl_SHA3_keccak((uint32_t)576U, + (uint32_t)1024U, + inputByteLen, + input, + (uint8_t)0x06U, + (uint32_t)64U, + output); +} + +static const +uint32_t +keccak_rotc[24U] = + { + (uint32_t)1U, (uint32_t)3U, (uint32_t)6U, (uint32_t)10U, (uint32_t)15U, (uint32_t)21U, + (uint32_t)28U, (uint32_t)36U, (uint32_t)45U, (uint32_t)55U, (uint32_t)2U, (uint32_t)14U, + (uint32_t)27U, (uint32_t)41U, (uint32_t)56U, (uint32_t)8U, (uint32_t)25U, (uint32_t)43U, + (uint32_t)62U, (uint32_t)18U, (uint32_t)39U, (uint32_t)61U, (uint32_t)20U, (uint32_t)44U + }; + +static const +uint32_t +keccak_piln[24U] = + { + (uint32_t)10U, (uint32_t)7U, (uint32_t)11U, (uint32_t)17U, (uint32_t)18U, (uint32_t)3U, + (uint32_t)5U, (uint32_t)16U, (uint32_t)8U, (uint32_t)21U, (uint32_t)24U, (uint32_t)4U, + (uint32_t)15U, (uint32_t)23U, (uint32_t)19U, (uint32_t)13U, (uint32_t)12U, (uint32_t)2U, + (uint32_t)20U, (uint32_t)14U, (uint32_t)22U, (uint32_t)9U, (uint32_t)6U, (uint32_t)1U + }; + +static const +uint64_t +keccak_rndc[24U] = + { + (uint64_t)0x0000000000000001U, (uint64_t)0x0000000000008082U, (uint64_t)0x800000000000808aU, + (uint64_t)0x8000000080008000U, (uint64_t)0x000000000000808bU, (uint64_t)0x0000000080000001U, + (uint64_t)0x8000000080008081U, (uint64_t)0x8000000000008009U, (uint64_t)0x000000000000008aU, + (uint64_t)0x0000000000000088U, (uint64_t)0x0000000080008009U, (uint64_t)0x000000008000000aU, + (uint64_t)0x000000008000808bU, (uint64_t)0x800000000000008bU, (uint64_t)0x8000000000008089U, + (uint64_t)0x8000000000008003U, (uint64_t)0x8000000000008002U, (uint64_t)0x8000000000000080U, + (uint64_t)0x000000000000800aU, (uint64_t)0x800000008000000aU, (uint64_t)0x8000000080008081U, + (uint64_t)0x8000000000008080U, (uint64_t)0x0000000080000001U, (uint64_t)0x8000000080008008U + }; + +void Hacl_Impl_SHA3_state_permute(uint64_t *s) +{ + for (uint32_t i0 = (uint32_t)0U; i0 < (uint32_t)24U; i0++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + (uint32_t)0U, + (uint32_t)5U, + (uint32_t)1U, + _C[i] = + s[i + + (uint32_t)0U] + ^ + (s[i + + (uint32_t)5U] + ^ (s[i + (uint32_t)10U] ^ (s[i + (uint32_t)15U] ^ s[i + (uint32_t)20U])));); + KRML_MAYBE_FOR5(i1, + (uint32_t)0U, + (uint32_t)5U, + (uint32_t)1U, + uint64_t uu____0 = _C[(i1 + (uint32_t)1U) % (uint32_t)5U]; + uint64_t + _D = + _C[(i1 + (uint32_t)4U) + % (uint32_t)5U] + ^ (uu____0 << (uint32_t)1U | uu____0 >> (uint32_t)63U); + KRML_MAYBE_FOR5(i, + (uint32_t)0U, + (uint32_t)5U, + (uint32_t)1U, + s[i1 + (uint32_t)5U * i] = s[i1 + (uint32_t)5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = (uint32_t)0U; i < (uint32_t)24U; i++) + { + uint32_t _Y = keccak_piln[i]; + uint32_t r = keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____1 = current; + s[_Y] = uu____1 << r | uu____1 >> ((uint32_t)64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + (uint32_t)0U, + (uint32_t)5U, + (uint32_t)1U, + uint64_t + v0 = + s[(uint32_t)0U + + (uint32_t)5U * i] + ^ (~s[(uint32_t)1U + (uint32_t)5U * i] & s[(uint32_t)2U + (uint32_t)5U * i]); + uint64_t + v1 = + s[(uint32_t)1U + + (uint32_t)5U * i] + ^ (~s[(uint32_t)2U + (uint32_t)5U * i] & s[(uint32_t)3U + (uint32_t)5U * i]); + uint64_t + v2 = + s[(uint32_t)2U + + (uint32_t)5U * i] + ^ (~s[(uint32_t)3U + (uint32_t)5U * i] & s[(uint32_t)4U + (uint32_t)5U * i]); + uint64_t + v3 = + s[(uint32_t)3U + + (uint32_t)5U * i] + ^ (~s[(uint32_t)4U + (uint32_t)5U * i] & s[(uint32_t)0U + (uint32_t)5U * i]); + uint64_t + v4 = + s[(uint32_t)4U + + (uint32_t)5U * i] + ^ (~s[(uint32_t)0U + (uint32_t)5U * i] & s[(uint32_t)1U + (uint32_t)5U * i]); + s[(uint32_t)0U + (uint32_t)5U * i] = v0; + s[(uint32_t)1U + (uint32_t)5U * i] = v1; + s[(uint32_t)2U + (uint32_t)5U * i] = v2; + s[(uint32_t)3U + (uint32_t)5U * i] = v3; + s[(uint32_t)4U + (uint32_t)5U * i] = v4;); + uint64_t c = keccak_rndc[i0]; + s[0U] = s[0U] ^ c; + } +} + +void Hacl_Impl_SHA3_loadState(uint32_t rateInBytes, uint8_t *input, uint64_t *s) +{ + uint8_t block[200U] = { 0U }; + memcpy(block, input, rateInBytes * sizeof (uint8_t)); + for (uint32_t i = (uint32_t)0U; i < (uint32_t)25U; i++) + { + uint64_t u = load64_le(block + i * (uint32_t)8U); + uint64_t x = u; + s[i] = s[i] ^ x; + } +} + +static void storeState(uint32_t rateInBytes, uint64_t *s, uint8_t *res) +{ + uint8_t block[200U] = { 0U }; + for (uint32_t i = (uint32_t)0U; i < (uint32_t)25U; i++) + { + uint64_t sj = s[i]; + store64_le(block + i * (uint32_t)8U, sj); + } + memcpy(res, block, rateInBytes * sizeof (uint8_t)); +} + +void Hacl_Impl_SHA3_absorb_inner(uint32_t rateInBytes, uint8_t *block, uint64_t *s) +{ + Hacl_Impl_SHA3_loadState(rateInBytes, block, s); + Hacl_Impl_SHA3_state_permute(s); +} + +static void +absorb( + uint64_t *s, + uint32_t rateInBytes, + uint32_t inputByteLen, + uint8_t *input, + uint8_t delimitedSuffix +) +{ + uint32_t n_blocks = inputByteLen / rateInBytes; + uint32_t rem = inputByteLen % rateInBytes; + for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) + { + uint8_t *block = input + i * rateInBytes; + Hacl_Impl_SHA3_absorb_inner(rateInBytes, block, s); + } + uint8_t *last = input + n_blocks * rateInBytes; + uint8_t lastBlock_[200U] = { 0U }; + uint8_t *lastBlock = lastBlock_; + memcpy(lastBlock, last, rem * sizeof (uint8_t)); + lastBlock[rem] = delimitedSuffix; + Hacl_Impl_SHA3_loadState(rateInBytes, lastBlock, s); + if (!((delimitedSuffix & (uint8_t)0x80U) == (uint8_t)0U) && rem == rateInBytes - (uint32_t)1U) + { + Hacl_Impl_SHA3_state_permute(s); + } + uint8_t nextBlock_[200U] = { 0U }; + uint8_t *nextBlock = nextBlock_; + nextBlock[rateInBytes - (uint32_t)1U] = (uint8_t)0x80U; + Hacl_Impl_SHA3_loadState(rateInBytes, nextBlock, s); + Hacl_Impl_SHA3_state_permute(s); +} + +void +Hacl_Impl_SHA3_squeeze( + uint64_t *s, + uint32_t rateInBytes, + uint32_t outputByteLen, + uint8_t *output +) +{ + uint32_t outBlocks = outputByteLen / rateInBytes; + uint32_t remOut = outputByteLen % rateInBytes; + uint8_t *last = output + outputByteLen - remOut; + uint8_t *blocks = output; + for (uint32_t i = (uint32_t)0U; i < outBlocks; i++) + { + storeState(rateInBytes, s, blocks + i * rateInBytes); + Hacl_Impl_SHA3_state_permute(s); + } + storeState(remOut, s, last); +} + +void +Hacl_Impl_SHA3_keccak( + uint32_t rate, + uint32_t capacity, + uint32_t inputByteLen, + uint8_t *input, + uint8_t delimitedSuffix, + uint32_t outputByteLen, + uint8_t *output +) +{ + uint32_t rateInBytes = rate / (uint32_t)8U; + uint64_t s[25U] = { 0U }; + absorb(s, rateInBytes, inputByteLen, input, delimitedSuffix); + Hacl_Impl_SHA3_squeeze(s, rateInBytes, outputByteLen, output); +} + diff --git a/Modules/_hacl/Hacl_Hash_SHA3.h b/Modules/_hacl/Hacl_Hash_SHA3.h new file mode 100644 index 00000000000000..2a5cf4b1844b9d --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_SHA3.h @@ -0,0 +1,136 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __Hacl_Hash_SHA3_H +#define __Hacl_Hash_SHA3_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "Hacl_Streaming_Types.h" + +typedef struct Hacl_Streaming_Keccak_hash_buf_s +{ + Spec_Hash_Definitions_hash_alg fst; + uint64_t *snd; +} +Hacl_Streaming_Keccak_hash_buf; + +typedef struct Hacl_Streaming_Keccak_state_s +{ + Hacl_Streaming_Keccak_hash_buf block_state; + uint8_t *buf; + uint64_t total_len; +} +Hacl_Streaming_Keccak_state; + +Spec_Hash_Definitions_hash_alg Hacl_Streaming_Keccak_get_alg(Hacl_Streaming_Keccak_state *s); + +Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_hash_alg a); + +void Hacl_Streaming_Keccak_free(Hacl_Streaming_Keccak_state *s); + +Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_copy(Hacl_Streaming_Keccak_state *s0); + +void Hacl_Streaming_Keccak_reset(Hacl_Streaming_Keccak_state *s); + +uint32_t +Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint32_t len); + +#define Hacl_Streaming_Keccak_Success 0 +#define Hacl_Streaming_Keccak_InvalidAlgorithm 1 +#define Hacl_Streaming_Keccak_InvalidLength 2 + +typedef uint8_t Hacl_Streaming_Keccak_error_code; + +Hacl_Streaming_Keccak_error_code +Hacl_Streaming_Keccak_finish(Hacl_Streaming_Keccak_state *s, uint8_t *dst); + +Hacl_Streaming_Keccak_error_code +Hacl_Streaming_Keccak_squeeze(Hacl_Streaming_Keccak_state *s, uint8_t *dst, uint32_t l); + +uint32_t Hacl_Streaming_Keccak_block_len(Hacl_Streaming_Keccak_state *s); + +uint32_t Hacl_Streaming_Keccak_hash_len(Hacl_Streaming_Keccak_state *s); + +bool Hacl_Streaming_Keccak_is_shake(Hacl_Streaming_Keccak_state *s); + +void +Hacl_SHA3_shake128_hacl( + uint32_t inputByteLen, + uint8_t *input, + uint32_t outputByteLen, + uint8_t *output +); + +void +Hacl_SHA3_shake256_hacl( + uint32_t inputByteLen, + uint8_t *input, + uint32_t outputByteLen, + uint8_t *output +); + +void Hacl_SHA3_sha3_224(uint32_t inputByteLen, uint8_t *input, uint8_t *output); + +void Hacl_SHA3_sha3_256(uint32_t inputByteLen, uint8_t *input, uint8_t *output); + +void Hacl_SHA3_sha3_384(uint32_t inputByteLen, uint8_t *input, uint8_t *output); + +void Hacl_SHA3_sha3_512(uint32_t inputByteLen, uint8_t *input, uint8_t *output); + +void Hacl_Impl_SHA3_absorb_inner(uint32_t rateInBytes, uint8_t *block, uint64_t *s); + +void +Hacl_Impl_SHA3_squeeze( + uint64_t *s, + uint32_t rateInBytes, + uint32_t outputByteLen, + uint8_t *output +); + +void +Hacl_Impl_SHA3_keccak( + uint32_t rate, + uint32_t capacity, + uint32_t inputByteLen, + uint8_t *input, + uint8_t delimitedSuffix, + uint32_t outputByteLen, + uint8_t *output +); + +#if defined(__cplusplus) +} +#endif + +#define __Hacl_Hash_SHA3_H_DEFINED +#endif diff --git a/Modules/_hacl/Hacl_Streaming_Types.h b/Modules/_hacl/Hacl_Streaming_Types.h index 51057611ca978d..8a60b707bc4958 100644 --- a/Modules/_hacl/Hacl_Streaming_Types.h +++ b/Modules/_hacl/Hacl_Streaming_Types.h @@ -35,6 +35,23 @@ extern "C" { #include "krml/lowstar_endianness.h" #include "krml/internal/target.h" +#define Spec_Hash_Definitions_SHA2_224 0 +#define Spec_Hash_Definitions_SHA2_256 1 +#define Spec_Hash_Definitions_SHA2_384 2 +#define Spec_Hash_Definitions_SHA2_512 3 +#define Spec_Hash_Definitions_SHA1 4 +#define Spec_Hash_Definitions_MD5 5 +#define Spec_Hash_Definitions_Blake2S 6 +#define Spec_Hash_Definitions_Blake2B 7 +#define Spec_Hash_Definitions_SHA3_256 8 +#define Spec_Hash_Definitions_SHA3_224 9 +#define Spec_Hash_Definitions_SHA3_384 10 +#define Spec_Hash_Definitions_SHA3_512 11 +#define Spec_Hash_Definitions_Shake128 12 +#define Spec_Hash_Definitions_Shake256 13 + +typedef uint8_t Spec_Hash_Definitions_hash_alg; + typedef struct Hacl_Streaming_MD_state_32_s { uint32_t *block_state; diff --git a/Modules/_hacl/include/krml/internal/target.h b/Modules/_hacl/include/krml/internal/target.h index dcbe7007b17be8..5a2f94eb2ec8da 100644 --- a/Modules/_hacl/include/krml/internal/target.h +++ b/Modules/_hacl/include/krml/internal/target.h @@ -19,6 +19,28 @@ # define inline __inline__ #endif +/******************************************************************************/ +/* Macros that KaRaMeL will generate. */ +/******************************************************************************/ + +/* For "bare" targets that do not have a C stdlib, the user might want to use + * [-add-early-include '"mydefinitions.h"'] and override these. */ +#ifndef KRML_HOST_PRINTF +# define KRML_HOST_PRINTF printf +#endif + +#if ( \ + (defined __STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \ + (!(defined KRML_HOST_EPRINTF))) +# define KRML_HOST_EPRINTF(...) fprintf(stderr, __VA_ARGS__) +#elif !(defined KRML_HOST_EPRINTF) && defined(_MSC_VER) +# define KRML_HOST_EPRINTF(...) fprintf(stderr, __VA_ARGS__) +#endif + +#ifndef KRML_HOST_EXIT +# define KRML_HOST_EXIT exit +#endif + #ifndef KRML_HOST_MALLOC # define KRML_HOST_MALLOC malloc #endif @@ -35,6 +57,28 @@ # define KRML_HOST_IGNORE(x) (void)(x) #endif +/* In FStar.Buffer.fst, the size of arrays is uint32_t, but it's a number of + * *elements*. Do an ugly, run-time check (some of which KaRaMeL can eliminate). + */ +#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 4)) +# define _KRML_CHECK_SIZE_PRAGMA \ + _Pragma("GCC diagnostic ignored \"-Wtype-limits\"") +#else +# define _KRML_CHECK_SIZE_PRAGMA +#endif + +#define KRML_CHECK_SIZE(size_elt, sz) \ + do { \ + _KRML_CHECK_SIZE_PRAGMA \ + if (((size_t)(sz)) > ((size_t)(SIZE_MAX / (size_elt)))) { \ + KRML_HOST_PRINTF( \ + "Maximum allocatable size exceeded, aborting before overflow at " \ + "%s:%d\n", \ + __FILE__, __LINE__); \ + KRML_HOST_EXIT(253); \ + } \ + } while (0) + /* Macros for prettier unrolling of loops */ #define KRML_LOOP1(i, n, x) { \ x \ diff --git a/Modules/_hacl/include/krml/lowstar_endianness.h b/Modules/_hacl/include/krml/lowstar_endianness.h index 32a7391e817ebb..1aa2ccd644c06f 100644 --- a/Modules/_hacl/include/krml/lowstar_endianness.h +++ b/Modules/_hacl/include/krml/lowstar_endianness.h @@ -77,7 +77,7 @@ # define le64toh(x) (x) /* ... for Windows (GCC-like, e.g. mingw or clang) */ -#elif (defined(_WIN32) || defined(_WIN64)) && \ +#elif (defined(_WIN32) || defined(_WIN64) || defined(__EMSCRIPTEN__)) && \ (defined(__GNUC__) || defined(__clang__)) # define htobe16(x) __builtin_bswap16(x) @@ -96,7 +96,8 @@ # define le64toh(x) (x) /* ... generic big-endian fallback code */ -#elif defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +/* ... AIX doesn't have __BYTE_ORDER__ (with XLC compiler) & is always big-endian */ +#elif (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) || defined(_AIX) /* byte swapping code inspired by: * https://github.com/rweather/arduinolibs/blob/master/libraries/Crypto/utility/EndianUtil.h diff --git a/Modules/_hacl/internal/Hacl_Hash_SHA3.h b/Modules/_hacl/internal/Hacl_Hash_SHA3.h new file mode 100644 index 00000000000000..1c9808b8dd497c --- /dev/null +++ b/Modules/_hacl/internal/Hacl_Hash_SHA3.h @@ -0,0 +1,65 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __internal_Hacl_Hash_SHA3_H +#define __internal_Hacl_Hash_SHA3_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "../Hacl_Hash_SHA3.h" + +void +Hacl_Hash_SHA3_update_multi_sha3( + Spec_Hash_Definitions_hash_alg a, + uint64_t *s, + uint8_t *blocks, + uint32_t n_blocks +); + +void +Hacl_Hash_SHA3_update_last_sha3( + Spec_Hash_Definitions_hash_alg a, + uint64_t *s, + uint8_t *input, + uint32_t input_len +); + +void Hacl_Impl_SHA3_state_permute(uint64_t *s); + +void Hacl_Impl_SHA3_loadState(uint32_t rateInBytes, uint8_t *input, uint64_t *s); + +#if defined(__cplusplus) +} +#endif + +#define __internal_Hacl_Hash_SHA3_H_DEFINED +#endif diff --git a/Modules/_hacl/python_hacl_namespaces.h b/Modules/_hacl/python_hacl_namespaces.h index ee28f244266b85..0df236282ac509 100644 --- a/Modules/_hacl/python_hacl_namespaces.h +++ b/Modules/_hacl/python_hacl_namespaces.h @@ -59,5 +59,28 @@ #define Hacl_Streaming_SHA1_legacy_copy python_hashlib_Hacl_Streaming_SHA1_legacy_copy #define Hacl_Streaming_SHA1_legacy_hash python_hashlib_Hacl_Streaming_SHA1_legacy_hash +#define Hacl_Hash_SHA3_update_last_sha3 python_hashlib_Hacl_Hash_SHA3_update_last_sha3 +#define Hacl_Hash_SHA3_update_multi_sha3 python_hashlib_Hacl_Hash_SHA3_update_multi_sha3 +#define Hacl_Impl_SHA3_absorb_inner python_hashlib_Hacl_Impl_SHA3_absorb_inner +#define Hacl_Impl_SHA3_keccak python_hashlib_Hacl_Impl_SHA3_keccak +#define Hacl_Impl_SHA3_loadState python_hashlib_Hacl_Impl_SHA3_loadState +#define Hacl_Impl_SHA3_squeeze python_hashlib_Hacl_Impl_SHA3_squeeze +#define Hacl_Impl_SHA3_state_permute python_hashlib_Hacl_Impl_SHA3_state_permute +#define Hacl_SHA3_sha3_224 python_hashlib_Hacl_SHA3_sha3_224 +#define Hacl_SHA3_sha3_256 python_hashlib_Hacl_SHA3_sha3_256 +#define Hacl_SHA3_sha3_384 python_hashlib_Hacl_SHA3_sha3_384 +#define Hacl_SHA3_sha3_512 python_hashlib_Hacl_SHA3_sha3_512 +#define Hacl_SHA3_shake128_hacl python_hashlib_Hacl_SHA3_shake128_hacl +#define Hacl_SHA3_shake256_hacl python_hashlib_Hacl_SHA3_shake256_hacl +#define Hacl_Streaming_Keccak_block_len python_hashlib_Hacl_Streaming_Keccak_block_len +#define Hacl_Streaming_Keccak_copy python_hashlib_Hacl_Streaming_Keccak_copy +#define Hacl_Streaming_Keccak_finish python_hashlib_Hacl_Streaming_Keccak_finish +#define Hacl_Streaming_Keccak_free python_hashlib_Hacl_Streaming_Keccak_free +#define Hacl_Streaming_Keccak_get_alg python_hashlib_Hacl_Streaming_Keccak_get_alg +#define Hacl_Streaming_Keccak_hash_len python_hashlib_Hacl_Streaming_Keccak_hash_len +#define Hacl_Streaming_Keccak_is_shake python_hashlib_Hacl_Streaming_Keccak_is_shake +#define Hacl_Streaming_Keccak_malloc python_hashlib_Hacl_Streaming_Keccak_malloc +#define Hacl_Streaming_Keccak_reset python_hashlib_Hacl_Streaming_Keccak_reset +#define Hacl_Streaming_Keccak_update python_hashlib_Hacl_Streaming_Keccak_update #endif // _PYTHON_HACL_NAMESPACES_H diff --git a/Modules/_hacl/refresh.sh b/Modules/_hacl/refresh.sh index 76b92ec4599102..220ebbe5561341 100755 --- a/Modules/_hacl/refresh.sh +++ b/Modules/_hacl/refresh.sh @@ -22,7 +22,7 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. -expected_hacl_star_rev=13e0c6721ac9206c4249ecc1dc04ed617ad1e262 +expected_hacl_star_rev=363eae2c2eb60e46f182ddd4bd1cd3f1d00b35c9 hacl_dir="$(realpath "$1")" cd "$(dirname "$0")" @@ -45,11 +45,14 @@ dist_files=( Hacl_Hash_SHA1.h internal/Hacl_Hash_SHA1.h Hacl_Hash_MD5.h + Hacl_Hash_SHA3.h internal/Hacl_Hash_MD5.h + internal/Hacl_Hash_SHA3.h internal/Hacl_SHA2_Generic.h Hacl_Streaming_SHA2.c Hacl_Hash_SHA1.c Hacl_Hash_MD5.c + Hacl_Hash_SHA3.c ) declare -a include_files @@ -134,9 +137,9 @@ $sed -i -z 's!#include \n!#include \n#include "python_hacl_n # Finally, we remove a bunch of ifdefs from target.h that are, again, useful in # the general case, but not exercised by the subset of HACL* that we vendor. -$sed -z -i 's!#ifndef KRML_\(HOST_PRINTF\|HOST_EXIT\|PRE_ALIGN\|POST_ALIGN\|ALIGNED_MALLOC\|ALIGNED_FREE\|HOST_TIME\)\n\(\n\|# [^\n]*\n\|[^#][^\n]*\n\)*#endif\n\n!!g' include/krml/internal/target.h -$sed -z -i 's!\n\n\([^#][^\n]*\n\)*#define KRML_\(EABORT\|EXIT\|CHECK_SIZE\)[^\n]*\(\n [^\n]*\)*!!g' include/krml/internal/target.h +$sed -z -i 's!#ifndef KRML_\(PRE_ALIGN\|POST_ALIGN\|ALIGNED_MALLOC\|ALIGNED_FREE\|HOST_TIME\)\n\(\n\|# [^\n]*\n\|[^#][^\n]*\n\)*#endif\n\n!!g' include/krml/internal/target.h +$sed -z -i 's!\n\n\([^#][^\n]*\n\)*#define KRML_\(EABORT\|EXIT\)[^\n]*\(\n [^\n]*\)*!!g' include/krml/internal/target.h $sed -z -i 's!\n\n\([^#][^\n]*\n\)*#if [^\n]*\n\( [^\n]*\n\)*#define KRML_\(EABORT\|EXIT\|CHECK_SIZE\)[^\n]*\(\n [^\n]*\)*!!g' include/krml/internal/target.h -$sed -z -i 's!\n\n\([^#][^\n]*\n\)*#if [^\n]*\n\( [^\n]*\n\)*# define _\?KRML_\(DEPRECATED\|CHECK_SIZE_PRAGMA\|HOST_EPRINTF\|HOST_SNPRINTF\)[^\n]*\n\([^#][^\n]*\n\|#el[^\n]*\n\|# [^\n]*\n\)*#endif!!g' include/krml/internal/target.h +$sed -z -i 's!\n\n\([^#][^\n]*\n\)*#if [^\n]*\n\( [^\n]*\n\)*# define _\?KRML_\(DEPRECATED\|HOST_SNPRINTF\)[^\n]*\n\([^#][^\n]*\n\|#el[^\n]*\n\|# [^\n]*\n\)*#endif!!g' include/krml/internal/target.h echo "Updated; verify all is okay using git diff and git status." diff --git a/Modules/_sha3/LICENSE b/Modules/_sha3/LICENSE deleted file mode 100644 index d2d484d8820dcf..00000000000000 --- a/Modules/_sha3/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015 Markku-Juhani O. Saarinen - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/Modules/_sha3/README.txt b/Modules/_sha3/README.txt deleted file mode 100644 index b35919b01677d3..00000000000000 --- a/Modules/_sha3/README.txt +++ /dev/null @@ -1,8 +0,0 @@ -tiny_sha3 -========= - -https://github.com/mjosaarinen/tiny_sha3 -commit dcbb3192047c2a721f5f851db591871d428036a9 - -- All functions have been converted to static functions. -- sha3() function is commented out. diff --git a/Modules/_sha3/sha3.c b/Modules/_sha3/sha3.c deleted file mode 100644 index e2d3fd7b8ad855..00000000000000 --- a/Modules/_sha3/sha3.c +++ /dev/null @@ -1,193 +0,0 @@ -// sha3.c -// 19-Nov-11 Markku-Juhani O. Saarinen - -// Revised 07-Aug-15 to match with official release of FIPS PUB 202 "SHA3" -// Revised 03-Sep-15 for portability + OpenSSL - style API - -#include "sha3.h" - -// update the state with given number of rounds - -static void sha3_keccakf(uint64_t st[25]) -{ - // constants - const uint64_t keccakf_rndc[24] = { - 0x0000000000000001, 0x0000000000008082, 0x800000000000808a, - 0x8000000080008000, 0x000000000000808b, 0x0000000080000001, - 0x8000000080008081, 0x8000000000008009, 0x000000000000008a, - 0x0000000000000088, 0x0000000080008009, 0x000000008000000a, - 0x000000008000808b, 0x800000000000008b, 0x8000000000008089, - 0x8000000000008003, 0x8000000000008002, 0x8000000000000080, - 0x000000000000800a, 0x800000008000000a, 0x8000000080008081, - 0x8000000000008080, 0x0000000080000001, 0x8000000080008008 - }; - const int keccakf_rotc[24] = { - 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14, - 27, 41, 56, 8, 25, 43, 62, 18, 39, 61, 20, 44 - }; - const int keccakf_piln[24] = { - 10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4, - 15, 23, 19, 13, 12, 2, 20, 14, 22, 9, 6, 1 - }; - - // variables - int i, j, r; - uint64_t t, bc[5]; - -#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ - uint8_t *v; - - // endianess conversion. this is redundant on little-endian targets - for (i = 0; i < 25; i++) { - v = (uint8_t *) &st[i]; - st[i] = ((uint64_t) v[0]) | (((uint64_t) v[1]) << 8) | - (((uint64_t) v[2]) << 16) | (((uint64_t) v[3]) << 24) | - (((uint64_t) v[4]) << 32) | (((uint64_t) v[5]) << 40) | - (((uint64_t) v[6]) << 48) | (((uint64_t) v[7]) << 56); - } -#endif - - // actual iteration - for (r = 0; r < KECCAKF_ROUNDS; r++) { - - // Theta - for (i = 0; i < 5; i++) - bc[i] = st[i] ^ st[i + 5] ^ st[i + 10] ^ st[i + 15] ^ st[i + 20]; - - for (i = 0; i < 5; i++) { - t = bc[(i + 4) % 5] ^ ROTL64(bc[(i + 1) % 5], 1); - for (j = 0; j < 25; j += 5) - st[j + i] ^= t; - } - - // Rho Pi - t = st[1]; - for (i = 0; i < 24; i++) { - j = keccakf_piln[i]; - bc[0] = st[j]; - st[j] = ROTL64(t, keccakf_rotc[i]); - t = bc[0]; - } - - // Chi - for (j = 0; j < 25; j += 5) { - for (i = 0; i < 5; i++) - bc[i] = st[j + i]; - for (i = 0; i < 5; i++) - st[j + i] ^= (~bc[(i + 1) % 5]) & bc[(i + 2) % 5]; - } - - // Iota - st[0] ^= keccakf_rndc[r]; - } - -#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ - // endianess conversion. this is redundant on little-endian targets - for (i = 0; i < 25; i++) { - v = (uint8_t *) &st[i]; - t = st[i]; - v[0] = t & 0xFF; - v[1] = (t >> 8) & 0xFF; - v[2] = (t >> 16) & 0xFF; - v[3] = (t >> 24) & 0xFF; - v[4] = (t >> 32) & 0xFF; - v[5] = (t >> 40) & 0xFF; - v[6] = (t >> 48) & 0xFF; - v[7] = (t >> 56) & 0xFF; - } -#endif -} - -// Initialize the context for SHA3 - -static int sha3_init(sha3_ctx_t *c, int mdlen) -{ - int i; - - for (i = 0; i < 25; i++) - c->st.q[i] = 0; - c->mdlen = mdlen; - c->rsiz = 200 - 2 * mdlen; - c->pt = 0; - - return 1; -} - -// update state with more data - -static int sha3_update(sha3_ctx_t *c, const void *data, size_t len) -{ - size_t i; - int j; - - j = c->pt; - for (i = 0; i < len; i++) { - c->st.b[j++] ^= ((const uint8_t *) data)[i]; - if (j >= c->rsiz) { - sha3_keccakf(c->st.q); - j = 0; - } - } - c->pt = j; - - return 1; -} - -// finalize and output a hash - -static int sha3_final(void *md, sha3_ctx_t *c) -{ - int i; - - c->st.b[c->pt] ^= 0x06; - c->st.b[c->rsiz - 1] ^= 0x80; - sha3_keccakf(c->st.q); - - for (i = 0; i < c->mdlen; i++) { - ((uint8_t *) md)[i] = c->st.b[i]; - } - - return 1; -} - -#if 0 -// compute a SHA-3 hash (md) of given byte length from "in" - -void *sha3(const void *in, size_t inlen, void *md, int mdlen) -{ - sha3_ctx_t sha3; - - sha3_init(&sha3, mdlen); - sha3_update(&sha3, in, inlen); - sha3_final(md, &sha3); - - return md; -} -#endif - -// SHAKE128 and SHAKE256 extensible-output functionality - -static void shake_xof(sha3_ctx_t *c) -{ - c->st.b[c->pt] ^= 0x1F; - c->st.b[c->rsiz - 1] ^= 0x80; - sha3_keccakf(c->st.q); - c->pt = 0; -} - -static void shake_out(sha3_ctx_t *c, void *out, size_t len) -{ - size_t i; - int j; - - j = c->pt; - for (i = 0; i < len; i++) { - if (j >= c->rsiz) { - sha3_keccakf(c->st.q); - j = 0; - } - ((uint8_t *) out)[i] = c->st.b[j++]; - } - c->pt = j; -} - diff --git a/Modules/_sha3/sha3.h b/Modules/_sha3/sha3.h deleted file mode 100644 index f973d6733ec2cc..00000000000000 --- a/Modules/_sha3/sha3.h +++ /dev/null @@ -1,49 +0,0 @@ -// sha3.h -// 19-Nov-11 Markku-Juhani O. Saarinen - -#ifndef SHA3_H -#define SHA3_H - -#include -#include - -#ifndef KECCAKF_ROUNDS -#define KECCAKF_ROUNDS 24 -#endif - -#ifndef ROTL64 -#define ROTL64(x, y) (((x) << (y)) | ((x) >> (64 - (y)))) -#endif - -// state context -typedef struct { - union { // state: - uint8_t b[200]; // 8-bit bytes - uint64_t q[25]; // 64-bit words - } st; - int pt, rsiz, mdlen; // these don't overflow -} sha3_ctx_t; - -// Compression function. -static void sha3_keccakf(uint64_t st[25]); - -// OpenSSL - like interfece -static int sha3_init(sha3_ctx_t *c, int mdlen); // mdlen = hash output in bytes -static int sha3_update(sha3_ctx_t *c, const void *data, size_t len); -static int sha3_final(void *md, sha3_ctx_t *c); // digest goes to md - -// compute a sha3 hash (md) of given byte length from "in" -#if 0 -static void *sha3(const void *in, size_t inlen, void *md, int mdlen); -#endif - -// SHAKE128 and SHAKE256 extensible-output functions -#define shake128_init(c) sha3_init(c, 16) -#define shake256_init(c) sha3_init(c, 32) -#define shake_update sha3_update - -static void shake_xof(sha3_ctx_t *c); -static void shake_out(sha3_ctx_t *c, void *out, size_t len); - -#endif - diff --git a/Modules/_sha3/clinic/sha3module.c.h b/Modules/clinic/sha3module.c.h similarity index 98% rename from Modules/_sha3/clinic/sha3module.c.h rename to Modules/clinic/sha3module.c.h index a0c7c1c043e515..299803a3420bf6 100644 --- a/Modules/_sha3/clinic/sha3module.c.h +++ b/Modules/clinic/sha3module.c.h @@ -12,7 +12,7 @@ PyDoc_STRVAR(py_sha3_new__doc__, "sha3_224(data=b\'\', /, *, usedforsecurity=True)\n" "--\n" "\n" -"Return a new BLAKE2b hash object."); +"Return a new SHA3 hash object."); static PyObject * py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity); @@ -193,4 +193,4 @@ _sha3_shake_128_hexdigest(SHA3object *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=747c3f34ddd14063 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=907cb475f3dc9ee0 input=a9049054013a1b77]*/ diff --git a/Modules/_sha3/sha3module.c b/Modules/sha3module.c similarity index 75% rename from Modules/_sha3/sha3module.c rename to Modules/sha3module.c index 93abc3b2710e55..f05187498a19b3 100644 --- a/Modules/_sha3/sha3module.c +++ b/Modules/sha3module.c @@ -22,23 +22,9 @@ #include "Python.h" #include "pycore_strhex.h" // _Py_strhex() #include "pycore_typeobject.h" // _PyType_GetModuleState() -#include "../hashlib.h" - -#include "sha3.c" +#include "hashlib.h" #define SHA3_MAX_DIGESTSIZE 64 /* 64 Bytes (512 Bits) for 224 to 512 */ -#define SHA3_LANESIZE 0 -#define SHA3_state sha3_ctx_t -#define SHA3_init sha3_init -#define SHA3_process sha3_update -#define SHA3_done(state, digest) sha3_final(digest, state) -#define SHA3_squeeze(state, out, len) shake_xof(state), shake_out(state, out, len) -#define SHA3_copystate(dest, src) memcpy(&(dest), &(src), sizeof(SHA3_state)) - -// no optimization -#define KeccakOpt 0 - -typedef enum { SUCCESS = 1, FAIL = 0, BAD_HASHLEN = 2 } HashReturn; typedef struct { PyTypeObject *sha3_224_type; @@ -70,10 +56,11 @@ class _sha3.shake_256 "SHA3object *" "&SHAKE256type" /* The structure for storing SHA3 info */ +#include "_hacl/Hacl_Hash_SHA3.h" + typedef struct { PyObject_HEAD - SHA3_state hash_state; - PyThread_type_lock lock; + Hacl_Streaming_Keccak_state *hash_state; } SHA3object; #include "clinic/sha3module.c.h" @@ -86,10 +73,23 @@ newSHA3object(PyTypeObject *type) if (newobj == NULL) { return NULL; } - newobj->lock = NULL; return newobj; } +static void sha3_update(Hacl_Streaming_Keccak_state *state, uint8_t *buf, Py_ssize_t len) { + /* Note: we explicitly ignore the error code on the basis that it would take > + * 1 billion years to hash more than 2^64 bytes. */ +#if PY_SSIZE_T_MAX > UINT32_MAX + while (len > UINT32_MAX) { + Hacl_Streaming_Keccak_update(state, buf, UINT32_MAX); + len -= UINT32_MAX; + buf += UINT32_MAX; + } +#endif + /* Cast to uint32_t is safe: len <= UINT32_MAX at this point. */ + Hacl_Streaming_Keccak_update(state, buf, (uint32_t) len); +} + /*[clinic input] @classmethod _sha3.sha3_224.__new__ as py_sha3_new @@ -98,14 +98,13 @@ _sha3.sha3_224.__new__ as py_sha3_new * usedforsecurity: bool = True -Return a new BLAKE2b hash object. +Return a new SHA3 hash object. [clinic start generated code]*/ static PyObject * py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) -/*[clinic end generated code: output=90409addc5d5e8b0 input=bcfcdf2e4368347a]*/ +/*[clinic end generated code: output=90409addc5d5e8b0 input=637e5f8f6a93982a]*/ { - HashReturn res; Py_buffer buf = {NULL, NULL}; SHA3State *state = _PyType_GetModuleState(type); SHA3object *self = newSHA3object(type); @@ -116,49 +115,29 @@ py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) assert(state != NULL); if (type == state->sha3_224_type) { - res = sha3_init(&self->hash_state, 28); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_224); } else if (type == state->sha3_256_type) { - res = sha3_init(&self->hash_state, 32); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_256); } else if (type == state->sha3_384_type) { - res = sha3_init(&self->hash_state, 48); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_384); } else if (type == state->sha3_512_type) { - res = sha3_init(&self->hash_state, 64); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_512); } else if (type == state->shake_128_type) { - res = sha3_init(&self->hash_state, 16); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_Shake128); } else if (type == state->shake_256_type) { - res = sha3_init(&self->hash_state, 32); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_Shake256); } else { PyErr_BadInternalCall(); goto error; } - if (res != SUCCESS) { - PyErr_SetString(PyExc_RuntimeError, - "internal error in SHA3 initialize()"); - goto error; - } - if (data) { GET_BUFFER_VIEW_OR_ERROR(data, &buf, goto error); - if (buf.len >= HASHLIB_GIL_MINSIZE) { - /* invariant: New objects can't be accessed by other code yet, - * thus it's safe to release the GIL without locking the object. - */ - Py_BEGIN_ALLOW_THREADS - res = SHA3_process(&self->hash_state, buf.buf, buf.len); - Py_END_ALLOW_THREADS - } - else { - res = SHA3_process(&self->hash_state, buf.buf, buf.len); - } - if (res != SUCCESS) { - PyErr_SetString(PyExc_RuntimeError, - "internal error in SHA3 Update()"); - goto error; - } - PyBuffer_Release(&buf); + sha3_update(self->hash_state, buf.buf, buf.len); } + PyBuffer_Release(&buf); + return (PyObject *)self; error: @@ -177,10 +156,7 @@ py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) static void SHA3_dealloc(SHA3object *self) { - if (self->lock) { - PyThread_free_lock(self->lock); - } - + Hacl_Streaming_Keccak_free(self->hash_state); PyTypeObject *tp = Py_TYPE(self); PyObject_Free(self); Py_DECREF(tp); @@ -205,9 +181,7 @@ _sha3_sha3_224_copy_impl(SHA3object *self) if ((newobj = newSHA3object(Py_TYPE(self))) == NULL) { return NULL; } - ENTER_HASHLIB(self); - SHA3_copystate(newobj->hash_state, self->hash_state); - LEAVE_HASHLIB(self); + newobj->hash_state = Hacl_Streaming_Keccak_copy(self->hash_state); return (PyObject *)newobj; } @@ -222,20 +196,12 @@ static PyObject * _sha3_sha3_224_digest_impl(SHA3object *self) /*[clinic end generated code: output=fd531842e20b2d5b input=5b2a659536bbd248]*/ { - unsigned char digest[SHA3_MAX_DIGESTSIZE + SHA3_LANESIZE]; - SHA3_state temp; - HashReturn res; - - ENTER_HASHLIB(self); - SHA3_copystate(temp, self->hash_state); - LEAVE_HASHLIB(self); - res = SHA3_done(&temp, digest); - if (res != SUCCESS) { - PyErr_SetString(PyExc_RuntimeError, "internal error in SHA3 Final()"); - return NULL; - } + unsigned char digest[SHA3_MAX_DIGESTSIZE]; + // This function errors out if the algorithm is Shake. Here, we know this + // not to be the case, and therefore do not perform error checking. + Hacl_Streaming_Keccak_finish(self->hash_state, digest); return PyBytes_FromStringAndSize((const char *)digest, - self->hash_state.mdlen); + Hacl_Streaming_Keccak_hash_len(self->hash_state)); } @@ -249,21 +215,10 @@ static PyObject * _sha3_sha3_224_hexdigest_impl(SHA3object *self) /*[clinic end generated code: output=75ad03257906918d input=2d91bb6e0d114ee3]*/ { - unsigned char digest[SHA3_MAX_DIGESTSIZE + SHA3_LANESIZE]; - SHA3_state temp; - HashReturn res; - - /* Get the raw (binary) digest value */ - ENTER_HASHLIB(self); - SHA3_copystate(temp, self->hash_state); - LEAVE_HASHLIB(self); - res = SHA3_done(&temp, digest); - if (res != SUCCESS) { - PyErr_SetString(PyExc_RuntimeError, "internal error in SHA3 Final()"); - return NULL; - } + unsigned char digest[SHA3_MAX_DIGESTSIZE]; + Hacl_Streaming_Keccak_finish(self->hash_state, digest); return _Py_strhex((const char *)digest, - self->hash_state.mdlen); + Hacl_Streaming_Keccak_hash_len(self->hash_state)); } @@ -281,36 +236,8 @@ _sha3_sha3_224_update(SHA3object *self, PyObject *data) /*[clinic end generated code: output=d3223352286ed357 input=a887f54dcc4ae227]*/ { Py_buffer buf; - HashReturn res; - GET_BUFFER_VIEW_OR_ERROUT(data, &buf); - - /* add new data, the function takes the length in bits not bytes */ - if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { - self->lock = PyThread_allocate_lock(); - } - /* Once a lock exists all code paths must be synchronized. We have to - * release the GIL even for small buffers as acquiring the lock may take - * an unlimited amount of time when another thread updates this object - * with lots of data. */ - if (self->lock) { - Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(self->lock, 1); - res = SHA3_process(&self->hash_state, buf.buf, buf.len); - PyThread_release_lock(self->lock); - Py_END_ALLOW_THREADS - } - else { - res = SHA3_process(&self->hash_state, buf.buf, buf.len); - } - - if (res != SUCCESS) { - PyBuffer_Release(&buf); - PyErr_SetString(PyExc_RuntimeError, - "internal error in SHA3 Update()"); - return NULL; - } - + sha3_update(self->hash_state, buf.buf, buf.len); PyBuffer_Release(&buf); Py_RETURN_NONE; } @@ -328,7 +255,7 @@ static PyMethodDef SHA3_methods[] = { static PyObject * SHA3_get_block_size(SHA3object *self, void *closure) { - int rate = self->hash_state.rsiz; + uint32_t rate = Hacl_Streaming_Keccak_block_len(self->hash_state); return PyLong_FromLong(rate); } @@ -363,14 +290,19 @@ SHA3_get_name(SHA3object *self, void *closure) static PyObject * SHA3_get_digest_size(SHA3object *self, void *closure) { - return PyLong_FromLong(self->hash_state.mdlen); + // Preserving previous behavior: variable-length algorithms return 0 + if (Hacl_Streaming_Keccak_is_shake(self->hash_state)) + return PyLong_FromLong(0); + else + return PyLong_FromLong(Hacl_Streaming_Keccak_hash_len(self->hash_state)); } static PyObject * SHA3_get_capacity_bits(SHA3object *self, void *closure) { - int capacity = 1600 - self->hash_state.rsiz * 8; + uint32_t rate = Hacl_Streaming_Keccak_block_len(self->hash_state) * 8; + int capacity = 1600 - rate; return PyLong_FromLong(capacity); } @@ -378,7 +310,7 @@ SHA3_get_capacity_bits(SHA3object *self, void *closure) static PyObject * SHA3_get_rate_bits(SHA3object *self, void *closure) { - unsigned int rate = self->hash_state.rsiz * 8; + uint32_t rate = Hacl_Streaming_Keccak_block_len(self->hash_state) * 8; return PyLong_FromLong(rate); } @@ -455,28 +387,26 @@ static PyObject * _SHAKE_digest(SHA3object *self, unsigned long digestlen, int hex) { unsigned char *digest = NULL; - SHA3_state temp; PyObject *result = NULL; if (digestlen >= (1 << 29)) { PyErr_SetString(PyExc_ValueError, "length is too large"); return NULL; } - /* ExtractLane needs at least SHA3_MAX_DIGESTSIZE + SHA3_LANESIZE and - * SHA3_LANESIZE extra space. - */ - digest = (unsigned char*)PyMem_Malloc(digestlen + SHA3_LANESIZE); + digest = (unsigned char*)PyMem_Malloc(digestlen); if (digest == NULL) { return PyErr_NoMemory(); } - /* Get the raw (binary) digest value */ - ENTER_HASHLIB(self); - SHA3_copystate(temp, self->hash_state); - LEAVE_HASHLIB(self); - SHA3_squeeze(&temp, digest, digestlen); + /* Get the raw (binary) digest value. The HACL functions errors out if: + * - the algorith is not shake -- not the case here + * - the output length is zero -- we follow the existing behavior and return + * an empty digest, without raising an error */ + if (digestlen > 0) { + Hacl_Streaming_Keccak_squeeze(self->hash_state, digest, digestlen); + } if (hex) { - result = _Py_strhex((const char *)digest, digestlen); + result = _Py_strhex((const char *)digest, digestlen); } else { result = PyBytes_FromStringAndSize((const char *)digest, digestlen); @@ -628,11 +558,8 @@ _sha3_exec(PyObject *m) init_sha3type(shake_256_type, SHAKE256_spec); #undef init_sha3type - if (PyModule_AddIntConstant(m, "keccakopt", KeccakOpt) < 0) { - return -1; - } if (PyModule_AddStringConstant(m, "implementation", - "tiny_sha3") < 0) { + "HACL") < 0) { return -1; } diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 8aafcb786a6064..28b1517c6f6b3a 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -375,13 +375,16 @@ + + + + - @@ -404,17 +407,15 @@ - - - + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 07476f30833372..75e6fbb13f98ba 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -776,6 +776,15 @@ Modules + + Modules + + + Modules + + + Modules + Modules @@ -794,9 +803,6 @@ Modules - - Modules - Modules @@ -875,16 +881,13 @@ Modules - - Modules - Modules - + Modules - + Modules diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 5924ab7860d8d5..9bd54db0f59c51 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -114,6 +114,7 @@ def clean_lines(text): Modules/md5module.c Modules/_hacl/include Modules/sha1module.c Modules/_hacl/include Modules/sha2module.c Modules/_hacl/include +Modules/sha3module.c Modules/_hacl/include Objects/stringlib/*.h Objects # possible system-installed headers, just in case @@ -271,13 +272,6 @@ def clean_lines(text): Modules/expat/xmlparse.c XML_POOR_ENTROPY 1 Modules/_dbmmodule.c HAVE_GDBM_DASH_NDBM_H 1 -# from Modules/_sha3/sha3module.c -Modules/_sha3/kcp/KeccakP-1600-inplace32BI.c PLATFORM_BYTE_ORDER 4321 # force big-endian -Modules/_sha3/kcp/*.c KeccakOpt 64 -Modules/_sha3/kcp/*.c KeccakP200_excluded 1 -Modules/_sha3/kcp/*.c KeccakP400_excluded 1 -Modules/_sha3/kcp/*.c KeccakP800_excluded 1 - # others Modules/_sre/sre_lib.h LOCAL(type) static inline type Modules/_sre/sre_lib.h SRE(F) sre_ucs2_##F From c2683fc46d775d6c4afcb23658c0fd1e328e3c53 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Mon, 8 May 2023 04:59:34 -0700 Subject: [PATCH 103/204] gh-97696: Improve and fix documentation for asyncio eager tasks (#104256) --- Doc/library/asyncio-task.rst | 19 ++++++++++++++++++- ...-05-03-16-51-53.gh-issue-104144.653Q0P.rst | 3 ++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index a46ebc1c3d25a9..b2d7362a9de213 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -560,6 +560,13 @@ Eager Task Factory using the provided *custom_task_constructor* when creating a new task instead of the default :class:`Task`. + *custom_task_constructor* must be a *callable* with the signature matching + the signature of :class:`Task.__init__ `. + The callable must return a :class:`asyncio.Task`-compatible object. + + This function returns a *callable* intended to be used as a task factory of an + event loop via :meth:`loop.set_task_factory(factory) `). + .. versionadded:: 3.12 @@ -1014,7 +1021,7 @@ Introspection Task Object =========== -.. class:: Task(coro, *, loop=None, name=None, context=None) +.. class:: Task(coro, *, loop=None, name=None, context=None, eager_start=False) A :class:`Future-like ` object that runs a Python :ref:`coroutine `. Not thread-safe. @@ -1054,6 +1061,13 @@ Task Object If no *context* is provided, the Task copies the current context and later runs its coroutine in the copied context. + An optional keyword-only *eager_start* argument allows eagerly starting + the execution of the :class:`asyncio.Task` at task creation time. + If set to ``True`` and the event loop is running, the task will start + executing the coroutine immediately, until the first time the coroutine + blocks. If the coroutine returns or raises without blocking, the task + will be finished eagerly and will skip scheduling to the event loop. + .. versionchanged:: 3.7 Added support for the :mod:`contextvars` module. @@ -1067,6 +1081,9 @@ Task Object .. versionchanged:: 3.11 Added the *context* parameter. + .. versionchanged:: 3.12 + Added the *eager_start* parameter. + .. method:: done() Return ``True`` if the Task is *done*. diff --git a/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst b/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst index 59870de3e02edd..ced3b7cea04954 100644 --- a/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst +++ b/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst @@ -1 +1,2 @@ -Optimize :class:`asyncio.TaskGroup` when using :func:`asyncio.eager_task_factory`. Skip scheduling done callbacks when all tasks finish without blocking. +Optimize :class:`asyncio.TaskGroup` when using :func:`asyncio.eager_task_factory`. +Skip scheduling a done callback if a TaskGroup task completes eagerly. From 1f5679540ca4aa5c0eae06d3a2d5eda34b47e041 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 8 May 2023 06:40:51 -0700 Subject: [PATCH 104/204] gh-102500: Remove mention of bytes shorthand (#104281) The bytes shorthand was removed in PEP 688: https://peps.python.org/pep-0688/#no-special-meaning-for-bytes I also remove the reference to `collections.abc.ByteString`, since that object is deprecated (#91896) and has different semantics (#102092) --- Doc/library/typing.rst | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 162041fc7a846e..ebab1389f07e58 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2130,15 +2130,10 @@ Corresponding to collections in :mod:`collections.abc` .. class:: ByteString(Sequence[int]) - A generic version of :class:`collections.abc.ByteString`. - This type represents the types :class:`bytes`, :class:`bytearray`, and :class:`memoryview` of byte sequences. - As a shorthand for this type, :class:`bytes` can be used to - annotate arguments of any of the types mentioned above. - - .. deprecated:: 3.9 + .. deprecated-removed:: 3.9 3.14 Prefer :class:`collections.abc.Buffer`, or a union like ``bytes | bytearray | memoryview``. .. class:: Collection(Sized, Iterable[T_co], Container[T_co]) @@ -2977,6 +2972,8 @@ convenience. This is subject to change, and not all deprecations are listed. | ``typing`` versions of standard | 3.9 | Undecided | :pep:`585` | | collections | | | | +----------------------------------+---------------+-------------------+----------------+ +| ``typing.ByteString`` | 3.9 | 3.14 | :gh:`91896` | ++----------------------------------+---------------+-------------------+----------------+ | ``typing.Text`` | 3.11 | Undecided | :gh:`92332` | +----------------------------------+---------------+-------------------+----------------+ | ``typing.Hashable`` and | 3.12 | Undecided | :gh:`94309` | From d513ddee94a05783b98f2b55f8fc0a4efbb9be82 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 8 May 2023 17:03:52 +0300 Subject: [PATCH 105/204] Trim trailing whitespace and test on CI (#104275) Co-authored-by: Alex Waygood --- .github/CODEOWNERS | 3 +++ .github/workflows/lint.yml | 22 ++++++++++++++++++++++ .pre-commit-config.yaml | 7 +++++++ Modules/_blake2/blake2module.h | 2 +- Modules/_blake2/impl/blake2b-round.h | 4 ++-- Modules/_blake2/impl/blake2s-load-xop.h | 2 +- Modules/_blake2/impl/blake2s-round.h | 2 +- Modules/_ctypes/_ctypes_test.c | 2 +- Modules/_testcapi/immortal.c | 2 +- Modules/termios.c | 4 ++-- Parser/tokenizer.c | 2 +- Tools/msi/bundle/bootstrap/pch.h | 2 +- Tools/msi/bundle/bootstrap/resource.h | 2 +- 13 files changed, 44 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/lint.yml create mode 100644 .pre-commit-config.yaml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3422ef835279bc..d40519e40d3cc2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -7,6 +7,9 @@ # GitHub .github/** @ezio-melotti @hugovk +# pre-commit +.pre-commit-config.yaml @hugovk @AlexWaygood + # Build system configure* @erlend-aasland @corona10 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000000000..4481ea80bfd936 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,22 @@ +name: Lint + +on: [push, pull_request, workflow_dispatch] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.x" + - uses: pre-commit/action@v3.0.0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000000000..808622f19a3dbf --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-yaml + - id: trailing-whitespace + types_or: [c, python, rst] diff --git a/Modules/_blake2/blake2module.h b/Modules/_blake2/blake2module.h index aa8f281178eadc..c8144ec9d48d29 100644 --- a/Modules/_blake2/blake2module.h +++ b/Modules/_blake2/blake2module.h @@ -38,6 +38,6 @@ #endif // HAVE_LIBB2 // for secure_zero_memory(), store32(), store48(), and store64() -#include "impl/blake2-impl.h" +#include "impl/blake2-impl.h" #endif // Py_BLAKE2MODULE_H diff --git a/Modules/_blake2/impl/blake2b-round.h b/Modules/_blake2/impl/blake2b-round.h index cebc22550da4cd..5b452c4d63babe 100644 --- a/Modules/_blake2/impl/blake2b-round.h +++ b/Modules/_blake2/impl/blake2b-round.h @@ -62,7 +62,7 @@ \ row2l = _mm_roti_epi64(row2l, -24); \ row2h = _mm_roti_epi64(row2h, -24); \ - + #define G2(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1) \ row1l = _mm_add_epi64(_mm_add_epi64(row1l, b0), row2l); \ row1h = _mm_add_epi64(_mm_add_epi64(row1h, b1), row2h); \ @@ -81,7 +81,7 @@ \ row2l = _mm_roti_epi64(row2l, -63); \ row2h = _mm_roti_epi64(row2h, -63); \ - + #if defined(HAVE_SSSE3) #define DIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h) \ t0 = _mm_alignr_epi8(row2h, row2l, 8); \ diff --git a/Modules/_blake2/impl/blake2s-load-xop.h b/Modules/_blake2/impl/blake2s-load-xop.h index ac591a77d191a7..14d9e7f7640672 100644 --- a/Modules/_blake2/impl/blake2s-load-xop.h +++ b/Modules/_blake2/impl/blake2s-load-xop.h @@ -166,7 +166,7 @@ buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(1),TOB(7)) ); #define LOAD_MSG_8_3(buf) \ t0 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(6),TOB(1),TOB(0),TOB(0)) ); \ buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(5),TOB(4)) ); \ - + #define LOAD_MSG_8_4(buf) \ buf = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(5),TOB(4),TOB(7),TOB(2)) ); diff --git a/Modules/_blake2/impl/blake2s-round.h b/Modules/_blake2/impl/blake2s-round.h index 1e2f2b7f59bd6c..3af4be35bee5d4 100644 --- a/Modules/_blake2/impl/blake2s-round.h +++ b/Modules/_blake2/impl/blake2s-round.h @@ -86,6 +86,6 @@ LOAD_MSG_ ##r ##_4(buf4); \ G2(row1,row2,row3,row4,buf4); \ UNDIAGONALIZE(row1,row2,row3,row4); \ - + #endif diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index ce652b362d5bb3..ddfb2c8a332a9e 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -1036,7 +1036,7 @@ EXPORT (HRESULT) KeepObject(IUnknown *punk) #ifdef MS_WIN32 -// i38748: c stub for testing stack corruption +// i38748: c stub for testing stack corruption // When executing a Python callback with a long and a long long typedef long(__stdcall *_test_i38748_funcType)(long, long long); diff --git a/Modules/_testcapi/immortal.c b/Modules/_testcapi/immortal.c index 10e1733d08a9ea..9f81389811c645 100644 --- a/Modules/_testcapi/immortal.c +++ b/Modules/_testcapi/immortal.c @@ -1,6 +1,6 @@ #include "parts.h" -int verify_immortality(PyObject *object) +int verify_immortality(PyObject *object) { assert(_Py_IsImmortal(object)); Py_ssize_t old_count = Py_REFCNT(object); diff --git a/Modules/termios.c b/Modules/termios.c index 169a36fc6477d8..6dc8200572bc0c 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -85,7 +85,7 @@ termios_tcgetattr_impl(PyObject *module, int fd) int r; Py_BEGIN_ALLOW_THREADS - r = tcgetattr(fd, &mode); + r = tcgetattr(fd, &mode); Py_END_ALLOW_THREADS if (r == -1) { return PyErr_SetFromErrno(state->TermiosError); @@ -372,7 +372,7 @@ termios_tcgetwinsize_impl(PyObject *module, int fd) #if defined(TIOCGWINSZ) termiosmodulestate *state = PyModule_GetState(module); struct winsize w; - int r; + int r; Py_BEGIN_ALLOW_THREADS r = ioctl(fd, TIOCGWINSZ, &w); diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 52d0d9a534cb6a..91ffabac56c7b3 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -48,7 +48,7 @@ static inline tokenizer_mode* TOK_GET_MODE(struct tok_state* tok) { } static inline tokenizer_mode* TOK_NEXT_MODE(struct tok_state* tok) { assert(tok->tok_mode_stack_index >= 0); - assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL); + assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL); return &(tok->tok_mode_stack[++tok->tok_mode_stack_index]); } #else diff --git a/Tools/msi/bundle/bootstrap/pch.h b/Tools/msi/bundle/bootstrap/pch.h index b0aa5111dabd0d..6d0974b34c61e7 100644 --- a/Tools/msi/bundle/bootstrap/pch.h +++ b/Tools/msi/bundle/bootstrap/pch.h @@ -5,7 +5,7 @@ // The license and further copyright text can be found in the file // LICENSE.TXT at the root directory of the distribution. // -// +// // // Precompiled header for standard bootstrapper application. // diff --git a/Tools/msi/bundle/bootstrap/resource.h b/Tools/msi/bundle/bootstrap/resource.h index 53c03c319f091f..d951e651f6d20d 100644 --- a/Tools/msi/bundle/bootstrap/resource.h +++ b/Tools/msi/bundle/bootstrap/resource.h @@ -14,7 +14,7 @@ // Next default values for new objects -// +// #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS #define _APS_NEXT_RESOURCE_VALUE 102 From 921185ed050efbca2f0adeab79f676b7f8cc3660 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 8 May 2023 15:18:36 +0100 Subject: [PATCH 106/204] gh-103193: Improve `getattr_static` test coverage (#104286) --- Lib/test/test_inspect.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index dd0325a43e0f58..d2b2f3171e785d 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -2187,6 +2187,35 @@ class Thing(metaclass=Meta): inspect.getattr_static(Thing, "spam") self.assertFalse(Thing.executed) + def test_custom___getattr__(self): + test = self + test.called = False + + class Foo: + def __getattr__(self, attr): + test.called = True + return {} + + with self.assertRaises(AttributeError): + inspect.getattr_static(Foo(), 'whatever') + + self.assertFalse(test.called) + + def test_custom___getattribute__(self): + test = self + test.called = False + + class Foo: + def __getattribute__(self, attr): + test.called = True + return {} + + with self.assertRaises(AttributeError): + inspect.getattr_static(Foo(), 'really_could_be_anything') + + self.assertFalse(test.called) + + class TestGetGeneratorState(unittest.TestCase): def setUp(self): From 76eef552f3653179782afcc5063f10560a6e1a80 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 8 May 2023 17:32:18 +0100 Subject: [PATCH 107/204] GH-104145: Use fully-qualified cross reference types for the bisect module (#104172) --- Doc/library/bisect.rst | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index e3c8c801904b61..8022c596f0af97 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -24,6 +24,8 @@ method to determine whether a value has been found. Instead, the functions only call the :meth:`__lt__` method and will return an insertion point between values in an array. +.. _bisect functions: + The following functions are provided: @@ -55,7 +57,7 @@ The following functions are provided: .. function:: bisect_right(a, x, lo=0, hi=len(a), *, key=None) bisect(a, x, lo=0, hi=len(a), *, key=None) - Similar to :func:`bisect_left`, but returns an insertion point which comes + Similar to :py:func:`~bisect.bisect_left`, but returns an insertion point which comes after (to the right of) any existing entries of *x* in *a*. The returned insertion point *ip* partitions the array *a* into two slices @@ -70,7 +72,7 @@ The following functions are provided: Insert *x* in *a* in sorted order. - This function first runs :func:`bisect_left` to locate an insertion point. + This function first runs :py:func:`~bisect.bisect_left` to locate an insertion point. Next, it runs the :meth:`insert` method on *a* to insert *x* at the appropriate position to maintain sort order. @@ -87,10 +89,10 @@ The following functions are provided: .. function:: insort_right(a, x, lo=0, hi=len(a), *, key=None) insort(a, x, lo=0, hi=len(a), *, key=None) - Similar to :func:`insort_left`, but inserting *x* in *a* after any existing + Similar to :py:func:`~bisect.insort_left`, but inserting *x* in *a* after any existing entries of *x*. - This function first runs :func:`bisect_right` to locate an insertion point. + This function first runs :py:func:`~bisect.bisect_right` to locate an insertion point. Next, it runs the :meth:`insert` method on *a* to insert *x* at the appropriate position to maintain sort order. @@ -120,7 +122,7 @@ thoughts in mind: they are used. Consequently, if the search functions are used in a loop, the key function may be called again and again on the same array elements. If the key function isn't fast, consider wrapping it with - :func:`functools.cache` to avoid duplicate computations. Alternatively, + :py:func:`functools.cache` to avoid duplicate computations. Alternatively, consider searching an array of precomputed keys to locate the insertion point (as shown in the examples section below). @@ -140,7 +142,7 @@ thoughts in mind: Searching Sorted Lists ---------------------- -The above :func:`bisect` functions are useful for finding insertion points but +The above `bisect functions`_ are useful for finding insertion points but can be tricky or awkward to use for common searching tasks. The following five functions show how to transform them into the standard lookups for sorted lists:: @@ -186,8 +188,8 @@ Examples .. _bisect-example: -The :func:`bisect` function can be useful for numeric table lookups. This -example uses :func:`bisect` to look up a letter grade for an exam score (say) +The :py:func:`~bisect.bisect` function can be useful for numeric table lookups. This +example uses :py:func:`~bisect.bisect` to look up a letter grade for an exam score (say) based on a set of ordered numeric breakpoints: 90 and up is an 'A', 80 to 89 is a 'B', and so on:: @@ -198,8 +200,8 @@ a 'B', and so on:: >>> [grade(score) for score in [33, 99, 77, 70, 89, 90, 100]] ['F', 'A', 'C', 'C', 'B', 'A', 'A'] -The :func:`bisect` and :func:`insort` functions also work with lists of -tuples. The *key* argument can serve to extract the field used for ordering +The :py:func:`~bisect.bisect` and :py:func:`~bisect.insort` functions also work with +lists of tuples. The *key* argument can serve to extract the field used for ordering records in a table:: >>> from collections import namedtuple From 874010c6cab2e079069767619af2e0eab05ad0b2 Mon Sep 17 00:00:00 2001 From: Jonathan Protzenko Date: Mon, 8 May 2023 09:52:11 -0700 Subject: [PATCH 108/204] gh-99108: fix typo in Modules/Setup (#104293) case sensitive filename --- Modules/Setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/Setup b/Modules/Setup index e5bc078af62c41..312e99fea530dc 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -166,7 +166,7 @@ PYTHONPATH=$(COREPYTHONPATH) #_md5 md5module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_MD5.c -D_BSD_SOURCE -D_DEFAULT_SOURCE #_sha1 sha1module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA1.c -D_BSD_SOURCE -D_DEFAULT_SOURCE #_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a -#_sha3 sha3module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_sha3.c -D_BSD_SOURCE -D_DEFAULT_SOURCE +#_sha3 sha3module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA3.c -D_BSD_SOURCE -D_DEFAULT_SOURCE # text encodings and unicode #_codecs_cn cjkcodecs/_codecs_cn.c From 405eacc1b87a42e19fd176131e70537f0539e05e Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 8 May 2023 09:52:41 -0700 Subject: [PATCH 109/204] gh-104223: Fix issues with inheriting from buffer classes (#104227) Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> --- Include/cpython/memoryobject.h | 1 + Include/internal/pycore_memoryobject.h | 3 +- Lib/test/test_buffer.py | 170 +++++++++++++++++++++++++ Objects/bytearrayobject.c | 1 + Objects/memoryobject.c | 45 ++++++- Objects/typeobject.c | 127 ++++++++++++++++-- 6 files changed, 334 insertions(+), 13 deletions(-) diff --git a/Include/cpython/memoryobject.h b/Include/cpython/memoryobject.h index deab3cc89f726e..3837fa8c6ab5aa 100644 --- a/Include/cpython/memoryobject.h +++ b/Include/cpython/memoryobject.h @@ -24,6 +24,7 @@ typedef struct { #define _Py_MEMORYVIEW_FORTRAN 0x004 /* Fortran contiguous layout */ #define _Py_MEMORYVIEW_SCALAR 0x008 /* scalar: ndim = 0 */ #define _Py_MEMORYVIEW_PIL 0x010 /* PIL-style layout */ +#define _Py_MEMORYVIEW_RESTRICTED 0x020 /* Disallow new references to the memoryview's buffer */ typedef struct { PyObject_VAR_HEAD diff --git a/Include/internal/pycore_memoryobject.h b/Include/internal/pycore_memoryobject.h index acc12c9275172c..fe19e3f9611a16 100644 --- a/Include/internal/pycore_memoryobject.h +++ b/Include/internal/pycore_memoryobject.h @@ -9,7 +9,8 @@ extern "C" { #endif PyObject * -PyMemoryView_FromObjectAndFlags(PyObject *v, int flags); +_PyMemoryView_FromBufferProc(PyObject *v, int flags, + getbufferproc bufferproc); #ifdef __cplusplus } diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index b6e82ad4db266a..2c65ae8114818f 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4579,6 +4579,176 @@ def test_c_buffer(self): buf.__release_buffer__(mv) self.assertEqual(buf.references, 0) + def test_inheritance(self): + class A(bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + + a = A(b"hello") + mv = memoryview(a) + self.assertEqual(mv.tobytes(), b"hello") + + def test_inheritance_releasebuffer(self): + rb_call_count = 0 + class B(bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + def __release_buffer__(self, view): + nonlocal rb_call_count + rb_call_count += 1 + super().__release_buffer__(view) + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(rb_call_count, 0) + self.assertEqual(rb_call_count, 1) + + def test_inherit_but_return_something_else(self): + class A(bytearray): + def __buffer__(self, flags): + return memoryview(b"hello") + + a = A(b"hello") + with memoryview(a) as mv: + self.assertEqual(mv.tobytes(), b"hello") + + rb_call_count = 0 + rb_raised = False + class B(bytearray): + def __buffer__(self, flags): + return memoryview(b"hello") + def __release_buffer__(self, view): + nonlocal rb_call_count + rb_call_count += 1 + try: + super().__release_buffer__(view) + except ValueError: + nonlocal rb_raised + rb_raised = True + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(rb_call_count, 0) + self.assertEqual(rb_call_count, 1) + self.assertIs(rb_raised, True) + + def test_override_only_release(self): + class C(bytearray): + def __release_buffer__(self, buffer): + super().__release_buffer__(buffer) + + c = C(b"hello") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + + def test_release_saves_reference(self): + smuggled_buffer = None + + class C(bytearray): + def __release_buffer__(s, buffer: memoryview): + with self.assertRaises(ValueError): + memoryview(buffer) + with self.assertRaises(ValueError): + buffer.cast("b") + with self.assertRaises(ValueError): + buffer.toreadonly() + with self.assertRaises(ValueError): + buffer[:1] + with self.assertRaises(ValueError): + buffer.__buffer__(0) + nonlocal smuggled_buffer + smuggled_buffer = buffer + self.assertEqual(buffer.tobytes(), b"hello") + super().__release_buffer__(buffer) + + c = C(b"hello") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + c.clear() + with self.assertRaises(ValueError): + smuggled_buffer.tobytes() + + def test_release_saves_reference_no_subclassing(self): + ba = bytearray(b"hello") + + class C: + def __buffer__(self, flags): + return memoryview(ba) + + def __release_buffer__(self, buffer): + self.buffer = buffer + + c = C() + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(c.buffer.tobytes(), b"hello") + + with self.assertRaises(BufferError): + ba.clear() + c.buffer.release() + ba.clear() + + def test_multiple_inheritance_buffer_last(self): + class A: + def __buffer__(self, flags): + return memoryview(b"hello A") + + class B(A, bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello A") + + class Releaser: + def __release_buffer__(self, buffer): + self.buffer = buffer + + class C(Releaser, bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + + c = C(b"hello C") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello C") + c.clear() + with self.assertRaises(ValueError): + c.buffer.tobytes() + + def test_multiple_inheritance_buffer_last(self): + class A: + def __buffer__(self, flags): + raise RuntimeError("should not be called") + + def __release_buffer__(self, buffer): + raise RuntimeError("should not be called") + + class B(bytearray, A): + def __buffer__(self, flags): + return super().__buffer__(flags) + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello") + + class Releaser: + buffer = None + def __release_buffer__(self, buffer): + self.buffer = buffer + + class C(bytearray, Releaser): + def __buffer__(self, flags): + return super().__buffer__(flags) + + c = C(b"hello") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + c.clear() + self.assertIs(c.buffer, None) + if __name__ == "__main__": unittest.main() diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 49d4dd524696a5..c36db59baaa10d 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -61,6 +61,7 @@ static void bytearray_releasebuffer(PyByteArrayObject *obj, Py_buffer *view) { obj->ob_exports--; + assert(obj->ob_exports >= 0); } static int diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index f008a8cc3e0474..b0168044d9f85a 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -193,6 +193,20 @@ PyTypeObject _PyManagedBuffer_Type = { return -1; \ } +#define CHECK_RESTRICTED(mv) \ + if (((PyMemoryViewObject *)(mv))->flags & _Py_MEMORYVIEW_RESTRICTED) { \ + PyErr_SetString(PyExc_ValueError, \ + "cannot create new view on restricted memoryview"); \ + return NULL; \ + } + +#define CHECK_RESTRICTED_INT(mv) \ + if (((PyMemoryViewObject *)(mv))->flags & _Py_MEMORYVIEW_RESTRICTED) { \ + PyErr_SetString(PyExc_ValueError, \ + "cannot create new view on restricted memoryview"); \ + return -1; \ + } + /* See gh-92888. These macros signal that we need to check the memoryview again due to possible read after frees. */ #define CHECK_RELEASED_AGAIN(mv) CHECK_RELEASED(mv) @@ -781,7 +795,7 @@ PyMemoryView_FromBuffer(const Py_buffer *info) using the given flags. If the object is a memoryview, the new memoryview must be registered with the same managed buffer. Otherwise, a new managed buffer is created. */ -PyObject * +static PyObject * PyMemoryView_FromObjectAndFlags(PyObject *v, int flags) { _PyManagedBufferObject *mbuf; @@ -789,6 +803,7 @@ PyMemoryView_FromObjectAndFlags(PyObject *v, int flags) if (PyMemoryView_Check(v)) { PyMemoryViewObject *mv = (PyMemoryViewObject *)v; CHECK_RELEASED(mv); + CHECK_RESTRICTED(mv); return mbuf_add_view(mv->mbuf, &mv->view); } else if (PyObject_CheckBuffer(v)) { @@ -806,6 +821,30 @@ PyMemoryView_FromObjectAndFlags(PyObject *v, int flags) Py_TYPE(v)->tp_name); return NULL; } + +/* Create a memoryview from an object that implements the buffer protocol, + using the given flags. + If the object is a memoryview, the new memoryview must be registered + with the same managed buffer. Otherwise, a new managed buffer is created. */ +PyObject * +_PyMemoryView_FromBufferProc(PyObject *v, int flags, getbufferproc bufferproc) +{ + _PyManagedBufferObject *mbuf = mbuf_alloc(); + if (mbuf == NULL) + return NULL; + + int res = bufferproc(v, &mbuf->master, flags); + if (res < 0) { + mbuf->master.obj = NULL; + Py_DECREF(mbuf); + return NULL; + } + + PyObject *ret = mbuf_add_view(mbuf, NULL); + Py_DECREF(mbuf); + return ret; +} + /* Create a memoryview from an object that implements the buffer protocol. If the object is a memoryview, the new memoryview must be registered with the same managed buffer. Otherwise, a new managed buffer is created. */ @@ -1397,6 +1436,7 @@ memoryview_cast_impl(PyMemoryViewObject *self, PyObject *format, Py_ssize_t ndim = 1; CHECK_RELEASED(self); + CHECK_RESTRICTED(self); if (!MV_C_CONTIGUOUS(self->flags)) { PyErr_SetString(PyExc_TypeError, @@ -1452,6 +1492,7 @@ memoryview_toreadonly_impl(PyMemoryViewObject *self) /*[clinic end generated code: output=2c7e056f04c99e62 input=dc06d20f19ba236f]*/ { CHECK_RELEASED(self); + CHECK_RESTRICTED(self); /* Even if self is already readonly, we still need to create a new * object for .release() to work correctly. */ @@ -1474,6 +1515,7 @@ memory_getbuf(PyMemoryViewObject *self, Py_buffer *view, int flags) int baseflags = self->flags; CHECK_RELEASED_INT(self); + CHECK_RESTRICTED_INT(self); /* start with complete information */ *view = *base; @@ -2535,6 +2577,7 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key) return memory_item(self, index); } else if (PySlice_Check(key)) { + CHECK_RESTRICTED(self); PyMemoryViewObject *sliced; sliced = (PyMemoryViewObject *)mbuf_add_view(self->mbuf, view); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 456b10ee01d6bc..98fac276a873e1 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6,7 +6,7 @@ #include "pycore_symtable.h" // _Py_Mangle() #include "pycore_dict.h" // _PyDict_KeysSize() #include "pycore_initconfig.h" // _PyStatus_OK() -#include "pycore_memoryobject.h" // PyMemoryView_FromObjectAndFlags() +#include "pycore_memoryobject.h" // _PyMemoryView_FromBufferProc() #include "pycore_moduleobject.h" // _PyModule_GetDef() #include "pycore_object.h" // _PyType_HasFeature() #include "pycore_long.h" // _PyLong_IsNegative() @@ -56,6 +56,11 @@ typedef struct PySlot_Offset { short slot_offset; } PySlot_Offset; +static void +slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer); + +static void +releasebuffer_call_python(PyObject *self, Py_buffer *buffer); static PyObject * slot_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds); @@ -8078,7 +8083,8 @@ wrap_buffer(PyObject *self, PyObject *args, void *wrapped) return NULL; } - return PyMemoryView_FromObjectAndFlags(self, Py_SAFE_DOWNCAST(flags, Py_ssize_t, int)); + return _PyMemoryView_FromBufferProc(self, Py_SAFE_DOWNCAST(flags, Py_ssize_t, int), + (getbufferproc)wrapped); } static PyObject * @@ -8094,6 +8100,10 @@ wrap_releasebuffer(PyObject *self, PyObject *args, void *wrapped) return NULL; } PyMemoryViewObject *mview = (PyMemoryViewObject *)arg; + if (mview->view.obj == NULL) { + // Already released, ignore + Py_RETURN_NONE; + } if (mview->view.obj != self) { PyErr_SetString(PyExc_ValueError, "memoryview's buffer is not this object"); @@ -8978,12 +8988,26 @@ bufferwrapper_releasebuf(PyObject *self, Py_buffer *view) { PyBufferWrapper *bw = (PyBufferWrapper *)self; - assert(PyMemoryView_Check(bw->mv)); - Py_TYPE(bw->mv)->tp_as_buffer->bf_releasebuffer(bw->mv, view); - if (Py_TYPE(bw->obj)->tp_as_buffer != NULL - && Py_TYPE(bw->obj)->tp_as_buffer->bf_releasebuffer != NULL) { - Py_TYPE(bw->obj)->tp_as_buffer->bf_releasebuffer(bw->obj, view); + if (bw->mv == NULL || bw->obj == NULL) { + // Already released + return; + } + + PyObject *mv = bw->mv; + PyObject *obj = bw->obj; + + assert(PyMemoryView_Check(mv)); + Py_TYPE(mv)->tp_as_buffer->bf_releasebuffer(mv, view); + // We only need to call bf_releasebuffer if it's a Python function. If it's a C + // bf_releasebuf, it will be called when the memoryview is released. + if (((PyMemoryViewObject *)mv)->view.obj != obj + && Py_TYPE(obj)->tp_as_buffer != NULL + && Py_TYPE(obj)->tp_as_buffer->bf_releasebuffer == slot_bf_releasebuffer) { + releasebuffer_call_python(obj, view); } + + Py_CLEAR(bw->mv); + Py_CLEAR(bw->obj); } static PyBufferProcs bufferwrapper_as_buffer = { @@ -9047,31 +9071,112 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) return -1; } +static int +releasebuffer_maybe_call_super(PyObject *self, Py_buffer *buffer) +{ + PyTypeObject *self_type = Py_TYPE(self); + PyObject *mro = lookup_tp_mro(self_type); + if (mro == NULL) { + return -1; + } + + assert(PyTuple_Check(mro)); + Py_ssize_t n = PyTuple_GET_SIZE(mro); + Py_ssize_t i; + + /* No need to check the last one: it's gonna be skipped anyway. */ + for (i = 0; i < n -1; i++) { + if ((PyObject *)(self_type) == PyTuple_GET_ITEM(mro, i)) + break; + } + i++; /* skip self_type */ + if (i >= n) + return -1; + + releasebufferproc base_releasebuffer = NULL; + for (; i < n; i++) { + PyObject *obj = PyTuple_GET_ITEM(mro, i); + if (!PyType_Check(obj)) { + continue; + } + PyTypeObject *base_type = (PyTypeObject *)obj; + if (base_type->tp_as_buffer != NULL + && base_type->tp_as_buffer->bf_releasebuffer != NULL + && base_type->tp_as_buffer->bf_releasebuffer != slot_bf_releasebuffer) { + base_releasebuffer = base_type->tp_as_buffer->bf_releasebuffer; + break; + } + } + + if (base_releasebuffer != NULL) { + base_releasebuffer(self, buffer); + } + return 0; +} + static void -slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) +releasebuffer_call_python(PyObject *self, Py_buffer *buffer) { PyObject *mv; - if (Py_TYPE(buffer->obj) == &_PyBufferWrapper_Type) { + bool is_buffer_wrapper = Py_TYPE(buffer->obj) == &_PyBufferWrapper_Type; + if (is_buffer_wrapper) { // Make sure we pass the same memoryview to // __release_buffer__() that __buffer__() returned. - mv = Py_NewRef(((PyBufferWrapper *)buffer->obj)->mv); + PyBufferWrapper *bw = (PyBufferWrapper *)buffer->obj; + if (bw->mv == NULL) { + return; + } + mv = Py_NewRef(bw->mv); } else { + // This means we are not dealing with a memoryview returned + // from a Python __buffer__ function. mv = PyMemoryView_FromBuffer(buffer); if (mv == NULL) { PyErr_WriteUnraisable(self); return; } + // Set the memoryview to restricted mode, which forbids + // users from saving any reference to the underlying buffer + // (e.g., by doing .cast()). This is necessary to ensure + // no Python code retains a reference to the to-be-released + // buffer. + ((PyMemoryViewObject *)mv)->flags |= _Py_MEMORYVIEW_RESTRICTED; } PyObject *stack[2] = {self, mv}; PyObject *ret = vectorcall_method(&_Py_ID(__release_buffer__), stack, 2); - Py_DECREF(mv); if (ret == NULL) { PyErr_WriteUnraisable(self); } else { Py_DECREF(ret); } + if (!is_buffer_wrapper) { + PyObject_CallMethodNoArgs(mv, &_Py_ID(release)); + } + Py_DECREF(mv); +} + +/* + * bf_releasebuffer is very delicate, because we need to ensure that + * C bf_releasebuffer slots are called correctly (or we'll leak memory), + * but we cannot trust any __release_buffer__ implemented in Python to + * do so correctly. Therefore, if a base class has a C bf_releasebuffer + * slot, we call it directly here. That is safe because this function + * only gets called from C callers of the bf_releasebuffer slot. Python + * code that calls __release_buffer__ directly instead goes through + * wrap_releasebuffer(), which doesn't call the bf_releasebuffer slot + * directly but instead simply releases the associated memoryview. + */ +static void +slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) +{ + releasebuffer_call_python(self, buffer); + if (releasebuffer_maybe_call_super(self, buffer) < 0) { + if (PyErr_Occurred()) { + PyErr_WriteUnraisable(self); + } + } } static PyObject * From 9af485436b83003b5705a6e54bdeb900c70e0c69 Mon Sep 17 00:00:00 2001 From: Arjun Date: Mon, 8 May 2023 10:55:59 -0700 Subject: [PATCH 110/204] gh-89550: Buffer GzipFile.write to reduce execution time by ~15% (#101251) Use `io.BufferedWriter` to buffer gzip writes. --------- Co-authored-by: Alex Waygood Co-authored-by: Gregory P. Smith --- Lib/gzip.py | 40 ++++++++++++++++--- ...3-01-22-14-53-12.gh-issue-89550.c1U23f.rst | 2 + 2 files changed, 37 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-01-22-14-53-12.gh-issue-89550.c1U23f.rst diff --git a/Lib/gzip.py b/Lib/gzip.py index 75c6ddc3f2cffb..8796c8d9fd9a2d 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -22,6 +22,7 @@ _COMPRESS_LEVEL_BEST = 9 READ_BUFFER_SIZE = 128 * 1024 +_WRITE_BUFFER_SIZE = 4 * io.DEFAULT_BUFFER_SIZE def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_BEST, @@ -120,6 +121,21 @@ class BadGzipFile(OSError): """Exception raised in some cases for invalid gzip files.""" +class _WriteBufferStream(io.RawIOBase): + """Minimal object to pass WriteBuffer flushes into GzipFile""" + def __init__(self, gzip_file): + self.gzip_file = gzip_file + + def write(self, data): + return self.gzip_file._write_raw(data) + + def seekable(self): + return False + + def writable(self): + return True + + class GzipFile(_compression.BaseStream): """The GzipFile class simulates most of the methods of a file object with the exception of the truncate() method. @@ -184,6 +200,7 @@ def __init__(self, filename=None, mode=None, if mode is None: mode = getattr(fileobj, 'mode', 'rb') + if mode.startswith('r'): self.mode = READ raw = _GzipReader(fileobj) @@ -206,6 +223,9 @@ def __init__(self, filename=None, mode=None, zlib.DEF_MEM_LEVEL, 0) self._write_mtime = mtime + self._buffer_size = _WRITE_BUFFER_SIZE + self._buffer = io.BufferedWriter(_WriteBufferStream(self), + buffer_size=self._buffer_size) else: raise ValueError("Invalid mode: {!r}".format(mode)) @@ -231,6 +251,11 @@ def _init_write(self, filename): self.bufsize = 0 self.offset = 0 # Current file offset for seek(), tell(), etc + def tell(self): + self._check_not_closed() + self._buffer.flush() + return super().tell() + def _write_gzip_header(self, compresslevel): self.fileobj.write(b'\037\213') # magic header self.fileobj.write(b'\010') # compression method @@ -272,6 +297,10 @@ def write(self,data): if self.fileobj is None: raise ValueError("write() on closed GzipFile object") + return self._buffer.write(data) + + def _write_raw(self, data): + # Called by our self._buffer underlying WriteBufferStream. if isinstance(data, (bytes, bytearray)): length = len(data) else: @@ -322,9 +351,9 @@ def close(self): fileobj = self.fileobj if fileobj is None: return - self.fileobj = None try: if self.mode == WRITE: + self._buffer.flush() fileobj.write(self.compress.flush()) write32u(fileobj, self.crc) # self.size may exceed 2 GiB, or even 4 GiB @@ -332,6 +361,7 @@ def close(self): elif self.mode == READ: self._buffer.close() finally: + self.fileobj = None myfileobj = self.myfileobj if myfileobj: self.myfileobj = None @@ -341,7 +371,7 @@ def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH): self._check_not_closed() if self.mode == WRITE: # Ensure the compressor's buffer is flushed - self.fileobj.write(self.compress.flush(zlib_mode)) + self._buffer.flush() self.fileobj.flush() def fileno(self): @@ -378,10 +408,10 @@ def seek(self, offset, whence=io.SEEK_SET): if offset < self.offset: raise OSError('Negative seek in write mode') count = offset - self.offset - chunk = b'\0' * 1024 - for i in range(count // 1024): + chunk = b'\0' * self._buffer_size + for i in range(count // self._buffer_size): self.write(chunk) - self.write(b'\0' * (count % 1024)) + self.write(b'\0' * (count % self._buffer_size)) elif self.mode == READ: self._check_not_closed() return self._buffer.seek(offset, whence) diff --git a/Misc/NEWS.d/next/Library/2023-01-22-14-53-12.gh-issue-89550.c1U23f.rst b/Misc/NEWS.d/next/Library/2023-01-22-14-53-12.gh-issue-89550.c1U23f.rst new file mode 100644 index 00000000000000..556db0eae00c0b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-01-22-14-53-12.gh-issue-89550.c1U23f.rst @@ -0,0 +1,2 @@ +Decrease execution time of some :mod:`gzip` file writes by 15% by +adding more appropriate buffering. From 942482c8e660765f68098eae347d84b93e37661a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 8 May 2023 20:01:25 +0100 Subject: [PATCH 111/204] GH-104284: Fix documentation gettext build (#104296) --- Doc/tools/extensions/pyspecific.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 4fe54e30b82b25..cd8d9febb0d13b 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -674,7 +674,14 @@ def process_audit_events(app, doctree, fromdocname): node.replace_self(table) -def patch_pairindextypes(app) -> None: +def patch_pairindextypes(app, _env) -> None: + """Remove all entries from ``pairindextypes`` before writing POT files. + + We want to run this just before writing output files, as the check to + circumvent is in ``I18nBuilder.write_doc()``. + As such, we link this to ``env-check-consistency``, even though it has + nothing to do with the environment consistency check. + """ if app.builder.name != 'gettext': return @@ -688,14 +695,7 @@ def patch_pairindextypes(app) -> None: # the Sphinx-translated pairindextypes values. As we intend to move # away from this, we need Sphinx to believe that these values don't # exist, by deleting them when using the gettext builder. - - pairindextypes.pop('module', None) - pairindextypes.pop('keyword', None) - pairindextypes.pop('operator', None) - pairindextypes.pop('object', None) - pairindextypes.pop('exception', None) - pairindextypes.pop('statement', None) - pairindextypes.pop('builtin', None) + pairindextypes.clear() def setup(app): @@ -719,7 +719,7 @@ def setup(app): app.add_directive_to_domain('py', 'awaitablemethod', PyAwaitableMethod) app.add_directive_to_domain('py', 'abstractmethod', PyAbstractMethod) app.add_directive('miscnews', MiscNews) - app.connect('builder-inited', patch_pairindextypes) + app.connect('env-check-consistency', patch_pairindextypes) app.connect('doctree-resolved', process_audit_events) app.connect('env-merge-info', audit_events_merge) app.connect('env-purge-doc', audit_events_purge) From 5c9ee498c6f4b75e0e020f17b6860309c3b7e11e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 8 May 2023 13:15:09 -0600 Subject: [PATCH 112/204] gh-99113: A Per-Interpreter GIL! (gh-104210) This is the culmination of PEP 684 (and of my 8-year long multi-core Python project)! Each subinterpreter may now be created with its own GIL (via Py_NewInterpreterFromConfig()). If not so configured then the interpreter will share with the main interpreter--the status quo since subinterpreters were added decades ago. The main interpreter always has its own GIL and subinterpreters from Py_NewInterpreter() will always share with the main interpreter. --- Include/internal/pycore_ceval.h | 3 +- Include/internal/pycore_ceval_state.h | 3 - Include/internal/pycore_interp.h | 3 + Include/internal/pycore_runtime.h | 2 - ...3-05-05-12-14-47.gh-issue-99113.-RAdnv.rst | 6 ++ Python/ceval_gil.c | 55 +++++-------------- Python/pystate.c | 4 +- 7 files changed, 24 insertions(+), 52 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-05-12-14-47.gh-issue-99113.-RAdnv.rst diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 9fd8571cbc87f4..3c8b368bd2af4e 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -21,8 +21,7 @@ struct _ceval_runtime_state; extern void _Py_FinishPendingCalls(PyThreadState *tstate); -extern void _PyEval_InitRuntimeState(struct _ceval_runtime_state *); -extern void _PyEval_InitState(struct _ceval_state *, PyThread_type_lock); +extern void _PyEval_InitState(PyInterpreterState *, PyThread_type_lock); extern void _PyEval_FiniState(struct _ceval_state *ceval); PyAPI_FUNC(void) _PyEval_SignalReceived(PyInterpreterState *interp); PyAPI_FUNC(int) _PyEval_AddPendingCall( diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 4781dd5735dcf6..b352801673c40a 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -49,9 +49,6 @@ struct _ceval_runtime_state { the main thread of the main interpreter can handle signals: see _Py_ThreadCanHandleSignals(). */ _Py_atomic_int signals_pending; - - /* This is (only) used indirectly through PyInterpreterState.ceval.gil. */ - struct _gil_runtime_state gil; }; #ifdef PY_HAVE_PERF_TRAMPOLINE diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 7276ce35ba68f0..527b2121148f4c 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -178,6 +178,9 @@ struct _is { basis. Also see _PyRuntimeState regarding the various mutex fields. */ + /* The per-interpreter GIL, which might not be used. */ + struct _gil_runtime_state _gil; + /* the initial PyInterpreterState.threads.head */ PyThreadState _initial_thread; }; diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index d1b165d0ab9c38..6e06e874711bc2 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -32,8 +32,6 @@ struct _getargs_runtime_state { struct _PyArg_Parser *static_parsers; }; -/* ceval state */ - /* GIL state */ struct _gilstate_runtime_state { diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-05-12-14-47.gh-issue-99113.-RAdnv.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-05-12-14-47.gh-issue-99113.-RAdnv.rst new file mode 100644 index 00000000000000..42e26cb27b6e01 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-05-12-14-47.gh-issue-99113.-RAdnv.rst @@ -0,0 +1,6 @@ +The GIL is now (optionally) per-interpreter. This is the fundamental change +for PEP 684. This is all made possible by virtue of the isolated state of +each interpreter in the process. The behavior of the main interpreter +remains unchanged. Likewise, interpreters created using +``Py_NewInterpreter()`` are not affected. To get an interpreter with its +own GIL, call ``Py_NewInterpreterFromConfig()``. diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 9958856bae8019..42e1436bc9130d 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -464,8 +464,7 @@ take_gil(PyThreadState *tstate) void _PyEval_SetSwitchInterval(unsigned long microseconds) { - /* XXX per-interpreter GIL */ - PyInterpreterState *interp = _PyInterpreterState_Main(); + PyInterpreterState *interp = _PyInterpreterState_Get(); struct _gil_runtime_state *gil = interp->ceval.gil; assert(gil != NULL); gil->interval = microseconds; @@ -473,8 +472,7 @@ void _PyEval_SetSwitchInterval(unsigned long microseconds) unsigned long _PyEval_GetSwitchInterval(void) { - /* XXX per-interpreter GIL */ - PyInterpreterState *interp = _PyInterpreterState_Main(); + PyInterpreterState *interp = _PyInterpreterState_Get(); struct _gil_runtime_state *gil = interp->ceval.gil; assert(gil != NULL); return gil->interval; @@ -484,7 +482,9 @@ unsigned long _PyEval_GetSwitchInterval(void) int _PyEval_ThreadsInitialized(void) { - /* XXX per-interpreter GIL */ + /* XXX This is only needed for an assert in PyGILState_Ensure(), + * which currently does not work with subinterpreters. + * Thus we only use the main interpreter. */ PyInterpreterState *interp = _PyInterpreterState_Main(); if (interp == NULL) { return 0; @@ -532,27 +532,16 @@ _PyEval_InitGIL(PyThreadState *tstate, int own_gil) assert(tstate->interp->ceval.gil == NULL); int locked; if (!own_gil) { + /* The interpreter will share the main interpreter's instead. */ PyInterpreterState *main_interp = _PyInterpreterState_Main(); assert(tstate->interp != main_interp); struct _gil_runtime_state *gil = main_interp->ceval.gil; init_shared_gil(tstate->interp, gil); locked = current_thread_holds_gil(gil, tstate); } - /* XXX per-interpreter GIL */ - else if (!_Py_IsMainInterpreter(tstate->interp)) { - /* Currently, the GIL is shared by all interpreters, - and only the main interpreter is responsible to create - and destroy it. */ - struct _gil_runtime_state *main_gil = _PyInterpreterState_Main()->ceval.gil; - init_shared_gil(tstate->interp, main_gil); - // XXX For now we lie. - tstate->interp->ceval.own_gil = 1; - locked = current_thread_holds_gil(main_gil, tstate); - } else { PyThread_init_thread(); - // XXX per-interpreter GIL: switch to interp->_gil. - init_own_gil(tstate->interp, &tstate->interp->runtime->ceval.gil); + init_own_gil(tstate->interp, &tstate->interp->_gil); locked = 0; } if (!locked) { @@ -565,7 +554,8 @@ _PyEval_InitGIL(PyThreadState *tstate, int own_gil) void _PyEval_FiniGIL(PyInterpreterState *interp) { - if (interp->ceval.gil == NULL) { + struct _gil_runtime_state *gil = interp->ceval.gil; + if (gil == NULL) { /* It was already finalized (or hasn't been initialized yet). */ assert(!interp->ceval.own_gil); return; @@ -573,24 +563,13 @@ _PyEval_FiniGIL(PyInterpreterState *interp) else if (!interp->ceval.own_gil) { #ifdef Py_DEBUG PyInterpreterState *main_interp = _PyInterpreterState_Main(); - assert(interp != main_interp); + assert(main_interp != NULL && interp != main_interp); assert(interp->ceval.gil == main_interp->ceval.gil); #endif interp->ceval.gil = NULL; return; } - /* XXX per-interpreter GIL */ - struct _gil_runtime_state *gil = &interp->runtime->ceval.gil; - if (!_Py_IsMainInterpreter(interp)) { - /* Currently, the GIL is shared by all interpreters, - and only the main interpreter is responsible to create - and destroy it. */ - assert(interp->ceval.gil == gil); - interp->ceval.gil = NULL; - return; - } - if (!gil_created(gil)) { /* First Py_InitializeFromConfig() call: the GIL doesn't exist yet: do nothing. */ @@ -974,21 +953,13 @@ Py_MakePendingCalls(void) return 0; } -/* The interpreter's recursion limit */ - void -_PyEval_InitRuntimeState(struct _ceval_runtime_state *ceval) +_PyEval_InitState(PyInterpreterState *interp, PyThread_type_lock pending_lock) { - /* XXX per-interpreter GIL */ - _gil_initialize(&ceval->gil); -} + _gil_initialize(&interp->_gil); -void -_PyEval_InitState(struct _ceval_state *ceval, PyThread_type_lock pending_lock) -{ - struct _pending_calls *pending = &ceval->pending; + struct _pending_calls *pending = &interp->ceval.pending; assert(pending->lock == NULL); - pending->lock = pending_lock; } diff --git a/Python/pystate.c b/Python/pystate.c index f14934361dab78..26debf1f88b94a 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -425,8 +425,6 @@ init_runtime(_PyRuntimeState *runtime, runtime->open_code_userdata = open_code_userdata; runtime->audit_hook_head = audit_hook_head; - _PyEval_InitRuntimeState(&runtime->ceval); - PyPreConfig_InitPythonConfig(&runtime->preconfig); PyThread_type_lock *lockptrs[NUMLOCKS] = { @@ -682,7 +680,7 @@ init_interpreter(PyInterpreterState *interp, memcpy(&interp->obmalloc.pools.used, temp, sizeof(temp)); } - _PyEval_InitState(&interp->ceval, pending_lock); + _PyEval_InitState(interp, pending_lock); _PyGC_InitState(&interp->gc); PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); From 4541d1a0dba3ef0c386991cf54c4c3c411a364c0 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 8 May 2023 16:56:01 -0600 Subject: [PATCH 113/204] gh-104310: Add importlib.util.allowing_all_extensions() (gh-104311) (I'll be adding docs for this separately.) --- Lib/importlib/util.py | 37 ++++++ Lib/test/support/import_helper.py | 2 + Lib/test/test_importlib/test_util.py | 121 ++++++++++++++++++ ...-05-08-15-50-59.gh-issue-104310.fXVSPY.rst | 3 + 4 files changed, 163 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-05-08-15-50-59.gh-issue-104310.fXVSPY.rst diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index 5294578cc26cf3..b1d9271f8e47ca 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -112,6 +112,43 @@ def find_spec(name, package=None): return spec +# Normally we would use contextlib.contextmanager. However, this module +# is imported by runpy, which means we want to avoid any unnecessary +# dependencies. Thus we use a class. + +class allowing_all_extensions: + """A context manager that lets users skip the compatibility check. + + Normally, extensions that do not support multiple interpreters + may not be imported in a subinterpreter. That implies modules + that do not implement multi-phase init. + + Likewise for modules import in a subinterpeter with its own GIL + when the extension does not support a per-interpreter GIL. This + implies the module does not have a Py_mod_multiple_interpreters slot + set to Py_MOD_PER_INTERPRETER_GIL_SUPPORTED. + + In both cases, this context manager may be used to temporarily + disable the check for compatible extension modules. + """ + + def __init__(self, disable_check=True): + self.disable_check = disable_check + + def __enter__(self): + self.old = _imp._override_multi_interp_extensions_check(self.override) + return self + + def __exit__(self, *args): + old = self.old + del self.old + _imp._override_multi_interp_extensions_check(old) + + @property + def override(self): + return -1 if self.disable_check else 1 + + class _LazyModule(types.ModuleType): """A subclass of the module type which triggers loading upon attribute access.""" diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 772c0987c2ebef..67f18e530edc4b 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -115,6 +115,8 @@ def multi_interp_extensions_check(enabled=True): It overrides the PyInterpreterConfig.check_multi_interp_extensions setting (see support.run_in_subinterp_with_config() and _xxsubinterpreters.create()). + + Also see importlib.utils.allowing_all_extensions(). """ old = _imp._override_multi_interp_extensions_check(1 if enabled else -1) try: diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 08a615ecf5288b..0be504925ecc6a 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -8,14 +8,29 @@ import importlib.util import os import pathlib +import re import string import sys from test import support +import textwrap import types import unittest import unittest.mock import warnings +try: + import _testsinglephase +except ImportError: + _testsinglephase = None +try: + import _testmultiphase +except ImportError: + _testmultiphase = None +try: + import _xxsubinterpreters as _interpreters +except ModuleNotFoundError: + _interpreters = None + class DecodeSourceBytesTests: @@ -637,5 +652,111 @@ def test_magic_number(self): self.assertEqual(EXPECTED_MAGIC_NUMBER, actual, msg) +@unittest.skipIf(_interpreters is None, 'subinterpreters required') +class AllowingAllExtensionsTests(unittest.TestCase): + + ERROR = re.compile("^: module (.*) does not support loading in subinterpreters") + + def run_with_own_gil(self, script): + interpid = _interpreters.create(isolated=True) + try: + _interpreters.run_string(interpid, script) + except _interpreters.RunFailedError as exc: + if m := self.ERROR.match(str(exc)): + modname, = m.groups() + raise ImportError(modname) + + def run_with_shared_gil(self, script): + interpid = _interpreters.create(isolated=False) + try: + _interpreters.run_string(interpid, script) + except _interpreters.RunFailedError as exc: + if m := self.ERROR.match(str(exc)): + modname, = m.groups() + raise ImportError(modname) + + @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") + def test_single_phase_init_module(self): + script = textwrap.dedent(''' + import importlib.util + with importlib.util.allowing_all_extensions(): + import _testsinglephase + ''') + with self.subTest('check disabled, shared GIL'): + self.run_with_shared_gil(script) + with self.subTest('check disabled, per-interpreter GIL'): + self.run_with_own_gil(script) + + script = textwrap.dedent(f''' + import importlib.util + with importlib.util.allowing_all_extensions(False): + import _testsinglephase + ''') + with self.subTest('check enabled, shared GIL'): + with self.assertRaises(ImportError): + self.run_with_shared_gil(script) + with self.subTest('check enabled, per-interpreter GIL'): + with self.assertRaises(ImportError): + self.run_with_own_gil(script) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_incomplete_multi_phase_init_module(self): + prescript = textwrap.dedent(f''' + from importlib.util import spec_from_loader, module_from_spec + from importlib.machinery import ExtensionFileLoader + + name = '_test_shared_gil_only' + filename = {_testmultiphase.__file__!r} + loader = ExtensionFileLoader(name, filename) + spec = spec_from_loader(name, loader) + + ''') + + script = prescript + textwrap.dedent(''' + import importlib.util + with importlib.util.allowing_all_extensions(): + module = module_from_spec(spec) + loader.exec_module(module) + ''') + with self.subTest('check disabled, shared GIL'): + self.run_with_shared_gil(script) + with self.subTest('check disabled, per-interpreter GIL'): + self.run_with_own_gil(script) + + script = prescript + textwrap.dedent(''' + import importlib.util + with importlib.util.allowing_all_extensions(False): + module = module_from_spec(spec) + loader.exec_module(module) + ''') + with self.subTest('check enabled, shared GIL'): + self.run_with_shared_gil(script) + with self.subTest('check enabled, per-interpreter GIL'): + with self.assertRaises(ImportError): + self.run_with_own_gil(script) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_complete_multi_phase_init_module(self): + script = textwrap.dedent(''' + import importlib.util + with importlib.util.allowing_all_extensions(): + import _testmultiphase + ''') + with self.subTest('check disabled, shared GIL'): + self.run_with_shared_gil(script) + with self.subTest('check disabled, per-interpreter GIL'): + self.run_with_own_gil(script) + + script = textwrap.dedent(f''' + import importlib.util + with importlib.util.allowing_all_extensions(False): + import _testmultiphase + ''') + with self.subTest('check enabled, shared GIL'): + self.run_with_shared_gil(script) + with self.subTest('check enabled, per-interpreter GIL'): + self.run_with_own_gil(script) + + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2023-05-08-15-50-59.gh-issue-104310.fXVSPY.rst b/Misc/NEWS.d/next/Library/2023-05-08-15-50-59.gh-issue-104310.fXVSPY.rst new file mode 100644 index 00000000000000..3743d569995f2e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-08-15-50-59.gh-issue-104310.fXVSPY.rst @@ -0,0 +1,3 @@ +Users may now use ``importlib.util.allowing_all_extensions()`` (a context +manager) to temporarily disable the strict compatibility checks for +importing extension modules in subinterpreters. From faf196213e60d8a90773e9e5680d3252bd294643 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 8 May 2023 16:27:20 -0700 Subject: [PATCH 114/204] GH-104308: socket.getnameinfo should release the GIL (#104307) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * socket.getnameinfo should release the GIL * 📜🤖 Added by blurb_it. --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- .../next/Library/2023-05-08-20-57-17.gh-issue-104307.DSB93G.rst | 1 + Modules/socketmodule.c | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-05-08-20-57-17.gh-issue-104307.DSB93G.rst diff --git a/Misc/NEWS.d/next/Library/2023-05-08-20-57-17.gh-issue-104307.DSB93G.rst b/Misc/NEWS.d/next/Library/2023-05-08-20-57-17.gh-issue-104307.DSB93G.rst new file mode 100644 index 00000000000000..03775845450caa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-08-20-57-17.gh-issue-104307.DSB93G.rst @@ -0,0 +1 @@ +:func:`socket.getnameinfo` now releases the GIL while contacting the DNS server diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 60219593be61e2..c11fb4400eab2f 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -6883,8 +6883,10 @@ socket_getnameinfo(PyObject *self, PyObject *args) } #endif } + Py_BEGIN_ALLOW_THREADS error = getnameinfo(res->ai_addr, (socklen_t) res->ai_addrlen, hbuf, sizeof(hbuf), pbuf, sizeof(pbuf), flags); + Py_END_ALLOW_THREADS if (error) { socket_state *state = get_module_state(self); set_gaierror(state, error); From bf89d4283a28dd00836f2c312a9255f543f93fc7 Mon Sep 17 00:00:00 2001 From: Jacob Bower <1978924+jbower-fb@users.noreply.github.com> Date: Mon, 8 May 2023 17:51:58 -0700 Subject: [PATCH 115/204] gh-97696 Remove unnecessary check for eager_start kwarg (#104188) Instead, add docstring to create_eager_task_factory. --- Lib/asyncio/tasks.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 7eb647bd129819..8d5bde09ea9b5b 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -942,18 +942,31 @@ def callback(): def create_eager_task_factory(custom_task_constructor): + """Create a function suitable for use as a task factory on an event-loop. - if "eager_start" not in inspect.signature(custom_task_constructor).parameters: - raise TypeError( - "Provided constructor does not support eager task execution") + Example usage: + + loop.set_task_factory( + asyncio.create_eager_task_factory(my_task_constructor)) + + Now, tasks created will be started immediately (rather than being first + scheduled to an event loop). The constructor argument can be any callable + that returns a Task-compatible object and has a signature compatible + with `Task.__init__`; it must have the `eager_start` keyword argument. + + Most applications will use `Task` for `custom_task_constructor` and in + this case there's no need to call `create_eager_task_factory()` + directly. Instead the global `eager_task_factory` instance can be + used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`. + """ def factory(loop, coro, *, name=None, context=None): return custom_task_constructor( coro, loop=loop, name=name, context=context, eager_start=True) - return factory + eager_task_factory = create_eager_task_factory(Task) From 9a9b176eb7e052af84c01c0cfb3231e51f980f2d Mon Sep 17 00:00:00 2001 From: samschott Date: Tue, 9 May 2023 10:24:29 +0200 Subject: [PATCH 116/204] gh-104180: Read SOCKS proxies from macOS System Configuration (#104181) read SOCKS proxies from macOS System Configuration in ``urllib.request``. --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Oleg Iarygin --- .../macOS/2023-05-04-21-47-59.gh-issue-104180.lEJCwd.rst | 2 ++ Modules/_scproxy.c | 5 +++++ 2 files changed, 7 insertions(+) create mode 100644 Misc/NEWS.d/next/macOS/2023-05-04-21-47-59.gh-issue-104180.lEJCwd.rst diff --git a/Misc/NEWS.d/next/macOS/2023-05-04-21-47-59.gh-issue-104180.lEJCwd.rst b/Misc/NEWS.d/next/macOS/2023-05-04-21-47-59.gh-issue-104180.lEJCwd.rst new file mode 100644 index 00000000000000..b6b18dcfd81394 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-05-04-21-47-59.gh-issue-104180.lEJCwd.rst @@ -0,0 +1,2 @@ +Support reading SOCKS proxy configuration from macOS System Configuration. +Patch by Sam Schott. diff --git a/Modules/_scproxy.c b/Modules/_scproxy.c index e66918016b8da6..0df0324df55f7d 100644 --- a/Modules/_scproxy.c +++ b/Modules/_scproxy.c @@ -206,6 +206,11 @@ get_proxies(PyObject* Py_UNUSED(mod), PyObject *Py_UNUSED(ignored)) kSCPropNetProxiesGopherProxy, kSCPropNetProxiesGopherPort); if (r == -1) goto error; + r = set_proxy(result, "socks", proxyDict, + kSCPropNetProxiesSOCKSEnable, + kSCPropNetProxiesSOCKSProxy, + kSCPropNetProxiesSOCKSPort); + if (r == -1) goto error; CFRelease(proxyDict); return result; From 03029ace924f7dac7ef49eee3417e0af6623060d Mon Sep 17 00:00:00 2001 From: Zhang Na <44800775+loongson-zn@users.noreply.github.com> Date: Tue, 9 May 2023 17:19:40 +0800 Subject: [PATCH 117/204] gh-90656: Add platform triplets for 64-bit LoongArch (LA64) (#30939) Signed-off-by: Zhang Na Co-authored-by: WANG Xuerui --- Doc/whatsnew/3.12.rst | 8 ++++++++ .../2022-06-20-15-15-11.gh-issue-90656.kFBbKe.rst | 7 +++++++ configure | 14 ++++++++++++++ configure.ac | 14 ++++++++++++++ 4 files changed, 43 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2022-06-20-15-15-11.gh-issue-90656.kFBbKe.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 65b3e9ffb8072d..fe6ad575f71720 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1169,6 +1169,14 @@ Build Changes optimization levels (0, 1, 2) at once. (Contributed by Victor Stinner in :gh:`99289`.) +* Add platform triplets for 64-bit LoongArch: + + * loongarch64-linux-gnusf + * loongarch64-linux-gnuf32 + * loongarch64-linux-gnu + + (Contributed by Zhang Na in :gh:`90656`.) + C API Changes ============= diff --git a/Misc/NEWS.d/next/Build/2022-06-20-15-15-11.gh-issue-90656.kFBbKe.rst b/Misc/NEWS.d/next/Build/2022-06-20-15-15-11.gh-issue-90656.kFBbKe.rst new file mode 100644 index 00000000000000..dfe71a5552070d --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-06-20-15-15-11.gh-issue-90656.kFBbKe.rst @@ -0,0 +1,7 @@ +Add platform triplets for 64-bit LoongArch: + +* loongarch64-linux-gnusf +* loongarch64-linux-gnuf32 +* loongarch64-linux-gnu + +Patch by Zhang Na. diff --git a/configure b/configure index b8fa9d66e735dd..c9ea72cf6efacf 100755 --- a/configure +++ b/configure @@ -6143,6 +6143,20 @@ cat > conftest.c <=6) && defined(_MIPSEL) diff --git a/configure.ac b/configure.ac index 0940b93c25f7bf..10672bd3761da8 100644 --- a/configure.ac +++ b/configure.ac @@ -959,6 +959,20 @@ cat > conftest.c <=6) && defined(_MIPSEL) From 41aff464cef83d2655029ddd180a51110e8d7f8e Mon Sep 17 00:00:00 2001 From: Sebastian Pipping Date: Tue, 9 May 2023 12:29:00 +0200 Subject: [PATCH 118/204] require-pr-label.yml: Add missing "permissions:" (#104309) --- .github/workflows/require-pr-label.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml index 916bbeb4352734..88aaea039f04f4 100644 --- a/.github/workflows/require-pr-label.yml +++ b/.github/workflows/require-pr-label.yml @@ -4,6 +4,10 @@ on: pull_request: types: [opened, reopened, labeled, unlabeled, synchronize] +permissions: + issues: read + pull-requests: read + jobs: label: name: DO-NOT-MERGE / unresolved review From 9196da417d20e1484e23b3c80483b0222abaadf2 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 9 May 2023 13:33:36 +0100 Subject: [PATCH 119/204] gh-103193: Fix refleaks in `test_inspect` and `test_typing` (#104320) --- Lib/test/libregrtest/utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index fb13fa0e243ba7..fd46819fd903fe 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -210,6 +210,13 @@ def clear_caches(): else: fractions._hash_algorithm.cache_clear() + try: + inspect = sys.modules['inspect'] + except KeyError: + pass + else: + inspect._shadowed_dict_from_mro_tuple.cache_clear() + def get_build_info(): # Get most important configure and build options as a list of strings. From 85f981880ae9591ba577e44d2945a771078a7c35 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Tue, 9 May 2023 05:39:59 -0700 Subject: [PATCH 120/204] gh-97696: Move around and update the whatsnew entry for asyncio eager task factory (#104298) --- Doc/whatsnew/3.12.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index fe6ad575f71720..12d357f117b1b6 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -282,6 +282,11 @@ asyncio writing to sockets and uses :meth:`~socket.socket.sendmsg` if the platform supports it. (Contributed by Kumar Aditya in :gh:`91166`.) +* Added :func:`asyncio.eager_task_factory` and :func:`asyncio.create_eager_task_factory` + functions to allow opting an event loop in to eager task execution, + making some use-cases 2x to 5x faster. + (Contributed by Jacob Bower & Itamar O in :gh:`102853`, :gh:`104140`, and :gh:`104138`) + * On Linux, :mod:`asyncio` uses :class:`~asyncio.PidfdChildWatcher` by default if :func:`os.pidfd_open` is available and functional instead of :class:`~asyncio.ThreadedChildWatcher`. @@ -644,11 +649,6 @@ Optimizations * Speed up :class:`asyncio.Task` creation by deferring expensive string formatting. (Contributed by Itamar O in :gh:`103793`.) -* Added :func:`asyncio.eager_task_factory` and :func:`asyncio.create_eager_task_factory` - functions to allow opting an event loop in to eager task execution, - speeding up some use-cases by up to 50%. - (Contributed by Jacob Bower & Itamar O in :gh:`102853`) - CPython bytecode changes ======================== From c21f82876089f3e9a7b1e706c029664b799fa659 Mon Sep 17 00:00:00 2001 From: chgnrdv <52372310+chgnrdv@users.noreply.github.com> Date: Tue, 9 May 2023 15:41:09 +0300 Subject: [PATCH 121/204] gh-104276: Make `_struct.unpack_iterator` type use type flag instead of custom constructor (#104277) --- Modules/_struct.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Modules/_struct.c b/Modules/_struct.c index 26434f714de5cc..4f9478bd98095d 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -1832,11 +1832,6 @@ unpackiter_iternext(unpackiterobject *self) return result; } -PyObject *unpackiter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - PyErr_Format(PyExc_TypeError, "Cannot create '%.200s objects", _PyType_Name(type)); - return NULL; -} - static PyType_Slot unpackiter_type_slots[] = { {Py_tp_dealloc, unpackiter_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -1844,7 +1839,6 @@ static PyType_Slot unpackiter_type_slots[] = { {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, unpackiter_iternext}, {Py_tp_methods, unpackiter_methods}, - {Py_tp_new, unpackiter_new}, {0, 0}, }; @@ -1853,7 +1847,7 @@ static PyType_Spec unpackiter_type_spec = { sizeof(unpackiterobject), 0, (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION), unpackiter_type_slots }; From ca95edf177e3c10e10d7011ed52619b1312cf15e Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 9 May 2023 14:33:40 +0100 Subject: [PATCH 122/204] gh-104240: return code unit metadata from codegen (#104300) --- Lib/test/support/bytecode_helper.py | 2 +- Lib/test/test_compiler_assemble.py | 2 +- Modules/_testinternalcapi.c | 2 +- Python/compile.c | 50 +++++++++++++++++++++++++++-- 4 files changed, 50 insertions(+), 6 deletions(-) diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index 357ec44dbc218d..7b577f54b8adc4 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -124,7 +124,7 @@ def complete_insts_info(self, insts): class CodegenTestCase(CompilationStepTestCase): def generate_code(self, ast): - insts = compiler_codegen(ast, "my_file.py", 0) + insts, _ = compiler_codegen(ast, "my_file.py", 0) return insts diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py index 96c1691e24a9c8..0bd7a09b001c1d 100644 --- a/Lib/test/test_compiler_assemble.py +++ b/Lib/test/test_compiler_assemble.py @@ -52,7 +52,7 @@ def test_simple_expr(self): 'filename' : 'avg.py', 'name' : 'avg', 'qualname' : 'stats.avg', - 'consts' : [2], + 'consts' : {2 : 0}, 'argcount' : 2, 'varnames' : {'x' : 0, 'y' : 1}, } diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 40ad6f88868daf..8009dca3d0b746 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -670,7 +670,7 @@ _testinternalcapi_assemble_code_object_impl(PyObject *module, umd.u_cellvars = PyDict_GetItemString(metadata, "cellvars"); umd.u_freevars = PyDict_GetItemString(metadata, "freevars"); - assert(PyList_Check(umd.u_consts)); + assert(PyDict_Check(umd.u_consts)); assert(PyDict_Check(umd.u_names)); assert(PyDict_Check(umd.u_varnames)); assert(PyDict_Check(umd.u_cellvars)); diff --git a/Python/compile.c b/Python/compile.c index f875e4e17e0a9f..12ae831361963d 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -7261,6 +7261,7 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, int optimize, int compile_mode) { PyObject *res = NULL; + PyObject *metadata = NULL; if (!PyAST_Check(ast)) { PyErr_SetString(PyExc_TypeError, "expected an AST"); @@ -7287,14 +7288,53 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, if (compiler_codegen(c, mod) < 0) { goto finally; } + + _PyCompile_CodeUnitMetadata *umd = &c->u->u_metadata; + metadata = PyDict_New(); + if (metadata == NULL) { + goto finally; + } +#define SET_MATADATA_ITEM(key, value) \ + if (value != NULL) { \ + if (PyDict_SetItemString(metadata, key, value) < 0) goto finally; \ + } + + SET_MATADATA_ITEM("name", umd->u_name); + SET_MATADATA_ITEM("qualname", umd->u_qualname); + SET_MATADATA_ITEM("consts", umd->u_consts); + SET_MATADATA_ITEM("names", umd->u_names); + SET_MATADATA_ITEM("varnames", umd->u_varnames); + SET_MATADATA_ITEM("cellvars", umd->u_cellvars); + SET_MATADATA_ITEM("freevars", umd->u_freevars); +#undef SET_MATADATA_ITEM + +#define SET_MATADATA_INT(key, value) do { \ + PyObject *v = PyLong_FromLong((long)value); \ + if (v == NULL) goto finally; \ + int res = PyDict_SetItemString(metadata, key, v); \ + Py_XDECREF(v); \ + if (res < 0) goto finally; \ + } while (0); + + SET_MATADATA_INT("argcount", umd->u_argcount); + SET_MATADATA_INT("posonlyargcount", umd->u_posonlyargcount); + SET_MATADATA_INT("kwonlyargcount", umd->u_kwonlyargcount); +#undef SET_MATADATA_INT + int addNone = mod->kind != Expression_kind; if (add_return_at_end(c, addNone) < 0) { - return NULL; + goto finally; } - res = instr_sequence_to_instructions(INSTR_SEQUENCE(c)); + PyObject *insts = instr_sequence_to_instructions(INSTR_SEQUENCE(c)); + if (insts == NULL) { + goto finally; + } + res = PyTuple_Pack(2, insts, metadata); + Py_DECREF(insts); finally: + Py_XDECREF(metadata); compiler_exit_scope(c); compiler_free(c); _PyArena_Free(arena); @@ -7375,10 +7415,14 @@ _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, goto error; } - PyObject *consts = umd->u_consts; + PyObject *consts = consts_dict_keys_inorder(umd->u_consts); + if (consts == NULL) { + goto error; + } co = _PyAssemble_MakeCodeObject(umd, const_cache, consts, maxdepth, &optimized_instrs, nlocalsplus, code_flags, filename); + Py_DECREF(consts); error: Py_DECREF(const_cache); From 82f789be3b15df5f6660f5fd0c563ad690ee00fb Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Tue, 9 May 2023 07:04:50 -0700 Subject: [PATCH 123/204] gh-104139: Add itms-services to uses_netloc urllib.parse. (#104312) Teach unsplit to retain the `"//"` when assembling `itms-services://?action=generate-bugs` style [Apple Platform Deployment](https://support.apple.com/en-gb/guide/deployment/depce7cefc4d/web) URLs. --- Lib/test/test_urlparse.py | 33 +++++++++++-------- Lib/urllib/parse.py | 2 +- ...-05-08-23-01-59.gh-issue-104139.83Tnt-.rst | 3 ++ 3 files changed, 23 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-08-23-01-59.gh-issue-104139.83Tnt-.rst diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 80fb9e5cd2a445..dcdbb1cc64fd28 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -72,20 +72,20 @@ class UrlParseTestCase(unittest.TestCase): def checkRoundtrips(self, url, parsed, split): result = urllib.parse.urlparse(url) - self.assertEqual(result, parsed) + self.assertSequenceEqual(result, parsed) t = (result.scheme, result.netloc, result.path, result.params, result.query, result.fragment) - self.assertEqual(t, parsed) + self.assertSequenceEqual(t, parsed) # put it back together and it should be the same result2 = urllib.parse.urlunparse(result) - self.assertEqual(result2, url) - self.assertEqual(result2, result.geturl()) + self.assertSequenceEqual(result2, url) + self.assertSequenceEqual(result2, result.geturl()) # the result of geturl() is a fixpoint; we can always parse it # again to get the same result: result3 = urllib.parse.urlparse(result.geturl()) self.assertEqual(result3.geturl(), result.geturl()) - self.assertEqual(result3, result) + self.assertSequenceEqual(result3, result) self.assertEqual(result3.scheme, result.scheme) self.assertEqual(result3.netloc, result.netloc) self.assertEqual(result3.path, result.path) @@ -99,18 +99,18 @@ def checkRoundtrips(self, url, parsed, split): # check the roundtrip using urlsplit() as well result = urllib.parse.urlsplit(url) - self.assertEqual(result, split) + self.assertSequenceEqual(result, split) t = (result.scheme, result.netloc, result.path, result.query, result.fragment) - self.assertEqual(t, split) + self.assertSequenceEqual(t, split) result2 = urllib.parse.urlunsplit(result) - self.assertEqual(result2, url) - self.assertEqual(result2, result.geturl()) + self.assertSequenceEqual(result2, url) + self.assertSequenceEqual(result2, result.geturl()) # check the fixpoint property of re-parsing the result of geturl() result3 = urllib.parse.urlsplit(result.geturl()) self.assertEqual(result3.geturl(), result.geturl()) - self.assertEqual(result3, result) + self.assertSequenceEqual(result3, result) self.assertEqual(result3.scheme, result.scheme) self.assertEqual(result3.netloc, result.netloc) self.assertEqual(result3.path, result.path) @@ -162,10 +162,15 @@ def test_roundtrips(self): ('svn+ssh', 'svn.zope.org', '/repos/main/ZConfig/trunk/', '', '')), ('git+ssh://git@github.com/user/project.git', - ('git+ssh', 'git@github.com','/user/project.git', - '','',''), - ('git+ssh', 'git@github.com','/user/project.git', - '', '')), + ('git+ssh', 'git@github.com','/user/project.git', + '','',''), + ('git+ssh', 'git@github.com','/user/project.git', + '', '')), + ('itms-services://?action=download-manifest&url=https://example.com/app', + ('itms-services', '', '', '', + 'action=download-manifest&url=https://example.com/app', ''), + ('itms-services', '', '', + 'action=download-manifest&url=https://example.com/app', '')), ] def _encode(t): return (t[0].encode('ascii'), diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 5f95c5ff7f9c1c..777b7c53efe565 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -54,7 +54,7 @@ 'imap', 'wais', 'file', 'mms', 'https', 'shttp', 'snews', 'prospero', 'rtsp', 'rtspu', 'rsync', 'svn', 'svn+ssh', 'sftp', 'nfs', 'git', 'git+ssh', - 'ws', 'wss'] + 'ws', 'wss', 'itms-services'] uses_params = ['', 'ftp', 'hdl', 'prospero', 'http', 'imap', 'https', 'shttp', 'rtsp', 'rtspu', 'sip', 'sips', diff --git a/Misc/NEWS.d/next/Library/2023-05-08-23-01-59.gh-issue-104139.83Tnt-.rst b/Misc/NEWS.d/next/Library/2023-05-08-23-01-59.gh-issue-104139.83Tnt-.rst new file mode 100644 index 00000000000000..145e75f6dea6f4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-08-23-01-59.gh-issue-104139.83Tnt-.rst @@ -0,0 +1,3 @@ +Teach :func:`urllib.parse.unsplit` to retain the ``"//"`` when assembling +``itms-services://?action=generate-bugs`` style `Apple Platform Deployment +`_ URLs. From afe7703744f813adb15719642444b5fd35888d86 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Tue, 9 May 2023 08:53:19 -0600 Subject: [PATCH 124/204] gh-104184: fix building --with-pydebug --enable-pystats (#104217) --- Python/specialize.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/specialize.c b/Python/specialize.c index b1cc66124cfa4a..2ccca3a2802c17 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1455,7 +1455,7 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins goto fail; } if (PyObject_CheckBuffer(container)) { - if (PyLong_CheckExact(sub) && (((size_t)Py_SIZE(sub)) > 1)) { + if (PyLong_CheckExact(sub) && (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub))) { SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); } else if (strcmp(container_type->tp_name, "array.array") == 0) { From 0aeda297931820436a50b78f4f7f0597274b5df4 Mon Sep 17 00:00:00 2001 From: Sam Carroll <70000253+samcarroll42@users.noreply.github.com> Date: Tue, 9 May 2023 12:01:58 -0400 Subject: [PATCH 125/204] gh-99889: Fix directory traversal security flaw in uu.decode() (#104096) * Fix directory traversal security flaw in uu.decode() * also check absolute paths and os.altsep * Add a regression test. --------- Co-authored-by: Gregory P. Smith [Google] --- Lib/test/test_uu.py | 28 +++++++++++++++++++ Lib/uu.py | 9 +++++- ...3-05-02-17-56-32.gh-issue-99889.l664SU.rst | 2 ++ 3 files changed, 38 insertions(+), 1 deletion(-) mode change 100755 => 100644 Lib/uu.py create mode 100644 Misc/NEWS.d/next/Security/2023-05-02-17-56-32.gh-issue-99889.l664SU.rst diff --git a/Lib/test/test_uu.py b/Lib/test/test_uu.py index 0493aae4fc67be..a189d6bc4b05d3 100644 --- a/Lib/test/test_uu.py +++ b/Lib/test/test_uu.py @@ -147,6 +147,34 @@ def test_newlines_escaped(self): uu.encode(inp, out, filename) self.assertIn(safefilename, out.getvalue()) + def test_no_directory_traversal(self): + relative_bad = b"""\ +begin 644 ../../../../../../../../tmp/test1 +$86)C"@`` +` +end +""" + with self.assertRaisesRegex(uu.Error, 'directory'): + uu.decode(io.BytesIO(relative_bad)) + if os.altsep: + relative_bad_bs = relative_bad.replace(b'/', b'\\') + with self.assertRaisesRegex(uu.Error, 'directory'): + uu.decode(io.BytesIO(relative_bad_bs)) + + absolute_bad = b"""\ +begin 644 /tmp/test2 +$86)C"@`` +` +end +""" + with self.assertRaisesRegex(uu.Error, 'directory'): + uu.decode(io.BytesIO(absolute_bad)) + if os.altsep: + absolute_bad_bs = absolute_bad.replace(b'/', b'\\') + with self.assertRaisesRegex(uu.Error, 'directory'): + uu.decode(io.BytesIO(absolute_bad_bs)) + + class UUStdIOTest(unittest.TestCase): def setUp(self): diff --git a/Lib/uu.py b/Lib/uu.py old mode 100755 new mode 100644 index 6f8805d8c5d0c6..26bb59ae073ec5 --- a/Lib/uu.py +++ b/Lib/uu.py @@ -133,7 +133,14 @@ def decode(in_file, out_file=None, mode=None, quiet=False): # If the filename isn't ASCII, what's up with that?!? out_file = hdrfields[2].rstrip(b' \t\r\n\f').decode("ascii") if os.path.exists(out_file): - raise Error('Cannot overwrite existing file: %s' % out_file) + raise Error(f'Cannot overwrite existing file: {out_file}') + if (out_file.startswith(os.sep) or + f'..{os.sep}' in out_file or ( + os.altsep and + (out_file.startswith(os.altsep) or + f'..{os.altsep}' in out_file)) + ): + raise Error(f'Refusing to write to {out_file} due to directory traversal') if mode is None: mode = int(hdrfields[1], 8) # diff --git a/Misc/NEWS.d/next/Security/2023-05-02-17-56-32.gh-issue-99889.l664SU.rst b/Misc/NEWS.d/next/Security/2023-05-02-17-56-32.gh-issue-99889.l664SU.rst new file mode 100644 index 00000000000000..b7002e81b6b677 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-05-02-17-56-32.gh-issue-99889.l664SU.rst @@ -0,0 +1,2 @@ +Fixed a security in flaw in :func:`uu.decode` that could allow for +directory traversal based on the input if no ``out_file`` was specified. From c3b595e73efac59360d6dc869802abc752092460 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Tue, 9 May 2023 11:02:14 -0600 Subject: [PATCH 126/204] gh-97933: (PEP 709) inline list/dict/set comprehensions (#101441) Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Co-authored-by: Erlend E. Aasland --- Doc/library/dis.rst | 8 + Doc/whatsnew/3.12.rst | 24 + Include/internal/pycore_code.h | 1 + Include/internal/pycore_compile.h | 3 + Include/internal/pycore_flowgraph.h | 2 +- Include/internal/pycore_opcode.h | 13 +- Include/internal/pycore_symtable.h | 1 + Include/opcode.h | 21 +- Lib/importlib/_bootstrap_external.py | 3 +- Lib/opcode.py | 4 + Lib/test/test_compile.py | 24 +- Lib/test/test_compiler_assemble.py | 5 +- Lib/test/test_dis.py | 33 +- Lib/test/test_inspect.py | 10 +- Lib/test/test_listcomps.py | 275 ++++- Lib/test/test_trace.py | 4 +- ...3-01-30-15-40-29.gh-issue-97933.nUlp3r.rst | 2 + Modules/_testinternalcapi.c | 2 + Objects/frameobject.c | 4 + Python/assemble.c | 3 + Python/bytecodes.c | 6 + Python/compile.c | 363 +++++-- Python/flowgraph.c | 16 +- Python/generated_cases.c.h | 962 +++++++++--------- Python/opcode_metadata.h | 5 + Python/opcode_targets.h | 8 +- Python/symtable.c | 136 ++- 27 files changed, 1243 insertions(+), 695 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 296d8a9c66faa4..248743b8fa0a87 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1196,6 +1196,14 @@ iterations of the loop. .. versionadded:: 3.12 +.. opcode:: LOAD_FAST_AND_CLEAR (var_num) + + Pushes a reference to the local ``co_varnames[var_num]`` onto the stack (or + pushes ``NULL`` onto the stack if the local variable has not been + initialized) and sets ``co_varnames[var_num]`` to ``NULL``. + + .. versionadded:: 3.12 + .. opcode:: STORE_FAST (var_num) Stores ``STACK.pop()`` into the local ``co_varnames[var_num]``. diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 12d357f117b1b6..eb13d4bf031c95 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -153,6 +153,30 @@ New Features In Python 3.14, the default will switch to ``'data'``. (Contributed by Petr Viktorin in :pep:`706`.) +.. _whatsnew312-pep709: + +PEP 709: Comprehension inlining +------------------------------- + +Dictionary, list, and set comprehensions are now inlined, rather than creating a +new single-use function object for each execution of the comprehension. This +speeds up execution of a comprehension by up to 2x. + +Comprehension iteration variables remain isolated; they don't overwrite a +variable of the same name in the outer scope, nor are they visible after the +comprehension. This isolation is now maintained via stack/locals manipulation, +not via separate function scope. + +Inlining does result in a few visible behavior changes: + +* There is no longer a separate frame for the comprehension in tracebacks, + and tracing/profiling no longer shows the comprehension as a function call. +* Calling :func:`locals` inside a comprehension now includes variables + from outside the comprehension, and no longer includes the synthetic ``.0`` + variable for the comprehension "argument". + +Contributed by Carl Meyer and Vladimir Matveev in :pep:`709`. + PEP 688: Making the buffer protocol accessible in Python -------------------------------------------------------- diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 86fd48b63ef8e4..c1f017fdb753c5 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -131,6 +131,7 @@ struct callable_cache { // Note that these all fit within a byte, as do combinations. // Later, we will use the smaller numbers to differentiate the different // kinds of locals (e.g. pos-only arg, varkwargs, local-only). +#define CO_FAST_HIDDEN 0x10 #define CO_FAST_LOCAL 0x20 #define CO_FAST_CELL 0x40 #define CO_FAST_FREE 0x80 diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index d2b12c91fe7a00..499f55f3e276be 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -70,6 +70,9 @@ typedef struct { PyObject *u_varnames; /* local variables */ PyObject *u_cellvars; /* cell variables */ PyObject *u_freevars; /* free variables */ + PyObject *u_fasthidden; /* dict; keys are names that are fast-locals only + temporarily within an inlined comprehension. When + value is True, treat as fast-local. */ Py_ssize_t u_argcount; /* number of arguments for block */ Py_ssize_t u_posonlyargcount; /* number of positional only arguments for block */ diff --git a/Include/internal/pycore_flowgraph.h b/Include/internal/pycore_flowgraph.h index 883334f4b182eb..720feb18636959 100644 --- a/Include/internal/pycore_flowgraph.h +++ b/Include/internal/pycore_flowgraph.h @@ -94,7 +94,7 @@ _PyCfgInstruction* _PyCfg_BasicblockLastInstr(const _PyCfgBasicblock *b); int _PyCfg_OptimizeCodeUnit(_PyCfgBuilder *g, PyObject *consts, PyObject *const_cache, int code_flags, int nlocals, int nparams, int firstlineno); int _PyCfg_Stackdepth(_PyCfgBasicblock *entryblock, int code_flags); -void _PyCfg_ConvertExceptionHandlersToNops(_PyCfgBasicblock *entryblock); +void _PyCfg_ConvertPseudoOps(_PyCfgBasicblock *entryblock); int _PyCfg_ResolveJumps(_PyCfgBuilder *g); diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index a82885463ab2e9..52ee70a7bfdc54 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -173,6 +173,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [LOAD_CONST__LOAD_FAST] = LOAD_CONST, [LOAD_DEREF] = LOAD_DEREF, [LOAD_FAST] = LOAD_FAST, + [LOAD_FAST_AND_CLEAR] = LOAD_FAST_AND_CLEAR, [LOAD_FAST_CHECK] = LOAD_FAST_CHECK, [LOAD_FAST__LOAD_CONST] = LOAD_FAST, [LOAD_FAST__LOAD_FAST] = LOAD_FAST, @@ -239,7 +240,7 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_TABLES #ifdef Py_DEBUG -static const char *const _PyOpcode_OpName[266] = { +static const char *const _PyOpcode_OpName[267] = { [CACHE] = "CACHE", [POP_TOP] = "POP_TOP", [PUSH_NULL] = "PUSH_NULL", @@ -383,7 +384,7 @@ static const char *const _PyOpcode_OpName[266] = { [JUMP_BACKWARD] = "JUMP_BACKWARD", [LOAD_SUPER_ATTR] = "LOAD_SUPER_ATTR", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", + [LOAD_FAST_AND_CLEAR] = "LOAD_FAST_AND_CLEAR", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -393,21 +394,21 @@ static const char *const _PyOpcode_OpName[266] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", + [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", - [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", + [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", - [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", + [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [SEND_GEN] = "SEND_GEN", - [167] = "<167>", [168] = "<168>", [169] = "<169>", [170] = "<170>", @@ -506,11 +507,11 @@ static const char *const _PyOpcode_OpName[266] = { [LOAD_SUPER_METHOD] = "LOAD_SUPER_METHOD", [LOAD_ZERO_SUPER_METHOD] = "LOAD_ZERO_SUPER_METHOD", [LOAD_ZERO_SUPER_ATTR] = "LOAD_ZERO_SUPER_ATTR", + [STORE_FAST_MAYBE_NULL] = "STORE_FAST_MAYBE_NULL", }; #endif #define EXTRA_CASES \ - case 167: \ case 168: \ case 169: \ case 170: \ diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 512c4c931f73e4..9a005be5402c4e 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -64,6 +64,7 @@ typedef struct _symtable_entry { unsigned ste_needs_class_closure : 1; /* for class scopes, true if a closure over __class__ should be created */ + unsigned ste_comp_inlined : 1; /* true if this comprehension is inlined */ unsigned ste_comp_iter_target : 1; /* true if visiting comprehension target */ int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */ int ste_lineno; /* first line of block */ diff --git a/Include/opcode.h b/Include/opcode.h index 37a9e9bffa4cb7..f6f4af8c793d8d 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -97,6 +97,7 @@ extern "C" { #define JUMP_BACKWARD 140 #define LOAD_SUPER_ATTR 141 #define CALL_FUNCTION_EX 142 +#define LOAD_FAST_AND_CLEAR 143 #define EXTENDED_ARG 144 #define LIST_APPEND 145 #define SET_ADD 146 @@ -146,7 +147,8 @@ extern "C" { #define LOAD_SUPER_METHOD 263 #define LOAD_ZERO_SUPER_METHOD 264 #define LOAD_ZERO_SUPER_ATTR 265 -#define MAX_PSEUDO_OPCODE 265 +#define STORE_FAST_MAYBE_NULL 266 +#define MAX_PSEUDO_OPCODE 266 #define BINARY_OP_ADD_FLOAT 6 #define BINARY_OP_ADD_INT 7 #define BINARY_OP_ADD_UNICODE 8 @@ -202,14 +204,14 @@ extern "C" { #define STORE_ATTR_INSTANCE_VALUE 111 #define STORE_ATTR_SLOT 112 #define STORE_ATTR_WITH_HINT 113 -#define STORE_FAST__LOAD_FAST 143 -#define STORE_FAST__STORE_FAST 153 -#define STORE_SUBSCR_DICT 154 -#define STORE_SUBSCR_LIST_INT 158 -#define UNPACK_SEQUENCE_LIST 159 -#define UNPACK_SEQUENCE_TUPLE 160 -#define UNPACK_SEQUENCE_TWO_TUPLE 161 -#define SEND_GEN 166 +#define STORE_FAST__LOAD_FAST 153 +#define STORE_FAST__STORE_FAST 154 +#define STORE_SUBSCR_DICT 158 +#define STORE_SUBSCR_LIST_INT 159 +#define UNPACK_SEQUENCE_LIST 160 +#define UNPACK_SEQUENCE_TUPLE 161 +#define UNPACK_SEQUENCE_TWO_TUPLE 166 +#define SEND_GEN 167 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ || ((op) == JUMP) \ @@ -218,6 +220,7 @@ extern "C" { || ((op) == LOAD_SUPER_METHOD) \ || ((op) == LOAD_ZERO_SUPER_METHOD) \ || ((op) == LOAD_ZERO_SUPER_ATTR) \ + || ((op) == STORE_FAST_MAYBE_NULL) \ ) #define HAS_CONST(op) (false\ diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 7a3fdbaebdf24e..d4a1593db2c874 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -442,6 +442,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12b1 3526 (Add instrumentation support) # Python 3.12b1 3527 (Add LOAD_SUPER_ATTR) # Python 3.12b1 3528 (Add LOAD_SUPER_ATTR_METHOD specialization) +# Python 3.12b1 3529 (Inline list/dict/set comprehensions) # Python 3.13 will start with 3550 @@ -458,7 +459,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3528).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3529).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index ad54bd27fba3d9..d6d478aa1f479d 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -198,6 +198,8 @@ def pseudo_op(name, op, real_ops): jrel_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards) name_op('LOAD_SUPER_ATTR', 141) def_op('CALL_FUNCTION_EX', 142) # Flags +def_op('LOAD_FAST_AND_CLEAR', 143) # Local variable number +haslocal.append(143) def_op('EXTENDED_ARG', 144) EXTENDED_ARG = 144 @@ -268,6 +270,8 @@ def pseudo_op(name, op, real_ops): pseudo_op('LOAD_ZERO_SUPER_METHOD', 264, ['LOAD_SUPER_ATTR']) pseudo_op('LOAD_ZERO_SUPER_ATTR', 265, ['LOAD_SUPER_ATTR']) +pseudo_op('STORE_FAST_MAYBE_NULL', 266, ['STORE_FAST']) + MAX_PSEUDO_OPCODE = MIN_PSEUDO_OPCODE + len(_pseudo_ops) - 1 del def_op, name_op, jrel_op, jabs_op, pseudo_op diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index dca38418935b76..c68b9ce388466e 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1352,14 +1352,11 @@ def test_multiline_list_comprehension(self): and x != 50)] """) compiled_code, _ = self.check_positions_against_ast(snippet) - compiled_code = compiled_code.co_consts[0] self.assertIsInstance(compiled_code, types.CodeType) self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND', line=1, end_line=2, column=1, end_column=8, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', line=1, end_line=2, column=1, end_column=8, occurrence=1) - self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', - line=1, end_line=6, column=0, end_column=32, occurrence=1) def test_multiline_async_list_comprehension(self): snippet = textwrap.dedent("""\ @@ -1374,13 +1371,13 @@ async def f(): compiled_code, _ = self.check_positions_against_ast(snippet) g = {} eval(compiled_code, g) - compiled_code = g['f'].__code__.co_consts[1] + compiled_code = g['f'].__code__ self.assertIsInstance(compiled_code, types.CodeType) self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND', line=2, end_line=3, column=5, end_column=12, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', line=2, end_line=3, column=5, end_column=12, occurrence=1) - self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST', line=2, end_line=7, column=4, end_column=36, occurrence=1) def test_multiline_set_comprehension(self): @@ -1393,14 +1390,11 @@ def test_multiline_set_comprehension(self): and x != 50)} """) compiled_code, _ = self.check_positions_against_ast(snippet) - compiled_code = compiled_code.co_consts[0] self.assertIsInstance(compiled_code, types.CodeType) self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD', line=1, end_line=2, column=1, end_column=8, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', line=1, end_line=2, column=1, end_column=8, occurrence=1) - self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', - line=1, end_line=6, column=0, end_column=32, occurrence=1) def test_multiline_async_set_comprehension(self): snippet = textwrap.dedent("""\ @@ -1415,13 +1409,13 @@ async def f(): compiled_code, _ = self.check_positions_against_ast(snippet) g = {} eval(compiled_code, g) - compiled_code = g['f'].__code__.co_consts[1] + compiled_code = g['f'].__code__ self.assertIsInstance(compiled_code, types.CodeType) self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD', line=2, end_line=3, column=5, end_column=12, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', line=2, end_line=3, column=5, end_column=12, occurrence=1) - self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST', line=2, end_line=7, column=4, end_column=36, occurrence=1) def test_multiline_dict_comprehension(self): @@ -1434,14 +1428,11 @@ def test_multiline_dict_comprehension(self): and x != 50)} """) compiled_code, _ = self.check_positions_against_ast(snippet) - compiled_code = compiled_code.co_consts[0] self.assertIsInstance(compiled_code, types.CodeType) self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD', line=1, end_line=2, column=1, end_column=7, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', line=1, end_line=2, column=1, end_column=7, occurrence=1) - self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', - line=1, end_line=6, column=0, end_column=32, occurrence=1) def test_multiline_async_dict_comprehension(self): snippet = textwrap.dedent("""\ @@ -1456,13 +1447,13 @@ async def f(): compiled_code, _ = self.check_positions_against_ast(snippet) g = {} eval(compiled_code, g) - compiled_code = g['f'].__code__.co_consts[1] + compiled_code = g['f'].__code__ self.assertIsInstance(compiled_code, types.CodeType) self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD', line=2, end_line=3, column=5, end_column=11, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', line=2, end_line=3, column=5, end_column=11, occurrence=1) - self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST', line=2, end_line=7, column=4, end_column=36, occurrence=1) def test_matchcase_sequence(self): @@ -1711,9 +1702,6 @@ def test_column_offset_deduplication(self): for source in [ "lambda: a", "(a for b in c)", - "[a for b in c]", - "{a for b in c}", - "{a: b for c in d}", ]: with self.subTest(source): code = compile(f"{source}, {source}", "", "eval") diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py index 0bd7a09b001c1d..3e2a127de728cd 100644 --- a/Lib/test/test_compiler_assemble.py +++ b/Lib/test/test_compiler_assemble.py @@ -16,7 +16,7 @@ def complete_metadata(self, metadata, filename="myfile.py"): metadata.setdefault(key, key) for key in ['consts']: metadata.setdefault(key, []) - for key in ['names', 'varnames', 'cellvars', 'freevars']: + for key in ['names', 'varnames', 'cellvars', 'freevars', 'fasthidden']: metadata.setdefault(key, {}) for key in ['argcount', 'posonlyargcount', 'kwonlyargcount']: metadata.setdefault(key, 0) @@ -33,6 +33,9 @@ def assemble_test(self, insts, metadata, expected): expected_metadata = {} for key, value in metadata.items(): + if key == "fasthidden": + # not exposed on code object + continue if isinstance(value, list): expected_metadata[key] = tuple(value) elif isinstance(value, dict): diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 2f5d67fde86170..c90702a408eb33 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -154,7 +154,7 @@ def bug708901(): def bug1333982(x=[]): - assert 0, ([s for s in x] + + assert 0, ((s for s in x) + 1) pass @@ -162,7 +162,7 @@ def bug1333982(x=[]): %3d RESUME 0 %3d LOAD_ASSERTION_ERROR - LOAD_CONST 1 ( at 0x..., file "%s", line %d>) + LOAD_CONST 1 ( at 0x..., file "%s", line %d>) MAKE_FUNCTION 0 LOAD_FAST 0 (x) GET_ITER @@ -675,7 +675,7 @@ async def _co(x): def _h(y): def foo(x): '''funcdoc''' - return [x + z for z in y] + return list(x + z for z in y) return foo dis_nested_0 = """\ @@ -705,13 +705,15 @@ def foo(x): %3d RESUME 0 -%3d LOAD_CLOSURE 0 (x) +%3d LOAD_GLOBAL 1 (NULL + list) + LOAD_CLOSURE 0 (x) BUILD_TUPLE 1 - LOAD_CONST 1 ( at 0x..., file "%s", line %d>) + LOAD_CONST 1 ( at 0x..., file "%s", line %d>) MAKE_FUNCTION 8 (closure) LOAD_DEREF 1 (y) GET_ITER CALL 0 + CALL 1 RETURN_VALUE """ % (dis_nested_0, __file__, @@ -723,21 +725,28 @@ def foo(x): ) dis_nested_2 = """%s -Disassembly of at 0x..., file "%s", line %d>: +Disassembly of at 0x..., file "%s", line %d>: COPY_FREE_VARS 1 -%3d RESUME 0 - BUILD_LIST 0 +%3d RETURN_GENERATOR + POP_TOP + RESUME 0 LOAD_FAST 0 (.0) - >> FOR_ITER 7 (to 26) + >> FOR_ITER 9 (to 32) STORE_FAST 1 (z) LOAD_DEREF 2 (x) LOAD_FAST 1 (z) BINARY_OP 0 (+) - LIST_APPEND 2 - JUMP_BACKWARD 9 (to 8) + YIELD_VALUE 1 + RESUME 1 + POP_TOP + JUMP_BACKWARD 11 (to 10) >> END_FOR - RETURN_VALUE + RETURN_CONST 0 (None) + >> CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) + RERAISE 1 +ExceptionTable: +1 row """ % (dis_nested_1, __file__, _h.__code__.co_firstlineno + 3, diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index d2b2f3171e785d..364f75db908b05 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -4280,14 +4280,14 @@ def test(*args, **kwargs): @cpython_only def test_signature_bind_implicit_arg(self): - # Issue #19611: getcallargs should work with set comprehensions + # Issue #19611: getcallargs should work with comprehensions def make_set(): - return {z * z for z in range(5)} - setcomp_code = make_set.__code__.co_consts[1] - setcomp_func = types.FunctionType(setcomp_code, {}) + return set(z * z for z in range(5)) + gencomp_code = make_set.__code__.co_consts[1] + gencomp_func = types.FunctionType(gencomp_code, {}) iterator = iter(range(5)) - self.assertEqual(self.call(setcomp_func, iterator), {0, 1, 4, 9, 16}) + self.assertEqual(set(self.call(gencomp_func, iterator)), {0, 1, 4, 9, 16}) def test_signature_bind_posonly_kwargs(self): def foo(bar, /, **kwargs): diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 91bf2547edc4ae..92fed98dd0004a 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -1,4 +1,5 @@ import doctest +import textwrap import unittest @@ -87,63 +88,227 @@ >>> [None for i in range(10)] [None, None, None, None, None, None, None, None, None, None] -########### Tests for various scoping corner cases ############ - -Return lambdas that use the iteration variable as a default argument - - >>> items = [(lambda i=i: i) for i in range(5)] - >>> [x() for x in items] - [0, 1, 2, 3, 4] - -Same again, only this time as a closure variable - - >>> items = [(lambda: i) for i in range(5)] - >>> [x() for x in items] - [4, 4, 4, 4, 4] - -Another way to test that the iteration variable is local to the list comp - - >>> items = [(lambda: i) for i in range(5)] - >>> i = 20 - >>> [x() for x in items] - [4, 4, 4, 4, 4] - -And confirm that a closure can jump over the list comp scope - - >>> items = [(lambda: y) for i in range(5)] - >>> y = 2 - >>> [x() for x in items] - [2, 2, 2, 2, 2] - -We also repeat each of the above scoping tests inside a function - - >>> def test_func(): - ... items = [(lambda i=i: i) for i in range(5)] - ... return [x() for x in items] - >>> test_func() - [0, 1, 2, 3, 4] +""" - >>> def test_func(): - ... items = [(lambda: i) for i in range(5)] - ... return [x() for x in items] - >>> test_func() - [4, 4, 4, 4, 4] - - >>> def test_func(): - ... items = [(lambda: i) for i in range(5)] - ... i = 20 - ... return [x() for x in items] - >>> test_func() - [4, 4, 4, 4, 4] - - >>> def test_func(): - ... items = [(lambda: y) for i in range(5)] - ... y = 2 - ... return [x() for x in items] - >>> test_func() - [2, 2, 2, 2, 2] -""" +class ListComprehensionTest(unittest.TestCase): + def _check_in_scopes(self, code, outputs=None, ns=None, scopes=None, raises=()): + code = textwrap.dedent(code) + scopes = scopes or ["module", "class", "function"] + for scope in scopes: + with self.subTest(scope=scope): + if scope == "class": + newcode = textwrap.dedent(""" + class _C: + {code} + """).format(code=textwrap.indent(code, " ")) + def get_output(moddict, name): + return getattr(moddict["_C"], name) + elif scope == "function": + newcode = textwrap.dedent(""" + def _f(): + {code} + return locals() + _out = _f() + """).format(code=textwrap.indent(code, " ")) + def get_output(moddict, name): + return moddict["_out"][name] + else: + newcode = code + def get_output(moddict, name): + return moddict[name] + ns = ns or {} + try: + exec(newcode, ns) + except raises as e: + # We care about e.g. NameError vs UnboundLocalError + self.assertIs(type(e), raises) + else: + for k, v in (outputs or {}).items(): + self.assertEqual(get_output(ns, k), v) + + def test_lambdas_with_iteration_var_as_default(self): + code = """ + items = [(lambda i=i: i) for i in range(5)] + y = [x() for x in items] + """ + outputs = {"y": [0, 1, 2, 3, 4]} + self._check_in_scopes(code, outputs) + + def test_lambdas_with_free_var(self): + code = """ + items = [(lambda: i) for i in range(5)] + y = [x() for x in items] + """ + outputs = {"y": [4, 4, 4, 4, 4]} + self._check_in_scopes(code, outputs) + + def test_class_scope_free_var_with_class_cell(self): + class C: + def method(self): + super() + return __class__ + items = [(lambda: i) for i in range(5)] + y = [x() for x in items] + + self.assertEqual(C.y, [4, 4, 4, 4, 4]) + self.assertIs(C().method(), C) + + def test_inner_cell_shadows_outer(self): + code = """ + items = [(lambda: i) for i in range(5)] + i = 20 + y = [x() for x in items] + """ + outputs = {"y": [4, 4, 4, 4, 4], "i": 20} + self._check_in_scopes(code, outputs) + + def test_closure_can_jump_over_comp_scope(self): + code = """ + items = [(lambda: y) for i in range(5)] + y = 2 + z = [x() for x in items] + """ + outputs = {"z": [2, 2, 2, 2, 2]} + self._check_in_scopes(code, outputs) + + def test_inner_cell_shadows_outer_redefined(self): + code = """ + y = 10 + items = [(lambda: y) for y in range(5)] + x = y + y = 20 + out = [z() for z in items] + """ + outputs = {"x": 10, "out": [4, 4, 4, 4, 4]} + self._check_in_scopes(code, outputs) + + def test_shadows_outer_cell(self): + code = """ + def inner(): + return g + [g for g in range(5)] + x = inner() + """ + outputs = {"x": -1} + self._check_in_scopes(code, outputs, ns={"g": -1}) + + def test_assignment_expression(self): + code = """ + x = -1 + items = [(x:=y) for y in range(3)] + """ + outputs = {"x": 2} + # assignment expression in comprehension is disallowed in class scope + self._check_in_scopes(code, outputs, scopes=["module", "function"]) + + def test_free_var_in_comp_child(self): + code = """ + lst = range(3) + funcs = [lambda: x for x in lst] + inc = [x + 1 for x in lst] + [x for x in inc] + x = funcs[0]() + """ + outputs = {"x": 2} + self._check_in_scopes(code, outputs) + + def test_shadow_with_free_and_local(self): + code = """ + lst = range(3) + x = -1 + funcs = [lambda: x for x in lst] + items = [x + 1 for x in lst] + """ + outputs = {"x": -1} + self._check_in_scopes(code, outputs) + + def test_shadow_comp_iterable_name(self): + code = """ + x = [1] + y = [x for x in x] + """ + outputs = {"x": [1]} + self._check_in_scopes(code, outputs) + + def test_nested_free(self): + code = """ + x = 1 + def g(): + [x for x in range(3)] + return x + g() + """ + outputs = {"x": 1} + self._check_in_scopes(code, outputs) + + def test_introspecting_frame_locals(self): + code = """ + import sys + [i for i in range(2)] + i = 20 + sys._getframe().f_locals + """ + outputs = {"i": 20} + self._check_in_scopes(code, outputs) + + def test_nested(self): + code = """ + l = [2, 3] + y = [[x ** 2 for x in range(x)] for x in l] + """ + outputs = {"y": [[0, 1], [0, 1, 4]]} + self._check_in_scopes(code, outputs) + + def test_nested_2(self): + code = """ + l = [1, 2, 3] + x = 3 + y = [x for [x ** x for x in range(x)][x - 1] in l] + """ + outputs = {"y": [3, 3, 3]} + self._check_in_scopes(code, outputs) + + def test_nested_3(self): + code = """ + l = [(1, 2), (3, 4), (5, 6)] + y = [x for (x, [x ** x for x in range(x)][x - 1]) in l] + """ + outputs = {"y": [1, 3, 5]} + self._check_in_scopes(code, outputs) + + def test_nameerror(self): + code = """ + [x for x in [1]] + x + """ + + self._check_in_scopes(code, raises=NameError) + + def test_dunder_name(self): + code = """ + y = [__x for __x in [1]] + """ + outputs = {"y": [1]} + self._check_in_scopes(code, outputs) + + def test_unbound_local_after_comprehension(self): + def f(): + if False: + x = 0 + [x for x in [1]] + return x + + with self.assertRaises(UnboundLocalError): + f() + + def test_unbound_local_inside_comprehension(self): + def f(): + l = [None] + return [1 for (l[0], l) in [[1, 2]]] + + with self.assertRaises(UnboundLocalError): + f() __test__ = {'doctests' : doctests} diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index fad2b3b8379ffc..73339ebdb7c4e9 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -187,9 +187,7 @@ def test_trace_list_comprehension(self): firstlineno_called = get_firstlineno(traced_doubler) expected = { (self.my_py_filename, firstlineno_calling + 1): 1, - # List comprehensions work differently in 3.x, so the count - # below changed compared to 2.x. - (self.my_py_filename, firstlineno_calling + 2): 12, + (self.my_py_filename, firstlineno_calling + 2): 11, (self.my_py_filename, firstlineno_calling + 3): 1, (self.my_py_filename, firstlineno_called + 1): 10, } diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst b/Misc/NEWS.d/next/Core and Builtins/2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst new file mode 100644 index 00000000000000..2eec05cb3ace5c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst @@ -0,0 +1,2 @@ +:pep:`709`: inline list, dict and set comprehensions to improve performance +and reduce bytecode size. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 8009dca3d0b746..ea9b6e72b3c924 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -669,12 +669,14 @@ _testinternalcapi_assemble_code_object_impl(PyObject *module, umd.u_varnames = PyDict_GetItemString(metadata, "varnames"); umd.u_cellvars = PyDict_GetItemString(metadata, "cellvars"); umd.u_freevars = PyDict_GetItemString(metadata, "freevars"); + umd.u_fasthidden = PyDict_GetItemString(metadata, "fasthidden"); assert(PyDict_Check(umd.u_consts)); assert(PyDict_Check(umd.u_names)); assert(PyDict_Check(umd.u_varnames)); assert(PyDict_Check(umd.u_cellvars)); assert(PyDict_Check(umd.u_freevars)); + assert(PyDict_Check(umd.u_fasthidden)); umd.u_argcount = get_nonnegative_int_from_dict(metadata, "argcount"); umd.u_posonlyargcount = get_nonnegative_int_from_dict(metadata, "posonlyargcount"); diff --git a/Objects/frameobject.c b/Objects/frameobject.c index d0eca447c012ea..d9aaea7831a380 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1224,6 +1224,10 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) } PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + if (kind & CO_FAST_HIDDEN) { + continue; + } if (value == NULL) { if (PyObject_DelItem(locals, name) != 0) { if (PyErr_ExceptionMatches(PyExc_KeyError)) { diff --git a/Python/assemble.c b/Python/assemble.c index 369dd8dcde9b9b..6889831ae3fe0c 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -456,6 +456,9 @@ compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, assert(offset < nlocalsplus); // For now we do not distinguish arg kinds. _PyLocals_Kind kind = CO_FAST_LOCAL; + if (PyDict_Contains(umd->u_fasthidden, k)) { + kind |= CO_FAST_HIDDEN; + } if (PyDict_GetItem(umd->u_cellvars, k) != NULL) { kind |= CO_FAST_CELL; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index e83894e8902872..82c10044418345 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -194,6 +194,12 @@ dummy_func( Py_INCREF(value); } + inst(LOAD_FAST_AND_CLEAR, (-- value)) { + value = GETLOCAL(oparg); + // do not use SETLOCAL here, it decrefs the old value + GETLOCAL(oparg) = NULL; + } + inst(LOAD_CONST, (-- value)) { value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); diff --git a/Python/compile.c b/Python/compile.c index 12ae831361963d..941c6e9d4fdbb7 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -381,7 +381,6 @@ struct compiler_unit { int u_scope_type; - PyObject *u_private; /* for private name mangling */ instr_sequence u_instr_sequence; /* codegen output */ @@ -485,13 +484,15 @@ static int compiler_sync_comprehension_generator( struct compiler *c, location loc, asdl_comprehension_seq *generators, int gen_index, int depth, - expr_ty elt, expr_ty val, int type); + expr_ty elt, expr_ty val, int type, + int iter_on_stack); static int compiler_async_comprehension_generator( struct compiler *c, location loc, asdl_comprehension_seq *generators, int gen_index, int depth, - expr_ty elt, expr_ty val, int type); + expr_ty elt, expr_ty val, int type, + int iter_on_stack); static int compiler_pattern(struct compiler *, pattern_ty, pattern_context *); static int compiler_match(struct compiler *, stmt_ty); @@ -689,6 +690,7 @@ compiler_unit_free(struct compiler_unit *u) Py_CLEAR(u->u_metadata.u_varnames); Py_CLEAR(u->u_metadata.u_freevars); Py_CLEAR(u->u_metadata.u_cellvars); + Py_CLEAR(u->u_metadata.u_fasthidden); Py_CLEAR(u->u_private); PyObject_Free(u); } @@ -837,6 +839,8 @@ stack_effect(int opcode, int oparg, int jump) * if an exception be raised. */ return jump ? 1 : 0; + case STORE_FAST_MAYBE_NULL: + return -1; case LOAD_METHOD: return 1; case LOAD_SUPER_METHOD: @@ -1239,11 +1243,9 @@ compiler_enter_scope(struct compiler *c, identifier name, } if (u->u_ste->ste_needs_class_closure) { /* Cook up an implicit __class__ cell. */ - int res; + Py_ssize_t res; assert(u->u_scope_type == COMPILER_SCOPE_CLASS); - assert(PyDict_GET_SIZE(u->u_metadata.u_cellvars) == 0); - res = PyDict_SetItem(u->u_metadata.u_cellvars, &_Py_ID(__class__), - _PyLong_GetZero()); + res = dict_add_o(u->u_metadata.u_cellvars, &_Py_ID(__class__)); if (res < 0) { compiler_unit_free(u); return ERROR; @@ -1257,6 +1259,12 @@ compiler_enter_scope(struct compiler *c, identifier name, return ERROR; } + u->u_metadata.u_fasthidden = PyDict_New(); + if (!u->u_metadata.u_fasthidden) { + compiler_unit_free(u); + return ERROR; + } + u->u_nfblocks = 0; u->u_metadata.u_firstlineno = lineno; u->u_metadata.u_consts = PyDict_New(); @@ -2235,7 +2243,6 @@ compiler_class(struct compiler *c, stmt_ty s) compiler_exit_scope(c); return ERROR; } - assert(i == 0); ADDOP_I(c, NO_LOCATION, LOAD_CLOSURE, i); ADDOP_I(c, NO_LOCATION, COPY, 1); if (compiler_nameop(c, NO_LOCATION, &_Py_ID(__classcell__), Store) < 0) { @@ -2245,7 +2252,6 @@ compiler_class(struct compiler *c, stmt_ty s) } else { /* No methods referenced __class__, so just return None */ - assert(PyDict_GET_SIZE(c->u->u_metadata.u_cellvars) == 0); ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); } ADDOP_IN_SCOPE(c, NO_LOCATION, RETURN_VALUE); @@ -3718,7 +3724,8 @@ compiler_nameop(struct compiler *c, location loc, optype = OP_DEREF; break; case LOCAL: - if (c->u->u_ste->ste_type == FunctionBlock) + if (c->u->u_ste->ste_type == FunctionBlock || + (PyDict_GetItem(c->u->u_metadata.u_fasthidden, mangled) == Py_True)) optype = OP_FAST; break; case GLOBAL_IMPLICIT: @@ -4742,16 +4749,19 @@ static int compiler_comprehension_generator(struct compiler *c, location loc, asdl_comprehension_seq *generators, int gen_index, int depth, - expr_ty elt, expr_ty val, int type) + expr_ty elt, expr_ty val, int type, + int iter_on_stack) { comprehension_ty gen; gen = (comprehension_ty)asdl_seq_GET(generators, gen_index); if (gen->is_async) { return compiler_async_comprehension_generator( - c, loc, generators, gen_index, depth, elt, val, type); + c, loc, generators, gen_index, depth, elt, val, type, + iter_on_stack); } else { return compiler_sync_comprehension_generator( - c, loc, generators, gen_index, depth, elt, val, type); + c, loc, generators, gen_index, depth, elt, val, type, + iter_on_stack); } } @@ -4759,7 +4769,8 @@ static int compiler_sync_comprehension_generator(struct compiler *c, location loc, asdl_comprehension_seq *generators, int gen_index, int depth, - expr_ty elt, expr_ty val, int type) + expr_ty elt, expr_ty val, int type, + int iter_on_stack) { /* generate code for the iterator, then each of the ifs, and then write to the element */ @@ -4771,37 +4782,39 @@ compiler_sync_comprehension_generator(struct compiler *c, location loc, comprehension_ty gen = (comprehension_ty)asdl_seq_GET(generators, gen_index); - if (gen_index == 0) { - /* Receive outermost iter as an implicit argument */ - c->u->u_metadata.u_argcount = 1; - ADDOP_I(c, loc, LOAD_FAST, 0); - } - else { - /* Sub-iter - calculate on the fly */ - /* Fast path for the temporary variable assignment idiom: - for y in [f(x)] - */ - asdl_expr_seq *elts; - switch (gen->iter->kind) { - case List_kind: - elts = gen->iter->v.List.elts; - break; - case Tuple_kind: - elts = gen->iter->v.Tuple.elts; - break; - default: - elts = NULL; - } - if (asdl_seq_LEN(elts) == 1) { - expr_ty elt = asdl_seq_GET(elts, 0); - if (elt->kind != Starred_kind) { - VISIT(c, expr, elt); - start = NO_LABEL; - } + if (!iter_on_stack) { + if (gen_index == 0) { + /* Receive outermost iter as an implicit argument */ + c->u->u_metadata.u_argcount = 1; + ADDOP_I(c, loc, LOAD_FAST, 0); } - if (IS_LABEL(start)) { - VISIT(c, expr, gen->iter); - ADDOP(c, loc, GET_ITER); + else { + /* Sub-iter - calculate on the fly */ + /* Fast path for the temporary variable assignment idiom: + for y in [f(x)] + */ + asdl_expr_seq *elts; + switch (gen->iter->kind) { + case List_kind: + elts = gen->iter->v.List.elts; + break; + case Tuple_kind: + elts = gen->iter->v.Tuple.elts; + break; + default: + elts = NULL; + } + if (asdl_seq_LEN(elts) == 1) { + expr_ty elt = asdl_seq_GET(elts, 0); + if (elt->kind != Starred_kind) { + VISIT(c, expr, elt); + start = NO_LABEL; + } + } + if (IS_LABEL(start)) { + VISIT(c, expr, gen->iter); + ADDOP(c, loc, GET_ITER); + } } } if (IS_LABEL(start)) { @@ -4822,7 +4835,7 @@ compiler_sync_comprehension_generator(struct compiler *c, location loc, RETURN_IF_ERROR( compiler_comprehension_generator(c, loc, generators, gen_index, depth, - elt, val, type)); + elt, val, type, 0)); } location elt_loc = LOC(elt); @@ -4875,7 +4888,8 @@ static int compiler_async_comprehension_generator(struct compiler *c, location loc, asdl_comprehension_seq *generators, int gen_index, int depth, - expr_ty elt, expr_ty val, int type) + expr_ty elt, expr_ty val, int type, + int iter_on_stack) { NEW_JUMP_TARGET_LABEL(c, start); NEW_JUMP_TARGET_LABEL(c, except); @@ -4884,15 +4898,17 @@ compiler_async_comprehension_generator(struct compiler *c, location loc, comprehension_ty gen = (comprehension_ty)asdl_seq_GET(generators, gen_index); - if (gen_index == 0) { - /* Receive outermost iter as an implicit argument */ - c->u->u_metadata.u_argcount = 1; - ADDOP_I(c, loc, LOAD_FAST, 0); - } - else { - /* Sub-iter - calculate on the fly */ - VISIT(c, expr, gen->iter); - ADDOP(c, loc, GET_AITER); + if (!iter_on_stack) { + if (gen_index == 0) { + /* Receive outermost iter as an implicit argument */ + c->u->u_metadata.u_argcount = 1; + ADDOP_I(c, loc, LOAD_FAST, 0); + } + else { + /* Sub-iter - calculate on the fly */ + VISIT(c, expr, gen->iter); + ADDOP(c, loc, GET_AITER); + } } USE_LABEL(c, start); @@ -4919,7 +4935,7 @@ compiler_async_comprehension_generator(struct compiler *c, location loc, RETURN_IF_ERROR( compiler_comprehension_generator(c, loc, generators, gen_index, depth, - elt, val, type)); + elt, val, type, 0)); } location elt_loc = LOC(elt); @@ -4968,26 +4984,212 @@ compiler_async_comprehension_generator(struct compiler *c, location loc, return SUCCESS; } +typedef struct { + PyObject *pushed_locals; + PyObject *temp_symbols; + PyObject *fast_hidden; +} inlined_comprehension_state; + +static int +push_inlined_comprehension_state(struct compiler *c, location loc, + PySTEntryObject *entry, + inlined_comprehension_state *state) +{ + // iterate over names bound in the comprehension and ensure we isolate + // them from the outer scope as needed + PyObject *k, *v; + Py_ssize_t pos = 0; + while (PyDict_Next(entry->ste_symbols, &pos, &k, &v)) { + assert(PyLong_Check(v)); + long symbol = PyLong_AS_LONG(v); + // only values bound in the comprehension (DEF_LOCAL) need to be handled + // at all; DEF_LOCAL | DEF_NONLOCAL can occur in the case of an + // assignment expression to a nonlocal in the comprehension, these don't + // need handling here since they shouldn't be isolated + if (symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) { + if (c->u->u_ste->ste_type != FunctionBlock) { + // non-function scope: override this name to use fast locals + PyObject *orig = PyDict_GetItem(c->u->u_metadata.u_fasthidden, k); + if (orig != Py_True) { + if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_True) < 0) { + return ERROR; + } + if (state->fast_hidden == NULL) { + state->fast_hidden = PySet_New(NULL); + if (state->fast_hidden == NULL) { + return ERROR; + } + } + if (PySet_Add(state->fast_hidden, k) < 0) { + return ERROR; + } + } + } + long scope = (symbol >> SCOPE_OFFSET) & SCOPE_MASK; + PyObject *outv = PyDict_GetItemWithError(c->u->u_ste->ste_symbols, k); + if (outv == NULL) { + return ERROR; + } + assert(PyLong_Check(outv)); + long outsc = (PyLong_AS_LONG(outv) >> SCOPE_OFFSET) & SCOPE_MASK; + if (scope != outsc) { + // If a name has different scope inside than outside the + // comprehension, we need to temporarily handle it with the + // right scope while compiling the comprehension. + if (state->temp_symbols == NULL) { + state->temp_symbols = PyDict_New(); + if (state->temp_symbols == NULL) { + return ERROR; + } + } + // update the symbol to the in-comprehension version and save + // the outer version; we'll restore it after running the + // comprehension + Py_INCREF(outv); + if (PyDict_SetItem(c->u->u_ste->ste_symbols, k, v) < 0) { + Py_DECREF(outv); + return ERROR; + } + if (PyDict_SetItem(state->temp_symbols, k, outv) < 0) { + Py_DECREF(outv); + return ERROR; + } + Py_DECREF(outv); + } + if (outsc == LOCAL || outsc == CELL || outsc == FREE) { + // local names bound in comprehension must be isolated from + // outer scope; push existing value (which may be NULL if + // not defined) on stack + if (state->pushed_locals == NULL) { + state->pushed_locals = PyList_New(0); + if (state->pushed_locals == NULL) { + return ERROR; + } + } + // in the case of a cell, this will actually push the cell + // itself to the stack, then we'll create a new one for the + // comprehension and restore the original one after + ADDOP_NAME(c, loc, LOAD_FAST_AND_CLEAR, k, varnames); + if (scope == CELL) { + ADDOP_NAME(c, loc, MAKE_CELL, k, cellvars); + } + if (PyList_Append(state->pushed_locals, k) < 0) { + return ERROR; + } + } + } + } + if (state->pushed_locals) { + // Outermost iterable expression was already evaluated and is on the + // stack, we need to swap it back to TOS. This also rotates the order of + // `pushed_locals` on the stack, but this will be reversed when we swap + // out the comprehension result in pop_inlined_comprehension_state + ADDOP_I(c, loc, SWAP, PyList_GET_SIZE(state->pushed_locals) + 1); + } + + return SUCCESS; +} + +static int +pop_inlined_comprehension_state(struct compiler *c, location loc, + inlined_comprehension_state state) +{ + PyObject *k, *v; + Py_ssize_t pos = 0; + if (state.temp_symbols) { + while (PyDict_Next(state.temp_symbols, &pos, &k, &v)) { + if (PyDict_SetItem(c->u->u_ste->ste_symbols, k, v)) { + return ERROR; + } + } + Py_CLEAR(state.temp_symbols); + } + if (state.pushed_locals) { + // pop names we pushed to stack earlier + Py_ssize_t npops = PyList_GET_SIZE(state.pushed_locals); + // Preserve the list/dict/set result of the comprehension as TOS. This + // reverses the SWAP we did in push_inlined_comprehension_state to get + // the outermost iterable to TOS, so we can still just iterate + // pushed_locals in simple reverse order + ADDOP_I(c, loc, SWAP, npops + 1); + for (Py_ssize_t i = npops - 1; i >= 0; --i) { + k = PyList_GetItem(state.pushed_locals, i); + if (k == NULL) { + return ERROR; + } + ADDOP_NAME(c, loc, STORE_FAST_MAYBE_NULL, k, varnames); + } + Py_CLEAR(state.pushed_locals); + } + if (state.fast_hidden) { + while (PySet_Size(state.fast_hidden) > 0) { + PyObject *k = PySet_Pop(state.fast_hidden); + if (k == NULL) { + return ERROR; + } + // we set to False instead of clearing, so we can track which names + // were temporarily fast-locals and should use CO_FAST_HIDDEN + if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_False)) { + Py_DECREF(k); + return ERROR; + } + Py_DECREF(k); + } + Py_CLEAR(state.fast_hidden); + } + return SUCCESS; +} + +static inline int +compiler_comprehension_iter(struct compiler *c, location loc, + comprehension_ty comp) +{ + VISIT(c, expr, comp->iter); + if (comp->is_async) { + ADDOP(c, loc, GET_AITER); + } + else { + ADDOP(c, loc, GET_ITER); + } + return SUCCESS; +} + static int compiler_comprehension(struct compiler *c, expr_ty e, int type, identifier name, asdl_comprehension_seq *generators, expr_ty elt, expr_ty val) { PyCodeObject *co = NULL; + inlined_comprehension_state inline_state = {NULL, NULL}; comprehension_ty outermost; int scope_type = c->u->u_scope_type; - int is_async_generator = 0; int is_top_level_await = IS_TOP_LEVEL_AWAIT(c); - - outermost = (comprehension_ty) asdl_seq_GET(generators, 0); - if (compiler_enter_scope(c, name, COMPILER_SCOPE_COMPREHENSION, - (void *)e, e->lineno) < 0) - { + PySTEntryObject *entry = PySymtable_Lookup(c->c_st, (void *)e); + if (entry == NULL) { goto error; } + int is_inlined = entry->ste_comp_inlined; + int is_async_generator = entry->ste_coroutine; + location loc = LOC(e); - is_async_generator = c->u->u_ste->ste_coroutine; + outermost = (comprehension_ty) asdl_seq_GET(generators, 0); + if (is_inlined) { + if (compiler_comprehension_iter(c, loc, outermost)) { + goto error; + } + if (push_inlined_comprehension_state(c, loc, entry, &inline_state)) { + goto error; + } + } + else { + if (compiler_enter_scope(c, name, COMPILER_SCOPE_COMPREHENSION, + (void *)e, e->lineno) < 0) + { + goto error; + } + } + Py_CLEAR(entry); if (is_async_generator && type != COMP_GENEXP && scope_type != COMPILER_SCOPE_ASYNC_FUNCTION && @@ -5018,13 +5220,23 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, } ADDOP_I(c, loc, op, 0); + if (is_inlined) { + ADDOP_I(c, loc, SWAP, 2); + } } if (compiler_comprehension_generator(c, loc, generators, 0, 0, - elt, val, type) < 0) { + elt, val, type, is_inlined) < 0) { goto error_in_scope; } + if (is_inlined) { + if (pop_inlined_comprehension_state(c, loc, inline_state)) { + goto error; + } + return SUCCESS; + } + if (type != COMP_GENEXP) { ADDOP(c, LOC(e), RETURN_VALUE); } @@ -5047,15 +5259,10 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, if (compiler_make_closure(c, loc, co, 0) < 0) { goto error; } - Py_DECREF(co); + Py_CLEAR(co); - VISIT(c, expr, outermost->iter); - - loc = LOC(e); - if (outermost->is_async) { - ADDOP(c, loc, GET_AITER); - } else { - ADDOP(c, loc, GET_ITER); + if (compiler_comprehension_iter(c, loc, outermost)) { + goto error; } ADDOP_I(c, loc, CALL, 0); @@ -5068,9 +5275,15 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, return SUCCESS; error_in_scope: - compiler_exit_scope(c); + if (!is_inlined) { + compiler_exit_scope(c); + } error: Py_XDECREF(co); + Py_XDECREF(entry); + Py_XDECREF(inline_state.pushed_locals); + Py_XDECREF(inline_state.temp_symbols); + Py_XDECREF(inline_state.fast_hidden); return ERROR; } @@ -6989,7 +7202,7 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, goto error; } - _PyCfg_ConvertExceptionHandlersToNops(g.g_entryblock); + _PyCfg_ConvertPseudoOps(g.g_entryblock); /* Order of basic blocks must have been determined by now */ @@ -7401,7 +7614,7 @@ _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, goto error; } - _PyCfg_ConvertExceptionHandlersToNops(g.g_entryblock); + _PyCfg_ConvertPseudoOps(g.g_entryblock); /* Order of basic blocks must have been determined by now */ diff --git a/Python/flowgraph.c b/Python/flowgraph.c index f79afb4c66cd8b..7f790b79d2844f 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -1289,7 +1289,9 @@ swaptimize(basicblock *block, int *ix) // - can't invoke arbitrary code (besides finalizers) // - only touch the TOS (and pop it when finished) #define SWAPPABLE(opcode) \ - ((opcode) == STORE_FAST || (opcode) == POP_TOP) + ((opcode) == STORE_FAST || \ + (opcode) == STORE_FAST_MAYBE_NULL || \ + (opcode) == POP_TOP) static int next_swappable_instruction(basicblock *block, int i, int lineno) @@ -1600,6 +1602,8 @@ scan_block_for_locals(basicblock *b, basicblock ***sp) uint64_t bit = (uint64_t)1 << instr->i_oparg; switch (instr->i_opcode) { case DELETE_FAST: + case LOAD_FAST_AND_CLEAR: + case STORE_FAST_MAYBE_NULL: unsafe_mask |= bit; break; case STORE_FAST: @@ -1639,7 +1643,8 @@ fast_scan_many_locals(basicblock *entryblock, int nlocals) Py_ssize_t blocknum = 0; // state[i - 64] == blocknum if local i is guaranteed to // be initialized, i.e., if it has had a previous LOAD_FAST or - // STORE_FAST within that basicblock (not followed by DELETE_FAST). + // STORE_FAST within that basicblock (not followed by + // DELETE_FAST/LOAD_FAST_AND_CLEAR/STORE_FAST_MAYBE_NULL). for (basicblock *b = entryblock; b != NULL; b = b->b_next) { blocknum++; for (int i = 0; i < b->b_iused; i++) { @@ -1653,6 +1658,8 @@ fast_scan_many_locals(basicblock *entryblock, int nlocals) assert(arg >= 0); switch (instr->i_opcode) { case DELETE_FAST: + case LOAD_FAST_AND_CLEAR: + case STORE_FAST_MAYBE_NULL: states[arg - 64] = blocknum - 1; break; case STORE_FAST: @@ -1975,7 +1982,7 @@ push_cold_blocks_to_end(cfg_builder *g, int code_flags) { } void -_PyCfg_ConvertExceptionHandlersToNops(basicblock *entryblock) +_PyCfg_ConvertPseudoOps(basicblock *entryblock) { for (basicblock *b = entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { @@ -1983,6 +1990,9 @@ _PyCfg_ConvertExceptionHandlersToNops(basicblock *entryblock) if (is_block_push(instr) || instr->i_opcode == POP_BLOCK) { INSTR_SET_OP0(instr, NOP); } + else if (instr->i_opcode == STORE_FAST_MAYBE_NULL) { + instr->i_opcode = STORE_FAST; + } } } for (basicblock *b = entryblock; b != NULL; b = b->b_next) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 069a7ced0a4c25..819c857c3c0119 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -92,13 +92,25 @@ DISPATCH(); } + TARGET(LOAD_FAST_AND_CLEAR) { + PyObject *value; + #line 198 "Python/bytecodes.c" + value = GETLOCAL(oparg); + // do not use SETLOCAL here, it decrefs the old value + GETLOCAL(oparg) = NULL; + #line 102 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = value; + DISPATCH(); + } + TARGET(LOAD_CONST) { PREDICTED(LOAD_CONST); PyObject *value; - #line 198 "Python/bytecodes.c" + #line 204 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 102 "Python/generated_cases.c.h" + #line 114 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -106,9 +118,9 @@ TARGET(STORE_FAST) { PyObject *value = stack_pointer[-1]; - #line 203 "Python/bytecodes.c" + #line 209 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 112 "Python/generated_cases.c.h" + #line 124 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -122,7 +134,7 @@ value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 126 "Python/generated_cases.c.h" + #line 138 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; @@ -132,7 +144,7 @@ value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 136 "Python/generated_cases.c.h" + #line 148 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -150,16 +162,16 @@ value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 154 "Python/generated_cases.c.h" + #line 166 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 198 "Python/bytecodes.c" + #line 204 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 163 "Python/generated_cases.c.h" + #line 175 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -172,9 +184,9 @@ PyObject *_tmp_1 = stack_pointer[-1]; { PyObject *value = _tmp_1; - #line 203 "Python/bytecodes.c" + #line 209 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 178 "Python/generated_cases.c.h" + #line 190 "Python/generated_cases.c.h" } oparg = (next_instr++)->op.arg; { @@ -183,7 +195,7 @@ value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 187 "Python/generated_cases.c.h" + #line 199 "Python/generated_cases.c.h" _tmp_1 = value; } stack_pointer[-1] = _tmp_1; @@ -195,16 +207,16 @@ PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; - #line 203 "Python/bytecodes.c" + #line 209 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 201 "Python/generated_cases.c.h" + #line 213 "Python/generated_cases.c.h" } oparg = (next_instr++)->op.arg; { PyObject *value = _tmp_2; - #line 203 "Python/bytecodes.c" + #line 209 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 208 "Python/generated_cases.c.h" + #line 220 "Python/generated_cases.c.h" } STACK_SHRINK(2); DISPATCH(); @@ -215,10 +227,10 @@ PyObject *_tmp_2; { PyObject *value; - #line 198 "Python/bytecodes.c" + #line 204 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 222 "Python/generated_cases.c.h" + #line 234 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; @@ -228,7 +240,7 @@ value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 232 "Python/generated_cases.c.h" + #line 244 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -239,8 +251,8 @@ TARGET(POP_TOP) { PyObject *value = stack_pointer[-1]; - #line 213 "Python/bytecodes.c" - #line 244 "Python/generated_cases.c.h" + #line 219 "Python/bytecodes.c" + #line 256 "Python/generated_cases.c.h" Py_DECREF(value); STACK_SHRINK(1); DISPATCH(); @@ -248,9 +260,9 @@ TARGET(PUSH_NULL) { PyObject *res; - #line 217 "Python/bytecodes.c" + #line 223 "Python/bytecodes.c" res = NULL; - #line 254 "Python/generated_cases.c.h" + #line 266 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -261,14 +273,14 @@ PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; - #line 213 "Python/bytecodes.c" - #line 266 "Python/generated_cases.c.h" + #line 219 "Python/bytecodes.c" + #line 278 "Python/generated_cases.c.h" Py_DECREF(value); } { PyObject *value = _tmp_2; - #line 213 "Python/bytecodes.c" - #line 272 "Python/generated_cases.c.h" + #line 219 "Python/bytecodes.c" + #line 284 "Python/generated_cases.c.h" Py_DECREF(value); } STACK_SHRINK(2); @@ -278,7 +290,7 @@ TARGET(INSTRUMENTED_END_FOR) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 223 "Python/bytecodes.c" + #line 229 "Python/bytecodes.c" /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { @@ -288,7 +300,7 @@ } PyErr_SetRaisedException(NULL); } - #line 292 "Python/generated_cases.c.h" + #line 304 "Python/generated_cases.c.h" Py_DECREF(receiver); Py_DECREF(value); STACK_SHRINK(2); @@ -298,9 +310,9 @@ TARGET(END_SEND) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 236 "Python/bytecodes.c" + #line 242 "Python/bytecodes.c" Py_DECREF(receiver); - #line 304 "Python/generated_cases.c.h" + #line 316 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; DISPATCH(); @@ -309,7 +321,7 @@ TARGET(INSTRUMENTED_END_SEND) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 240 "Python/bytecodes.c" + #line 246 "Python/bytecodes.c" if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, next_instr-1)) { @@ -318,7 +330,7 @@ PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); - #line 322 "Python/generated_cases.c.h" + #line 334 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; DISPATCH(); @@ -327,13 +339,13 @@ TARGET(UNARY_NEGATIVE) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 251 "Python/bytecodes.c" + #line 257 "Python/bytecodes.c" res = PyNumber_Negative(value); - #line 333 "Python/generated_cases.c.h" + #line 345 "Python/generated_cases.c.h" Py_DECREF(value); - #line 253 "Python/bytecodes.c" + #line 259 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 337 "Python/generated_cases.c.h" + #line 349 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -341,11 +353,11 @@ TARGET(UNARY_NOT) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 257 "Python/bytecodes.c" + #line 263 "Python/bytecodes.c" int err = PyObject_IsTrue(value); - #line 347 "Python/generated_cases.c.h" + #line 359 "Python/generated_cases.c.h" Py_DECREF(value); - #line 259 "Python/bytecodes.c" + #line 265 "Python/bytecodes.c" if (err < 0) goto pop_1_error; if (err == 0) { res = Py_True; @@ -354,7 +366,7 @@ res = Py_False; } Py_INCREF(res); - #line 358 "Python/generated_cases.c.h" + #line 370 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -362,13 +374,13 @@ TARGET(UNARY_INVERT) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 270 "Python/bytecodes.c" + #line 276 "Python/bytecodes.c" res = PyNumber_Invert(value); - #line 368 "Python/generated_cases.c.h" + #line 380 "Python/generated_cases.c.h" Py_DECREF(value); - #line 272 "Python/bytecodes.c" + #line 278 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 372 "Python/generated_cases.c.h" + #line 384 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -377,7 +389,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *prod; - #line 289 "Python/bytecodes.c" + #line 295 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -385,7 +397,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (prod == NULL) goto pop_2_error; - #line 389 "Python/generated_cases.c.h" + #line 401 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -396,14 +408,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *prod; - #line 299 "Python/bytecodes.c" + #line 305 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dprod = ((PyFloatObject *)left)->ob_fval * ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dprod, prod); - #line 407 "Python/generated_cases.c.h" + #line 419 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -414,7 +426,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sub; - #line 308 "Python/bytecodes.c" + #line 314 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -422,7 +434,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sub == NULL) goto pop_2_error; - #line 426 "Python/generated_cases.c.h" + #line 438 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -433,13 +445,13 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sub; - #line 318 "Python/bytecodes.c" + #line 324 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsub, sub); - #line 443 "Python/generated_cases.c.h" + #line 455 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -450,7 +462,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 326 "Python/bytecodes.c" + #line 332 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -458,7 +470,7 @@ _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res == NULL) goto pop_2_error; - #line 462 "Python/generated_cases.c.h" + #line 474 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -468,7 +480,7 @@ TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; - #line 342 "Python/bytecodes.c" + #line 348 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; @@ -495,7 +507,7 @@ if (*target_local == NULL) goto pop_2_error; // The STORE_FAST is already done. JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1); - #line 499 "Python/generated_cases.c.h" + #line 511 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -504,14 +516,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sum; - #line 371 "Python/bytecodes.c" + #line 377 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsum = ((PyFloatObject *)left)->ob_fval + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsum, sum); - #line 515 "Python/generated_cases.c.h" + #line 527 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; @@ -522,7 +534,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sum; - #line 380 "Python/bytecodes.c" + #line 386 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -530,7 +542,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sum == NULL) goto pop_2_error; - #line 534 "Python/generated_cases.c.h" + #line 546 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; @@ -543,7 +555,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; PyObject *res; - #line 398 "Python/bytecodes.c" + #line 404 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -555,12 +567,12 @@ DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ res = PyObject_GetItem(container, sub); - #line 559 "Python/generated_cases.c.h" + #line 571 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); - #line 410 "Python/bytecodes.c" + #line 416 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 564 "Python/generated_cases.c.h" + #line 576 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -572,7 +584,7 @@ PyObject *start = stack_pointer[-2]; PyObject *container = stack_pointer[-3]; PyObject *res; - #line 414 "Python/bytecodes.c" + #line 420 "Python/bytecodes.c" PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. @@ -585,7 +597,7 @@ } Py_DECREF(container); if (res == NULL) goto pop_3_error; - #line 589 "Python/generated_cases.c.h" + #line 601 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = res; DISPATCH(); @@ -596,7 +608,7 @@ PyObject *start = stack_pointer[-2]; PyObject *container = stack_pointer[-3]; PyObject *v = stack_pointer[-4]; - #line 429 "Python/bytecodes.c" + #line 435 "Python/bytecodes.c" PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); int err; if (slice == NULL) { @@ -609,7 +621,7 @@ Py_DECREF(v); Py_DECREF(container); if (err) goto pop_4_error; - #line 613 "Python/generated_cases.c.h" + #line 625 "Python/generated_cases.c.h" STACK_SHRINK(4); DISPATCH(); } @@ -618,7 +630,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *res; - #line 444 "Python/bytecodes.c" + #line 450 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); @@ -632,7 +644,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - #line 636 "Python/generated_cases.c.h" + #line 648 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -643,7 +655,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *tuple = stack_pointer[-2]; PyObject *res; - #line 460 "Python/bytecodes.c" + #line 466 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); @@ -657,7 +669,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(tuple); - #line 661 "Python/generated_cases.c.h" + #line 673 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -668,7 +680,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *res; - #line 476 "Python/bytecodes.c" + #line 482 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = PyDict_GetItemWithError(dict, sub); @@ -676,14 +688,14 @@ if (!_PyErr_Occurred(tstate)) { _PyErr_SetKeyError(sub); } - #line 680 "Python/generated_cases.c.h" + #line 692 "Python/generated_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); - #line 484 "Python/bytecodes.c" + #line 490 "Python/bytecodes.c" if (true) goto pop_2_error; } Py_INCREF(res); // Do this before DECREF'ing dict, sub - #line 687 "Python/generated_cases.c.h" + #line 699 "Python/generated_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); STACK_SHRINK(1); @@ -695,7 +707,7 @@ TARGET(BINARY_SUBSCR_GETITEM) { PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; - #line 491 "Python/bytecodes.c" + #line 497 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; @@ -717,15 +729,15 @@ JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 721 "Python/generated_cases.c.h" + #line 733 "Python/generated_cases.c.h" } TARGET(LIST_APPEND) { PyObject *v = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 515 "Python/bytecodes.c" + #line 521 "Python/bytecodes.c" if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; - #line 729 "Python/generated_cases.c.h" + #line 741 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -734,13 +746,13 @@ TARGET(SET_ADD) { PyObject *v = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 520 "Python/bytecodes.c" + #line 526 "Python/bytecodes.c" int err = PySet_Add(set, v); - #line 740 "Python/generated_cases.c.h" + #line 752 "Python/generated_cases.c.h" Py_DECREF(v); - #line 522 "Python/bytecodes.c" + #line 528 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 744 "Python/generated_cases.c.h" + #line 756 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -753,7 +765,7 @@ PyObject *container = stack_pointer[-2]; PyObject *v = stack_pointer[-3]; uint16_t counter = read_u16(&next_instr[0].cache); - #line 533 "Python/bytecodes.c" + #line 539 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr--; @@ -768,13 +780,13 @@ #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); - #line 772 "Python/generated_cases.c.h" + #line 784 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); - #line 548 "Python/bytecodes.c" + #line 554 "Python/bytecodes.c" if (err) goto pop_3_error; - #line 778 "Python/generated_cases.c.h" + #line 790 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -784,7 +796,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; - #line 552 "Python/bytecodes.c" + #line 558 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -801,7 +813,7 @@ Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - #line 805 "Python/generated_cases.c.h" + #line 817 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -811,13 +823,13 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; - #line 571 "Python/bytecodes.c" + #line 577 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); if (err) goto pop_3_error; - #line 821 "Python/generated_cases.c.h" + #line 833 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -826,15 +838,15 @@ TARGET(DELETE_SUBSCR) { PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; - #line 579 "Python/bytecodes.c" + #line 585 "Python/bytecodes.c" /* del container[sub] */ int err = PyObject_DelItem(container, sub); - #line 833 "Python/generated_cases.c.h" + #line 845 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); - #line 582 "Python/bytecodes.c" + #line 588 "Python/bytecodes.c" if (err) goto pop_2_error; - #line 838 "Python/generated_cases.c.h" + #line 850 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -842,14 +854,14 @@ TARGET(CALL_INTRINSIC_1) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 586 "Python/bytecodes.c" + #line 592 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg](tstate, value); - #line 849 "Python/generated_cases.c.h" + #line 861 "Python/generated_cases.c.h" Py_DECREF(value); - #line 589 "Python/bytecodes.c" + #line 595 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 853 "Python/generated_cases.c.h" + #line 865 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -858,15 +870,15 @@ PyObject *value1 = stack_pointer[-1]; PyObject *value2 = stack_pointer[-2]; PyObject *res; - #line 593 "Python/bytecodes.c" + #line 599 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_2); res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1); - #line 865 "Python/generated_cases.c.h" + #line 877 "Python/generated_cases.c.h" Py_DECREF(value2); Py_DECREF(value1); - #line 596 "Python/bytecodes.c" + #line 602 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 870 "Python/generated_cases.c.h" + #line 882 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -874,7 +886,7 @@ TARGET(RAISE_VARARGS) { PyObject **args = (stack_pointer - oparg); - #line 600 "Python/bytecodes.c" + #line 606 "Python/bytecodes.c" PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -892,12 +904,12 @@ break; } if (true) { STACK_SHRINK(oparg); goto error; } - #line 896 "Python/generated_cases.c.h" + #line 908 "Python/generated_cases.c.h" } TARGET(INTERPRETER_EXIT) { PyObject *retval = stack_pointer[-1]; - #line 620 "Python/bytecodes.c" + #line 626 "Python/bytecodes.c" assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); STACK_SHRINK(1); // Since we're not going to DISPATCH() @@ -908,12 +920,12 @@ assert(!_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallTstate(tstate); return retval; - #line 912 "Python/generated_cases.c.h" + #line 924 "Python/generated_cases.c.h" } TARGET(RETURN_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 633 "Python/bytecodes.c" + #line 639 "Python/bytecodes.c" STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -926,12 +938,12 @@ frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 930 "Python/generated_cases.c.h" + #line 942 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_RETURN_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 648 "Python/bytecodes.c" + #line 654 "Python/bytecodes.c" int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, next_instr-1, retval); @@ -948,11 +960,11 @@ frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 952 "Python/generated_cases.c.h" + #line 964 "Python/generated_cases.c.h" } TARGET(RETURN_CONST) { - #line 667 "Python/bytecodes.c" + #line 673 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(retval); assert(EMPTY()); @@ -966,11 +978,11 @@ frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 970 "Python/generated_cases.c.h" + #line 982 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_RETURN_CONST) { - #line 683 "Python/bytecodes.c" + #line 689 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, @@ -988,13 +1000,13 @@ frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 992 "Python/generated_cases.c.h" + #line 1004 "Python/generated_cases.c.h" } TARGET(GET_AITER) { PyObject *obj = stack_pointer[-1]; PyObject *iter; - #line 703 "Python/bytecodes.c" + #line 709 "Python/bytecodes.c" unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); @@ -1007,16 +1019,16 @@ "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - #line 1011 "Python/generated_cases.c.h" + #line 1023 "Python/generated_cases.c.h" Py_DECREF(obj); - #line 716 "Python/bytecodes.c" + #line 722 "Python/bytecodes.c" if (true) goto pop_1_error; } iter = (*getter)(obj); - #line 1018 "Python/generated_cases.c.h" + #line 1030 "Python/generated_cases.c.h" Py_DECREF(obj); - #line 721 "Python/bytecodes.c" + #line 727 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; if (Py_TYPE(iter)->tp_as_async == NULL || @@ -1029,7 +1041,7 @@ Py_DECREF(iter); if (true) goto pop_1_error; } - #line 1033 "Python/generated_cases.c.h" + #line 1045 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -1037,7 +1049,7 @@ TARGET(GET_ANEXT) { PyObject *aiter = stack_pointer[-1]; PyObject *awaitable; - #line 736 "Python/bytecodes.c" + #line 742 "Python/bytecodes.c" unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); @@ -1081,7 +1093,7 @@ } } - #line 1085 "Python/generated_cases.c.h" + #line 1097 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = awaitable; PREDICT(LOAD_CONST); @@ -1092,16 +1104,16 @@ PREDICTED(GET_AWAITABLE); PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 783 "Python/bytecodes.c" + #line 789 "Python/bytecodes.c" iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { format_awaitable_error(tstate, Py_TYPE(iterable), oparg); } - #line 1103 "Python/generated_cases.c.h" + #line 1115 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 790 "Python/bytecodes.c" + #line 796 "Python/bytecodes.c" if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); @@ -1119,7 +1131,7 @@ if (iter == NULL) goto pop_1_error; - #line 1123 "Python/generated_cases.c.h" + #line 1135 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -1130,7 +1142,7 @@ PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; PyObject *retval; - #line 816 "Python/bytecodes.c" + #line 822 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PySendCache *cache = (_PySendCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1176,7 +1188,7 @@ } } Py_DECREF(v); - #line 1180 "Python/generated_cases.c.h" + #line 1192 "Python/generated_cases.c.h" stack_pointer[-1] = retval; next_instr += 1; DISPATCH(); @@ -1185,7 +1197,7 @@ TARGET(SEND_GEN) { PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 864 "Python/bytecodes.c" + #line 870 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)receiver; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); @@ -1200,12 +1212,12 @@ tstate->exc_info = &gen->gi_exc_state; JUMPBY(INLINE_CACHE_ENTRIES_SEND); DISPATCH_INLINED(gen_frame); - #line 1204 "Python/generated_cases.c.h" + #line 1216 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_YIELD_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 881 "Python/bytecodes.c" + #line 887 "Python/bytecodes.c" assert(frame != &entry_frame); PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; @@ -1222,12 +1234,12 @@ gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 1226 "Python/generated_cases.c.h" + #line 1238 "Python/generated_cases.c.h" } TARGET(YIELD_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 900 "Python/bytecodes.c" + #line 906 "Python/bytecodes.c" // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -1243,15 +1255,15 @@ gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 1247 "Python/generated_cases.c.h" + #line 1259 "Python/generated_cases.c.h" } TARGET(POP_EXCEPT) { PyObject *exc_value = stack_pointer[-1]; - #line 918 "Python/bytecodes.c" + #line 924 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); - #line 1255 "Python/generated_cases.c.h" + #line 1267 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1259,7 +1271,7 @@ TARGET(RERAISE) { PyObject *exc = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); - #line 923 "Python/bytecodes.c" + #line 929 "Python/bytecodes.c" assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1277,26 +1289,26 @@ Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; - #line 1281 "Python/generated_cases.c.h" + #line 1293 "Python/generated_cases.c.h" } TARGET(END_ASYNC_FOR) { PyObject *exc = stack_pointer[-1]; PyObject *awaitable = stack_pointer[-2]; - #line 943 "Python/bytecodes.c" + #line 949 "Python/bytecodes.c" assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { - #line 1290 "Python/generated_cases.c.h" + #line 1302 "Python/generated_cases.c.h" Py_DECREF(awaitable); Py_DECREF(exc); - #line 946 "Python/bytecodes.c" + #line 952 "Python/bytecodes.c" } else { Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; } - #line 1300 "Python/generated_cases.c.h" + #line 1312 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -1307,23 +1319,23 @@ PyObject *sub_iter = stack_pointer[-3]; PyObject *none; PyObject *value; - #line 955 "Python/bytecodes.c" + #line 961 "Python/bytecodes.c" assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); - #line 1316 "Python/generated_cases.c.h" + #line 1328 "Python/generated_cases.c.h" Py_DECREF(sub_iter); Py_DECREF(last_sent_val); Py_DECREF(exc_value); - #line 960 "Python/bytecodes.c" + #line 966 "Python/bytecodes.c" none = Py_NewRef(Py_None); } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); goto exception_unwind; } - #line 1327 "Python/generated_cases.c.h" + #line 1339 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; stack_pointer[-2] = none; @@ -1332,9 +1344,9 @@ TARGET(LOAD_ASSERTION_ERROR) { PyObject *value; - #line 969 "Python/bytecodes.c" + #line 975 "Python/bytecodes.c" value = Py_NewRef(PyExc_AssertionError); - #line 1338 "Python/generated_cases.c.h" + #line 1350 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1342,7 +1354,7 @@ TARGET(LOAD_BUILD_CLASS) { PyObject *bc; - #line 973 "Python/bytecodes.c" + #line 979 "Python/bytecodes.c" if (PyDict_CheckExact(BUILTINS())) { bc = _PyDict_GetItemWithError(BUILTINS(), &_Py_ID(__build_class__)); @@ -1364,7 +1376,7 @@ if (true) goto error; } } - #line 1368 "Python/generated_cases.c.h" + #line 1380 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = bc; DISPATCH(); @@ -1372,33 +1384,33 @@ TARGET(STORE_NAME) { PyObject *v = stack_pointer[-1]; - #line 997 "Python/bytecodes.c" + #line 1003 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - #line 1383 "Python/generated_cases.c.h" + #line 1395 "Python/generated_cases.c.h" Py_DECREF(v); - #line 1004 "Python/bytecodes.c" + #line 1010 "Python/bytecodes.c" if (true) goto pop_1_error; } if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); - #line 1392 "Python/generated_cases.c.h" + #line 1404 "Python/generated_cases.c.h" Py_DECREF(v); - #line 1011 "Python/bytecodes.c" + #line 1017 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1396 "Python/generated_cases.c.h" + #line 1408 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_NAME) { - #line 1015 "Python/bytecodes.c" + #line 1021 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; @@ -1415,7 +1427,7 @@ name); goto error; } - #line 1419 "Python/generated_cases.c.h" + #line 1431 "Python/generated_cases.c.h" DISPATCH(); } @@ -1423,7 +1435,7 @@ PREDICTED(UNPACK_SEQUENCE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq = stack_pointer[-1]; - #line 1041 "Python/bytecodes.c" + #line 1047 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1436,11 +1448,11 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = unpack_iterable(tstate, seq, oparg, -1, top); - #line 1440 "Python/generated_cases.c.h" + #line 1452 "Python/generated_cases.c.h" Py_DECREF(seq); - #line 1054 "Python/bytecodes.c" + #line 1060 "Python/bytecodes.c" if (res == 0) goto pop_1_error; - #line 1444 "Python/generated_cases.c.h" + #line 1456 "Python/generated_cases.c.h" STACK_SHRINK(1); STACK_GROW(oparg); next_instr += 1; @@ -1450,14 +1462,14 @@ TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1058 "Python/bytecodes.c" + #line 1064 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - #line 1461 "Python/generated_cases.c.h" + #line 1473 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1468,7 +1480,7 @@ TARGET(UNPACK_SEQUENCE_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1068 "Python/bytecodes.c" + #line 1074 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1476,7 +1488,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - #line 1480 "Python/generated_cases.c.h" + #line 1492 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1487,7 +1499,7 @@ TARGET(UNPACK_SEQUENCE_LIST) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1079 "Python/bytecodes.c" + #line 1085 "Python/bytecodes.c" DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1495,7 +1507,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - #line 1499 "Python/generated_cases.c.h" + #line 1511 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1505,15 +1517,15 @@ TARGET(UNPACK_EX) { PyObject *seq = stack_pointer[-1]; - #line 1090 "Python/bytecodes.c" + #line 1096 "Python/bytecodes.c" int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - #line 1513 "Python/generated_cases.c.h" + #line 1525 "Python/generated_cases.c.h" Py_DECREF(seq); - #line 1094 "Python/bytecodes.c" + #line 1100 "Python/bytecodes.c" if (res == 0) goto pop_1_error; - #line 1517 "Python/generated_cases.c.h" + #line 1529 "Python/generated_cases.c.h" STACK_GROW((oparg & 0xFF) + (oparg >> 8)); DISPATCH(); } @@ -1524,7 +1536,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *v = stack_pointer[-2]; uint16_t counter = read_u16(&next_instr[0].cache); - #line 1105 "Python/bytecodes.c" + #line 1111 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(frame->f_code->co_names, oparg); @@ -1540,12 +1552,12 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, v); - #line 1544 "Python/generated_cases.c.h" + #line 1556 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(owner); - #line 1121 "Python/bytecodes.c" + #line 1127 "Python/bytecodes.c" if (err) goto pop_2_error; - #line 1549 "Python/generated_cases.c.h" + #line 1561 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -1553,34 +1565,34 @@ TARGET(DELETE_ATTR) { PyObject *owner = stack_pointer[-1]; - #line 1125 "Python/bytecodes.c" + #line 1131 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); - #line 1560 "Python/generated_cases.c.h" + #line 1572 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1128 "Python/bytecodes.c" + #line 1134 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1564 "Python/generated_cases.c.h" + #line 1576 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(STORE_GLOBAL) { PyObject *v = stack_pointer[-1]; - #line 1132 "Python/bytecodes.c" + #line 1138 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); - #line 1574 "Python/generated_cases.c.h" + #line 1586 "Python/generated_cases.c.h" Py_DECREF(v); - #line 1135 "Python/bytecodes.c" + #line 1141 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1578 "Python/generated_cases.c.h" + #line 1590 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_GLOBAL) { - #line 1139 "Python/bytecodes.c" + #line 1145 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err; err = PyDict_DelItem(GLOBALS(), name); @@ -1592,13 +1604,13 @@ } goto error; } - #line 1596 "Python/generated_cases.c.h" + #line 1608 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_NAME) { PyObject *v; - #line 1153 "Python/bytecodes.c" + #line 1159 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *locals = LOCALS(); if (locals == NULL) { @@ -1657,7 +1669,7 @@ } } } - #line 1661 "Python/generated_cases.c.h" + #line 1673 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = v; DISPATCH(); @@ -1668,7 +1680,7 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *null = NULL; PyObject *v; - #line 1220 "Python/bytecodes.c" + #line 1226 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1720,7 +1732,7 @@ } } null = NULL; - #line 1724 "Python/generated_cases.c.h" + #line 1736 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = v; @@ -1734,7 +1746,7 @@ PyObject *res; uint16_t index = read_u16(&next_instr[1].cache); uint16_t version = read_u16(&next_instr[2].cache); - #line 1274 "Python/bytecodes.c" + #line 1280 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); @@ -1745,7 +1757,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - #line 1749 "Python/generated_cases.c.h" + #line 1761 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1760,7 +1772,7 @@ uint16_t index = read_u16(&next_instr[1].cache); uint16_t mod_version = read_u16(&next_instr[2].cache); uint16_t bltn_version = read_u16(&next_instr[3].cache); - #line 1287 "Python/bytecodes.c" + #line 1293 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); PyDictObject *mdict = (PyDictObject *)GLOBALS(); @@ -1775,7 +1787,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - #line 1779 "Python/generated_cases.c.h" + #line 1791 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1785,16 +1797,16 @@ } TARGET(DELETE_FAST) { - #line 1304 "Python/bytecodes.c" + #line 1310 "Python/bytecodes.c" PyObject *v = GETLOCAL(oparg); if (v == NULL) goto unbound_local_error; SETLOCAL(oparg, NULL); - #line 1793 "Python/generated_cases.c.h" + #line 1805 "Python/generated_cases.c.h" DISPATCH(); } TARGET(MAKE_CELL) { - #line 1310 "Python/bytecodes.c" + #line 1316 "Python/bytecodes.c" // "initial" is probably NULL but not if it's an arg (or set // via PyFrame_LocalsToFast() before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); @@ -1803,12 +1815,12 @@ goto resume_with_error; } SETLOCAL(oparg, cell); - #line 1807 "Python/generated_cases.c.h" + #line 1819 "Python/generated_cases.c.h" DISPATCH(); } TARGET(DELETE_DEREF) { - #line 1321 "Python/bytecodes.c" + #line 1327 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); // Can't use ERROR_IF here. @@ -1819,13 +1831,13 @@ } PyCell_SET(cell, NULL); Py_DECREF(oldobj); - #line 1823 "Python/generated_cases.c.h" + #line 1835 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLASSDEREF) { PyObject *value; - #line 1334 "Python/bytecodes.c" + #line 1340 "Python/bytecodes.c" PyObject *name, *locals = LOCALS(); assert(locals); assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus); @@ -1857,7 +1869,7 @@ } Py_INCREF(value); } - #line 1861 "Python/generated_cases.c.h" + #line 1873 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1865,7 +1877,7 @@ TARGET(LOAD_DEREF) { PyObject *value; - #line 1368 "Python/bytecodes.c" + #line 1374 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { @@ -1873,7 +1885,7 @@ if (true) goto error; } Py_INCREF(value); - #line 1877 "Python/generated_cases.c.h" + #line 1889 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1881,18 +1893,18 @@ TARGET(STORE_DEREF) { PyObject *v = stack_pointer[-1]; - #line 1378 "Python/bytecodes.c" + #line 1384 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); - #line 1890 "Python/generated_cases.c.h" + #line 1902 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(COPY_FREE_VARS) { - #line 1385 "Python/bytecodes.c" + #line 1391 "Python/bytecodes.c" /* Copy closure variables to free variables */ PyCodeObject *co = frame->f_code; assert(PyFunction_Check(frame->f_funcobj)); @@ -1903,22 +1915,22 @@ PyObject *o = PyTuple_GET_ITEM(closure, i); frame->localsplus[offset + i] = Py_NewRef(o); } - #line 1907 "Python/generated_cases.c.h" + #line 1919 "Python/generated_cases.c.h" DISPATCH(); } TARGET(BUILD_STRING) { PyObject **pieces = (stack_pointer - oparg); PyObject *str; - #line 1398 "Python/bytecodes.c" + #line 1404 "Python/bytecodes.c" str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - #line 1916 "Python/generated_cases.c.h" + #line 1928 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } - #line 1400 "Python/bytecodes.c" + #line 1406 "Python/bytecodes.c" if (str == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1922 "Python/generated_cases.c.h" + #line 1934 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = str; @@ -1928,10 +1940,10 @@ TARGET(BUILD_TUPLE) { PyObject **values = (stack_pointer - oparg); PyObject *tup; - #line 1404 "Python/bytecodes.c" + #line 1410 "Python/bytecodes.c" tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1935 "Python/generated_cases.c.h" + #line 1947 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = tup; @@ -1941,10 +1953,10 @@ TARGET(BUILD_LIST) { PyObject **values = (stack_pointer - oparg); PyObject *list; - #line 1409 "Python/bytecodes.c" + #line 1415 "Python/bytecodes.c" list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1948 "Python/generated_cases.c.h" + #line 1960 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = list; @@ -1954,7 +1966,7 @@ TARGET(LIST_EXTEND) { PyObject *iterable = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 1414 "Python/bytecodes.c" + #line 1420 "Python/bytecodes.c" PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1965,13 +1977,13 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - #line 1969 "Python/generated_cases.c.h" + #line 1981 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1425 "Python/bytecodes.c" + #line 1431 "Python/bytecodes.c" if (true) goto pop_1_error; } Py_DECREF(none_val); - #line 1975 "Python/generated_cases.c.h" + #line 1987 "Python/generated_cases.c.h" Py_DECREF(iterable); STACK_SHRINK(1); DISPATCH(); @@ -1980,13 +1992,13 @@ TARGET(SET_UPDATE) { PyObject *iterable = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 1432 "Python/bytecodes.c" + #line 1438 "Python/bytecodes.c" int err = _PySet_Update(set, iterable); - #line 1986 "Python/generated_cases.c.h" + #line 1998 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1434 "Python/bytecodes.c" + #line 1440 "Python/bytecodes.c" if (err < 0) goto pop_1_error; - #line 1990 "Python/generated_cases.c.h" + #line 2002 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1994,7 +2006,7 @@ TARGET(BUILD_SET) { PyObject **values = (stack_pointer - oparg); PyObject *set; - #line 1438 "Python/bytecodes.c" + #line 1444 "Python/bytecodes.c" set = PySet_New(NULL); if (set == NULL) goto error; @@ -2009,7 +2021,7 @@ Py_DECREF(set); if (true) { STACK_SHRINK(oparg); goto error; } } - #line 2013 "Python/generated_cases.c.h" + #line 2025 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = set; @@ -2019,7 +2031,7 @@ TARGET(BUILD_MAP) { PyObject **values = (stack_pointer - oparg*2); PyObject *map; - #line 1455 "Python/bytecodes.c" + #line 1461 "Python/bytecodes.c" map = _PyDict_FromItems( values, 2, values+1, 2, @@ -2027,13 +2039,13 @@ if (map == NULL) goto error; - #line 2031 "Python/generated_cases.c.h" + #line 2043 "Python/generated_cases.c.h" for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } - #line 1463 "Python/bytecodes.c" + #line 1469 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } - #line 2037 "Python/generated_cases.c.h" + #line 2049 "Python/generated_cases.c.h" STACK_SHRINK(oparg*2); STACK_GROW(1); stack_pointer[-1] = map; @@ -2041,7 +2053,7 @@ } TARGET(SETUP_ANNOTATIONS) { - #line 1467 "Python/bytecodes.c" + #line 1473 "Python/bytecodes.c" int err; PyObject *ann_dict; if (LOCALS() == NULL) { @@ -2081,7 +2093,7 @@ Py_DECREF(ann_dict); } } - #line 2085 "Python/generated_cases.c.h" + #line 2097 "Python/generated_cases.c.h" DISPATCH(); } @@ -2089,7 +2101,7 @@ PyObject *keys = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); PyObject *map; - #line 1509 "Python/bytecodes.c" + #line 1515 "Python/bytecodes.c" if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -2099,14 +2111,14 @@ map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); - #line 2103 "Python/generated_cases.c.h" + #line 2115 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); - #line 1519 "Python/bytecodes.c" + #line 1525 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } - #line 2110 "Python/generated_cases.c.h" + #line 2122 "Python/generated_cases.c.h" STACK_SHRINK(oparg); stack_pointer[-1] = map; DISPATCH(); @@ -2114,7 +2126,7 @@ TARGET(DICT_UPDATE) { PyObject *update = stack_pointer[-1]; - #line 1523 "Python/bytecodes.c" + #line 1529 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { @@ -2122,12 +2134,12 @@ "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } - #line 2126 "Python/generated_cases.c.h" + #line 2138 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1531 "Python/bytecodes.c" + #line 1537 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2131 "Python/generated_cases.c.h" + #line 2143 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); DISPATCH(); @@ -2135,17 +2147,17 @@ TARGET(DICT_MERGE) { PyObject *update = stack_pointer[-1]; - #line 1537 "Python/bytecodes.c" + #line 1543 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { format_kwargs_error(tstate, PEEK(3 + oparg), update); - #line 2144 "Python/generated_cases.c.h" + #line 2156 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1542 "Python/bytecodes.c" + #line 1548 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2149 "Python/generated_cases.c.h" + #line 2161 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); PREDICT(CALL_FUNCTION_EX); @@ -2155,13 +2167,13 @@ TARGET(MAP_ADD) { PyObject *value = stack_pointer[-1]; PyObject *key = stack_pointer[-2]; - #line 1549 "Python/bytecodes.c" + #line 1555 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - #line 2165 "Python/generated_cases.c.h" + #line 2177 "Python/generated_cases.c.h" STACK_SHRINK(2); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -2175,7 +2187,7 @@ PyObject *global_super = stack_pointer[-3]; PyObject *res2 = NULL; PyObject *res; - #line 1563 "Python/bytecodes.c" + #line 1569 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 2); int load_method = oparg & 1; #if ENABLE_SPECIALIZATION @@ -2193,16 +2205,16 @@ // handle any case whose performance we care about PyObject *stack[] = {class, self}; PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - #line 2197 "Python/generated_cases.c.h" + #line 2209 "Python/generated_cases.c.h" Py_DECREF(global_super); Py_DECREF(class); Py_DECREF(self); - #line 1581 "Python/bytecodes.c" + #line 1587 "Python/bytecodes.c" if (super == NULL) goto pop_3_error; res = PyObject_GetAttr(super, name); Py_DECREF(super); if (res == NULL) goto pop_3_error; - #line 2206 "Python/generated_cases.c.h" + #line 2218 "Python/generated_cases.c.h" STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2220,7 +2232,7 @@ uint32_t class_version = read_u32(&next_instr[1].cache); uint32_t self_type_version = read_u32(&next_instr[3].cache); PyObject *method = read_obj(&next_instr[5].cache); - #line 1588 "Python/bytecodes.c" + #line 1594 "Python/bytecodes.c" DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); DEOPT_IF(((PyTypeObject *)class)->tp_version_tag != class_version, LOAD_SUPER_ATTR); @@ -2231,7 +2243,7 @@ Py_INCREF(res2); Py_DECREF(global_super); Py_DECREF(class); - #line 2235 "Python/generated_cases.c.h" + #line 2247 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; stack_pointer[-2] = res2; @@ -2245,7 +2257,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; - #line 1615 "Python/bytecodes.c" + #line 1621 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2279,9 +2291,9 @@ NULL | meth | arg1 | ... | argN */ - #line 2283 "Python/generated_cases.c.h" + #line 2295 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1649 "Python/bytecodes.c" + #line 1655 "Python/bytecodes.c" if (meth == NULL) goto pop_1_error; res2 = NULL; res = meth; @@ -2290,12 +2302,12 @@ else { /* Classic, pushes one value. */ res = PyObject_GetAttr(owner, name); - #line 2294 "Python/generated_cases.c.h" + #line 2306 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1658 "Python/bytecodes.c" + #line 1664 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; } - #line 2299 "Python/generated_cases.c.h" + #line 2311 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -2309,7 +2321,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1663 "Python/bytecodes.c" + #line 1669 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2322,7 +2334,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2326 "Python/generated_cases.c.h" + #line 2338 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2337,7 +2349,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1679 "Python/bytecodes.c" + #line 1685 "Python/bytecodes.c" DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -2350,7 +2362,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2354 "Python/generated_cases.c.h" + #line 2366 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2365,7 +2377,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1695 "Python/bytecodes.c" + #line 1701 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2392,7 +2404,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2396 "Python/generated_cases.c.h" + #line 2408 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2407,7 +2419,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1725 "Python/bytecodes.c" + #line 1731 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2417,7 +2429,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2421 "Python/generated_cases.c.h" + #line 2433 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2432,7 +2444,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 1738 "Python/bytecodes.c" + #line 1744 "Python/bytecodes.c" DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, @@ -2444,7 +2456,7 @@ res = descr; assert(res != NULL); Py_INCREF(res); - #line 2448 "Python/generated_cases.c.h" + #line 2460 "Python/generated_cases.c.h" Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2458,7 +2470,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *fget = read_obj(&next_instr[5].cache); - #line 1753 "Python/bytecodes.c" + #line 1759 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -2482,7 +2494,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 2486 "Python/generated_cases.c.h" + #line 2498 "Python/generated_cases.c.h" } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { @@ -2490,7 +2502,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *getattribute = read_obj(&next_instr[5].cache); - #line 1779 "Python/bytecodes.c" + #line 1785 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -2516,7 +2528,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 2520 "Python/generated_cases.c.h" + #line 2532 "Python/generated_cases.c.h" } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -2524,7 +2536,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1807 "Python/bytecodes.c" + #line 1813 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2542,7 +2554,7 @@ Py_DECREF(old_value); } Py_DECREF(owner); - #line 2546 "Python/generated_cases.c.h" + #line 2558 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2553,7 +2565,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t hint = read_u16(&next_instr[3].cache); - #line 1827 "Python/bytecodes.c" + #line 1833 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2592,7 +2604,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - #line 2596 "Python/generated_cases.c.h" + #line 2608 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2603,7 +2615,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1868 "Python/bytecodes.c" + #line 1874 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2613,7 +2625,7 @@ *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - #line 2617 "Python/generated_cases.c.h" + #line 2629 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2625,7 +2637,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1887 "Python/bytecodes.c" + #line 1893 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2638,12 +2650,12 @@ #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); - #line 2642 "Python/generated_cases.c.h" + #line 2654 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1900 "Python/bytecodes.c" + #line 1906 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2647 "Python/generated_cases.c.h" + #line 2659 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2654,7 +2666,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1904 "Python/bytecodes.c" + #line 1910 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2666,7 +2678,7 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2670 "Python/generated_cases.c.h" + #line 2682 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2677,7 +2689,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1919 "Python/bytecodes.c" + #line 1925 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -2693,7 +2705,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2697 "Python/generated_cases.c.h" + #line 2709 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2704,7 +2716,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1938 "Python/bytecodes.c" + #line 1944 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2717,7 +2729,7 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2721 "Python/generated_cases.c.h" + #line 2733 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2728,14 +2740,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1953 "Python/bytecodes.c" + #line 1959 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; - #line 2734 "Python/generated_cases.c.h" + #line 2746 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1955 "Python/bytecodes.c" + #line 1961 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2739 "Python/generated_cases.c.h" + #line 2751 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2745,15 +2757,15 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1959 "Python/bytecodes.c" + #line 1965 "Python/bytecodes.c" int res = PySequence_Contains(right, left); - #line 2751 "Python/generated_cases.c.h" + #line 2763 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1961 "Python/bytecodes.c" + #line 1967 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = Py_NewRef((res^oparg) ? Py_True : Py_False); - #line 2757 "Python/generated_cases.c.h" + #line 2769 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2764,12 +2776,12 @@ PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; - #line 1966 "Python/bytecodes.c" + #line 1972 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { - #line 2770 "Python/generated_cases.c.h" + #line 2782 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1968 "Python/bytecodes.c" + #line 1974 "Python/bytecodes.c" if (true) goto pop_2_error; } @@ -2777,10 +2789,10 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); - #line 2781 "Python/generated_cases.c.h" + #line 2793 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1976 "Python/bytecodes.c" + #line 1982 "Python/bytecodes.c" if (res < 0) goto pop_2_error; assert((match == NULL) == (rest == NULL)); @@ -2789,7 +2801,7 @@ if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } - #line 2793 "Python/generated_cases.c.h" + #line 2805 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -2799,21 +2811,21 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1987 "Python/bytecodes.c" + #line 1993 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - #line 2806 "Python/generated_cases.c.h" + #line 2818 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1990 "Python/bytecodes.c" + #line 1996 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - #line 2813 "Python/generated_cases.c.h" + #line 2825 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1995 "Python/bytecodes.c" + #line 2001 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2817 "Python/generated_cases.c.h" + #line 2829 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -2822,15 +2834,15 @@ PyObject *fromlist = stack_pointer[-1]; PyObject *level = stack_pointer[-2]; PyObject *res; - #line 1999 "Python/bytecodes.c" + #line 2005 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); - #line 2829 "Python/generated_cases.c.h" + #line 2841 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); - #line 2002 "Python/bytecodes.c" + #line 2008 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2834 "Python/generated_cases.c.h" + #line 2846 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -2839,29 +2851,29 @@ TARGET(IMPORT_FROM) { PyObject *from = stack_pointer[-1]; PyObject *res; - #line 2006 "Python/bytecodes.c" + #line 2012 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - #line 2847 "Python/generated_cases.c.h" + #line 2859 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); } TARGET(JUMP_FORWARD) { - #line 2012 "Python/bytecodes.c" + #line 2018 "Python/bytecodes.c" JUMPBY(oparg); - #line 2856 "Python/generated_cases.c.h" + #line 2868 "Python/generated_cases.c.h" DISPATCH(); } TARGET(JUMP_BACKWARD) { PREDICTED(JUMP_BACKWARD); - #line 2016 "Python/bytecodes.c" + #line 2022 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); - #line 2865 "Python/generated_cases.c.h" + #line 2877 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2869,7 +2881,7 @@ TARGET(POP_JUMP_IF_FALSE) { PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = stack_pointer[-1]; - #line 2022 "Python/bytecodes.c" + #line 2028 "Python/bytecodes.c" if (Py_IsTrue(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2879,9 +2891,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2883 "Python/generated_cases.c.h" + #line 2895 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 2032 "Python/bytecodes.c" + #line 2038 "Python/bytecodes.c" if (err == 0) { JUMPBY(oparg); } @@ -2889,14 +2901,14 @@ if (err < 0) goto pop_1_error; } } - #line 2893 "Python/generated_cases.c.h" + #line 2905 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = stack_pointer[-1]; - #line 2042 "Python/bytecodes.c" + #line 2048 "Python/bytecodes.c" if (Py_IsFalse(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2906,9 +2918,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2910 "Python/generated_cases.c.h" + #line 2922 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 2052 "Python/bytecodes.c" + #line 2058 "Python/bytecodes.c" if (err > 0) { JUMPBY(oparg); } @@ -2916,67 +2928,67 @@ if (err < 0) goto pop_1_error; } } - #line 2920 "Python/generated_cases.c.h" + #line 2932 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = stack_pointer[-1]; - #line 2062 "Python/bytecodes.c" + #line 2068 "Python/bytecodes.c" if (!Py_IsNone(value)) { - #line 2929 "Python/generated_cases.c.h" + #line 2941 "Python/generated_cases.c.h" Py_DECREF(value); - #line 2064 "Python/bytecodes.c" + #line 2070 "Python/bytecodes.c" JUMPBY(oparg); } else { _Py_DECREF_NO_DEALLOC(value); } - #line 2937 "Python/generated_cases.c.h" + #line 2949 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = stack_pointer[-1]; - #line 2072 "Python/bytecodes.c" + #line 2078 "Python/bytecodes.c" if (Py_IsNone(value)) { _Py_DECREF_NO_DEALLOC(value); JUMPBY(oparg); } else { - #line 2950 "Python/generated_cases.c.h" + #line 2962 "Python/generated_cases.c.h" Py_DECREF(value); - #line 2078 "Python/bytecodes.c" + #line 2084 "Python/bytecodes.c" } - #line 2954 "Python/generated_cases.c.h" + #line 2966 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - #line 2082 "Python/bytecodes.c" + #line 2088 "Python/bytecodes.c" /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ JUMPBY(-oparg); - #line 2967 "Python/generated_cases.c.h" + #line 2979 "Python/generated_cases.c.h" DISPATCH(); } TARGET(GET_LEN) { PyObject *obj = stack_pointer[-1]; PyObject *len_o; - #line 2091 "Python/bytecodes.c" + #line 2097 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 2980 "Python/generated_cases.c.h" + #line 2992 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -2987,16 +2999,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; - #line 2099 "Python/bytecodes.c" + #line 2105 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 2996 "Python/generated_cases.c.h" + #line 3008 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); - #line 2104 "Python/bytecodes.c" + #line 2110 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -3004,7 +3016,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_NewRef(Py_None); // Failure! } - #line 3008 "Python/generated_cases.c.h" + #line 3020 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -3013,10 +3025,10 @@ TARGET(MATCH_MAPPING) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2114 "Python/bytecodes.c" + #line 2120 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = Py_NewRef(match ? Py_True : Py_False); - #line 3020 "Python/generated_cases.c.h" + #line 3032 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -3026,10 +3038,10 @@ TARGET(MATCH_SEQUENCE) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2120 "Python/bytecodes.c" + #line 2126 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = Py_NewRef(match ? Py_True : Py_False); - #line 3033 "Python/generated_cases.c.h" + #line 3045 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -3040,11 +3052,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; - #line 2126 "Python/bytecodes.c" + #line 2132 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 3048 "Python/generated_cases.c.h" + #line 3060 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -3053,14 +3065,14 @@ TARGET(GET_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2132 "Python/bytecodes.c" + #line 2138 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 3060 "Python/generated_cases.c.h" + #line 3072 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2135 "Python/bytecodes.c" + #line 2141 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 3064 "Python/generated_cases.c.h" + #line 3076 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -3068,7 +3080,7 @@ TARGET(GET_YIELD_FROM_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2139 "Python/bytecodes.c" + #line 2145 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -3091,11 +3103,11 @@ if (iter == NULL) { goto error; } - #line 3095 "Python/generated_cases.c.h" + #line 3107 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2162 "Python/bytecodes.c" + #line 2168 "Python/bytecodes.c" } - #line 3099 "Python/generated_cases.c.h" + #line 3111 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -3106,7 +3118,7 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2181 "Python/bytecodes.c" + #line 2187 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -3137,7 +3149,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator - #line 3141 "Python/generated_cases.c.h" + #line 3153 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3145,7 +3157,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - #line 2214 "Python/bytecodes.c" + #line 2220 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -3171,14 +3183,14 @@ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - #line 3175 "Python/generated_cases.c.h" + #line 3187 "Python/generated_cases.c.h" DISPATCH(); } TARGET(FOR_ITER_LIST) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2242 "Python/bytecodes.c" + #line 2248 "Python/bytecodes.c" DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; STAT_INC(FOR_ITER, hit); @@ -3198,7 +3210,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator - #line 3202 "Python/generated_cases.c.h" + #line 3214 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3208,7 +3220,7 @@ TARGET(FOR_ITER_TUPLE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2264 "Python/bytecodes.c" + #line 2270 "Python/bytecodes.c" _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3228,7 +3240,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator - #line 3232 "Python/generated_cases.c.h" + #line 3244 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3238,7 +3250,7 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2286 "Python/bytecodes.c" + #line 2292 "Python/bytecodes.c" _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3256,7 +3268,7 @@ if (next == NULL) { goto error; } - #line 3260 "Python/generated_cases.c.h" + #line 3272 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3265,7 +3277,7 @@ TARGET(FOR_ITER_GEN) { PyObject *iter = stack_pointer[-1]; - #line 2306 "Python/bytecodes.c" + #line 2312 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -3280,14 +3292,14 @@ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); - #line 3284 "Python/generated_cases.c.h" + #line 3296 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2323 "Python/bytecodes.c" + #line 2329 "Python/bytecodes.c" PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -3310,16 +3322,16 @@ Py_DECREF(enter); goto error; } - #line 3314 "Python/generated_cases.c.h" + #line 3326 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2346 "Python/bytecodes.c" + #line 2352 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3323 "Python/generated_cases.c.h" + #line 3335 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3331,7 +3343,7 @@ PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2356 "Python/bytecodes.c" + #line 2362 "Python/bytecodes.c" /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -3357,16 +3369,16 @@ Py_DECREF(enter); goto error; } - #line 3361 "Python/generated_cases.c.h" + #line 3373 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2382 "Python/bytecodes.c" + #line 2388 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3370 "Python/generated_cases.c.h" + #line 3382 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3378,7 +3390,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; - #line 2391 "Python/bytecodes.c" + #line 2397 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3399,7 +3411,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 3403 "Python/generated_cases.c.h" + #line 3415 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3408,7 +3420,7 @@ TARGET(PUSH_EXC_INFO) { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; - #line 2414 "Python/bytecodes.c" + #line 2420 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3418,7 +3430,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 3422 "Python/generated_cases.c.h" + #line 3434 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3432,7 +3444,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2426 "Python/bytecodes.c" + #line 2432 "Python/bytecodes.c" /* Cached method object */ PyTypeObject *self_cls = Py_TYPE(self); assert(type_version != 0); @@ -3449,7 +3461,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); - #line 3453 "Python/generated_cases.c.h" + #line 3465 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3463,7 +3475,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2445 "Python/bytecodes.c" + #line 2451 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); @@ -3473,7 +3485,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3477 "Python/generated_cases.c.h" + #line 3489 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3487,7 +3499,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2457 "Python/bytecodes.c" + #line 2463 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; @@ -3501,7 +3513,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3505 "Python/generated_cases.c.h" + #line 3517 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3510,16 +3522,16 @@ } TARGET(KW_NAMES) { - #line 2473 "Python/bytecodes.c" + #line 2479 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); - #line 3518 "Python/generated_cases.c.h" + #line 3530 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_CALL) { - #line 2479 "Python/bytecodes.c" + #line 2485 "Python/bytecodes.c" int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); @@ -3532,7 +3544,7 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(CALL); - #line 3536 "Python/generated_cases.c.h" + #line 3548 "Python/generated_cases.c.h" } TARGET(CALL) { @@ -3542,7 +3554,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2524 "Python/bytecodes.c" + #line 2530 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3624,7 +3636,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3628 "Python/generated_cases.c.h" + #line 3640 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3636,7 +3648,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2612 "Python/bytecodes.c" + #line 2618 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3646,7 +3658,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3650 "Python/generated_cases.c.h" + #line 3662 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3655,7 +3667,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2624 "Python/bytecodes.c" + #line 2630 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3681,7 +3693,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 3685 "Python/generated_cases.c.h" + #line 3697 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3689,7 +3701,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2652 "Python/bytecodes.c" + #line 2658 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3725,7 +3737,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 3729 "Python/generated_cases.c.h" + #line 3741 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3733,7 +3745,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2690 "Python/bytecodes.c" + #line 2696 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3743,7 +3755,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 3747 "Python/generated_cases.c.h" + #line 3759 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3756,7 +3768,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2702 "Python/bytecodes.c" + #line 2708 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3767,7 +3779,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3771 "Python/generated_cases.c.h" + #line 3783 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3781,7 +3793,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2716 "Python/bytecodes.c" + #line 2722 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3792,7 +3804,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3796 "Python/generated_cases.c.h" + #line 3808 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3806,7 +3818,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2730 "Python/bytecodes.c" + #line 2736 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3828,7 +3840,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3832 "Python/generated_cases.c.h" + #line 3844 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3842,7 +3854,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2755 "Python/bytecodes.c" + #line 2761 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3870,7 +3882,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3874 "Python/generated_cases.c.h" + #line 3886 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3884,7 +3896,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2786 "Python/bytecodes.c" + #line 2792 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3916,7 +3928,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 3920 "Python/generated_cases.c.h" + #line 3932 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3930,7 +3942,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2821 "Python/bytecodes.c" + #line 2827 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -3962,7 +3974,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3966 "Python/generated_cases.c.h" + #line 3978 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3976,7 +3988,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2856 "Python/bytecodes.c" + #line 2862 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -4001,7 +4013,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4005 "Python/generated_cases.c.h" + #line 4017 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4014,7 +4026,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2883 "Python/bytecodes.c" + #line 2889 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -4041,7 +4053,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4045 "Python/generated_cases.c.h" + #line 4057 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4053,7 +4065,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2913 "Python/bytecodes.c" + #line 2919 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); assert(method != NULL); @@ -4071,14 +4083,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 4075 "Python/generated_cases.c.h" + #line 4087 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2933 "Python/bytecodes.c" + #line 2939 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4109,7 +4121,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4113 "Python/generated_cases.c.h" + #line 4125 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4122,7 +4134,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2967 "Python/bytecodes.c" + #line 2973 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4151,7 +4163,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4155 "Python/generated_cases.c.h" + #line 4167 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4164,7 +4176,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2999 "Python/bytecodes.c" + #line 3005 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4193,7 +4205,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4197 "Python/generated_cases.c.h" + #line 4209 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4206,7 +4218,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3031 "Python/bytecodes.c" + #line 3037 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4234,7 +4246,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4238 "Python/generated_cases.c.h" + #line 4250 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4244,9 +4256,9 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #line 3062 "Python/bytecodes.c" + #line 3068 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4250 "Python/generated_cases.c.h" + #line 4262 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4255,7 +4267,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 3066 "Python/bytecodes.c" + #line 3072 "Python/bytecodes.c" // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4317,14 +4329,14 @@ } result = PyObject_Call(func, callargs, kwargs); } - #line 4321 "Python/generated_cases.c.h" + #line 4333 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3128 "Python/bytecodes.c" + #line 3134 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4328 "Python/generated_cases.c.h" + #line 4340 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4339,7 +4351,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 3138 "Python/bytecodes.c" + #line 3144 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4368,14 +4380,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4372 "Python/generated_cases.c.h" + #line 4384 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3169 "Python/bytecodes.c" + #line 3175 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4396,7 +4408,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4400 "Python/generated_cases.c.h" + #line 4412 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4404,15 +4416,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3192 "Python/bytecodes.c" + #line 3198 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4410 "Python/generated_cases.c.h" + #line 4422 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3194 "Python/bytecodes.c" + #line 3200 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4416 "Python/generated_cases.c.h" + #line 4428 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4423,7 +4435,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 3198 "Python/bytecodes.c" + #line 3204 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4458,7 +4470,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4462 "Python/generated_cases.c.h" + #line 4474 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4467,10 +4479,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3235 "Python/bytecodes.c" + #line 3241 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4474 "Python/generated_cases.c.h" + #line 4486 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4482,7 +4494,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3240 "Python/bytecodes.c" + #line 3246 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4497,12 +4509,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4501 "Python/generated_cases.c.h" + #line 4513 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3255 "Python/bytecodes.c" + #line 3261 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4506 "Python/generated_cases.c.h" + #line 4518 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4512,16 +4524,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3260 "Python/bytecodes.c" + #line 3266 "Python/bytecodes.c" assert(oparg >= 2); - #line 4518 "Python/generated_cases.c.h" + #line 4530 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_LINE) { - #line 3264 "Python/bytecodes.c" + #line 3270 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( @@ -4541,11 +4553,11 @@ } opcode = original_opcode; DISPATCH_GOTO(); - #line 4545 "Python/generated_cases.c.h" + #line 4557 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3286 "Python/bytecodes.c" + #line 3292 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4557,26 +4569,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4561 "Python/generated_cases.c.h" + #line 4573 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3300 "Python/bytecodes.c" + #line 3306 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4567 "Python/generated_cases.c.h" + #line 4579 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3304 "Python/bytecodes.c" + #line 3310 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4574 "Python/generated_cases.c.h" + #line 4586 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3309 "Python/bytecodes.c" + #line 3315 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4585,12 +4597,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4589 "Python/generated_cases.c.h" + #line 4601 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3320 "Python/bytecodes.c" + #line 3326 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4599,12 +4611,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4603 "Python/generated_cases.c.h" + #line 4615 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3331 "Python/bytecodes.c" + #line 3337 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4617,12 +4629,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4621 "Python/generated_cases.c.h" + #line 4633 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3346 "Python/bytecodes.c" + #line 3352 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4635,30 +4647,30 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4639 "Python/generated_cases.c.h" + #line 4651 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3361 "Python/bytecodes.c" + #line 3367 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4650 "Python/generated_cases.c.h" + #line 4662 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3369 "Python/bytecodes.c" + #line 3375 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); - #line 4657 "Python/generated_cases.c.h" + #line 4669 "Python/generated_cases.c.h" } TARGET(RESERVED) { - #line 3374 "Python/bytecodes.c" + #line 3380 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); - #line 4664 "Python/generated_cases.c.h" + #line 4676 "Python/generated_cases.c.h" } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 77f0ae0c1a4c30..c1a6ed4c18ab0a 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -21,6 +21,8 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 0; case LOAD_FAST: return 0; + case LOAD_FAST_AND_CLEAR: + return 0; case LOAD_CONST: return 0; case STORE_FAST: @@ -409,6 +411,8 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case LOAD_FAST: return 1; + case LOAD_FAST_AND_CLEAR: + return 1; case LOAD_CONST: return 1; case STORE_FAST: @@ -795,6 +799,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [LOAD_CLOSURE] = { true, INSTR_FMT_IB }, [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB }, [LOAD_FAST] = { true, INSTR_FMT_IB }, + [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB }, [LOAD_CONST] = { true, INSTR_FMT_IB }, [STORE_FAST] = { true, INSTR_FMT_IB }, [LOAD_FAST__LOAD_FAST] = { true, INSTR_FMT_IBIB }, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 042cee222f705c..00f15ff98da418 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -142,7 +142,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_BACKWARD, &&TARGET_LOAD_SUPER_ATTR, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_STORE_FAST__LOAD_FAST, + &&TARGET_LOAD_FAST_AND_CLEAR, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,24 +152,24 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, + &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_STORE_FAST__STORE_FAST, - &&TARGET_STORE_SUBSCR_DICT, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, + &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_UNPACK_SEQUENCE_TUPLE, - &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, + &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&TARGET_SEND_GEN, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_CALL, &&TARGET_KW_NAMES, &&TARGET_CALL_INTRINSIC_1, diff --git a/Python/symtable.c b/Python/symtable.c index df7473943f3fc1..6e74d764245a57 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -103,6 +103,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_comprehension = NoComprehension; ste->ste_returns_value = 0; ste->ste_needs_class_closure = 0; + ste->ste_comp_inlined = 0; ste->ste_comp_iter_target = 0; ste->ste_comp_iter_expr = 0; @@ -558,6 +559,67 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, return 1; } +static int +is_free_in_any_child(PySTEntryObject *entry, PyObject *key) +{ + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(entry->ste_children); i++) { + PySTEntryObject *child_ste = (PySTEntryObject *)PyList_GET_ITEM( + entry->ste_children, i); + long scope = _PyST_GetScope(child_ste, key); + if (scope == FREE) { + return 1; + } + } + return 0; +} + +static int +inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, + PyObject *scopes, PyObject *comp_free) +{ + PyObject *k, *v; + Py_ssize_t pos = 0; + while (PyDict_Next(comp->ste_symbols, &pos, &k, &v)) { + // skip comprehension parameter + long comp_flags = PyLong_AS_LONG(v); + if (comp_flags & DEF_PARAM) { + assert(_PyUnicode_EqualToASCIIString(k, ".0")); + continue; + } + int scope = (comp_flags >> SCOPE_OFFSET) & SCOPE_MASK; + int only_flags = comp_flags & ((1 << SCOPE_OFFSET) - 1); + PyObject *existing = PyDict_GetItemWithError(ste->ste_symbols, k); + if (existing == NULL && PyErr_Occurred()) { + return 0; + } + if (!existing) { + // name does not exist in scope, copy from comprehension + assert(scope != FREE || PySet_Contains(comp_free, k) == 1); + PyObject *v_flags = PyLong_FromLong(only_flags); + if (v_flags == NULL) { + return 0; + } + int ok = PyDict_SetItem(ste->ste_symbols, k, v_flags); + Py_DECREF(v_flags); + if (ok < 0) { + return 0; + } + SET_SCOPE(scopes, k, scope); + } + else { + // free vars in comprehension that are locals in outer scope can + // now simply be locals, unless they are free in comp children + if ((PyLong_AsLong(existing) & DEF_BOUND) && + !is_free_in_any_child(comp, k)) { + if (PySet_Discard(comp_free, k) < 0) { + return 0; + } + } + } + } + return 1; +} + #undef SET_SCOPE /* If a name is defined in free and also in locals, then this block @@ -727,17 +789,17 @@ update_symbols(PyObject *symbols, PyObject *scopes, static int analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free, - PyObject *global, PyObject* child_free); + PyObject *global, PyObject **child_free); static int analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, PyObject *global) { PyObject *name, *v, *local = NULL, *scopes = NULL, *newbound = NULL; - PyObject *newglobal = NULL, *newfree = NULL, *allfree = NULL; + PyObject *newglobal = NULL, *newfree = NULL; PyObject *temp; - int i, success = 0; - Py_ssize_t pos = 0; + int success = 0; + Py_ssize_t i, pos = 0; local = PySet_New(NULL); /* collect new names bound in block */ if (!local) @@ -746,8 +808,8 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, if (!scopes) goto error; - /* Allocate new global and bound variable dictionaries. These - dictionaries hold the names visible in nested blocks. For + /* Allocate new global, bound and free variable sets. These + sets hold the names visible in nested blocks. For ClassBlocks, the bound and global names are initialized before analyzing names, because class bindings aren't visible in methods. For other blocks, they are initialized @@ -826,28 +888,55 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, newbound, newglobal now contain the names visible in nested blocks. The free variables in the children will - be collected in allfree. + be added to newfree. */ - allfree = PySet_New(NULL); - if (!allfree) - goto error; for (i = 0; i < PyList_GET_SIZE(ste->ste_children); ++i) { + PyObject *child_free = NULL; PyObject *c = PyList_GET_ITEM(ste->ste_children, i); PySTEntryObject* entry; assert(c && PySTEntry_Check(c)); entry = (PySTEntryObject*)c; + + // we inline all non-generator-expression comprehensions + int inline_comp = + entry->ste_comprehension && + !entry->ste_generator; + if (!analyze_child_block(entry, newbound, newfree, newglobal, - allfree)) + &child_free)) + { goto error; + } + if (inline_comp) { + if (!inline_comprehension(ste, entry, scopes, child_free)) { + Py_DECREF(child_free); + goto error; + } + entry->ste_comp_inlined = 1; + } + temp = PyNumber_InPlaceOr(newfree, child_free); + Py_DECREF(child_free); + if (!temp) + goto error; + Py_DECREF(temp); /* Check if any children have free variables */ if (entry->ste_free || entry->ste_child_free) ste->ste_child_free = 1; } - temp = PyNumber_InPlaceOr(newfree, allfree); - if (!temp) - goto error; - Py_DECREF(temp); + /* Splice children of inlined comprehensions into our children list */ + for (i = PyList_GET_SIZE(ste->ste_children) - 1; i >= 0; --i) { + PyObject* c = PyList_GET_ITEM(ste->ste_children, i); + PySTEntryObject* entry; + assert(c && PySTEntry_Check(c)); + entry = (PySTEntryObject*)c; + if (entry->ste_comp_inlined && + PyList_SetSlice(ste->ste_children, i, i + 1, + entry->ste_children) < 0) + { + goto error; + } + } /* Check if any local variables must be converted to cell variables */ if (ste->ste_type == FunctionBlock && !analyze_cells(scopes, newfree)) @@ -870,7 +959,6 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, Py_XDECREF(newbound); Py_XDECREF(newglobal); Py_XDECREF(newfree); - Py_XDECREF(allfree); if (!success) assert(PyErr_Occurred()); return success; @@ -878,16 +966,15 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, static int analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free, - PyObject *global, PyObject* child_free) + PyObject *global, PyObject** child_free) { PyObject *temp_bound = NULL, *temp_global = NULL, *temp_free = NULL; - PyObject *temp; - /* Copy the bound and global dictionaries. + /* Copy the bound/global/free sets. - These dictionaries are used by all blocks enclosed by the + These sets are used by all blocks enclosed by the current block. The analyze_block() call modifies these - dictionaries. + sets. */ temp_bound = PySet_New(bound); @@ -902,12 +989,8 @@ analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free, if (!analyze_block(entry, temp_bound, temp_free, temp_global)) goto error; - temp = PyNumber_InPlaceOr(child_free, temp_free); - if (!temp) - goto error; - Py_DECREF(temp); + *child_free = temp_free; Py_DECREF(temp_bound); - Py_DECREF(temp_free); Py_DECREF(temp_global); return 1; error: @@ -2216,4 +2299,3 @@ _Py_Mangle(PyObject *privateobj, PyObject *ident) assert(_PyUnicode_CheckConsistency(result, 1)); return result; } - From 2866e030f01dc3ff08de32857fa77d52468b676b Mon Sep 17 00:00:00 2001 From: Jacob Bower <1978924+jbower-fb@users.noreply.github.com> Date: Tue, 9 May 2023 10:09:16 -0700 Subject: [PATCH 127/204] gh-97696 Add documentation for get_coro() behavior with eager tasks (#104304) --- Doc/library/asyncio-task.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index b2d7362a9de213..fe8d028150403d 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -527,6 +527,8 @@ Running Tasks Concurrently and there is no running event loop. +.. _eager-task-factory: + Eager Task Factory ================== @@ -1174,8 +1176,17 @@ Task Object Return the coroutine object wrapped by the :class:`Task`. + .. note:: + + This will return ``None`` for Tasks which have already + completed eagerly. See the :ref:`Eager Task Factory `. + .. versionadded:: 3.8 + .. versionchanged:: 3.12 + + Newly added eager task execution means result may be ``None``. + .. method:: get_context() Return the :class:`contextvars.Context` object From 01c321ca34d99f35f174768c6f8c500801d4ef4c Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Tue, 9 May 2023 22:18:15 +0300 Subject: [PATCH 128/204] gh-104328: Fix typo in ``typing.Generic`` multiple inheritance error message (#104335) --- Lib/typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/typing.py b/Lib/typing.py index 0dacdd9031a776..4f8cba88632d6a 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1896,7 +1896,7 @@ def __init_subclass__(cls, *args, **kwargs): base.__origin__ is Generic): if gvars is not None: raise TypeError( - "Cannot inherit from Generic[...] multiple types.") + "Cannot inherit from Generic[...] multiple times.") gvars = base.__parameters__ if gvars is not None: tvarset = set(tvars) From 7ba6288feb961fcd60a29415c6371d2d3eb80bec Mon Sep 17 00:00:00 2001 From: David Foster Date: Tue, 9 May 2023 15:57:17 -0400 Subject: [PATCH 129/204] gh-102327: Extend docs for "url" and "headers" parameters to HTTPConnection.request() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added example on how to use the HTTPConnection object for making GET request. Original issue: https://github.com/python/cpython/issues/102327 --------- Co-authored-by: Éric --- Doc/library/http.client.rst | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index eb8c1e198e2b09..bf1f2e3920783d 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -264,7 +264,10 @@ HTTPConnection Objects encode_chunked=False) This will send a request to the server using the HTTP request - method *method* and the selector *url*. + method *method* and the request URI *url*. The provided *url* must be + an absolute path to conform with :rfc:`RFC 2616 §5.1.2 <2616#section-5.1.2>` + (unless connecting to an HTTP proxy server or using the ``OPTIONS`` or + ``CONNECT`` methods). If *body* is specified, the specified data is sent after the headers are finished. It may be a :class:`str`, a :term:`bytes-like object`, an @@ -279,7 +282,10 @@ HTTPConnection Objects iterable are sent as is until the iterable is exhausted. The *headers* argument should be a mapping of extra HTTP headers to send - with the request. + with the request. A :rfc:`Host header <2616#section-14.23>` + must be provided to conform with :rfc:`RFC 2616 §5.1.2 <2616#section-5.1.2>` + (unless connecting to an HTTP proxy server or using the ``OPTIONS`` or + ``CONNECT`` methods). If *headers* contains neither Content-Length nor Transfer-Encoding, but there is a request body, one of those @@ -298,6 +304,16 @@ HTTPConnection Objects HTTPConnection object assumes that all encoding is handled by the calling code. If it is ``True``, the body will be chunk-encoded. + For example, to perform a ``GET`` request to ``https://docs.python.org/3/``:: + + >>> import http.client + >>> host = "docs.python.org" + >>> conn = http.client.HTTPSConnection(host) + >>> conn.request("GET", "/3/", headers={"Host": host}) + >>> response = conn.getresponse() + >>> print(response.status, response.reason) + 200 OK + .. note:: Chunked transfer encoding has been added to the HTTP protocol version 1.1. Unless the HTTP server is known to handle HTTP 1.1, From fe694a6db620062f467469bd2bb987315d72fd62 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 9 May 2023 22:16:22 +0100 Subject: [PATCH 130/204] gh-90953: Don't use deprecated AST nodes in clinic.py (#104322) --- Tools/clinic/clinic.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index a6f330d1502dad..2ae8a02aa6d285 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4701,10 +4701,8 @@ def bad_node(self, node): c_default = "NULL" elif (isinstance(expr, ast.BinOp) or (isinstance(expr, ast.UnaryOp) and - not (isinstance(expr.operand, ast.Num) or - (hasattr(ast, 'Constant') and - isinstance(expr.operand, ast.Constant) and - type(expr.operand.value) in (int, float, complex))) + not (isinstance(expr.operand, ast.Constant) and + type(expr.operand.value) in {int, float, complex}) )): c_default = kwargs.get("c_default") if not (isinstance(c_default, str) and c_default): @@ -4806,14 +4804,10 @@ def bad_node(self, node): self.function.parameters[key] = p def parse_converter(self, annotation): - if (hasattr(ast, 'Constant') and - isinstance(annotation, ast.Constant) and + if (isinstance(annotation, ast.Constant) and type(annotation.value) is str): return annotation.value, True, {} - if isinstance(annotation, ast.Str): - return annotation.s, True, {} - if isinstance(annotation, ast.Name): return annotation.id, False, {} From 235b82721dbbe2bc5bf1f45a74e22b7185e5783e Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 10 May 2023 01:09:03 +0200 Subject: [PATCH 131/204] gh-101819: Refactor _io in preparation for module isolation (#104334) - Replace query with parameter in bufferediobase_unsupported() - Replace query with parameter in iobase_unsupported() - Hide delegate: Add method wrapper for _PyIOBase_check_seekable - Hide delegate: Add method wraper for _PyIOBase_check_readable - Hide delegate: Add method wraper for _PyIOBase_check_writable - Replace query with parameter in _PyIOBase_check_seekable() - Replace query with parameter in _PyIOBase_check_readable() - Replace query with parameter in _PyIOBase_check_writable() --- Modules/_io/_iomodule.h | 14 +++++++---- Modules/_io/bufferedio.c | 54 ++++++++++++++++++++++++---------------- Modules/_io/iobase.c | 54 ++++++++++++++++++++++++++++------------ 3 files changed, 80 insertions(+), 42 deletions(-) diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index 1bf301c9cf0a94..b3873ddf7e0847 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -34,9 +34,13 @@ extern PyType_Spec winconsoleio_spec; * with args=NULL, and return a new reference. * BUT when args=Py_True is passed, they return a borrowed reference. */ -extern PyObject* _PyIOBase_check_readable(PyObject *self, PyObject *args); -extern PyObject* _PyIOBase_check_writable(PyObject *self, PyObject *args); -extern PyObject* _PyIOBase_check_seekable(PyObject *self, PyObject *args); +typedef struct _io_state _PyIO_State; // Forward decl. +extern PyObject* _PyIOBase_check_readable(_PyIO_State *state, + PyObject *self, PyObject *args); +extern PyObject* _PyIOBase_check_writable(_PyIO_State *state, + PyObject *self, PyObject *args); +extern PyObject* _PyIOBase_check_seekable(_PyIO_State *state, + PyObject *self, PyObject *args); extern PyObject* _PyIOBase_check_closed(PyObject *self, PyObject *args); /* Helper for finalization. @@ -140,7 +144,7 @@ extern Py_off_t PyNumber_AsOff_t(PyObject *item, PyObject *err); extern PyModuleDef _PyIO_Module; -typedef struct { +struct _io_state { int initialized; PyObject *unsupported_operation; @@ -161,7 +165,7 @@ typedef struct { #ifdef MS_WINDOWS PyTypeObject *PyWindowsConsoleIO_Type; #endif -} _PyIO_State; +}; #define IO_MOD_STATE(mod) ((_PyIO_State *)PyModule_GetState(mod)) #define IO_STATE() _PyIO_get_module_state() diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 723d16b47fef9f..172fafe6db8a3d 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -106,11 +106,9 @@ _io__BufferedIOBase_readinto1_impl(PyObject *self, Py_buffer *buffer) } static PyObject * -bufferediobase_unsupported(const char *message) +bufferediobase_unsupported(_PyIO_State *state, const char *message) { - _PyIO_State *state = IO_STATE(); - if (state != NULL) - PyErr_SetString(state->unsupported_operation, message); + PyErr_SetString(state->unsupported_operation, message); return NULL; } @@ -127,7 +125,8 @@ static PyObject * _io__BufferedIOBase_detach_impl(PyObject *self) /*[clinic end generated code: output=754977c8d10ed88c input=822427fb58fe4169]*/ { - return bufferediobase_unsupported("detach"); + _PyIO_State *state = IO_STATE(); + return bufferediobase_unsupported(state, "detach"); } PyDoc_STRVAR(bufferediobase_read_doc, @@ -151,7 +150,8 @@ PyDoc_STRVAR(bufferediobase_read_doc, static PyObject * bufferediobase_read(PyObject *self, PyObject *args) { - return bufferediobase_unsupported("read"); + _PyIO_State *state = IO_STATE(); + return bufferediobase_unsupported(state, "read"); } PyDoc_STRVAR(bufferediobase_read1_doc, @@ -164,7 +164,8 @@ PyDoc_STRVAR(bufferediobase_read1_doc, static PyObject * bufferediobase_read1(PyObject *self, PyObject *args) { - return bufferediobase_unsupported("read1"); + _PyIO_State *state = IO_STATE(); + return bufferediobase_unsupported(state, "read1"); } PyDoc_STRVAR(bufferediobase_write_doc, @@ -179,7 +180,8 @@ PyDoc_STRVAR(bufferediobase_write_doc, static PyObject * bufferediobase_write(PyObject *self, PyObject *args) { - return bufferediobase_unsupported("write"); + _PyIO_State *state = IO_STATE(); + return bufferediobase_unsupported(state, "write"); } @@ -1222,8 +1224,10 @@ _io__Buffered_seek_impl(buffered *self, PyObject *targetobj, int whence) CHECK_CLOSED(self, "seek of closed file") - if (_PyIOBase_check_seekable(self->raw, Py_True) == NULL) + _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); + if (_PyIOBase_check_seekable(state, self->raw, Py_True) == NULL) { return NULL; + } target = PyNumber_AsOff_t(targetobj, PyExc_ValueError); if (target == -1 && PyErr_Occurred()) @@ -1298,7 +1302,8 @@ _io__Buffered_truncate_impl(buffered *self, PyObject *pos) CHECK_INITIALIZED(self) CHECK_CLOSED(self, "truncate of closed file") if (!self->writable) { - return bufferediobase_unsupported("truncate"); + _PyIO_State *state = IO_STATE(); + return bufferediobase_unsupported(state, "truncate"); } if (!ENTER_BUFFERED(self)) return NULL; @@ -1419,8 +1424,10 @@ _io_BufferedReader___init___impl(buffered *self, PyObject *raw, self->ok = 0; self->detached = 0; - if (_PyIOBase_check_readable(raw, Py_True) == NULL) + _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); + if (_PyIOBase_check_readable(state, raw, Py_True) == NULL) { return -1; + } Py_XSETREF(self->raw, Py_NewRef(raw)); self->buffer_size = buffer_size; @@ -1431,7 +1438,6 @@ _io_BufferedReader___init___impl(buffered *self, PyObject *raw, return -1; _bufferedreader_reset_buf(self); - _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); self->fast_closed_checks = ( Py_IS_TYPE(self, state->PyBufferedReader_Type) && Py_IS_TYPE(raw, state->PyFileIO_Type) @@ -1774,8 +1780,10 @@ _io_BufferedWriter___init___impl(buffered *self, PyObject *raw, self->ok = 0; self->detached = 0; - if (_PyIOBase_check_writable(raw, Py_True) == NULL) + _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); + if (_PyIOBase_check_writable(state, raw, Py_True) == NULL) { return -1; + } Py_INCREF(raw); Py_XSETREF(self->raw, raw); @@ -1788,7 +1796,6 @@ _io_BufferedWriter___init___impl(buffered *self, PyObject *raw, _bufferedwriter_reset_buf(self); self->pos = 0; - _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); self->fast_closed_checks = ( Py_IS_TYPE(self, state->PyBufferedWriter_Type) && Py_IS_TYPE(raw, state->PyFileIO_Type) @@ -2092,12 +2099,14 @@ _io_BufferedRWPair___init___impl(rwpair *self, PyObject *reader, PyObject *writer, Py_ssize_t buffer_size) /*[clinic end generated code: output=327e73d1aee8f984 input=620d42d71f33a031]*/ { - if (_PyIOBase_check_readable(reader, Py_True) == NULL) + _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); + if (_PyIOBase_check_readable(state, reader, Py_True) == NULL) { return -1; - if (_PyIOBase_check_writable(writer, Py_True) == NULL) + } + if (_PyIOBase_check_writable(state, writer, Py_True) == NULL) { return -1; + } - _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); self->reader = (buffered *) PyObject_CallFunction( (PyObject *)state->PyBufferedReader_Type, "On", reader, buffer_size); @@ -2290,12 +2299,16 @@ _io_BufferedRandom___init___impl(buffered *self, PyObject *raw, self->ok = 0; self->detached = 0; - if (_PyIOBase_check_seekable(raw, Py_True) == NULL) + _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); + if (_PyIOBase_check_seekable(state, raw, Py_True) == NULL) { return -1; - if (_PyIOBase_check_readable(raw, Py_True) == NULL) + } + if (_PyIOBase_check_readable(state, raw, Py_True) == NULL) { return -1; - if (_PyIOBase_check_writable(raw, Py_True) == NULL) + } + if (_PyIOBase_check_writable(state, raw, Py_True) == NULL) { return -1; + } Py_INCREF(raw); Py_XSETREF(self->raw, raw); @@ -2309,7 +2322,6 @@ _io_BufferedRandom___init___impl(buffered *self, PyObject *raw, _bufferedwriter_reset_buf(self); self->pos = 0; - _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); self->fast_closed_checks = (Py_IS_TYPE(self, state->PyBufferedRandom_Type) && Py_IS_TYPE(raw, state->PyFileIO_Type)); diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index 682ed000eb1fd9..ca13866c33fba6 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -71,11 +71,9 @@ PyDoc_STRVAR(iobase_doc, /* Internal methods */ static PyObject * -iobase_unsupported(const char *message) +iobase_unsupported(_PyIO_State *state, const char *message) { - _PyIO_State *state = IO_STATE(); - if (state != NULL) - PyErr_SetString(state->unsupported_operation, message); + PyErr_SetString(state->unsupported_operation, message); return NULL; } @@ -97,7 +95,8 @@ PyDoc_STRVAR(iobase_seek_doc, static PyObject * iobase_seek(PyObject *self, PyObject *args) { - return iobase_unsupported("seek"); + _PyIO_State *state = IO_STATE(); + return iobase_unsupported(state, "seek"); } /*[clinic input] @@ -122,7 +121,8 @@ PyDoc_STRVAR(iobase_truncate_doc, static PyObject * iobase_truncate(PyObject *self, PyObject *args) { - return iobase_unsupported("truncate"); + _PyIO_State *state = IO_STATE(); + return iobase_unsupported(state, "truncate"); } static int @@ -204,6 +204,27 @@ _PyIOBase_check_closed(PyObject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject * +iobase_check_seekable(PyObject *self, PyObject *args) +{ + _PyIO_State *state = IO_STATE(); + return _PyIOBase_check_seekable(state, self, args); +} + +static PyObject * +iobase_check_readable(PyObject *self, PyObject *args) +{ + _PyIO_State *state = IO_STATE(); + return _PyIOBase_check_readable(state, self, args); +} + +static PyObject * +iobase_check_writable(PyObject *self, PyObject *args) +{ + _PyIO_State *state = IO_STATE(); + return _PyIOBase_check_writable(state, self, args); +} + /* XXX: IOBase thinks it has to maintain its own internal state in `__IOBase_closed` and call flush() by itself, but it is redundant with whatever behaviour a non-trivial derived class will implement. */ @@ -372,14 +393,14 @@ _io__IOBase_seekable_impl(PyObject *self) } PyObject * -_PyIOBase_check_seekable(PyObject *self, PyObject *args) +_PyIOBase_check_seekable(_PyIO_State *state, PyObject *self, PyObject *args) { PyObject *res = PyObject_CallMethodNoArgs(self, &_Py_ID(seekable)); if (res == NULL) return NULL; if (res != Py_True) { Py_CLEAR(res); - iobase_unsupported("File or stream is not seekable."); + iobase_unsupported(state, "File or stream is not seekable."); return NULL; } if (args == Py_True) { @@ -405,14 +426,14 @@ _io__IOBase_readable_impl(PyObject *self) /* May be called with any object */ PyObject * -_PyIOBase_check_readable(PyObject *self, PyObject *args) +_PyIOBase_check_readable(_PyIO_State *state, PyObject *self, PyObject *args) { PyObject *res = PyObject_CallMethodNoArgs(self, &_Py_ID(readable)); if (res == NULL) return NULL; if (res != Py_True) { Py_CLEAR(res); - iobase_unsupported("File or stream is not readable."); + iobase_unsupported(state, "File or stream is not readable."); return NULL; } if (args == Py_True) { @@ -438,14 +459,14 @@ _io__IOBase_writable_impl(PyObject *self) /* May be called with any object */ PyObject * -_PyIOBase_check_writable(PyObject *self, PyObject *args) +_PyIOBase_check_writable(_PyIO_State *state, PyObject *self, PyObject *args) { PyObject *res = PyObject_CallMethodNoArgs(self, &_Py_ID(writable)); if (res == NULL) return NULL; if (res != Py_True) { Py_CLEAR(res); - iobase_unsupported("File or stream is not writable."); + iobase_unsupported(state, "File or stream is not writable."); return NULL; } if (args == Py_True) { @@ -487,7 +508,8 @@ static PyObject * _io__IOBase_fileno_impl(PyObject *self) /*[clinic end generated code: output=7cc0973f0f5f3b73 input=4e37028947dc1cc8]*/ { - return iobase_unsupported("fileno"); + _PyIO_State *state = IO_STATE(); + return iobase_unsupported(state, "fileno"); } /*[clinic input] @@ -798,9 +820,9 @@ static PyMethodDef iobase_methods[] = { _IO__IOBASE_WRITABLE_METHODDEF {"_checkClosed", _PyIOBase_check_closed, METH_NOARGS}, - {"_checkSeekable", _PyIOBase_check_seekable, METH_NOARGS}, - {"_checkReadable", _PyIOBase_check_readable, METH_NOARGS}, - {"_checkWritable", _PyIOBase_check_writable, METH_NOARGS}, + {"_checkSeekable", iobase_check_seekable, METH_NOARGS}, + {"_checkReadable", iobase_check_readable, METH_NOARGS}, + {"_checkWritable", iobase_check_writable, METH_NOARGS}, _IO__IOBASE_FILENO_METHODDEF _IO__IOBASE_ISATTY_METHODDEF From 2c863b3871c6127a80aa7229033219f1cdcc8711 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kul=C3=ADk?= Date: Wed, 10 May 2023 01:49:55 +0200 Subject: [PATCH 132/204] gh-74895: adjust tests to work on Solaris (#104326) Solaris is unusual here, but apparently everyone is happy when SOCK_STREAM is explicitly specified. --- Lib/test/test_socket.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index bb7bf436d2d721..68cdc6eaa91375 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1622,7 +1622,7 @@ def test_getaddrinfo_int_port_overflow(self): from _testcapi import ULONG_MAX, LONG_MAX, LONG_MIN try: - socket.getaddrinfo(None, ULONG_MAX + 1) + socket.getaddrinfo(None, ULONG_MAX + 1, type=socket.SOCK_STREAM) except OverflowError: # Platforms differ as to what values consitute a getaddrinfo() error # return. Some fail for LONG_MAX+1, others ULONG_MAX+1, and Windows @@ -1632,28 +1632,28 @@ def test_getaddrinfo_int_port_overflow(self): pass try: - socket.getaddrinfo(None, LONG_MAX + 1) + socket.getaddrinfo(None, LONG_MAX + 1, type=socket.SOCK_STREAM) except OverflowError: self.fail("Either no error or socket.gaierror expected.") except socket.gaierror: pass try: - socket.getaddrinfo(None, LONG_MAX - 0xffff + 1) + socket.getaddrinfo(None, LONG_MAX - 0xffff + 1, type=socket.SOCK_STREAM) except OverflowError: self.fail("Either no error or socket.gaierror expected.") except socket.gaierror: pass try: - socket.getaddrinfo(None, LONG_MIN - 1) + socket.getaddrinfo(None, LONG_MIN - 1, type=socket.SOCK_STREAM) except OverflowError: self.fail("Either no error or socket.gaierror expected.") except socket.gaierror: pass - socket.getaddrinfo(None, 0) # No error expected. - socket.getaddrinfo(None, 0xffff) # No error expected. + socket.getaddrinfo(None, 0, type=socket.SOCK_STREAM) # No error expected. + socket.getaddrinfo(None, 0xffff, type=socket.SOCK_STREAM) # No error expected. def test_getnameinfo(self): # only IP addresses are allowed From 29f348e232e82938ba2165843c448c2b291504c5 Mon Sep 17 00:00:00 2001 From: JohnJamesUtley <81572567+JohnJamesUtley@users.noreply.github.com> Date: Tue, 9 May 2023 20:18:35 -0400 Subject: [PATCH 133/204] gh-103848: Adds checks to ensure that bracketed hosts found by urlsplit are of IPv6 or IPvFuture format (#103849) * Adds checks to ensure that bracketed hosts found by urlsplit are of IPv6 or IPvFuture format --------- Co-authored-by: Gregory P. Smith --- Lib/test/test_urlparse.py | 26 +++++++++++++++++++ Lib/urllib/parse.py | 16 +++++++++++- ...-04-26-09-54-25.gh-issue-103848.aDSnpR.rst | 2 ++ 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index dcdbb1cc64fd28..e324babdd5ed0b 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -1042,6 +1042,32 @@ def test_issue14072(self): self.assertEqual(p2.scheme, 'tel') self.assertEqual(p2.path, '+31641044153') + def test_invalid_bracketed_hosts(self): + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[192.0.2.146]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[important.com:8000]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v123r.IP]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v12ae]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v.IP]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v123.]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[0439:23af::2309::fae7:1234]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[0439:23af:2309::fae7:1234:2342:438e:192.0.2.146]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@]v6a.ip[/Path') + + def test_splitting_bracketed_hosts(self): + p1 = urllib.parse.urlsplit('scheme://user@[v6a.ip]/path?query') + self.assertEqual(p1.hostname, 'v6a.ip') + self.assertEqual(p1.username, 'user') + self.assertEqual(p1.path, '/path') + p2 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7%test]/path?query') + self.assertEqual(p2.hostname, '0439:23af:2309::fae7%test') + self.assertEqual(p2.username, 'user') + self.assertEqual(p2.path, '/path') + p3 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7:1234:192.0.2.146%test]/path?query') + self.assertEqual(p3.hostname, '0439:23af:2309::fae7:1234:192.0.2.146%test') + self.assertEqual(p3.username, 'user') + self.assertEqual(p3.path, '/path') + def test_port_casting_failure_message(self): message = "Port could not be cast to integer value as 'oracle'" p1 = urllib.parse.urlparse('http://Server=sde; Service=sde:oracle') diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 777b7c53efe565..da0073969bff0b 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -33,6 +33,7 @@ import re import types import warnings +import ipaddress __all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag", "urlsplit", "urlunsplit", "urlencode", "parse_qs", @@ -427,6 +428,17 @@ def _checknetloc(netloc): raise ValueError("netloc '" + netloc + "' contains invalid " + "characters under NFKC normalization") +# Valid bracketed hosts are defined in +# https://www.rfc-editor.org/rfc/rfc3986#page-49 and https://url.spec.whatwg.org/ +def _check_bracketed_host(hostname): + if hostname.startswith('v'): + if not re.match(r"\Av[a-fA-F0-9]+\..+\Z", hostname): + raise ValueError(f"IPvFuture address is invalid") + else: + ip = ipaddress.ip_address(hostname) # Throws Value Error if not IPv6 or IPv4 + if isinstance(ip, ipaddress.IPv4Address): + raise ValueError(f"An IPv4 address cannot be in brackets") + # typed=True avoids BytesWarnings being emitted during cache key # comparison since this API supports both bytes and str input. @functools.lru_cache(typed=True) @@ -466,12 +478,14 @@ def urlsplit(url, scheme='', allow_fragments=True): break else: scheme, url = url[:i].lower(), url[i+1:] - if url[:2] == '//': netloc, url = _splitnetloc(url, 2) if (('[' in netloc and ']' not in netloc) or (']' in netloc and '[' not in netloc)): raise ValueError("Invalid IPv6 URL") + if '[' in netloc and ']' in netloc: + bracketed_host = netloc.partition('[')[2].partition(']')[0] + _check_bracketed_host(bracketed_host) if allow_fragments and '#' in url: url, fragment = url.split('#', 1) if '?' in url: diff --git a/Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst b/Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst new file mode 100644 index 00000000000000..81e5904aa6cca2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst @@ -0,0 +1,2 @@ +Add checks to ensure that ``[`` bracketed ``]`` hosts found by +:func:`urllib.parse.urlsplit` are of IPv6 or IPvFuture format. From 22f3425c3d3d896be0917d80d55e8abb08d99b18 Mon Sep 17 00:00:00 2001 From: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> Date: Wed, 10 May 2023 08:59:04 +0800 Subject: [PATCH 134/204] gh-103247: clear the module cache in a test in test_importlib/extensions/test_loader.py (GH-104226) --- .../test_importlib/extension/test_loader.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index 3a74b821eaee49..a7c6245825ff86 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -262,15 +262,16 @@ def test_reload(self): def test_try_registration(self): # Assert that the PyState_{Find,Add,Remove}Module C API doesn't work. - module = self.load_module() - with self.subTest('PyState_FindModule'): - self.assertEqual(module.call_state_registration_func(0), None) - with self.subTest('PyState_AddModule'): - with self.assertRaises(SystemError): - module.call_state_registration_func(1) - with self.subTest('PyState_RemoveModule'): - with self.assertRaises(SystemError): - module.call_state_registration_func(2) + with util.uncache(self.name): + module = self.load_module() + with self.subTest('PyState_FindModule'): + self.assertEqual(module.call_state_registration_func(0), None) + with self.subTest('PyState_AddModule'): + with self.assertRaises(SystemError): + module.call_state_registration_func(1) + with self.subTest('PyState_RemoveModule'): + with self.assertRaises(SystemError): + module.call_state_registration_func(2) def test_load_submodule(self): # Test loading a simulated submodule. From 68a8ca6dc10bdceb4efaac569081b78ec01c3a99 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 10 May 2023 12:59:03 +0200 Subject: [PATCH 135/204] gh-101819: Harden _io init (#104352) Fix potential refleak if PyModule_AddObject() fails. --- Modules/_io/_iomodule.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index c05407b5d61815..6c5ea287964294 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -730,9 +730,11 @@ PyInit__io(void) "UnsupportedOperation", PyExc_OSError, PyExc_ValueError); if (state->unsupported_operation == NULL) goto fail; - if (PyModule_AddObject(m, "UnsupportedOperation", - Py_NewRef(state->unsupported_operation)) < 0) + if (PyModule_AddObjectRef(m, "UnsupportedOperation", + state->unsupported_operation) < 0) + { goto fail; + } /* BlockingIOError, for compatibility */ if (PyModule_AddObjectRef(m, "BlockingIOError", @@ -785,7 +787,6 @@ PyInit__io(void) return m; fail: - Py_XDECREF(state->unsupported_operation); Py_DECREF(m); return NULL; } From 2dcb289ed08980c8f97d538060b4ad8d5e82b56a Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 10 May 2023 12:59:31 +0200 Subject: [PATCH 136/204] gh-101819: Clean up _io windows console io after gh-104197 (#104354) --- Modules/_io/_iomodule.c | 2 +- Modules/_io/_iomodule.h | 2 +- Modules/_io/winconsoleio.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 6c5ea287964294..ee4eca70e2af8f 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -772,7 +772,7 @@ PyInit__io(void) // PyRawIOBase_Type(PyIOBase_Type) subclasses ADD_TYPE(m, state->PyFileIO_Type, &fileio_spec, state->PyRawIOBase_Type); -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO ADD_TYPE(m, state->PyWindowsConsoleIO_Type, &winconsoleio_spec, state->PyRawIOBase_Type); #endif diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index b3873ddf7e0847..44d651338e6975 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -162,7 +162,7 @@ struct _io_state { PyTypeObject *PyStringIO_Type; PyTypeObject *PyTextIOBase_Type; PyTypeObject *PyTextIOWrapper_Type; -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO PyTypeObject *PyWindowsConsoleIO_Type; #endif }; diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index fdb57cff7c04d6..58d9f2963aa93c 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -263,7 +263,7 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, int fd_is_own = 0; HANDLE handle = NULL; -#ifdef Py_DEBUG +#ifdef NDEBUG _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); assert(PyObject_TypeCheck(self, state->PyWindowsConsoleIO_Type)); #endif From b8f7ab5783b370004757af5a4c6e70c63dc5fe7a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 10 May 2023 07:28:40 -0600 Subject: [PATCH 137/204] gh-104252: Immortalize Py_EMPTY_KEYS (gh-104253) This was missed in gh-19474. It matters for with a per-interpreter GIL since PyDictKeysObject.dk_refcnt breaks isolation and leads to races. --- Include/internal/pycore_dict_state.h | 5 ++++ Include/internal/pycore_runtime_init.h | 4 +--- Objects/dictobject.c | 32 +++++++++++++++++--------- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index d608088ed2d7ce..ece0f10ca25170 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -38,6 +38,11 @@ struct _Py_dict_state { PyDict_WatchCallback watchers[DICT_MAX_WATCHERS]; }; +#define _dict_state_INIT \ + { \ + .next_keys_version = 2, \ + } + #ifdef __cplusplus } diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index a48461c0742872..3b1444f3429b9a 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -107,9 +107,7 @@ extern PyTypeObject _PyExc_MemoryError; }, \ }, \ .dtoa = _dtoa_state_INIT(&(INTERP)), \ - .dict_state = { \ - .next_keys_version = 2, \ - }, \ + .dict_state = _dict_state_INIT, \ .func_state = { \ .next_version = 1, \ }, \ diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 2ef520044340ee..7436c113f37c4a 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -300,9 +300,19 @@ _PyDict_DebugMallocStats(FILE *out) static void free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys); +/* PyDictKeysObject has refcounts like PyObject does, so we have the + following two functions to mirror what Py_INCREF() and Py_DECREF() do. + (Keep in mind that PyDictKeysObject isn't actually a PyObject.) + Likewise a PyDictKeysObject can be immortal (e.g. Py_EMPTY_KEYS), + so we apply a naive version of what Py_INCREF() and Py_DECREF() do + for immortal objects. */ + static inline void dictkeys_incref(PyDictKeysObject *dk) { + if (dk->dk_refcnt == _Py_IMMORTAL_REFCNT) { + return; + } #ifdef Py_REF_DEBUG _Py_IncRefTotal(_PyInterpreterState_GET()); #endif @@ -312,6 +322,9 @@ dictkeys_incref(PyDictKeysObject *dk) static inline void dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk) { + if (dk->dk_refcnt == _Py_IMMORTAL_REFCNT) { + return; + } assert(dk->dk_refcnt > 0); #ifdef Py_REF_DEBUG _Py_DecRefTotal(_PyInterpreterState_GET()); @@ -447,7 +460,7 @@ estimate_log2_keysize(Py_ssize_t n) * (which cannot fail and thus can do no allocation). */ static PyDictKeysObject empty_keys_struct = { - 1, /* dk_refcnt */ + _Py_IMMORTAL_REFCNT, /* dk_refcnt */ 0, /* dk_log2_size */ 0, /* dk_log2_index_bytes */ DICT_KEYS_UNICODE, /* dk_kind */ @@ -779,6 +792,7 @@ clone_combined_dict_keys(PyDictObject *orig) assert(PyDict_Check(orig)); assert(Py_TYPE(orig)->tp_iter == (getiterfunc)dict_iter); assert(orig->ma_values == NULL); + assert(orig->ma_keys != Py_EMPTY_KEYS); assert(orig->ma_keys->dk_refcnt == 1); size_t keys_size = _PyDict_KeysSize(orig->ma_keys); @@ -833,7 +847,7 @@ PyObject * PyDict_New(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); - dictkeys_incref(Py_EMPTY_KEYS); + /* We don't incref Py_EMPTY_KEYS here because it is immortal. */ return new_dict(interp, Py_EMPTY_KEYS, NULL, 0, 0); } @@ -1331,7 +1345,7 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, Py_DECREF(value); return -1; } - dictkeys_decref(interp, Py_EMPTY_KEYS); + /* We don't decref Py_EMPTY_KEYS here because it is immortal. */ mp->ma_keys = newkeys; mp->ma_values = NULL; @@ -1529,14 +1543,10 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, // We can not use free_keys_object here because key's reference // are moved already. + if (oldkeys != Py_EMPTY_KEYS) { #ifdef Py_REF_DEBUG - _Py_DecRefTotal(_PyInterpreterState_GET()); + _Py_DecRefTotal(_PyInterpreterState_GET()); #endif - if (oldkeys == Py_EMPTY_KEYS) { - oldkeys->dk_refcnt--; - assert(oldkeys->dk_refcnt > 0); - } - else { assert(oldkeys->dk_kind != DICT_KEYS_SPLIT); assert(oldkeys->dk_refcnt == 1); #if PyDict_MAXFREELIST > 0 @@ -2080,8 +2090,8 @@ PyDict_Clear(PyObject *op) dictkeys_decref(interp, oldkeys); } else { - assert(oldkeys->dk_refcnt == 1); - dictkeys_decref(interp, oldkeys); + assert(oldkeys->dk_refcnt == 1); + dictkeys_decref(interp, oldkeys); } ASSERT_CONSISTENT(mp); } From 13ac1766bca7969a6c142c9176db901dd29c3519 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 10 May 2023 16:46:37 +0300 Subject: [PATCH 138/204] gh-103960: Dark mode: invert image brightness (#103983) --- Doc/howto/logging.rst | 1 + Doc/library/hashlib.rst | 1 + Doc/library/pathlib.rst | 1 + 3 files changed, 3 insertions(+) diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 145449b2dfbd9f..a72e9a820ef347 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -418,6 +418,7 @@ The flow of log event information in loggers and handlers is illustrated in the following diagram. .. image:: logging_flow.png + :class: invert-in-dark-mode Loggers ^^^^^^^ diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 6275f96f7d4d19..797870b9d7e260 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -430,6 +430,7 @@ Constructor functions also accept the following tree hashing parameters: .. figure:: hashlib-blake2-tree.png :alt: Explanation of tree mode parameters. + :class: invert-in-dark-mode See section 2.10 in `BLAKE2 specification `_ for comprehensive review of tree diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 5ffa33d4e61f19..93af07ae5ac10f 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -21,6 +21,7 @@ inherit from pure paths but also provide I/O operations. .. image:: pathlib-inheritance.png :align: center + :class: invert-in-dark-mode If you've never used this module before or just aren't sure which class is right for your task, :class:`Path` is most likely what you need. It instantiates From ce8d3db25660b029fa589a2072f4daf2a8723c50 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 10 May 2023 16:22:55 +0200 Subject: [PATCH 139/204] gh-101819: Adapt _io._BufferedIOBase_Type methods to Argument Clinic (#104355) Make sure the defining class is passed to all methods, so we can easily fetch module state from them in the future. --- Modules/_io/bufferedio.c | 107 +++++++++++------- Modules/_io/clinic/bufferedio.c.h | 173 +++++++++++++++++++++++++++++- 2 files changed, 236 insertions(+), 44 deletions(-) diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 172fafe6db8a3d..c7ae60281c2f58 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -115,6 +115,9 @@ bufferediobase_unsupported(_PyIO_State *state, const char *message) /*[clinic input] _io._BufferedIOBase.detach + cls: defining_class + / + Disconnect this buffer from its underlying raw stream and return it. After the raw stream has been detached, the buffer is in an unusable @@ -122,63 +125,89 @@ state. [clinic start generated code]*/ static PyObject * -_io__BufferedIOBase_detach_impl(PyObject *self) -/*[clinic end generated code: output=754977c8d10ed88c input=822427fb58fe4169]*/ +_io__BufferedIOBase_detach_impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=b87b135d67cd4448 input=0b61a7b4357c1ea7]*/ { _PyIO_State *state = IO_STATE(); return bufferediobase_unsupported(state, "detach"); } -PyDoc_STRVAR(bufferediobase_read_doc, - "Read and return up to n bytes.\n" - "\n" - "If the argument is omitted, None, or negative, reads and\n" - "returns all data until EOF.\n" - "\n" - "If the argument is positive, and the underlying raw stream is\n" - "not 'interactive', multiple raw reads may be issued to satisfy\n" - "the byte count (unless EOF is reached first). But for\n" - "interactive raw streams (as well as sockets and pipes), at most\n" - "one raw read will be issued, and a short result does not imply\n" - "that EOF is imminent.\n" - "\n" - "Returns an empty bytes object on EOF.\n" - "\n" - "Returns None if the underlying raw stream was open in non-blocking\n" - "mode and no data is available at the moment.\n"); +/*[clinic input] +_io._BufferedIOBase.read + + cls: defining_class + / + *args: object + +Read and return up to n bytes. + +If the argument is omitted, None, or negative, read and +return all data until EOF. + +If the argument is positive, and the underlying raw stream is +not 'interactive', multiple raw reads may be issued to satisfy +the byte count (unless EOF is reached first). +However, for interactive raw streams (as well as sockets and pipes), +at most one raw read will be issued, and a short result does not +imply that EOF is imminent. + +Return an empty bytes object on EOF. + +Return None if the underlying raw stream was open in non-blocking +mode and no data is available at the moment. +[clinic start generated code]*/ static PyObject * -bufferediobase_read(PyObject *self, PyObject *args) +_io__BufferedIOBase_read_impl(PyObject *self, PyTypeObject *cls, + PyObject *args) +/*[clinic end generated code: output=4521b30940fd7b67 input=390205758adc8510]*/ { _PyIO_State *state = IO_STATE(); return bufferediobase_unsupported(state, "read"); } -PyDoc_STRVAR(bufferediobase_read1_doc, - "Read and return up to n bytes, with at most one read() call\n" - "to the underlying raw stream. A short result does not imply\n" - "that EOF is imminent.\n" - "\n" - "Returns an empty bytes object on EOF.\n"); +/*[clinic input] +_io._BufferedIOBase.read1 + + cls: defining_class + / + *args: object + +Read and return up to n bytes, with at most one read() call to the underlying raw stream. + +Return an empty bytes object on EOF. +A short result does not imply that EOF is imminent. +[clinic start generated code]*/ static PyObject * -bufferediobase_read1(PyObject *self, PyObject *args) +_io__BufferedIOBase_read1_impl(PyObject *self, PyTypeObject *cls, + PyObject *args) +/*[clinic end generated code: output=636fd241c21e050a input=ef546a1238c5b41c]*/ { _PyIO_State *state = IO_STATE(); return bufferediobase_unsupported(state, "read1"); } -PyDoc_STRVAR(bufferediobase_write_doc, - "Write the given buffer to the IO stream.\n" - "\n" - "Returns the number of bytes written, which is always the length of b\n" - "in bytes.\n" - "\n" - "Raises BlockingIOError if the buffer is full and the\n" - "underlying raw stream cannot accept more data at the moment.\n"); +/*[clinic input] +_io._BufferedIOBase.write + + cls: defining_class + / + *args: object + +Write the given buffer to the IO stream. + +Return the number of bytes written, which is always +the length of b in bytes. + +Raise BlockingIOError if the buffer is full and the +underlying raw stream cannot accept more data at the moment. +[clinic start generated code]*/ static PyObject * -bufferediobase_write(PyObject *self, PyObject *args) +_io__BufferedIOBase_write_impl(PyObject *self, PyTypeObject *cls, + PyObject *args) +/*[clinic end generated code: output=d51feea4bcac9892 input=f79b72c4dccb3dc2]*/ { _PyIO_State *state = IO_STATE(); return bufferediobase_unsupported(state, "write"); @@ -2336,11 +2365,11 @@ _io_BufferedRandom___init___impl(buffered *self, PyObject *raw, static PyMethodDef bufferediobase_methods[] = { _IO__BUFFEREDIOBASE_DETACH_METHODDEF - {"read", bufferediobase_read, METH_VARARGS, bufferediobase_read_doc}, - {"read1", bufferediobase_read1, METH_VARARGS, bufferediobase_read1_doc}, + _IO__BUFFEREDIOBASE_READ_METHODDEF + _IO__BUFFEREDIOBASE_READ1_METHODDEF _IO__BUFFEREDIOBASE_READINTO_METHODDEF _IO__BUFFEREDIOBASE_READINTO1_METHODDEF - {"write", bufferediobase_write, METH_VARARGS, bufferediobase_write_doc}, + _IO__BUFFEREDIOBASE_WRITE_METHODDEF {NULL, NULL} }; diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index d44321bb8b960e..a898b01899babe 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -92,15 +92,178 @@ PyDoc_STRVAR(_io__BufferedIOBase_detach__doc__, "state."); #define _IO__BUFFEREDIOBASE_DETACH_METHODDEF \ - {"detach", (PyCFunction)_io__BufferedIOBase_detach, METH_NOARGS, _io__BufferedIOBase_detach__doc__}, + {"detach", _PyCFunction_CAST(_io__BufferedIOBase_detach), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__BufferedIOBase_detach__doc__}, static PyObject * -_io__BufferedIOBase_detach_impl(PyObject *self); +_io__BufferedIOBase_detach_impl(PyObject *self, PyTypeObject *cls); static PyObject * -_io__BufferedIOBase_detach(PyObject *self, PyObject *Py_UNUSED(ignored)) +_io__BufferedIOBase_detach(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _io__BufferedIOBase_detach_impl(self); + if (nargs) { + PyErr_SetString(PyExc_TypeError, "detach() takes no arguments"); + return NULL; + } + return _io__BufferedIOBase_detach_impl(self, cls); +} + +PyDoc_STRVAR(_io__BufferedIOBase_read__doc__, +"read($self, /, *args)\n" +"--\n" +"\n" +"Read and return up to n bytes.\n" +"\n" +"If the argument is omitted, None, or negative, read and\n" +"return all data until EOF.\n" +"\n" +"If the argument is positive, and the underlying raw stream is\n" +"not \'interactive\', multiple raw reads may be issued to satisfy\n" +"the byte count (unless EOF is reached first).\n" +"However, for interactive raw streams (as well as sockets and pipes),\n" +"at most one raw read will be issued, and a short result does not\n" +"imply that EOF is imminent.\n" +"\n" +"Return an empty bytes object on EOF.\n" +"\n" +"Return None if the underlying raw stream was open in non-blocking\n" +"mode and no data is available at the moment."); + +#define _IO__BUFFEREDIOBASE_READ_METHODDEF \ + {"read", _PyCFunction_CAST(_io__BufferedIOBase_read), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__BufferedIOBase_read__doc__}, + +static PyObject * +_io__BufferedIOBase_read_impl(PyObject *self, PyTypeObject *cls, + PyObject *args); + +static PyObject * +_io__BufferedIOBase_read(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = { NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "read", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *__clinic_args = NULL; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); + if (!args) { + goto exit; + } + __clinic_args = args[0]; + return_value = _io__BufferedIOBase_read_impl(self, cls, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(_io__BufferedIOBase_read1__doc__, +"read1($self, /, *args)\n" +"--\n" +"\n" +"Read and return up to n bytes, with at most one read() call to the underlying raw stream.\n" +"\n" +"Return an empty bytes object on EOF.\n" +"A short result does not imply that EOF is imminent."); + +#define _IO__BUFFEREDIOBASE_READ1_METHODDEF \ + {"read1", _PyCFunction_CAST(_io__BufferedIOBase_read1), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__BufferedIOBase_read1__doc__}, + +static PyObject * +_io__BufferedIOBase_read1_impl(PyObject *self, PyTypeObject *cls, + PyObject *args); + +static PyObject * +_io__BufferedIOBase_read1(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = { NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "read1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *__clinic_args = NULL; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); + if (!args) { + goto exit; + } + __clinic_args = args[0]; + return_value = _io__BufferedIOBase_read1_impl(self, cls, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(_io__BufferedIOBase_write__doc__, +"write($self, /, *args)\n" +"--\n" +"\n" +"Write the given buffer to the IO stream.\n" +"\n" +"Return the number of bytes written, which is always\n" +"the length of b in bytes.\n" +"\n" +"Raise BlockingIOError if the buffer is full and the\n" +"underlying raw stream cannot accept more data at the moment."); + +#define _IO__BUFFEREDIOBASE_WRITE_METHODDEF \ + {"write", _PyCFunction_CAST(_io__BufferedIOBase_write), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__BufferedIOBase_write__doc__}, + +static PyObject * +_io__BufferedIOBase_write_impl(PyObject *self, PyTypeObject *cls, + PyObject *args); + +static PyObject * +_io__BufferedIOBase_write(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = { NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "write", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *__clinic_args = NULL; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); + if (!args) { + goto exit; + } + __clinic_args = args[0]; + return_value = _io__BufferedIOBase_write_impl(self, cls, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; } PyDoc_STRVAR(_io__Buffered_peek__doc__, @@ -714,4 +877,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=8412b10c04259bb8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c4ea041ccc91b5d2 input=a9049054013a1b77]*/ From a7a2dbbf72aceef61bfb50901bfa39bfb8d6d229 Mon Sep 17 00:00:00 2001 From: chgnrdv <52372310+chgnrdv@users.noreply.github.com> Date: Wed, 10 May 2023 17:48:55 +0300 Subject: [PATCH 140/204] gh-104010: Separate and improve docs for `typing.get_origin` and `typing.get_args` (#104013) * separate documentation and examples for both functions * add examples demonstrating behaviour with unsupported types * document return value of `get_origin` for `ParamSpecArgs` and `ParamSpecKwargs` instances Co-authored-by: Jelle Zijlstra --- Doc/library/typing.rst | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index ebab1389f07e58..fb1c916b141a25 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2876,24 +2876,37 @@ Introspection helpers if a default value equal to ``None`` was set. Now the annotation is returned unchanged. -.. function:: get_args(tp) .. function:: get_origin(tp) - Provide basic introspection for generic types and special typing forms. - - For a typing object of the form ``X[Y, Z, ...]`` these functions return - ``X`` and ``(Y, Z, ...)``. If ``X`` is a generic alias for a builtin or + Get the unsubscripted version of a type: for a typing object of the form + ``X[Y, Z, ...]`` return ``X``. If ``X`` is a generic alias for a builtin or :mod:`collections` class, it gets normalized to the original class. + If ``X`` is an instance of :class:`ParamSpecArgs` or :class:`ParamSpecKwargs`, + return the underlying :class:`ParamSpec`. + Return ``None`` for unsupported objects. + Examples:: + + assert get_origin(str) is None + assert get_origin(Dict[str, int]) is dict + assert get_origin(Union[int, str]) is Union + P = ParamSpec('P') + assert get_origin(P.args) is P + assert get_origin(P.kwargs) is P + + .. versionadded:: 3.8 + +.. function:: get_args(tp) + + Get type arguments with all substitutions performed: for a typing object + of the form ``X[Y, Z, ...]`` return ``(Y, Z, ...)``. If ``X`` is a union or :class:`Literal` contained in another generic type, the order of ``(Y, Z, ...)`` may be different from the order of the original arguments ``[Y, Z, ...]`` due to type caching. - For unsupported objects return ``None`` and ``()`` correspondingly. + Return ``()`` for unsupported objects. Examples:: - assert get_origin(Dict[str, int]) is dict + assert get_args(int) == () assert get_args(Dict[int, str]) == (int, str) - - assert get_origin(Union[int, str]) is Union assert get_args(Union[int, str]) == (int, str) .. versionadded:: 3.8 From 7a3b03509e5e3e72d8c47137579cccb52548a318 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 10 May 2023 18:44:52 +0200 Subject: [PATCH 141/204] gh-104263: Rely on Py_NAN and introduce Py_INFINITY (GH-104202) This PR removes `_Py_dg_stdnan` and `_Py_dg_infinity` in favour of using the standard `NAN` and `INFINITY` macros provided by C99. This change has the side-effect of fixing a bug on MIPS where the hard-coded value used by `_Py_dg_stdnan` gave a signalling NaN rather than a quiet NaN. --------- Co-authored-by: Mark Dickinson --- Include/internal/pycore_dtoa.h | 2 - Include/pymath.h | 25 ++++---- Lib/test/test_cmath.py | 5 ++ Lib/test/test_complex.py | 7 +++ Lib/test/test_float.py | 5 +- Lib/test/test_math.py | 4 +- ...-05-08-10-34-55.gh-issue-104263.ctHWI8.rst | 6 ++ Modules/cmathmodule.c | 61 ++----------------- Modules/mathmodule.c | 39 ++---------- Objects/floatobject.c | 13 +--- Python/dtoa.c | 34 ----------- Python/pystrtod.c | 41 +------------ 12 files changed, 44 insertions(+), 198 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-08-10-34-55.gh-issue-104263.ctHWI8.rst diff --git a/Include/internal/pycore_dtoa.h b/Include/internal/pycore_dtoa.h index fb524770efed7c..4d9681d59a64f7 100644 --- a/Include/internal/pycore_dtoa.h +++ b/Include/internal/pycore_dtoa.h @@ -64,8 +64,6 @@ PyAPI_FUNC(double) _Py_dg_strtod(const char *str, char **ptr); PyAPI_FUNC(char *) _Py_dg_dtoa(double d, int mode, int ndigits, int *decpt, int *sign, char **rve); PyAPI_FUNC(void) _Py_dg_freedtoa(char *s); -PyAPI_FUNC(double) _Py_dg_stdnan(int sign); -PyAPI_FUNC(double) _Py_dg_infinity(int sign); #endif // _PY_SHORT_FLOAT_REPR == 1 diff --git a/Include/pymath.h b/Include/pymath.h index 772b67e4977563..4c1e3d9984894b 100644 --- a/Include/pymath.h +++ b/Include/pymath.h @@ -39,27 +39,24 @@ // Return 1 if float or double arg is neither infinite nor NAN, else 0. #define Py_IS_FINITE(X) isfinite(X) -/* HUGE_VAL is supposed to expand to a positive double infinity. Python - * uses Py_HUGE_VAL instead because some platforms are broken in this - * respect. We used to embed code in pyport.h to try to worm around that, - * but different platforms are broken in conflicting ways. If you're on - * a platform where HUGE_VAL is defined incorrectly, fiddle your Python - * config to #define Py_HUGE_VAL to something that works on your platform. +// Py_INFINITY: Value that evaluates to a positive double infinity. +#ifndef Py_INFINITY +# define Py_INFINITY ((double)INFINITY) +#endif + +/* Py_HUGE_VAL should always be the same as Py_INFINITY. But historically + * this was not reliable and Python did not require IEEE floats and C99 + * conformity. Prefer Py_INFINITY for new code. */ #ifndef Py_HUGE_VAL # define Py_HUGE_VAL HUGE_VAL #endif -// Py_NAN: Value that evaluates to a quiet Not-a-Number (NaN). +/* Py_NAN: Value that evaluates to a quiet Not-a-Number (NaN). The sign is + * undefined and normally not relevant, but e.g. fixed for float("nan"). + */ #if !defined(Py_NAN) -# if _Py__has_builtin(__builtin_nan) - // Built-in implementation of the ISO C99 function nan(): quiet NaN. -# define Py_NAN (__builtin_nan("")) -#else - // Use C99 NAN constant: quiet Not-A-Number. - // NAN is a float, Py_NAN is a double: cast to double. # define Py_NAN ((double)NAN) -# endif #endif #endif /* Py_PYMATH_H */ diff --git a/Lib/test/test_cmath.py b/Lib/test/test_cmath.py index cd2c6939105d40..57f80d5d8cd016 100644 --- a/Lib/test/test_cmath.py +++ b/Lib/test/test_cmath.py @@ -166,6 +166,11 @@ def test_infinity_and_nan_constants(self): self.assertEqual(cmath.nan.imag, 0.0) self.assertEqual(cmath.nanj.real, 0.0) self.assertTrue(math.isnan(cmath.nanj.imag)) + # Also check that the sign of all of these is positive: + self.assertEqual(math.copysign(1., cmath.nan.real), 1.) + self.assertEqual(math.copysign(1., cmath.nan.imag), 1.) + self.assertEqual(math.copysign(1., cmath.nanj.real), 1.) + self.assertEqual(math.copysign(1., cmath.nanj.imag), 1.) # Check consistency with reprs. self.assertEqual(repr(cmath.inf), "inf") diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index 51ba151505fb54..9180cca62b28b8 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -529,6 +529,12 @@ class complex2(complex): self.assertFloatsAreIdentical(z.real, x) self.assertFloatsAreIdentical(z.imag, y) + def test_constructor_negative_nans_from_string(self): + self.assertEqual(copysign(1., complex("-nan").real), -1.) + self.assertEqual(copysign(1., complex("-nanj").imag), -1.) + self.assertEqual(copysign(1., complex("-nan-nanj").real), -1.) + self.assertEqual(copysign(1., complex("-nan-nanj").imag), -1.) + def test_underscores(self): # check underscores for lit in VALID_UNDERSCORE_LITERALS: @@ -569,6 +575,7 @@ def test(v, expected, test_fn=self.assertEqual): test(complex(NAN, 1), "(nan+1j)") test(complex(1, NAN), "(1+nanj)") test(complex(NAN, NAN), "(nan+nanj)") + test(complex(-NAN, -NAN), "(nan+nanj)") test(complex(0, INF), "infj") test(complex(0, -INF), "-infj") diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index f8350c1e4caa27..c4ee1e08251d63 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -1040,11 +1040,8 @@ def test_inf_signs(self): self.assertEqual(copysign(1.0, float('inf')), 1.0) self.assertEqual(copysign(1.0, float('-inf')), -1.0) - @unittest.skipUnless(getattr(sys, 'float_repr_style', '') == 'short', - "applies only when using short float repr style") def test_nan_signs(self): - # When using the dtoa.c code, the sign of float('nan') should - # be predictable. + # The sign of float('nan') should be predictable. self.assertEqual(copysign(1.0, float('nan')), 1.0) self.assertEqual(copysign(1.0, float('-nan')), -1.0) diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 433161c2dd4145..f282434c9a3359 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -1881,11 +1881,11 @@ def testIsinf(self): self.assertFalse(math.isinf(0.)) self.assertFalse(math.isinf(1.)) - @requires_IEEE_754 def test_nan_constant(self): + # `math.nan` must be a quiet NaN with positive sign bit self.assertTrue(math.isnan(math.nan)) + self.assertEqual(math.copysign(1., math.nan), 1.) - @requires_IEEE_754 def test_inf_constant(self): self.assertTrue(math.isinf(math.inf)) self.assertGreater(math.inf, 0.0) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-08-10-34-55.gh-issue-104263.ctHWI8.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-08-10-34-55.gh-issue-104263.ctHWI8.rst new file mode 100644 index 00000000000000..342467cfcd4e75 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-08-10-34-55.gh-issue-104263.ctHWI8.rst @@ -0,0 +1,6 @@ +Fix ``float("nan")`` to produce a quiet NaN on platforms (like MIPS) where +the meaning of the signalling / quiet bit is inverted from its usual +meaning. Also introduce a new macro ``Py_INFINITY`` matching C99's +``INFINITY``, and refactor internals to rely on C99's ``NAN`` and +``INFINITY`` macros instead of hard-coding bit patterns for infinities and +NaNs. Thanks Sebastian Berg. diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index 914a697f8e173b..1a31bdc824bb03 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -8,7 +8,6 @@ #include "Python.h" #include "pycore_pymath.h" // _PY_SHORT_FLOAT_REPR -#include "pycore_dtoa.h" // _Py_dg_stdnan() /* we need DBL_MAX, DBL_MIN, DBL_EPSILON, DBL_MANT_DIG and FLT_RADIX from float.h. We assume that FLT_RADIX is either 2 or 16. */ #include @@ -88,53 +87,6 @@ else { #endif #define CM_SCALE_DOWN (-(CM_SCALE_UP+1)/2) -/* Constants cmath.inf, cmath.infj, cmath.nan, cmath.nanj. - cmath.nan and cmath.nanj are defined only when either - _PY_SHORT_FLOAT_REPR is 1 (which should be - the most common situation on machines using an IEEE 754 - representation), or Py_NAN is defined. */ - -static double -m_inf(void) -{ -#if _PY_SHORT_FLOAT_REPR == 1 - return _Py_dg_infinity(0); -#else - return Py_HUGE_VAL; -#endif -} - -static Py_complex -c_infj(void) -{ - Py_complex r; - r.real = 0.0; - r.imag = m_inf(); - return r; -} - -#if _PY_SHORT_FLOAT_REPR == 1 - -static double -m_nan(void) -{ -#if _PY_SHORT_FLOAT_REPR == 1 - return _Py_dg_stdnan(0); -#else - return Py_NAN; -#endif -} - -static Py_complex -c_nanj(void) -{ - Py_complex r; - r.real = 0.0; - r.imag = m_nan(); - return r; -} - -#endif /* forward declarations */ static Py_complex cmath_asinh_impl(PyObject *, Py_complex); @@ -1274,23 +1226,22 @@ cmath_exec(PyObject *mod) if (PyModule_AddObject(mod, "tau", PyFloat_FromDouble(Py_MATH_TAU)) < 0) { return -1; } - if (PyModule_AddObject(mod, "inf", PyFloat_FromDouble(m_inf())) < 0) { + if (PyModule_AddObject(mod, "inf", PyFloat_FromDouble(Py_INFINITY)) < 0) { return -1; } + Py_complex infj = {0.0, Py_INFINITY}; if (PyModule_AddObject(mod, "infj", - PyComplex_FromCComplex(c_infj())) < 0) { + PyComplex_FromCComplex(infj)) < 0) { return -1; } -#if _PY_SHORT_FLOAT_REPR == 1 - if (PyModule_AddObject(mod, "nan", PyFloat_FromDouble(m_nan())) < 0) { + if (PyModule_AddObject(mod, "nan", PyFloat_FromDouble(fabs(Py_NAN))) < 0) { return -1; } - if (PyModule_AddObject(mod, "nanj", - PyComplex_FromCComplex(c_nanj())) < 0) { + Py_complex nanj = {0.0, fabs(Py_NAN)}; + if (PyModule_AddObject(mod, "nanj", PyComplex_FromCComplex(nanj)) < 0) { return -1; } -#endif /* initialize special value tables */ diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 3737a9654575ab..f369b2c45ce3ba 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -59,7 +59,6 @@ raised for division by zero and mod by zero. #include "Python.h" #include "pycore_bitutils.h" // _Py_bit_length() #include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_dtoa.h" // _Py_dg_infinity() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_object.h" // _PyObject_LookupSpecial() @@ -389,34 +388,6 @@ lanczos_sum(double x) return num/den; } -/* Constant for +infinity, generated in the same way as float('inf'). */ - -static double -m_inf(void) -{ -#if _PY_SHORT_FLOAT_REPR == 1 - return _Py_dg_infinity(0); -#else - return Py_HUGE_VAL; -#endif -} - -/* Constant nan value, generated in the same way as float('nan'). */ -/* We don't currently assume that Py_NAN is defined everywhere. */ - -#if _PY_SHORT_FLOAT_REPR == 1 - -static double -m_nan(void) -{ -#if _PY_SHORT_FLOAT_REPR == 1 - return _Py_dg_stdnan(0); -#else - return Py_NAN; -#endif -} - -#endif static double m_tgamma(double x) @@ -435,7 +406,7 @@ m_tgamma(double x) if (x == 0.0) { errno = EDOM; /* tgamma(+-0.0) = +-inf, divide-by-zero */ - return copysign(Py_HUGE_VAL, x); + return copysign(Py_INFINITY, x); } /* integer arguments */ @@ -3938,7 +3909,7 @@ math_ulp_impl(PyObject *module, double x) if (Py_IS_INFINITY(x)) { return x; } - double inf = m_inf(); + double inf = Py_INFINITY; double x2 = nextafter(x, inf); if (Py_IS_INFINITY(x2)) { /* special case: x is the largest positive representable float */ @@ -3975,14 +3946,12 @@ math_exec(PyObject *module) if (PyModule_AddObject(module, "tau", PyFloat_FromDouble(Py_MATH_TAU)) < 0) { return -1; } - if (PyModule_AddObject(module, "inf", PyFloat_FromDouble(m_inf())) < 0) { + if (PyModule_AddObject(module, "inf", PyFloat_FromDouble(Py_INFINITY)) < 0) { return -1; } -#if _PY_SHORT_FLOAT_REPR == 1 - if (PyModule_AddObject(module, "nan", PyFloat_FromDouble(m_nan())) < 0) { + if (PyModule_AddObject(module, "nan", PyFloat_FromDouble(fabs(Py_NAN))) < 0) { return -1; } -#endif return 0; } diff --git a/Objects/floatobject.c b/Objects/floatobject.c index d257857d9c619c..83a263c0d9c67e 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -2424,25 +2424,14 @@ PyFloat_Unpack2(const char *data, int le) f |= *p; if (e == 0x1f) { -#if _PY_SHORT_FLOAT_REPR == 0 if (f == 0) { /* Infinity */ return sign ? -Py_HUGE_VAL : Py_HUGE_VAL; } else { /* NaN */ - return sign ? -Py_NAN : Py_NAN; + return sign ? -fabs(Py_NAN) : fabs(Py_NAN); } -#else // _PY_SHORT_FLOAT_REPR == 1 - if (f == 0) { - /* Infinity */ - return _Py_dg_infinity(sign); - } - else { - /* NaN */ - return _Py_dg_stdnan(sign); - } -#endif // _PY_SHORT_FLOAT_REPR == 1 } x = (double)f / 1024.0; diff --git a/Python/dtoa.c b/Python/dtoa.c index 6ea60ac9946e0f..c5e343b82f74c5 100644 --- a/Python/dtoa.c +++ b/Python/dtoa.c @@ -273,11 +273,6 @@ typedef union { double d; ULong L[2]; } U; #define Big0 (Frac_mask1 | Exp_msk1*(DBL_MAX_EXP+Bias-1)) #define Big1 0xffffffff -/* Standard NaN used by _Py_dg_stdnan. */ - -#define NAN_WORD0 0x7ff80000 -#define NAN_WORD1 0 - /* Bits of the representation of positive infinity. */ #define POSINF_WORD0 0x7ff00000 @@ -1399,35 +1394,6 @@ bigcomp(U *rv, const char *s0, BCinfo *bc) return 0; } -/* Return a 'standard' NaN value. - - There are exactly two quiet NaNs that don't arise by 'quieting' signaling - NaNs (see IEEE 754-2008, section 6.2.1). If sign == 0, return the one whose - sign bit is cleared. Otherwise, return the one whose sign bit is set. -*/ - -double -_Py_dg_stdnan(int sign) -{ - U rv; - word0(&rv) = NAN_WORD0; - word1(&rv) = NAN_WORD1; - if (sign) - word0(&rv) |= Sign_bit; - return dval(&rv); -} - -/* Return positive or negative infinity, according to the given sign (0 for - * positive infinity, 1 for negative infinity). */ - -double -_Py_dg_infinity(int sign) -{ - U rv; - word0(&rv) = POSINF_WORD0; - word1(&rv) = POSINF_WORD1; - return sign ? -dval(&rv) : dval(&rv); -} double _Py_dg_strtod(const char *s00, char **se) diff --git a/Python/pystrtod.c b/Python/pystrtod.c index d77b846f0403f0..9bb060e3d11979 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -23,44 +23,6 @@ case_insensitive_match(const char *s, const char *t) return the NaN or Infinity as a double and set *endptr to point just beyond the successfully parsed portion of the string. On failure, return -1.0 and set *endptr to point to the start of the string. */ - -#if _PY_SHORT_FLOAT_REPR == 1 - -double -_Py_parse_inf_or_nan(const char *p, char **endptr) -{ - double retval; - const char *s; - int negate = 0; - - s = p; - if (*s == '-') { - negate = 1; - s++; - } - else if (*s == '+') { - s++; - } - if (case_insensitive_match(s, "inf")) { - s += 3; - if (case_insensitive_match(s, "inity")) - s += 5; - retval = _Py_dg_infinity(negate); - } - else if (case_insensitive_match(s, "nan")) { - s += 3; - retval = _Py_dg_stdnan(negate); - } - else { - s = p; - retval = -1.0; - } - *endptr = (char *)s; - return retval; -} - -#else - double _Py_parse_inf_or_nan(const char *p, char **endptr) { @@ -84,7 +46,7 @@ _Py_parse_inf_or_nan(const char *p, char **endptr) } else if (case_insensitive_match(s, "nan")) { s += 3; - retval = negate ? -Py_NAN : Py_NAN; + retval = negate ? -fabs(Py_NAN) : fabs(Py_NAN); } else { s = p; @@ -94,7 +56,6 @@ _Py_parse_inf_or_nan(const char *p, char **endptr) return retval; } -#endif /** * _PyOS_ascii_strtod: From a33ce66dca57d4c36b1022fdf3b7e322f3203468 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Wed, 10 May 2023 18:17:08 +0100 Subject: [PATCH 142/204] GH-87695: Fix OSError from `pathlib.Path.glob()` (GH-104292) Fix issue where `pathlib.Path.glob()` raised `OSError` when it encountered a symlink to an overly long path. --- Lib/pathlib.py | 4 ++-- Lib/test/test_pathlib.py | 9 +++++++++ .../2023-05-08-15-39-00.gh-issue-87695.f6iO7v.rst | 2 ++ 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-08-15-39-00.gh-issue-87695.f6iO7v.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 20ec1ce9d80374..25863c76f3086d 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -178,11 +178,11 @@ def _iterate_directories(self, parent_path, scandir): for entry in entries: entry_is_dir = False try: - entry_is_dir = entry.is_dir() + entry_is_dir = entry.is_dir(follow_symlinks=False) except OSError as e: if not _ignore_error(e): raise - if entry_is_dir and not entry.is_symlink(): + if entry_is_dir: path = parent_path._make_child_relpath(entry.name) for p in self._iterate_directories(path, scandir): yield p diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index ee0ef9a34c385c..10dd604138a659 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1977,6 +1977,15 @@ def my_scandir(path): subdir.chmod(000) self.assertEqual(len(set(base.glob("*"))), 4) + @os_helper.skip_unless_symlink + def test_glob_long_symlink(self): + # See gh-87695 + base = self.cls(BASE) / 'long_symlink' + base.mkdir() + bad_link = base / 'bad_link' + bad_link.symlink_to("bad" * 200) + self.assertEqual(sorted(base.glob('**/*')), [bad_link]) + def _check_resolve(self, p, expected, strict=True): q = p.resolve(strict) self.assertEqual(q, expected) diff --git a/Misc/NEWS.d/next/Library/2023-05-08-15-39-00.gh-issue-87695.f6iO7v.rst b/Misc/NEWS.d/next/Library/2023-05-08-15-39-00.gh-issue-87695.f6iO7v.rst new file mode 100644 index 00000000000000..7b677656bea308 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-08-15-39-00.gh-issue-87695.f6iO7v.rst @@ -0,0 +1,2 @@ +Fix issue where :meth:`pathlib.Path.glob` raised :exc:`OSError` when it +encountered a symlink to an overly long path. From e464ec9f4c4af6c2fdb6db9a3fa70ffd703ae01f Mon Sep 17 00:00:00 2001 From: Christopher Chavez Date: Wed, 10 May 2023 13:53:13 -0500 Subject: [PATCH 143/204] gh-103538: Remove unused TK_AQUA code (GH-103539) --- ...-04-14-21-12-32.gh-issue-103538.M4FK_v.rst | 3 ++ Modules/_tkinter.c | 14 ------ Modules/tkappinit.c | 50 ------------------- 3 files changed, 3 insertions(+), 64 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-14-21-12-32.gh-issue-103538.M4FK_v.rst diff --git a/Misc/NEWS.d/next/Library/2023-04-14-21-12-32.gh-issue-103538.M4FK_v.rst b/Misc/NEWS.d/next/Library/2023-04-14-21-12-32.gh-issue-103538.M4FK_v.rst new file mode 100644 index 00000000000000..32788307d6f33b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-14-21-12-32.gh-issue-103538.M4FK_v.rst @@ -0,0 +1,3 @@ +Remove ``_tkinter`` module code guarded by definition of the ``TK_AQUA`` macro +which was only needed for Tk 8.4.7 or earlier and was never actually defined by +any build system or documented for manual use. diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 385a05932a77ed..4dada0b28f0559 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -3283,20 +3283,6 @@ PyInit__tkinter(void) } PyTclObject_Type = o; -#ifdef TK_AQUA - /* Tk_MacOSXSetupTkNotifier must be called before Tcl's subsystems - * start waking up. Note that Tcl_FindExecutable will do this, this - * code must be above it! The original warning from - * tkMacOSXAppInit.c is copied below. - * - * NB - You have to swap in the Tk Notifier BEFORE you start up the - * Tcl interpreter for now. It probably should work to do this - * in the other order, but for now it doesn't seem to. - * - */ - Tk_MacOSXSetupTkNotifier(); -#endif - /* This helps the dynamic loader; in Unicode aware Tcl versions it also helps Tcl find its encodings. */ diff --git a/Modules/tkappinit.c b/Modules/tkappinit.c index 67d6250318c616..4c4081e43a8e3d 100644 --- a/Modules/tkappinit.c +++ b/Modules/tkappinit.c @@ -23,54 +23,9 @@ Tcl_AppInit(Tcl_Interp *interp) { const char *_tkinter_skip_tk_init; -#ifdef TK_AQUA -#ifndef MAX_PATH_LEN -#define MAX_PATH_LEN 1024 -#endif - char tclLibPath[MAX_PATH_LEN], tkLibPath[MAX_PATH_LEN]; - Tcl_Obj* pathPtr; - - /* pre- Tcl_Init code copied from tkMacOSXAppInit.c */ - Tk_MacOSXOpenBundleResources (interp, "com.tcltk.tcllibrary", - tclLibPath, MAX_PATH_LEN, 0); - - if (tclLibPath[0] != '\0') { - Tcl_SetVar(interp, "tcl_library", tclLibPath, TCL_GLOBAL_ONLY); - Tcl_SetVar(interp, "tclDefaultLibrary", tclLibPath, TCL_GLOBAL_ONLY); - Tcl_SetVar(interp, "tcl_pkgPath", tclLibPath, TCL_GLOBAL_ONLY); - } - - if (tclLibPath[0] != '\0') { - Tcl_SetVar(interp, "tcl_library", tclLibPath, TCL_GLOBAL_ONLY); - Tcl_SetVar(interp, "tclDefaultLibrary", tclLibPath, TCL_GLOBAL_ONLY); - Tcl_SetVar(interp, "tcl_pkgPath", tclLibPath, TCL_GLOBAL_ONLY); - } -#endif if (Tcl_Init (interp) == TCL_ERROR) return TCL_ERROR; -#ifdef TK_AQUA - /* pre- Tk_Init code copied from tkMacOSXAppInit.c */ - Tk_MacOSXOpenBundleResources (interp, "com.tcltk.tklibrary", - tkLibPath, MAX_PATH_LEN, 1); - - if (tclLibPath[0] != '\0') { - pathPtr = Tcl_NewStringObj(tclLibPath, -1); - } else { - Tcl_Obj *pathPtr = TclGetLibraryPath(); - } - - if (tkLibPath[0] != '\0') { - Tcl_Obj *objPtr; - - Tcl_SetVar(interp, "tk_library", tkLibPath, TCL_GLOBAL_ONLY); - objPtr = Tcl_NewStringObj(tkLibPath, -1); - Tcl_ListObjAppendElement(NULL, pathPtr, objPtr); - } - - TclSetLibraryPath(pathPtr); -#endif - #ifdef WITH_XXX /* Initialize modules that don't require Tk */ #endif @@ -88,11 +43,6 @@ Tcl_AppInit(Tcl_Interp *interp) Tk_MainWindow(interp); -#ifdef TK_AQUA - TkMacOSXInitAppleEvents(interp); - TkMacOSXInitMenus(interp); -#endif - #ifdef WITH_PIL /* 0.2b5 and later -- not yet released as of May 14 */ { extern void TkImaging_Init(Tcl_Interp *); From 7b8d7f56b64ab4370fea77e77ea4984dd2a73979 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 10 May 2023 22:43:51 +0100 Subject: [PATCH 144/204] gh-103000: Optimise `dataclasses.asdict` for the common case (#104364) Co-authored-by: David Ellis --- Lib/dataclasses.py | 17 ++++++++++++----- ...23-05-10-19-33-36.gh-issue-103000.j0KSfD.rst | 2 ++ 2 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-10-19-33-36.gh-issue-103000.j0KSfD.rst diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index b0b8a773b7594f..3eacba840db426 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1324,11 +1324,18 @@ def _asdict_inner(obj, dict_factory): if type(obj) in _ATOMIC_TYPES: return obj elif _is_dataclass_instance(obj): - result = [] - for f in fields(obj): - value = _asdict_inner(getattr(obj, f.name), dict_factory) - result.append((f.name, value)) - return dict_factory(result) + # fast path for the common case + if dict_factory is dict: + return { + f.name: _asdict_inner(getattr(obj, f.name), dict) + for f in fields(obj) + } + else: + result = [] + for f in fields(obj): + value = _asdict_inner(getattr(obj, f.name), dict_factory) + result.append((f.name, value)) + return dict_factory(result) elif isinstance(obj, tuple) and hasattr(obj, '_fields'): # obj is a namedtuple. Recurse into it, but the returned # object is another namedtuple of the same type. This is diff --git a/Misc/NEWS.d/next/Library/2023-05-10-19-33-36.gh-issue-103000.j0KSfD.rst b/Misc/NEWS.d/next/Library/2023-05-10-19-33-36.gh-issue-103000.j0KSfD.rst new file mode 100644 index 00000000000000..f84ec5c8b3caf6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-10-19-33-36.gh-issue-103000.j0KSfD.rst @@ -0,0 +1,2 @@ +Improve performance of :func:`dataclasses.asdict` for the common case where +*dict_factory* is ``dict``. Patch by David C Ellis. From 373bca0cc5256dc512ffc22bdff4424f7ee8baa2 Mon Sep 17 00:00:00 2001 From: penguin_wwy <940375606@qq.com> Date: Thu, 11 May 2023 06:40:59 +0800 Subject: [PATCH 145/204] GH-102181: Improve specialization stats for SEND (GH-102182) --- Include/cpython/genobject.h | 2 + Objects/genobject.c | 3 -- Python/specialize.c | 86 +++++++++++++++++--------------- Tools/scripts/summarize_stats.py | 2 + 4 files changed, 49 insertions(+), 44 deletions(-) diff --git a/Include/cpython/genobject.h b/Include/cpython/genobject.h index 18b8ce913e6e31..7856481b5db300 100644 --- a/Include/cpython/genobject.h +++ b/Include/cpython/genobject.h @@ -77,6 +77,8 @@ PyAPI_FUNC(PyObject *) PyAsyncGen_New(PyFrameObject *, #define PyAsyncGen_CheckExact(op) Py_IS_TYPE((op), &PyAsyncGen_Type) +#define PyAsyncGenASend_CheckExact(op) Py_IS_TYPE((op), &_PyAsyncGenASend_Type) + #undef _PyGenObject_HEAD diff --git a/Objects/genobject.c b/Objects/genobject.c index 937d497753e970..9252c654934565 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1406,9 +1406,6 @@ typedef struct _PyAsyncGenWrappedValue { #define _PyAsyncGenWrappedValue_CheckExact(o) \ Py_IS_TYPE(o, &_PyAsyncGenWrappedValue_Type) -#define PyAsyncGenASend_CheckExact(o) \ - Py_IS_TYPE(o, &_PyAsyncGenASend_Type) - static int async_gen_traverse(PyAsyncGenObject *gen, visitproc visit, void *arg) diff --git a/Python/specialize.c b/Python/specialize.c index 2ccca3a2802c17..45f1cb412db241 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -436,27 +436,28 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_COMPARE_OP_FLOAT_LONG 21 #define SPEC_FAIL_COMPARE_OP_LONG_FLOAT 22 -/* FOR_ITER */ -#define SPEC_FAIL_FOR_ITER_GENERATOR 10 -#define SPEC_FAIL_FOR_ITER_COROUTINE 11 -#define SPEC_FAIL_FOR_ITER_ASYNC_GENERATOR 12 -#define SPEC_FAIL_FOR_ITER_LIST 13 -#define SPEC_FAIL_FOR_ITER_TUPLE 14 -#define SPEC_FAIL_FOR_ITER_SET 15 -#define SPEC_FAIL_FOR_ITER_STRING 16 -#define SPEC_FAIL_FOR_ITER_BYTES 17 -#define SPEC_FAIL_FOR_ITER_RANGE 18 -#define SPEC_FAIL_FOR_ITER_ITERTOOLS 19 -#define SPEC_FAIL_FOR_ITER_DICT_KEYS 20 -#define SPEC_FAIL_FOR_ITER_DICT_ITEMS 21 -#define SPEC_FAIL_FOR_ITER_DICT_VALUES 22 -#define SPEC_FAIL_FOR_ITER_ENUMERATE 23 -#define SPEC_FAIL_FOR_ITER_MAP 24 -#define SPEC_FAIL_FOR_ITER_ZIP 25 -#define SPEC_FAIL_FOR_ITER_SEQ_ITER 26 -#define SPEC_FAIL_FOR_ITER_REVERSED_LIST 27 -#define SPEC_FAIL_FOR_ITER_CALLABLE 28 -#define SPEC_FAIL_FOR_ITER_ASCII_STRING 29 +/* FOR_ITER and SEND */ +#define SPEC_FAIL_ITER_GENERATOR 10 +#define SPEC_FAIL_ITER_COROUTINE 11 +#define SPEC_FAIL_ITER_ASYNC_GENERATOR 12 +#define SPEC_FAIL_ITER_LIST 13 +#define SPEC_FAIL_ITER_TUPLE 14 +#define SPEC_FAIL_ITER_SET 15 +#define SPEC_FAIL_ITER_STRING 16 +#define SPEC_FAIL_ITER_BYTES 17 +#define SPEC_FAIL_ITER_RANGE 18 +#define SPEC_FAIL_ITER_ITERTOOLS 19 +#define SPEC_FAIL_ITER_DICT_KEYS 20 +#define SPEC_FAIL_ITER_DICT_ITEMS 21 +#define SPEC_FAIL_ITER_DICT_VALUES 22 +#define SPEC_FAIL_ITER_ENUMERATE 23 +#define SPEC_FAIL_ITER_MAP 24 +#define SPEC_FAIL_ITER_ZIP 25 +#define SPEC_FAIL_ITER_SEQ_ITER 26 +#define SPEC_FAIL_ITER_REVERSED_LIST 27 +#define SPEC_FAIL_ITER_CALLABLE 28 +#define SPEC_FAIL_ITER_ASCII_STRING 29 +#define SPEC_FAIL_ITER_ASYNC_GENERATOR_SEND 30 // UNPACK_SEQUENCE @@ -2122,66 +2123,69 @@ int _PySpecialization_ClassifyIterator(PyObject *iter) { if (PyGen_CheckExact(iter)) { - return SPEC_FAIL_FOR_ITER_GENERATOR; + return SPEC_FAIL_ITER_GENERATOR; } if (PyCoro_CheckExact(iter)) { - return SPEC_FAIL_FOR_ITER_COROUTINE; + return SPEC_FAIL_ITER_COROUTINE; } if (PyAsyncGen_CheckExact(iter)) { - return SPEC_FAIL_FOR_ITER_ASYNC_GENERATOR; + return SPEC_FAIL_ITER_ASYNC_GENERATOR; + } + if (PyAsyncGenASend_CheckExact(iter)) { + return SPEC_FAIL_ITER_ASYNC_GENERATOR_SEND; } PyTypeObject *t = Py_TYPE(iter); if (t == &PyListIter_Type) { - return SPEC_FAIL_FOR_ITER_LIST; + return SPEC_FAIL_ITER_LIST; } if (t == &PyTupleIter_Type) { - return SPEC_FAIL_FOR_ITER_TUPLE; + return SPEC_FAIL_ITER_TUPLE; } if (t == &PyDictIterKey_Type) { - return SPEC_FAIL_FOR_ITER_DICT_KEYS; + return SPEC_FAIL_ITER_DICT_KEYS; } if (t == &PyDictIterValue_Type) { - return SPEC_FAIL_FOR_ITER_DICT_VALUES; + return SPEC_FAIL_ITER_DICT_VALUES; } if (t == &PyDictIterItem_Type) { - return SPEC_FAIL_FOR_ITER_DICT_ITEMS; + return SPEC_FAIL_ITER_DICT_ITEMS; } if (t == &PySetIter_Type) { - return SPEC_FAIL_FOR_ITER_SET; + return SPEC_FAIL_ITER_SET; } if (t == &PyUnicodeIter_Type) { - return SPEC_FAIL_FOR_ITER_STRING; + return SPEC_FAIL_ITER_STRING; } if (t == &PyBytesIter_Type) { - return SPEC_FAIL_FOR_ITER_BYTES; + return SPEC_FAIL_ITER_BYTES; } if (t == &PyRangeIter_Type) { - return SPEC_FAIL_FOR_ITER_RANGE; + return SPEC_FAIL_ITER_RANGE; } if (t == &PyEnum_Type) { - return SPEC_FAIL_FOR_ITER_ENUMERATE; + return SPEC_FAIL_ITER_ENUMERATE; } if (t == &PyMap_Type) { - return SPEC_FAIL_FOR_ITER_MAP; + return SPEC_FAIL_ITER_MAP; } if (t == &PyZip_Type) { - return SPEC_FAIL_FOR_ITER_ZIP; + return SPEC_FAIL_ITER_ZIP; } if (t == &PySeqIter_Type) { - return SPEC_FAIL_FOR_ITER_SEQ_ITER; + return SPEC_FAIL_ITER_SEQ_ITER; } if (t == &PyListRevIter_Type) { - return SPEC_FAIL_FOR_ITER_REVERSED_LIST; + return SPEC_FAIL_ITER_REVERSED_LIST; } if (t == &_PyUnicodeASCIIIter_Type) { - return SPEC_FAIL_FOR_ITER_ASCII_STRING; + return SPEC_FAIL_ITER_ASCII_STRING; } const char *name = t->tp_name; if (strncmp(name, "itertools", 9) == 0) { - return SPEC_FAIL_FOR_ITER_ITERTOOLS; + return SPEC_FAIL_ITER_ITERTOOLS; } if (strncmp(name, "callable_iterator", 17) == 0) { - return SPEC_FAIL_FOR_ITER_CALLABLE; + return SPEC_FAIL_ITER_CALLABLE; } return SPEC_FAIL_OTHER; } diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index ce25374f3a9a52..4f25ba36d86689 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -228,6 +228,8 @@ def kind_to_text(kind, defines, opname): return pretty(defines[kind][0]) if opname.endswith("ATTR"): opname = "ATTR" + if opname in ("FOR_ITER", "SEND"): + opname = "ITER" if opname.endswith("SUBSCR"): opname = "SUBSCR" for name in defines[kind]: From 94f30c75576bb8a20724b2ac758fa33af089a522 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Thu, 11 May 2023 01:01:39 +0100 Subject: [PATCH 146/204] GH-90208: Suppress OSError exceptions from `pathlib.Path.glob()` (GH-104141) `pathlib.Path.glob()` now suppresses all OSError exceptions, except those raised from calling `is_dir()` on the top-level path. Previously, `glob()` suppressed ENOENT, ENOTDIR, EBADF and ELOOP errors and their Windows equivalents. PermissionError was also suppressed unless it occurred when calling `is_dir()` on the top-level path. However, the selector would abort prematurely if a PermissionError was raised, and so `glob()` could return incomplete results. --- Lib/pathlib.py | 33 +++++++--------- Lib/test/test_pathlib.py | 38 ++++++------------- ...3-05-03-19-22-24.gh-issue-90208.tI00da.rst | 4 ++ 3 files changed, 29 insertions(+), 46 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-03-19-22-24.gh-issue-90208.tI00da.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 25863c76f3086d..40b72930e1f08a 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -142,25 +142,21 @@ def _select_from(self, parent_path, scandir): # avoid exhausting file descriptors when globbing deep trees. with scandir(parent_path) as scandir_it: entries = list(scandir_it) + except OSError: + pass + else: for entry in entries: if self.dironly: try: - # "entry.is_dir()" can raise PermissionError - # in some cases (see bpo-38894), which is not - # among the errors ignored by _ignore_error() if not entry.is_dir(): continue - except OSError as e: - if not _ignore_error(e): - raise + except OSError: continue name = entry.name if self.match(name): path = parent_path._make_child_relpath(name) for p in self.successor._select_from(path, scandir): yield p - except PermissionError: - return class _RecursiveWildcardSelector(_Selector): @@ -175,28 +171,25 @@ def _iterate_directories(self, parent_path, scandir): # avoid exhausting file descriptors when globbing deep trees. with scandir(parent_path) as scandir_it: entries = list(scandir_it) + except OSError: + pass + else: for entry in entries: entry_is_dir = False try: entry_is_dir = entry.is_dir(follow_symlinks=False) - except OSError as e: - if not _ignore_error(e): - raise + except OSError: + pass if entry_is_dir: path = parent_path._make_child_relpath(entry.name) for p in self._iterate_directories(path, scandir): yield p - except PermissionError: - return def _select_from(self, parent_path, scandir): - try: - successor_select = self.successor._select_from - for starting_point in self._iterate_directories(parent_path, scandir): - for p in successor_select(starting_point, scandir): - yield p - except PermissionError: - return + successor_select = self.successor._select_from + for starting_point in self._iterate_directories(parent_path, scandir): + for p in successor_select(starting_point, scandir): + yield p class _DoubleRecursiveWildcardSelector(_RecursiveWildcardSelector): diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 10dd604138a659..67ca4795962b3e 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1949,33 +1949,19 @@ def test_glob_permissions(self): P = self.cls base = P(BASE) / 'permissions' base.mkdir() + self.addCleanup(os_helper.rmtree, base) - file1 = base / "file1" - file1.touch() - file2 = base / "file2" - file2.touch() - - subdir = base / "subdir" - - file3 = base / "file3" - file3.symlink_to(subdir / "other") - - # Patching is needed to avoid relying on the filesystem - # to return the order of the files as the error will not - # happen if the symlink is the last item. - real_scandir = os.scandir - def my_scandir(path): - with real_scandir(path) as scandir_it: - entries = list(scandir_it) - entries.sort(key=lambda entry: entry.name) - return contextlib.nullcontext(entries) - - with mock.patch("os.scandir", my_scandir): - self.assertEqual(len(set(base.glob("*"))), 3) - subdir.mkdir() - self.assertEqual(len(set(base.glob("*"))), 4) - subdir.chmod(000) - self.assertEqual(len(set(base.glob("*"))), 4) + for i in range(100): + link = base / f"link{i}" + if i % 2: + link.symlink_to(P(BASE, "dirE", "nonexistent")) + else: + link.symlink_to(P(BASE, "dirC")) + + self.assertEqual(len(set(base.glob("*"))), 100) + self.assertEqual(len(set(base.glob("*/"))), 50) + self.assertEqual(len(set(base.glob("*/fileC"))), 50) + self.assertEqual(len(set(base.glob("*/file*"))), 50) @os_helper.skip_unless_symlink def test_glob_long_symlink(self): diff --git a/Misc/NEWS.d/next/Library/2023-05-03-19-22-24.gh-issue-90208.tI00da.rst b/Misc/NEWS.d/next/Library/2023-05-03-19-22-24.gh-issue-90208.tI00da.rst new file mode 100644 index 00000000000000..1fd9588bebd5d8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-03-19-22-24.gh-issue-90208.tI00da.rst @@ -0,0 +1,4 @@ +Fixed issue where :meth:`pathlib.Path.glob` returned incomplete results when +it encountered a :exc:`PermissionError`. This method now suppresses all +:exc:`OSError` exceptions, except those raised from calling +:meth:`~pathlib.Path.is_dir` on the top-level path. From fcd5fb49b1d71165f3c503c3d2e74a082ddb2f21 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Wed, 10 May 2023 19:08:40 -0600 Subject: [PATCH 147/204] gh-104357: fix inlined comprehensions that close over iteration var (#104368) --- Lib/test/test_listcomps.py | 10 ++++++++++ Python/symtable.c | 19 +++++++++++++------ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 92fed98dd0004a..1cc202bb599ae6 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -163,6 +163,16 @@ def test_inner_cell_shadows_outer(self): outputs = {"y": [4, 4, 4, 4, 4], "i": 20} self._check_in_scopes(code, outputs) + def test_inner_cell_shadows_outer_no_store(self): + code = """ + def f(x): + return [lambda: x for x in range(x)], x + fns, x = f(2) + y = [fn() for fn in fns] + """ + outputs = {"y": [1, 1], "x": 2} + self._check_in_scopes(code, outputs) + def test_closure_can_jump_over_comp_scope(self): code = """ items = [(lambda: y) for i in range(5)] diff --git a/Python/symtable.c b/Python/symtable.c index 6e74d764245a57..9361674bf16594 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -607,12 +607,19 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, SET_SCOPE(scopes, k, scope); } else { - // free vars in comprehension that are locals in outer scope can - // now simply be locals, unless they are free in comp children - if ((PyLong_AsLong(existing) & DEF_BOUND) && - !is_free_in_any_child(comp, k)) { - if (PySet_Discard(comp_free, k) < 0) { - return 0; + if (PyLong_AsLong(existing) & DEF_BOUND) { + // cell vars in comprehension that are locals in outer scope + // must be promoted to cell so u_cellvars isn't wrong + if (scope == CELL && ste->ste_type == FunctionBlock) { + SET_SCOPE(scopes, k, scope); + } + + // free vars in comprehension that are locals in outer scope can + // now simply be locals, unless they are free in comp children + if (!is_free_in_any_child(comp, k)) { + if (PySet_Discard(comp_free, k) < 0) { + return 0; + } } } } From 4abfe6a14b5be5decbaa3142d9e2549cf2d86c34 Mon Sep 17 00:00:00 2001 From: Carey Metcalfe Date: Thu, 11 May 2023 01:25:16 -0600 Subject: [PATCH 148/204] GH-92184: Convert os.altsep to '/' in filenames when creating ZipInfo objects (#92185) This causes the zipfile module to also consider the character defined by `os.altsep` (if there is one) to be a path separator and convert it to a forward slash, as defined by the zip specification. A logical no-op on all known platforms today as os.altsep is currently only set to a meaningful value on Windows (where it is "/"). --- Lib/zipfile/__init__.py | 2 ++ .../next/Library/2022-05-02-16-21-05.gh-issue-92184.hneGVW.rst | 3 +++ 2 files changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2022-05-02-16-21-05.gh-issue-92184.hneGVW.rst diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 95c047991f872b..116b939e55fe70 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -352,6 +352,8 @@ def _sanitize_filename(filename): # ZIP format specification. if os.sep != "/" and os.sep in filename: filename = filename.replace(os.sep, "/") + if os.altsep and os.altsep != "/" and os.altsep in filename: + filename = filename.replace(os.altsep, "/") return filename diff --git a/Misc/NEWS.d/next/Library/2022-05-02-16-21-05.gh-issue-92184.hneGVW.rst b/Misc/NEWS.d/next/Library/2022-05-02-16-21-05.gh-issue-92184.hneGVW.rst new file mode 100644 index 00000000000000..65dbdc9f60371e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-05-02-16-21-05.gh-issue-92184.hneGVW.rst @@ -0,0 +1,3 @@ +When creating zip files using :mod:`zipfile`, ``os.altsep``, if not ``None``, +will always be treated as a path separator even when it is not ``/``. +Patch by Carey Metcalfe. From 7470321f8171dce96a604ba2c24176837f999ca0 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 11 May 2023 10:44:31 +0200 Subject: [PATCH 149/204] gh-101117: Improve accuracy of sqlite3.Cursor.rowcount docs (#104287) The SQLite C API sqlite3_changes() can only be relied upon when the current active statement has been run to completion. --- Doc/library/sqlite3.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 89673b8006ae77..e7129fb3e4de6d 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1694,7 +1694,10 @@ Cursor objects ``INSERT``, ``UPDATE``, ``DELETE``, and ``REPLACE`` statements; is ``-1`` for other statements, including :abbr:`CTE (Common Table Expression)` queries. - It is only updated by the :meth:`execute` and :meth:`executemany` methods. + It is only updated by the :meth:`execute` and :meth:`executemany` methods, + after the statement has run to completion. + This means that any resulting rows must be fetched in order for + :attr:`!rowcount` to be updated. .. attribute:: row_factory From 7dabb35f833b50bdca0afc1e065b44eb3b7954c8 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 11 May 2023 12:19:22 +0200 Subject: [PATCH 150/204] gh-101819: Adapt _io.TextIOBase methods to Argument Clinic (#104383) --- Modules/_io/clinic/textio.c.h | 167 +++++++++++++++++++++++++++++++++- Modules/_io/textio.c | 106 +++++++++++++-------- 2 files changed, 233 insertions(+), 40 deletions(-) diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index db968e884cc805..01965013ec6abc 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -8,6 +8,171 @@ preserve #endif +PyDoc_STRVAR(_io__TextIOBase_detach__doc__, +"detach($self, /)\n" +"--\n" +"\n" +"Separate the underlying buffer from the TextIOBase and return it.\n" +"\n" +"After the underlying buffer has been detached, the TextIO is in an unusable state."); + +#define _IO__TEXTIOBASE_DETACH_METHODDEF \ + {"detach", _PyCFunction_CAST(_io__TextIOBase_detach), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__TextIOBase_detach__doc__}, + +static PyObject * +_io__TextIOBase_detach_impl(PyObject *self, PyTypeObject *cls); + +static PyObject * +_io__TextIOBase_detach(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs) { + PyErr_SetString(PyExc_TypeError, "detach() takes no arguments"); + return NULL; + } + return _io__TextIOBase_detach_impl(self, cls); +} + +PyDoc_STRVAR(_io__TextIOBase_read__doc__, +"read($self, /, *args)\n" +"--\n" +"\n" +"Read at most size characters from stream.\n" +"\n" +"Read from underlying buffer until we have size characters or we hit EOF.\n" +"If size is negative or omitted, read until EOF."); + +#define _IO__TEXTIOBASE_READ_METHODDEF \ + {"read", _PyCFunction_CAST(_io__TextIOBase_read), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__TextIOBase_read__doc__}, + +static PyObject * +_io__TextIOBase_read_impl(PyObject *self, PyTypeObject *cls, PyObject *args); + +static PyObject * +_io__TextIOBase_read(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = { NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "read", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *__clinic_args = NULL; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); + if (!args) { + goto exit; + } + __clinic_args = args[0]; + return_value = _io__TextIOBase_read_impl(self, cls, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(_io__TextIOBase_readline__doc__, +"readline($self, /, *args)\n" +"--\n" +"\n" +"Read until newline or EOF.\n" +"\n" +"Return an empty string if EOF is hit immediately."); + +#define _IO__TEXTIOBASE_READLINE_METHODDEF \ + {"readline", _PyCFunction_CAST(_io__TextIOBase_readline), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__TextIOBase_readline__doc__}, + +static PyObject * +_io__TextIOBase_readline_impl(PyObject *self, PyTypeObject *cls, + PyObject *args); + +static PyObject * +_io__TextIOBase_readline(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = { NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "readline", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *__clinic_args = NULL; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); + if (!args) { + goto exit; + } + __clinic_args = args[0]; + return_value = _io__TextIOBase_readline_impl(self, cls, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(_io__TextIOBase_write__doc__, +"write($self, /, *args)\n" +"--\n" +"\n" +"Write string to stream.\n" +"\n" +"Return the number of characters written\n" +"(which is always equal to the length of the string)."); + +#define _IO__TEXTIOBASE_WRITE_METHODDEF \ + {"write", _PyCFunction_CAST(_io__TextIOBase_write), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__TextIOBase_write__doc__}, + +static PyObject * +_io__TextIOBase_write_impl(PyObject *self, PyTypeObject *cls, PyObject *args); + +static PyObject * +_io__TextIOBase_write(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = { NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "write", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *__clinic_args = NULL; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); + if (!args) { + goto exit; + } + __clinic_args = args[0]; + return_value = _io__TextIOBase_write_impl(self, cls, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + PyDoc_STRVAR(_io_IncrementalNewlineDecoder___init____doc__, "IncrementalNewlineDecoder(decoder, translate, errors=\'strict\')\n" "--\n" @@ -769,4 +934,4 @@ _io_TextIOWrapper_close(textio *self, PyObject *Py_UNUSED(ignored)) { return _io_TextIOWrapper_close_impl(self); } -/*[clinic end generated code: output=73f84b13c343b34b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d800e5a8a50d6720 input=a9049054013a1b77]*/ diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 070687a83d1bc2..96f5e0c45f51b8 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -20,8 +20,16 @@ module _io class _io.IncrementalNewlineDecoder "nldecoder_object *" "clinic_state()->PyIncrementalNewlineDecoder_Type" class _io.TextIOWrapper "textio *" "clinic_state()->TextIOWrapper_Type" +class _io._TextIOBase "PyObject *" "&PyTextIOBase_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=81f67cf54eaa6001]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=8b7f24fa13bfdd7f]*/ + +typedef struct nldecoder_object nldecoder_object; +typedef struct textio textio; + +#define clinic_state() (find_io_state_by_def(Py_TYPE(self))) +#include "clinic/textio.c.h" +#undef clinic_state /* TextIOBase */ @@ -42,52 +50,76 @@ _unsupported(const char *message) return NULL; } -PyDoc_STRVAR(textiobase_detach_doc, - "Separate the underlying buffer from the TextIOBase and return it.\n" - "\n" - "After the underlying buffer has been detached, the TextIO is in an\n" - "unusable state.\n" - ); +/*[clinic input] +_io._TextIOBase.detach + cls: defining_class + / + +Separate the underlying buffer from the TextIOBase and return it. + +After the underlying buffer has been detached, the TextIO is in an unusable state. +[clinic start generated code]*/ static PyObject * -textiobase_detach(PyObject *self, PyObject *Py_UNUSED(ignored)) +_io__TextIOBase_detach_impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=50915f40c609eaa4 input=987ca3640d0a3776]*/ { return _unsupported("detach"); } -PyDoc_STRVAR(textiobase_read_doc, - "Read at most size characters from stream.\n" - "\n" - "Read from underlying buffer until we have size characters or we hit EOF.\n" - "If size is negative or omitted, read until EOF.\n" - ); +/*[clinic input] +_io._TextIOBase.read + cls: defining_class + / + *args: object + +Read at most size characters from stream. + +Read from underlying buffer until we have size characters or we hit EOF. +If size is negative or omitted, read until EOF. +[clinic start generated code]*/ static PyObject * -textiobase_read(PyObject *self, PyObject *args) +_io__TextIOBase_read_impl(PyObject *self, PyTypeObject *cls, PyObject *args) +/*[clinic end generated code: output=3adf28998831f461 input=cee1e84664a20de0]*/ { return _unsupported("read"); } -PyDoc_STRVAR(textiobase_readline_doc, - "Read until newline or EOF.\n" - "\n" - "Returns an empty string if EOF is hit immediately.\n" - ); +/*[clinic input] +_io._TextIOBase.readline + cls: defining_class + / + *args: object + +Read until newline or EOF. + +Return an empty string if EOF is hit immediately. +[clinic start generated code]*/ static PyObject * -textiobase_readline(PyObject *self, PyObject *args) +_io__TextIOBase_readline_impl(PyObject *self, PyTypeObject *cls, + PyObject *args) +/*[clinic end generated code: output=3073a948d02319f3 input=58f801259f7ff3ef]*/ { return _unsupported("readline"); } -PyDoc_STRVAR(textiobase_write_doc, - "Write string to stream.\n" - "Returns the number of characters written (which is always equal to\n" - "the length of the string).\n" - ); +/*[clinic input] +_io._TextIOBase.write + cls: defining_class + / + *args: object + +Write string to stream. + +Return the number of characters written +(which is always equal to the length of the string). +[clinic start generated code]*/ static PyObject * -textiobase_write(PyObject *self, PyObject *args) +_io__TextIOBase_write_impl(PyObject *self, PyTypeObject *cls, PyObject *args) +/*[clinic end generated code: output=5d985eb529472bc4 input=21b6961b5cba9496]*/ { return _unsupported("write"); } @@ -132,10 +164,10 @@ textiobase_errors_get(PyObject *self, void *context) static PyMethodDef textiobase_methods[] = { - {"detach", textiobase_detach, METH_NOARGS, textiobase_detach_doc}, - {"read", textiobase_read, METH_VARARGS, textiobase_read_doc}, - {"readline", textiobase_readline, METH_VARARGS, textiobase_readline_doc}, - {"write", textiobase_write, METH_VARARGS, textiobase_write_doc}, + _IO__TEXTIOBASE_DETACH_METHODDEF + _IO__TEXTIOBASE_READ_METHODDEF + _IO__TEXTIOBASE_READLINE_METHODDEF + _IO__TEXTIOBASE_WRITE_METHODDEF {NULL, NULL} }; @@ -200,14 +232,14 @@ PyTypeObject PyTextIOBase_Type = { /* IncrementalNewlineDecoder */ -typedef struct { +struct nldecoder_object { PyObject_HEAD PyObject *decoder; PyObject *errors; unsigned int pendingcr: 1; unsigned int translate: 1; unsigned int seennl: 3; -} nldecoder_object; +}; /*[clinic input] _io.IncrementalNewlineDecoder.__init__ @@ -645,7 +677,7 @@ incrementalnewlinedecoder_newlines_get(nldecoder_object *self, void *context) typedef PyObject * (*encodefunc_t)(PyObject *, PyObject *); -typedef struct +struct textio { PyObject_HEAD int ok; /* initialized? */ @@ -704,7 +736,7 @@ typedef struct PyObject *dict; _PyIO_State *state; -} textio; +}; static void textiowrapper_set_decoded_chars(textio *self, PyObject *chars); @@ -3179,10 +3211,6 @@ textiowrapper_chunk_size_set(textio *self, PyObject *arg, void *context) return 0; } -#define clinic_state() (find_io_state_by_def(Py_TYPE(self))) -#include "clinic/textio.c.h" -#undef clinic_state - static PyMethodDef incrementalnewlinedecoder_methods[] = { _IO_INCREMENTALNEWLINEDECODER_DECODE_METHODDEF _IO_INCREMENTALNEWLINEDECODER_GETSTATE_METHODDEF From d0a738c6df6f3f2d9e82e48733c64d9fbd810043 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 11 May 2023 12:26:30 +0200 Subject: [PATCH 151/204] gh-101819: Refactor `_io` futher in preparation for module isolation (#104369) --- Modules/_io/clinic/fileio.c.h | 122 +++++++++++++++++++++------- Modules/_io/clinic/winconsoleio.c.h | 108 ++++++++++++++++++------ Modules/_io/fileio.c | 53 +++++++----- Modules/_io/textio.c | 41 ++++++---- Modules/_io/winconsoleio.c | 60 ++++++++------ 5 files changed, 268 insertions(+), 116 deletions(-) diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index dfad8a58c4723e..33a37a389d223d 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -215,27 +215,45 @@ PyDoc_STRVAR(_io_FileIO_readinto__doc__, "Same as RawIOBase.readinto()."); #define _IO_FILEIO_READINTO_METHODDEF \ - {"readinto", (PyCFunction)_io_FileIO_readinto, METH_O, _io_FileIO_readinto__doc__}, + {"readinto", _PyCFunction_CAST(_io_FileIO_readinto), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_readinto__doc__}, static PyObject * -_io_FileIO_readinto_impl(fileio *self, Py_buffer *buffer); +_io_FileIO_readinto_impl(fileio *self, PyTypeObject *cls, Py_buffer *buffer); static PyObject * -_io_FileIO_readinto(fileio *self, PyObject *arg) +_io_FileIO_readinto(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "readinto", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; Py_buffer buffer = {NULL, NULL}; - if (PyObject_GetBuffer(arg, &buffer, PyBUF_WRITABLE) < 0) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (PyObject_GetBuffer(args[0], &buffer, PyBUF_WRITABLE) < 0) { PyErr_Clear(); - _PyArg_BadArgument("readinto", "argument", "read-write bytes-like object", arg); + _PyArg_BadArgument("readinto", "argument 1", "read-write bytes-like object", args[0]); goto exit; } if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("readinto", "argument", "contiguous buffer", arg); + _PyArg_BadArgument("readinto", "argument 1", "contiguous buffer", args[0]); goto exit; } - return_value = _io_FileIO_readinto_impl(self, &buffer); + return_value = _io_FileIO_readinto_impl(self, cls, &buffer); exit: /* Cleanup for buffer */ @@ -278,28 +296,43 @@ PyDoc_STRVAR(_io_FileIO_read__doc__, "Return an empty bytes object at EOF."); #define _IO_FILEIO_READ_METHODDEF \ - {"read", _PyCFunction_CAST(_io_FileIO_read), METH_FASTCALL, _io_FileIO_read__doc__}, + {"read", _PyCFunction_CAST(_io_FileIO_read), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_read__doc__}, static PyObject * -_io_FileIO_read_impl(fileio *self, Py_ssize_t size); +_io_FileIO_read_impl(fileio *self, PyTypeObject *cls, Py_ssize_t size); static PyObject * -_io_FileIO_read(fileio *self, PyObject *const *args, Py_ssize_t nargs) +_io_FileIO_read(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "read", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; Py_ssize_t size = -1; - if (!_PyArg_CheckPositional("read", nargs, 0, 1)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { goto exit; } if (nargs < 1) { - goto skip_optional; + goto skip_optional_posonly; } if (!_Py_convert_optional_to_ssize_t(args[0], &size)) { goto exit; } -skip_optional: - return_value = _io_FileIO_read_impl(self, size); +skip_optional_posonly: + return_value = _io_FileIO_read_impl(self, cls, size); exit: return return_value; @@ -316,25 +349,43 @@ PyDoc_STRVAR(_io_FileIO_write__doc__, "returns None if the write would block."); #define _IO_FILEIO_WRITE_METHODDEF \ - {"write", (PyCFunction)_io_FileIO_write, METH_O, _io_FileIO_write__doc__}, + {"write", _PyCFunction_CAST(_io_FileIO_write), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_write__doc__}, static PyObject * -_io_FileIO_write_impl(fileio *self, Py_buffer *b); +_io_FileIO_write_impl(fileio *self, PyTypeObject *cls, Py_buffer *b); static PyObject * -_io_FileIO_write(fileio *self, PyObject *arg) +_io_FileIO_write(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "write", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; Py_buffer b = {NULL, NULL}; - if (PyObject_GetBuffer(arg, &b, PyBUF_SIMPLE) != 0) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (PyObject_GetBuffer(args[0], &b, PyBUF_SIMPLE) != 0) { goto exit; } if (!PyBuffer_IsContiguous(&b, 'C')) { - _PyArg_BadArgument("write", "argument", "contiguous buffer", arg); + _PyArg_BadArgument("write", "argument 1", "contiguous buffer", args[0]); goto exit; } - return_value = _io_FileIO_write_impl(self, &b); + return_value = _io_FileIO_write_impl(self, cls, &b); exit: /* Cleanup for b */ @@ -422,26 +473,41 @@ PyDoc_STRVAR(_io_FileIO_truncate__doc__, "The current file position is changed to the value of size."); #define _IO_FILEIO_TRUNCATE_METHODDEF \ - {"truncate", _PyCFunction_CAST(_io_FileIO_truncate), METH_FASTCALL, _io_FileIO_truncate__doc__}, + {"truncate", _PyCFunction_CAST(_io_FileIO_truncate), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_truncate__doc__}, static PyObject * -_io_FileIO_truncate_impl(fileio *self, PyObject *posobj); +_io_FileIO_truncate_impl(fileio *self, PyTypeObject *cls, PyObject *posobj); static PyObject * -_io_FileIO_truncate(fileio *self, PyObject *const *args, Py_ssize_t nargs) +_io_FileIO_truncate(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "truncate", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; PyObject *posobj = Py_None; - if (!_PyArg_CheckPositional("truncate", nargs, 0, 1)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { goto exit; } if (nargs < 1) { - goto skip_optional; + goto skip_optional_posonly; } posobj = args[0]; -skip_optional: - return_value = _io_FileIO_truncate_impl(self, posobj); +skip_optional_posonly: + return_value = _io_FileIO_truncate_impl(self, cls, posobj); exit: return return_value; @@ -470,4 +536,4 @@ _io_FileIO_isatty(fileio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=29ed2ae6c451c139 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bef47b31b644996a input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index 4c5cd0892c4a6d..cd3348dc1227cd 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -20,15 +20,19 @@ PyDoc_STRVAR(_io__WindowsConsoleIO_close__doc__, "close() may be called more than once without error."); #define _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF \ - {"close", (PyCFunction)_io__WindowsConsoleIO_close, METH_NOARGS, _io__WindowsConsoleIO_close__doc__}, + {"close", _PyCFunction_CAST(_io__WindowsConsoleIO_close), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__WindowsConsoleIO_close__doc__}, static PyObject * -_io__WindowsConsoleIO_close_impl(winconsoleio *self); +_io__WindowsConsoleIO_close_impl(winconsoleio *self, PyTypeObject *cls); static PyObject * -_io__WindowsConsoleIO_close(winconsoleio *self, PyObject *Py_UNUSED(ignored)) +_io__WindowsConsoleIO_close(winconsoleio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _io__WindowsConsoleIO_close_impl(self); + if (nargs) { + PyErr_SetString(PyExc_TypeError, "close() takes no arguments"); + return NULL; + } + return _io__WindowsConsoleIO_close_impl(self, cls); } #endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ @@ -208,27 +212,46 @@ PyDoc_STRVAR(_io__WindowsConsoleIO_readinto__doc__, "Same as RawIOBase.readinto()."); #define _IO__WINDOWSCONSOLEIO_READINTO_METHODDEF \ - {"readinto", (PyCFunction)_io__WindowsConsoleIO_readinto, METH_O, _io__WindowsConsoleIO_readinto__doc__}, + {"readinto", _PyCFunction_CAST(_io__WindowsConsoleIO_readinto), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__WindowsConsoleIO_readinto__doc__}, static PyObject * -_io__WindowsConsoleIO_readinto_impl(winconsoleio *self, Py_buffer *buffer); +_io__WindowsConsoleIO_readinto_impl(winconsoleio *self, PyTypeObject *cls, + Py_buffer *buffer); static PyObject * -_io__WindowsConsoleIO_readinto(winconsoleio *self, PyObject *arg) +_io__WindowsConsoleIO_readinto(winconsoleio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "readinto", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; Py_buffer buffer = {NULL, NULL}; - if (PyObject_GetBuffer(arg, &buffer, PyBUF_WRITABLE) < 0) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (PyObject_GetBuffer(args[0], &buffer, PyBUF_WRITABLE) < 0) { PyErr_Clear(); - _PyArg_BadArgument("readinto", "argument", "read-write bytes-like object", arg); + _PyArg_BadArgument("readinto", "argument 1", "read-write bytes-like object", args[0]); goto exit; } if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("readinto", "argument", "contiguous buffer", arg); + _PyArg_BadArgument("readinto", "argument 1", "contiguous buffer", args[0]); goto exit; } - return_value = _io__WindowsConsoleIO_readinto_impl(self, &buffer); + return_value = _io__WindowsConsoleIO_readinto_impl(self, cls, &buffer); exit: /* Cleanup for buffer */ @@ -278,28 +301,44 @@ PyDoc_STRVAR(_io__WindowsConsoleIO_read__doc__, "Return an empty bytes object at EOF."); #define _IO__WINDOWSCONSOLEIO_READ_METHODDEF \ - {"read", _PyCFunction_CAST(_io__WindowsConsoleIO_read), METH_FASTCALL, _io__WindowsConsoleIO_read__doc__}, + {"read", _PyCFunction_CAST(_io__WindowsConsoleIO_read), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__WindowsConsoleIO_read__doc__}, static PyObject * -_io__WindowsConsoleIO_read_impl(winconsoleio *self, Py_ssize_t size); +_io__WindowsConsoleIO_read_impl(winconsoleio *self, PyTypeObject *cls, + Py_ssize_t size); static PyObject * -_io__WindowsConsoleIO_read(winconsoleio *self, PyObject *const *args, Py_ssize_t nargs) +_io__WindowsConsoleIO_read(winconsoleio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "read", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; Py_ssize_t size = -1; - if (!_PyArg_CheckPositional("read", nargs, 0, 1)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { goto exit; } if (nargs < 1) { - goto skip_optional; + goto skip_optional_posonly; } if (!_Py_convert_optional_to_ssize_t(args[0], &size)) { goto exit; } -skip_optional: - return_value = _io__WindowsConsoleIO_read_impl(self, size); +skip_optional_posonly: + return_value = _io__WindowsConsoleIO_read_impl(self, cls, size); exit: return return_value; @@ -319,25 +358,44 @@ PyDoc_STRVAR(_io__WindowsConsoleIO_write__doc__, "The number of bytes actually written is returned."); #define _IO__WINDOWSCONSOLEIO_WRITE_METHODDEF \ - {"write", (PyCFunction)_io__WindowsConsoleIO_write, METH_O, _io__WindowsConsoleIO_write__doc__}, + {"write", _PyCFunction_CAST(_io__WindowsConsoleIO_write), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__WindowsConsoleIO_write__doc__}, static PyObject * -_io__WindowsConsoleIO_write_impl(winconsoleio *self, Py_buffer *b); +_io__WindowsConsoleIO_write_impl(winconsoleio *self, PyTypeObject *cls, + Py_buffer *b); static PyObject * -_io__WindowsConsoleIO_write(winconsoleio *self, PyObject *arg) +_io__WindowsConsoleIO_write(winconsoleio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "write", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; Py_buffer b = {NULL, NULL}; - if (PyObject_GetBuffer(arg, &b, PyBUF_SIMPLE) != 0) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (PyObject_GetBuffer(args[0], &b, PyBUF_SIMPLE) != 0) { goto exit; } if (!PyBuffer_IsContiguous(&b, 'C')) { - _PyArg_BadArgument("write", "argument", "contiguous buffer", arg); + _PyArg_BadArgument("write", "argument 1", "contiguous buffer", args[0]); goto exit; } - return_value = _io__WindowsConsoleIO_write_impl(self, &b); + return_value = _io__WindowsConsoleIO_write_impl(self, cls, &b); exit: /* Cleanup for b */ @@ -407,4 +465,4 @@ _io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ -/*[clinic end generated code: output=163e934aa9b0ef16 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=235393758365c229 input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index cc0e7307b9da77..473f0a8a6befcb 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -549,13 +549,10 @@ err_closed(void) } static PyObject * -err_mode(const char *action) +err_mode(_PyIO_State *state, const char *action) { - _PyIO_State *state = IO_STATE(); - if (state != NULL) - PyErr_Format(state->unsupported_operation, - "File not open for %s", action); - return NULL; + return PyErr_Format(state->unsupported_operation, + "File not open for %s", action); } /*[clinic input] @@ -631,6 +628,7 @@ _io_FileIO_seekable_impl(fileio *self) /*[clinic input] _io.FileIO.readinto + cls: defining_class buffer: Py_buffer(accept={rwbuffer}) / @@ -638,16 +636,18 @@ Same as RawIOBase.readinto(). [clinic start generated code]*/ static PyObject * -_io_FileIO_readinto_impl(fileio *self, Py_buffer *buffer) -/*[clinic end generated code: output=b01a5a22c8415cb4 input=4721d7b68b154eaf]*/ +_io_FileIO_readinto_impl(fileio *self, PyTypeObject *cls, Py_buffer *buffer) +/*[clinic end generated code: output=97f0f3d69534db34 input=fd20323e18ce1ec8]*/ { Py_ssize_t n; int err; if (self->fd < 0) return err_closed(); - if (!self->readable) - return err_mode("reading"); + if (!self->readable) { + _PyIO_State *state = get_io_state_by_cls(cls); + return err_mode(state, "reading"); + } n = _Py_read(self->fd, buffer->buf, buffer->len); /* copy errno because PyBuffer_Release() can indirectly modify it */ @@ -784,6 +784,7 @@ _io_FileIO_readall_impl(fileio *self) /*[clinic input] _io.FileIO.read + cls: defining_class size: Py_ssize_t(accept={int, NoneType}) = -1 / @@ -795,8 +796,8 @@ Return an empty bytes object at EOF. [clinic start generated code]*/ static PyObject * -_io_FileIO_read_impl(fileio *self, Py_ssize_t size) -/*[clinic end generated code: output=42528d39dd0ca641 input=bec9a2c704ddcbc9]*/ +_io_FileIO_read_impl(fileio *self, PyTypeObject *cls, Py_ssize_t size) +/*[clinic end generated code: output=bbd749c7c224143e input=f613d2057e4a1918]*/ { char *ptr; Py_ssize_t n; @@ -804,8 +805,10 @@ _io_FileIO_read_impl(fileio *self, Py_ssize_t size) if (self->fd < 0) return err_closed(); - if (!self->readable) - return err_mode("reading"); + if (!self->readable) { + _PyIO_State *state = get_io_state_by_cls(cls); + return err_mode(state, "reading"); + } if (size < 0) return _io_FileIO_readall_impl(self); @@ -843,6 +846,7 @@ _io_FileIO_read_impl(fileio *self, Py_ssize_t size) /*[clinic input] _io.FileIO.write + cls: defining_class b: Py_buffer / @@ -854,16 +858,18 @@ returns None if the write would block. [clinic start generated code]*/ static PyObject * -_io_FileIO_write_impl(fileio *self, Py_buffer *b) -/*[clinic end generated code: output=b4059db3d363a2f7 input=6e7908b36f0ce74f]*/ +_io_FileIO_write_impl(fileio *self, PyTypeObject *cls, Py_buffer *b) +/*[clinic end generated code: output=927e25be80f3b77b input=2776314f043088f5]*/ { Py_ssize_t n; int err; if (self->fd < 0) return err_closed(); - if (!self->writable) - return err_mode("writing"); + if (!self->writable) { + _PyIO_State *state = get_io_state_by_cls(cls); + return err_mode(state, "writing"); + } n = _Py_write(self->fd, b->buf, b->len); /* copy errno because PyBuffer_Release() can indirectly modify it */ @@ -994,6 +1000,7 @@ _io_FileIO_tell_impl(fileio *self) #ifdef HAVE_FTRUNCATE /*[clinic input] _io.FileIO.truncate + cls: defining_class size as posobj: object = None / @@ -1004,8 +1011,8 @@ The current file position is changed to the value of size. [clinic start generated code]*/ static PyObject * -_io_FileIO_truncate_impl(fileio *self, PyObject *posobj) -/*[clinic end generated code: output=e49ca7a916c176fa input=b0ac133939823875]*/ +_io_FileIO_truncate_impl(fileio *self, PyTypeObject *cls, PyObject *posobj) +/*[clinic end generated code: output=d936732a49e8d5a2 input=c367fb45d6bb2c18]*/ { Py_off_t pos; int ret; @@ -1014,8 +1021,10 @@ _io_FileIO_truncate_impl(fileio *self, PyObject *posobj) fd = self->fd; if (fd < 0) return err_closed(); - if (!self->writable) - return err_mode("writing"); + if (!self->writable) { + _PyIO_State *state = get_io_state_by_cls(cls); + return err_mode(state, "writing"); + } if (posobj == Py_None) { /* Get the current position. */ diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 96f5e0c45f51b8..3cc292cc35102e 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -42,11 +42,9 @@ PyDoc_STRVAR(textiobase_doc, ); static PyObject * -_unsupported(const char *message) +_unsupported(_PyIO_State *state, const char *message) { - _PyIO_State *state = IO_STATE(); - if (state != NULL) - PyErr_SetString(state->unsupported_operation, message); + PyErr_SetString(state->unsupported_operation, message); return NULL; } @@ -64,7 +62,8 @@ static PyObject * _io__TextIOBase_detach_impl(PyObject *self, PyTypeObject *cls) /*[clinic end generated code: output=50915f40c609eaa4 input=987ca3640d0a3776]*/ { - return _unsupported("detach"); + _PyIO_State *state = IO_STATE(); + return _unsupported(state, "detach"); } /*[clinic input] @@ -83,7 +82,8 @@ static PyObject * _io__TextIOBase_read_impl(PyObject *self, PyTypeObject *cls, PyObject *args) /*[clinic end generated code: output=3adf28998831f461 input=cee1e84664a20de0]*/ { - return _unsupported("read"); + _PyIO_State *state = IO_STATE(); + return _unsupported(state, "read"); } /*[clinic input] @@ -102,7 +102,8 @@ _io__TextIOBase_readline_impl(PyObject *self, PyTypeObject *cls, PyObject *args) /*[clinic end generated code: output=3073a948d02319f3 input=58f801259f7ff3ef]*/ { - return _unsupported("readline"); + _PyIO_State *state = IO_STATE(); + return _unsupported(state, "readline"); } /*[clinic input] @@ -121,7 +122,8 @@ static PyObject * _io__TextIOBase_write_impl(PyObject *self, PyTypeObject *cls, PyObject *args) /*[clinic end generated code: output=5d985eb529472bc4 input=21b6961b5cba9496]*/ { - return _unsupported("write"); + _PyIO_State *state = IO_STATE(); + return _unsupported(state, "write"); } PyDoc_STRVAR(textiobase_encoding_doc, @@ -1381,7 +1383,8 @@ _io_TextIOWrapper_reconfigure_impl(textio *self, PyObject *encoding, /* Check if something is in the read buffer */ if (self->decoded_chars != NULL) { if (encoding != Py_None || errors != Py_None || newline_obj != NULL) { - _unsupported("It is not possible to set the encoding or newline " + _unsupported(self->state, + "It is not possible to set the encoding or newline " "of stream after the first read"); return NULL; } @@ -1648,8 +1651,9 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text) CHECK_ATTACHED(self); CHECK_CLOSED(self); - if (self->encoder == NULL) - return _unsupported("not writable"); + if (self->encoder == NULL) { + return _unsupported(self->state, "not writable"); + } Py_INCREF(text); @@ -1830,7 +1834,7 @@ textiowrapper_read_chunk(textio *self, Py_ssize_t size_hint) */ if (self->decoder == NULL) { - _unsupported("not readable"); + _unsupported(self->state, "not readable"); return -1; } @@ -1955,8 +1959,9 @@ _io_TextIOWrapper_read_impl(textio *self, Py_ssize_t n) CHECK_ATTACHED(self); CHECK_CLOSED(self); - if (self->decoder == NULL) - return _unsupported("not readable"); + if (self->decoder == NULL) { + return _unsupported(self->state, "not readable"); + } if (_textiowrapper_writeflush(self) < 0) return NULL; @@ -2487,7 +2492,7 @@ _io_TextIOWrapper_seek_impl(textio *self, PyObject *cookieObj, int whence) Py_INCREF(cookieObj); if (!self->seekable) { - _unsupported("underlying stream is not seekable"); + _unsupported(self->state, "underlying stream is not seekable"); goto fail; } @@ -2501,7 +2506,7 @@ _io_TextIOWrapper_seek_impl(textio *self, PyObject *cookieObj, int whence) goto fail; if (cmp == 0) { - _unsupported("can't do nonzero cur-relative seeks"); + _unsupported(self->state, "can't do nonzero cur-relative seeks"); goto fail; } @@ -2521,7 +2526,7 @@ _io_TextIOWrapper_seek_impl(textio *self, PyObject *cookieObj, int whence) goto fail; if (cmp == 0) { - _unsupported("can't do nonzero end-relative seeks"); + _unsupported(self->state, "can't do nonzero end-relative seeks"); goto fail; } @@ -2684,7 +2689,7 @@ _io_TextIOWrapper_tell_impl(textio *self) CHECK_CLOSED(self); if (!self->seekable) { - _unsupported("underlying stream is not seekable"); + _unsupported(self->state, "underlying stream is not seekable"); goto fail; } if (!self->telling) { diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 58d9f2963aa93c..f341cb238ce85e 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -180,6 +180,8 @@ internal_close(winconsoleio *self) /*[clinic input] _io._WindowsConsoleIO.close + cls: defining_class + / Close the console object. @@ -188,13 +190,15 @@ close() may be called more than once without error. [clinic start generated code]*/ static PyObject * -_io__WindowsConsoleIO_close_impl(winconsoleio *self) -/*[clinic end generated code: output=27ef95b66c29057b input=68c4e5754f8136c2]*/ +_io__WindowsConsoleIO_close_impl(winconsoleio *self, PyTypeObject *cls) +/*[clinic end generated code: output=e50c1808c063e1e2 input=161001bd2a649a4b]*/ { PyObject *res; PyObject *exc; int rc; - res = PyObject_CallMethodOneArg((PyObject*)&PyRawIOBase_Type, + + _PyIO_State *state = get_io_state_by_cls(cls); + res = PyObject_CallMethodOneArg((PyObject*)state->PyRawIOBase_Type, &_Py_ID(close), (PyObject*)self); if (!self->closefd) { self->fd = -1; @@ -453,13 +457,10 @@ err_closed(void) } static PyObject * -err_mode(const char *action) +err_mode(_PyIO_State *state, const char *action) { - _PyIO_State *state = IO_STATE(); - if (state != NULL) - PyErr_Format(state->unsupported_operation, - "Console buffer does not support %s", action); - return NULL; + return PyErr_Format(state->unsupported_operation, + "Console buffer does not support %s", action); } /*[clinic input] @@ -636,14 +637,14 @@ read_console_w(HANDLE handle, DWORD maxlen, DWORD *readlen) { static Py_ssize_t -readinto(winconsoleio *self, char *buf, Py_ssize_t len) +readinto(_PyIO_State *state, winconsoleio *self, char *buf, Py_ssize_t len) { if (self->fd == -1) { err_closed(); return -1; } if (!self->readable) { - err_mode("reading"); + err_mode(state, "reading"); return -1; } if (len == 0) @@ -733,6 +734,7 @@ readinto(winconsoleio *self, char *buf, Py_ssize_t len) /*[clinic input] _io._WindowsConsoleIO.readinto + cls: defining_class buffer: Py_buffer(accept={rwbuffer}) / @@ -740,10 +742,12 @@ Same as RawIOBase.readinto(). [clinic start generated code]*/ static PyObject * -_io__WindowsConsoleIO_readinto_impl(winconsoleio *self, Py_buffer *buffer) -/*[clinic end generated code: output=66d1bdfa3f20af39 input=4ed68da48a6baffe]*/ +_io__WindowsConsoleIO_readinto_impl(winconsoleio *self, PyTypeObject *cls, + Py_buffer *buffer) +/*[clinic end generated code: output=96717c74f6204b79 input=4b0627c3b1645f78]*/ { - Py_ssize_t len = readinto(self, buffer->buf, buffer->len); + _PyIO_State *state = get_io_state_by_cls(cls); + Py_ssize_t len = readinto(state, self, buffer->buf, buffer->len); if (len < 0) return NULL; @@ -897,6 +901,7 @@ _io__WindowsConsoleIO_readall_impl(winconsoleio *self) /*[clinic input] _io._WindowsConsoleIO.read + cls: defining_class size: Py_ssize_t(accept={int, NoneType}) = -1 / @@ -908,16 +913,19 @@ Return an empty bytes object at EOF. [clinic start generated code]*/ static PyObject * -_io__WindowsConsoleIO_read_impl(winconsoleio *self, Py_ssize_t size) -/*[clinic end generated code: output=57df68af9f4b22d0 input=8bc73bc15d0fa072]*/ +_io__WindowsConsoleIO_read_impl(winconsoleio *self, PyTypeObject *cls, + Py_ssize_t size) +/*[clinic end generated code: output=7e569a586537c0ae input=a14570a5da273365]*/ { PyObject *bytes; Py_ssize_t bytes_size; if (self->fd == -1) return err_closed(); - if (!self->readable) - return err_mode("reading"); + if (!self->readable) { + _PyIO_State *state = get_io_state_by_cls(cls); + return err_mode(state, "reading"); + } if (size < 0) return _io__WindowsConsoleIO_readall_impl(self); @@ -930,7 +938,9 @@ _io__WindowsConsoleIO_read_impl(winconsoleio *self, Py_ssize_t size) if (bytes == NULL) return NULL; - bytes_size = readinto(self, PyBytes_AS_STRING(bytes), PyBytes_GET_SIZE(bytes)); + _PyIO_State *state = get_io_state_by_cls(cls); + bytes_size = readinto(state, self, PyBytes_AS_STRING(bytes), + PyBytes_GET_SIZE(bytes)); if (bytes_size < 0) { Py_CLEAR(bytes); return NULL; @@ -948,6 +958,7 @@ _io__WindowsConsoleIO_read_impl(winconsoleio *self, Py_ssize_t size) /*[clinic input] _io._WindowsConsoleIO.write + cls: defining_class b: Py_buffer / @@ -958,8 +969,9 @@ The number of bytes actually written is returned. [clinic start generated code]*/ static PyObject * -_io__WindowsConsoleIO_write_impl(winconsoleio *self, Py_buffer *b) -/*[clinic end generated code: output=775bdb16fbf9137b input=be35fb624f97c941]*/ +_io__WindowsConsoleIO_write_impl(winconsoleio *self, PyTypeObject *cls, + Py_buffer *b) +/*[clinic end generated code: output=e8019f480243cb29 input=10ac37c19339dfbe]*/ { BOOL res = TRUE; wchar_t *wbuf; @@ -968,8 +980,10 @@ _io__WindowsConsoleIO_write_impl(winconsoleio *self, Py_buffer *b) if (self->fd == -1) return err_closed(); - if (!self->writable) - return err_mode("writing"); + if (!self->writable) { + _PyIO_State *state = get_io_state_by_cls(cls); + return err_mode(state, "writing"); + } handle = _Py_get_osfhandle(self->fd); if (handle == INVALID_HANDLE_VALUE) From ed41124bb55cba61739e5180fd12d1285c3baa3d Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 11 May 2023 12:29:23 +0200 Subject: [PATCH 152/204] gh-101819: Adapt _io._Buffered* methods to Argument Clinic (#104367) --- Modules/_io/bufferedio.c | 158 ++++++++++++++------- Modules/_io/clinic/bufferedio.c.h | 226 ++++++++++++++++++++++++++++-- 2 files changed, 328 insertions(+), 56 deletions(-) diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index c7ae60281c2f58..00e228bca4375b 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -419,8 +419,13 @@ buffered_dealloc(buffered *self) Py_DECREF(tp); } +/*[clinic input] +_io._Buffered.__sizeof__ +[clinic start generated code]*/ + static PyObject * -buffered_sizeof(buffered *self, PyObject *Py_UNUSED(ignored)) +_io__Buffered___sizeof___impl(buffered *self) +/*[clinic end generated code: output=0231ef7f5053134e input=753c782d808d34df]*/ { size_t res = _PyObject_SIZE(Py_TYPE(self)); if (self->buffer) { @@ -450,8 +455,17 @@ buffered_clear(buffered *self) /* Because this can call arbitrary code, it shouldn't be called when the refcount is 0 (that is, not directly from tp_dealloc unless the refcount has been temporarily re-incremented). */ +/*[clinic input] +_io._Buffered._dealloc_warn + + source: object + / + +[clinic start generated code]*/ + static PyObject * -buffered_dealloc_warn(buffered *self, PyObject *source) +_io__Buffered__dealloc_warn(buffered *self, PyObject *source) +/*[clinic end generated code: output=690dcc3df8967162 input=8f845f2a4786391c]*/ { if (self->ok && self->raw) { PyObject *r; @@ -471,9 +485,13 @@ buffered_dealloc_warn(buffered *self, PyObject *source) */ /* Flush and close */ +/*[clinic input] +_io._Buffered.flush as _io__Buffered_simple_flush +[clinic start generated code]*/ static PyObject * -buffered_simple_flush(buffered *self, PyObject *args) +_io__Buffered_simple_flush_impl(buffered *self) +/*[clinic end generated code: output=29ebb3820db1bdfd input=f33ef045e7250767]*/ { CHECK_INITIALIZED(self) return PyObject_CallMethodNoArgs(self->raw, &_Py_ID(flush)); @@ -500,8 +518,13 @@ buffered_closed_get(buffered *self, void *context) return PyObject_GetAttr(self->raw, &_Py_ID(closed)); } +/*[clinic input] +_io._Buffered.close +[clinic start generated code]*/ + static PyObject * -buffered_close(buffered *self, PyObject *args) +_io__Buffered_close_impl(buffered *self) +/*[clinic end generated code: output=7280b7b42033be0c input=d20b83d1ddd7d805]*/ { PyObject *res = NULL; int r; @@ -520,7 +543,7 @@ buffered_close(buffered *self, PyObject *args) } if (self->finalizing) { - PyObject *r = buffered_dealloc_warn(self, (PyObject *) self); + PyObject *r = _io__Buffered__dealloc_warn(self, (PyObject *) self); if (r) Py_DECREF(r); else @@ -560,10 +583,13 @@ buffered_close(buffered *self, PyObject *args) return res; } -/* detach */ +/*[clinic input] +_io._Buffered.detach +[clinic start generated code]*/ static PyObject * -buffered_detach(buffered *self, PyObject *Py_UNUSED(ignored)) +_io__Buffered_detach_impl(buffered *self) +/*[clinic end generated code: output=dd0fc057b8b779f7 input=482762a345cc9f44]*/ { PyObject *raw, *res; CHECK_INITIALIZED(self) @@ -580,22 +606,37 @@ buffered_detach(buffered *self, PyObject *Py_UNUSED(ignored)) /* Inquiries */ +/*[clinic input] +_io._Buffered.seekable +[clinic start generated code]*/ + static PyObject * -buffered_seekable(buffered *self, PyObject *Py_UNUSED(ignored)) +_io__Buffered_seekable_impl(buffered *self) +/*[clinic end generated code: output=90172abb5ceb6e8f input=7d35764f5fb5262b]*/ { CHECK_INITIALIZED(self) return PyObject_CallMethodNoArgs(self->raw, &_Py_ID(seekable)); } +/*[clinic input] +_io._Buffered.readable +[clinic start generated code]*/ + static PyObject * -buffered_readable(buffered *self, PyObject *Py_UNUSED(ignored)) +_io__Buffered_readable_impl(buffered *self) +/*[clinic end generated code: output=92afa07661ecb698 input=640619addb513b8b]*/ { CHECK_INITIALIZED(self) return PyObject_CallMethodNoArgs(self->raw, &_Py_ID(readable)); } +/*[clinic input] +_io._Buffered.writable +[clinic start generated code]*/ + static PyObject * -buffered_writable(buffered *self, PyObject *Py_UNUSED(ignored)) +_io__Buffered_writable_impl(buffered *self) +/*[clinic end generated code: output=4e3eee8d6f9d8552 input=b35ea396b2201554]*/ { CHECK_INITIALIZED(self) return PyObject_CallMethodNoArgs(self->raw, &_Py_ID(writable)); @@ -617,15 +658,25 @@ buffered_mode_get(buffered *self, void *context) /* Lower-level APIs */ +/*[clinic input] +_io._Buffered.fileno +[clinic start generated code]*/ + static PyObject * -buffered_fileno(buffered *self, PyObject *Py_UNUSED(ignored)) +_io__Buffered_fileno_impl(buffered *self) +/*[clinic end generated code: output=b717648d58a95ee3 input=768ea30b3f6314a7]*/ { CHECK_INITIALIZED(self) return PyObject_CallMethodNoArgs(self->raw, &_Py_ID(fileno)); } +/*[clinic input] +_io._Buffered.isatty +[clinic start generated code]*/ + static PyObject * -buffered_isatty(buffered *self, PyObject *Py_UNUSED(ignored)) +_io__Buffered_isatty_impl(buffered *self) +/*[clinic end generated code: output=c20e55caae67baea input=9ea007b11559bee4]*/ { CHECK_INITIALIZED(self) return PyObject_CallMethodNoArgs(self->raw, &_Py_ID(isatty)); @@ -830,8 +881,13 @@ buffered_flush_and_rewind_unlocked(buffered *self) Py_RETURN_NONE; } +/*[clinic input] +_io._Buffered.flush +[clinic start generated code]*/ + static PyObject * -buffered_flush(buffered *self, PyObject *args) +_io__Buffered_flush_impl(buffered *self) +/*[clinic end generated code: output=da2674ef1ce71f3a input=fda63444697c6bf4]*/ { PyObject *res; @@ -1205,8 +1261,13 @@ _io__Buffered_readline_impl(buffered *self, Py_ssize_t size) } +/*[clinic input] +_io._Buffered.tell +[clinic start generated code]*/ + static PyObject * -buffered_tell(buffered *self, PyObject *Py_UNUSED(ignored)) +_io__Buffered_tell_impl(buffered *self) +/*[clinic end generated code: output=386972ae84716c1e input=ad61e04a6b349573]*/ { Py_off_t pos; @@ -1318,20 +1379,21 @@ _io__Buffered_seek_impl(buffered *self, PyObject *targetobj, int whence) /*[clinic input] _io._Buffered.truncate + cls: defining_class pos: object = None / [clinic start generated code]*/ static PyObject * -_io__Buffered_truncate_impl(buffered *self, PyObject *pos) -/*[clinic end generated code: output=667ca03c60c270de input=8a1be34d57cca2d3]*/ +_io__Buffered_truncate_impl(buffered *self, PyTypeObject *cls, PyObject *pos) +/*[clinic end generated code: output=fe3882fbffe79f1a input=f5b737d97d76303f]*/ { PyObject *res = NULL; CHECK_INITIALIZED(self) CHECK_CLOSED(self, "truncate of closed file") if (!self->writable) { - _PyIO_State *state = IO_STATE(); + _PyIO_State *state = get_io_state_by_cls(cls); return bufferediobase_unsupported(state, "truncate"); } if (!ENTER_BUFFERED(self)) @@ -2427,14 +2489,14 @@ PyTypeObject PyBufferedIOBase_Type = { static PyMethodDef bufferedreader_methods[] = { /* BufferedIOMixin methods */ - {"detach", (PyCFunction)buffered_detach, METH_NOARGS}, - {"flush", (PyCFunction)buffered_simple_flush, METH_NOARGS}, - {"close", (PyCFunction)buffered_close, METH_NOARGS}, - {"seekable", (PyCFunction)buffered_seekable, METH_NOARGS}, - {"readable", (PyCFunction)buffered_readable, METH_NOARGS}, - {"fileno", (PyCFunction)buffered_fileno, METH_NOARGS}, - {"isatty", (PyCFunction)buffered_isatty, METH_NOARGS}, - {"_dealloc_warn", (PyCFunction)buffered_dealloc_warn, METH_O}, + _IO__BUFFERED_DETACH_METHODDEF + _IO__BUFFERED_SIMPLE_FLUSH_METHODDEF + _IO__BUFFERED_CLOSE_METHODDEF + _IO__BUFFERED_SEEKABLE_METHODDEF + _IO__BUFFERED_READABLE_METHODDEF + _IO__BUFFERED_FILENO_METHODDEF + _IO__BUFFERED_ISATTY_METHODDEF + _IO__BUFFERED__DEALLOC_WARN_METHODDEF _IO__BUFFERED_READ_METHODDEF _IO__BUFFERED_PEEK_METHODDEF @@ -2443,9 +2505,9 @@ static PyMethodDef bufferedreader_methods[] = { _IO__BUFFERED_READINTO1_METHODDEF _IO__BUFFERED_READLINE_METHODDEF _IO__BUFFERED_SEEK_METHODDEF - {"tell", (PyCFunction)buffered_tell, METH_NOARGS}, + _IO__BUFFERED_TELL_METHODDEF _IO__BUFFERED_TRUNCATE_METHODDEF - {"__sizeof__", (PyCFunction)buffered_sizeof, METH_NOARGS}, + _IO__BUFFERED___SIZEOF___METHODDEF {NULL, NULL} }; @@ -2489,20 +2551,20 @@ PyType_Spec bufferedreader_spec = { static PyMethodDef bufferedwriter_methods[] = { /* BufferedIOMixin methods */ - {"close", (PyCFunction)buffered_close, METH_NOARGS}, - {"detach", (PyCFunction)buffered_detach, METH_NOARGS}, - {"seekable", (PyCFunction)buffered_seekable, METH_NOARGS}, - {"writable", (PyCFunction)buffered_writable, METH_NOARGS}, - {"fileno", (PyCFunction)buffered_fileno, METH_NOARGS}, - {"isatty", (PyCFunction)buffered_isatty, METH_NOARGS}, - {"_dealloc_warn", (PyCFunction)buffered_dealloc_warn, METH_O}, + _IO__BUFFERED_CLOSE_METHODDEF + _IO__BUFFERED_DETACH_METHODDEF + _IO__BUFFERED_SEEKABLE_METHODDEF + _IO__BUFFERED_WRITABLE_METHODDEF + _IO__BUFFERED_FILENO_METHODDEF + _IO__BUFFERED_ISATTY_METHODDEF + _IO__BUFFERED__DEALLOC_WARN_METHODDEF _IO_BUFFEREDWRITER_WRITE_METHODDEF _IO__BUFFERED_TRUNCATE_METHODDEF - {"flush", (PyCFunction)buffered_flush, METH_NOARGS}, + _IO__BUFFERED_FLUSH_METHODDEF _IO__BUFFERED_SEEK_METHODDEF - {"tell", (PyCFunction)buffered_tell, METH_NOARGS}, - {"__sizeof__", (PyCFunction)buffered_sizeof, METH_NOARGS}, + _IO__BUFFERED_TELL_METHODDEF + _IO__BUFFERED___SIZEOF___METHODDEF {NULL, NULL} }; @@ -2596,19 +2658,19 @@ PyType_Spec bufferedrwpair_spec = { static PyMethodDef bufferedrandom_methods[] = { /* BufferedIOMixin methods */ - {"close", (PyCFunction)buffered_close, METH_NOARGS}, - {"detach", (PyCFunction)buffered_detach, METH_NOARGS}, - {"seekable", (PyCFunction)buffered_seekable, METH_NOARGS}, - {"readable", (PyCFunction)buffered_readable, METH_NOARGS}, - {"writable", (PyCFunction)buffered_writable, METH_NOARGS}, - {"fileno", (PyCFunction)buffered_fileno, METH_NOARGS}, - {"isatty", (PyCFunction)buffered_isatty, METH_NOARGS}, - {"_dealloc_warn", (PyCFunction)buffered_dealloc_warn, METH_O}, + _IO__BUFFERED_CLOSE_METHODDEF + _IO__BUFFERED_DETACH_METHODDEF + _IO__BUFFERED_SEEKABLE_METHODDEF + _IO__BUFFERED_READABLE_METHODDEF + _IO__BUFFERED_WRITABLE_METHODDEF + _IO__BUFFERED_FILENO_METHODDEF + _IO__BUFFERED_ISATTY_METHODDEF + _IO__BUFFERED__DEALLOC_WARN_METHODDEF - {"flush", (PyCFunction)buffered_flush, METH_NOARGS}, + _IO__BUFFERED_FLUSH_METHODDEF _IO__BUFFERED_SEEK_METHODDEF - {"tell", (PyCFunction)buffered_tell, METH_NOARGS}, + _IO__BUFFERED_TELL_METHODDEF _IO__BUFFERED_TRUNCATE_METHODDEF _IO__BUFFERED_READ_METHODDEF _IO__BUFFERED_READ1_METHODDEF @@ -2617,7 +2679,7 @@ static PyMethodDef bufferedrandom_methods[] = { _IO__BUFFERED_READLINE_METHODDEF _IO__BUFFERED_PEEK_METHODDEF _IO_BUFFEREDWRITER_WRITE_METHODDEF - {"__sizeof__", (PyCFunction)buffered_sizeof, METH_NOARGS}, + _IO__BUFFERED___SIZEOF___METHODDEF {NULL, NULL} }; diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index a898b01899babe..e2ac90c54a10f0 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -266,6 +266,184 @@ _io__BufferedIOBase_write(PyObject *self, PyTypeObject *cls, PyObject *const *ar return return_value; } +PyDoc_STRVAR(_io__Buffered___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n"); + +#define _IO__BUFFERED___SIZEOF___METHODDEF \ + {"__sizeof__", (PyCFunction)_io__Buffered___sizeof__, METH_NOARGS, _io__Buffered___sizeof____doc__}, + +static PyObject * +_io__Buffered___sizeof___impl(buffered *self); + +static PyObject * +_io__Buffered___sizeof__(buffered *self, PyObject *Py_UNUSED(ignored)) +{ + return _io__Buffered___sizeof___impl(self); +} + +PyDoc_STRVAR(_io__Buffered__dealloc_warn__doc__, +"_dealloc_warn($self, source, /)\n" +"--\n" +"\n"); + +#define _IO__BUFFERED__DEALLOC_WARN_METHODDEF \ + {"_dealloc_warn", (PyCFunction)_io__Buffered__dealloc_warn, METH_O, _io__Buffered__dealloc_warn__doc__}, + +PyDoc_STRVAR(_io__Buffered_simple_flush__doc__, +"flush($self, /)\n" +"--\n" +"\n"); + +#define _IO__BUFFERED_SIMPLE_FLUSH_METHODDEF \ + {"flush", (PyCFunction)_io__Buffered_simple_flush, METH_NOARGS, _io__Buffered_simple_flush__doc__}, + +static PyObject * +_io__Buffered_simple_flush_impl(buffered *self); + +static PyObject * +_io__Buffered_simple_flush(buffered *self, PyObject *Py_UNUSED(ignored)) +{ + return _io__Buffered_simple_flush_impl(self); +} + +PyDoc_STRVAR(_io__Buffered_close__doc__, +"close($self, /)\n" +"--\n" +"\n"); + +#define _IO__BUFFERED_CLOSE_METHODDEF \ + {"close", (PyCFunction)_io__Buffered_close, METH_NOARGS, _io__Buffered_close__doc__}, + +static PyObject * +_io__Buffered_close_impl(buffered *self); + +static PyObject * +_io__Buffered_close(buffered *self, PyObject *Py_UNUSED(ignored)) +{ + return _io__Buffered_close_impl(self); +} + +PyDoc_STRVAR(_io__Buffered_detach__doc__, +"detach($self, /)\n" +"--\n" +"\n"); + +#define _IO__BUFFERED_DETACH_METHODDEF \ + {"detach", (PyCFunction)_io__Buffered_detach, METH_NOARGS, _io__Buffered_detach__doc__}, + +static PyObject * +_io__Buffered_detach_impl(buffered *self); + +static PyObject * +_io__Buffered_detach(buffered *self, PyObject *Py_UNUSED(ignored)) +{ + return _io__Buffered_detach_impl(self); +} + +PyDoc_STRVAR(_io__Buffered_seekable__doc__, +"seekable($self, /)\n" +"--\n" +"\n"); + +#define _IO__BUFFERED_SEEKABLE_METHODDEF \ + {"seekable", (PyCFunction)_io__Buffered_seekable, METH_NOARGS, _io__Buffered_seekable__doc__}, + +static PyObject * +_io__Buffered_seekable_impl(buffered *self); + +static PyObject * +_io__Buffered_seekable(buffered *self, PyObject *Py_UNUSED(ignored)) +{ + return _io__Buffered_seekable_impl(self); +} + +PyDoc_STRVAR(_io__Buffered_readable__doc__, +"readable($self, /)\n" +"--\n" +"\n"); + +#define _IO__BUFFERED_READABLE_METHODDEF \ + {"readable", (PyCFunction)_io__Buffered_readable, METH_NOARGS, _io__Buffered_readable__doc__}, + +static PyObject * +_io__Buffered_readable_impl(buffered *self); + +static PyObject * +_io__Buffered_readable(buffered *self, PyObject *Py_UNUSED(ignored)) +{ + return _io__Buffered_readable_impl(self); +} + +PyDoc_STRVAR(_io__Buffered_writable__doc__, +"writable($self, /)\n" +"--\n" +"\n"); + +#define _IO__BUFFERED_WRITABLE_METHODDEF \ + {"writable", (PyCFunction)_io__Buffered_writable, METH_NOARGS, _io__Buffered_writable__doc__}, + +static PyObject * +_io__Buffered_writable_impl(buffered *self); + +static PyObject * +_io__Buffered_writable(buffered *self, PyObject *Py_UNUSED(ignored)) +{ + return _io__Buffered_writable_impl(self); +} + +PyDoc_STRVAR(_io__Buffered_fileno__doc__, +"fileno($self, /)\n" +"--\n" +"\n"); + +#define _IO__BUFFERED_FILENO_METHODDEF \ + {"fileno", (PyCFunction)_io__Buffered_fileno, METH_NOARGS, _io__Buffered_fileno__doc__}, + +static PyObject * +_io__Buffered_fileno_impl(buffered *self); + +static PyObject * +_io__Buffered_fileno(buffered *self, PyObject *Py_UNUSED(ignored)) +{ + return _io__Buffered_fileno_impl(self); +} + +PyDoc_STRVAR(_io__Buffered_isatty__doc__, +"isatty($self, /)\n" +"--\n" +"\n"); + +#define _IO__BUFFERED_ISATTY_METHODDEF \ + {"isatty", (PyCFunction)_io__Buffered_isatty, METH_NOARGS, _io__Buffered_isatty__doc__}, + +static PyObject * +_io__Buffered_isatty_impl(buffered *self); + +static PyObject * +_io__Buffered_isatty(buffered *self, PyObject *Py_UNUSED(ignored)) +{ + return _io__Buffered_isatty_impl(self); +} + +PyDoc_STRVAR(_io__Buffered_flush__doc__, +"flush($self, /)\n" +"--\n" +"\n"); + +#define _IO__BUFFERED_FLUSH_METHODDEF \ + {"flush", (PyCFunction)_io__Buffered_flush, METH_NOARGS, _io__Buffered_flush__doc__}, + +static PyObject * +_io__Buffered_flush_impl(buffered *self); + +static PyObject * +_io__Buffered_flush(buffered *self, PyObject *Py_UNUSED(ignored)) +{ + return _io__Buffered_flush_impl(self); +} + PyDoc_STRVAR(_io__Buffered_peek__doc__, "peek($self, size=0, /)\n" "--\n" @@ -490,6 +668,23 @@ _io__Buffered_readline(buffered *self, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(_io__Buffered_tell__doc__, +"tell($self, /)\n" +"--\n" +"\n"); + +#define _IO__BUFFERED_TELL_METHODDEF \ + {"tell", (PyCFunction)_io__Buffered_tell, METH_NOARGS, _io__Buffered_tell__doc__}, + +static PyObject * +_io__Buffered_tell_impl(buffered *self); + +static PyObject * +_io__Buffered_tell(buffered *self, PyObject *Py_UNUSED(ignored)) +{ + return _io__Buffered_tell_impl(self); +} + PyDoc_STRVAR(_io__Buffered_seek__doc__, "seek($self, target, whence=0, /)\n" "--\n" @@ -532,26 +727,41 @@ PyDoc_STRVAR(_io__Buffered_truncate__doc__, "\n"); #define _IO__BUFFERED_TRUNCATE_METHODDEF \ - {"truncate", _PyCFunction_CAST(_io__Buffered_truncate), METH_FASTCALL, _io__Buffered_truncate__doc__}, + {"truncate", _PyCFunction_CAST(_io__Buffered_truncate), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__Buffered_truncate__doc__}, static PyObject * -_io__Buffered_truncate_impl(buffered *self, PyObject *pos); +_io__Buffered_truncate_impl(buffered *self, PyTypeObject *cls, PyObject *pos); static PyObject * -_io__Buffered_truncate(buffered *self, PyObject *const *args, Py_ssize_t nargs) +_io__Buffered_truncate(buffered *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "truncate", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; PyObject *pos = Py_None; - if (!_PyArg_CheckPositional("truncate", nargs, 0, 1)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { goto exit; } if (nargs < 1) { - goto skip_optional; + goto skip_optional_posonly; } pos = args[0]; -skip_optional: - return_value = _io__Buffered_truncate_impl(self, pos); +skip_optional_posonly: + return_value = _io__Buffered_truncate_impl(self, cls, pos); exit: return return_value; @@ -877,4 +1087,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=c4ea041ccc91b5d2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d770e392e8702e12 input=a9049054013a1b77]*/ From e629ab6adf19544d5ee3f87bd1a9e9ff90808a08 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 11 May 2023 12:34:26 +0200 Subject: [PATCH 153/204] gh-101819: Adapt _io.IOBase.seek and _io.IOBase.truncate to Argument Clinic (#104384) --- Modules/_io/clinic/iobase.c.h | 102 +++++++++++++++++++++++++++++++++- Modules/_io/iobase.c | 55 +++++++++++------- 2 files changed, 135 insertions(+), 22 deletions(-) diff --git a/Modules/_io/clinic/iobase.c.h b/Modules/_io/clinic/iobase.c.h index 01c035dad2641e..727398800bec01 100644 --- a/Modules/_io/clinic/iobase.c.h +++ b/Modules/_io/clinic/iobase.c.h @@ -8,6 +8,59 @@ preserve #endif +PyDoc_STRVAR(_io__IOBase_seek__doc__, +"seek($self, /, *args)\n" +"--\n" +"\n" +"Change the stream position to the given byte offset.\n" +"\n" +"The offset is interpreted relative to the position indicated by whence.\n" +"Values for whence are:\n" +"\n" +"* 0 -- start of stream (the default); offset should be zero or positive\n" +"* 1 -- current stream position; offset may be negative\n" +"* 2 -- end of stream; offset is usually negative\n" +"\n" +"Return the new absolute position."); + +#define _IO__IOBASE_SEEK_METHODDEF \ + {"seek", _PyCFunction_CAST(_io__IOBase_seek), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__IOBase_seek__doc__}, + +static PyObject * +_io__IOBase_seek_impl(PyObject *self, PyTypeObject *cls, PyObject *args); + +static PyObject * +_io__IOBase_seek(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = { NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "seek", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *__clinic_args = NULL; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); + if (!args) { + goto exit; + } + __clinic_args = args[0]; + return_value = _io__IOBase_seek_impl(self, cls, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + PyDoc_STRVAR(_io__IOBase_tell__doc__, "tell($self, /)\n" "--\n" @@ -26,6 +79,53 @@ _io__IOBase_tell(PyObject *self, PyObject *Py_UNUSED(ignored)) return _io__IOBase_tell_impl(self); } +PyDoc_STRVAR(_io__IOBase_truncate__doc__, +"truncate($self, /, *args)\n" +"--\n" +"\n" +"Truncate file to size bytes.\n" +"\n" +"File pointer is left unchanged. Size defaults to the current IO position\n" +"as reported by tell(). Return the new size."); + +#define _IO__IOBASE_TRUNCATE_METHODDEF \ + {"truncate", _PyCFunction_CAST(_io__IOBase_truncate), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__IOBase_truncate__doc__}, + +static PyObject * +_io__IOBase_truncate_impl(PyObject *self, PyTypeObject *cls, PyObject *args); + +static PyObject * +_io__IOBase_truncate(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = { NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "truncate", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *__clinic_args = NULL; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); + if (!args) { + goto exit; + } + __clinic_args = args[0]; + return_value = _io__IOBase_truncate_impl(self, cls, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + PyDoc_STRVAR(_io__IOBase_flush__doc__, "flush($self, /)\n" "--\n" @@ -316,4 +416,4 @@ _io__RawIOBase_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _io__RawIOBase_readall_impl(self); } -/*[clinic end generated code: output=b7246a2087eb966b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b6d4845254da1da2 input=a9049054013a1b77]*/ diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index ca13866c33fba6..a74e46cc8dc536 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -79,21 +79,27 @@ iobase_unsupported(_PyIO_State *state, const char *message) /* Positioning */ -PyDoc_STRVAR(iobase_seek_doc, - "Change stream position.\n" - "\n" - "Change the stream position to the given byte offset. The offset is\n" - "interpreted relative to the position indicated by whence. Values\n" - "for whence are:\n" - "\n" - "* 0 -- start of stream (the default); offset should be zero or positive\n" - "* 1 -- current stream position; offset may be negative\n" - "* 2 -- end of stream; offset is usually negative\n" - "\n" - "Return the new absolute position."); +/*[clinic input] +_io._IOBase.seek + cls: defining_class + / + *args: object + +Change the stream position to the given byte offset. + +The offset is interpreted relative to the position indicated by whence. +Values for whence are: + +* 0 -- start of stream (the default); offset should be zero or positive +* 1 -- current stream position; offset may be negative +* 2 -- end of stream; offset is usually negative + +Return the new absolute position. +[clinic start generated code]*/ static PyObject * -iobase_seek(PyObject *self, PyObject *args) +_io__IOBase_seek_impl(PyObject *self, PyTypeObject *cls, PyObject *args) +/*[clinic end generated code: output=1dd694ac9de260fa input=ebb5476eb22fc5d4]*/ { _PyIO_State *state = IO_STATE(); return iobase_unsupported(state, "seek"); @@ -112,14 +118,21 @@ _io__IOBase_tell_impl(PyObject *self) return _PyObject_CallMethod(self, &_Py_ID(seek), "ii", 0, 1); } -PyDoc_STRVAR(iobase_truncate_doc, - "Truncate file to size bytes.\n" - "\n" - "File pointer is left unchanged. Size defaults to the current IO\n" - "position as reported by tell(). Returns the new size."); +/*[clinic input] +_io._IOBase.truncate + cls: defining_class + / + *args: object + +Truncate file to size bytes. + +File pointer is left unchanged. Size defaults to the current IO position +as reported by tell(). Return the new size. +[clinic start generated code]*/ static PyObject * -iobase_truncate(PyObject *self, PyObject *args) +_io__IOBase_truncate_impl(PyObject *self, PyTypeObject *cls, PyObject *args) +/*[clinic end generated code: output=b7eed4649cbe22c1 input=ad90582a1d8b5cc9]*/ { _PyIO_State *state = IO_STATE(); return iobase_unsupported(state, "truncate"); @@ -809,9 +822,9 @@ _io__IOBase_writelines(PyObject *self, PyObject *lines) #include "clinic/iobase.c.h" static PyMethodDef iobase_methods[] = { - {"seek", iobase_seek, METH_VARARGS, iobase_seek_doc}, + _IO__IOBASE_SEEK_METHODDEF _IO__IOBASE_TELL_METHODDEF - {"truncate", iobase_truncate, METH_VARARGS, iobase_truncate_doc}, + _IO__IOBASE_TRUNCATE_METHODDEF _IO__IOBASE_FLUSH_METHODDEF _IO__IOBASE_CLOSE_METHODDEF From 167072938342981b96d06d739cd97185207b64dd Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Thu, 11 May 2023 05:52:06 -0600 Subject: [PATCH 154/204] gh-87849: fix SEND specialization family definition (GH-104268) --- Python/bytecodes.c | 4 +- Python/generated_cases.c.h | 347 +++++++++++++++++++------------------ Python/opcode_metadata.h | 2 +- 3 files changed, 177 insertions(+), 176 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 82c10044418345..1c09dc6be3d687 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -813,7 +813,7 @@ dummy_func( PREDICT(LOAD_CONST); } - family(for_iter, INLINE_CACHE_ENTRIES_FOR_ITER) = { + family(send, INLINE_CACHE_ENTRIES_SEND) = { SEND, SEND_GEN, }; @@ -866,7 +866,7 @@ dummy_func( Py_DECREF(v); } - inst(SEND_GEN, (unused/1, receiver, v -- receiver)) { + inst(SEND_GEN, (unused/1, receiver, v -- receiver, unused)) { PyGenObject *gen = (PyGenObject *)receiver; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 819c857c3c0119..a86b895dd43d82 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1139,6 +1139,7 @@ TARGET(SEND) { PREDICTED(SEND); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; PyObject *retval; @@ -1188,7 +1189,7 @@ } } Py_DECREF(v); - #line 1192 "Python/generated_cases.c.h" + #line 1193 "Python/generated_cases.c.h" stack_pointer[-1] = retval; next_instr += 1; DISPATCH(); @@ -1212,7 +1213,7 @@ tstate->exc_info = &gen->gi_exc_state; JUMPBY(INLINE_CACHE_ENTRIES_SEND); DISPATCH_INLINED(gen_frame); - #line 1216 "Python/generated_cases.c.h" + #line 1217 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_YIELD_VALUE) { @@ -1234,7 +1235,7 @@ gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 1238 "Python/generated_cases.c.h" + #line 1239 "Python/generated_cases.c.h" } TARGET(YIELD_VALUE) { @@ -1255,7 +1256,7 @@ gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 1259 "Python/generated_cases.c.h" + #line 1260 "Python/generated_cases.c.h" } TARGET(POP_EXCEPT) { @@ -1263,7 +1264,7 @@ #line 924 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); - #line 1267 "Python/generated_cases.c.h" + #line 1268 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1289,7 +1290,7 @@ Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; - #line 1293 "Python/generated_cases.c.h" + #line 1294 "Python/generated_cases.c.h" } TARGET(END_ASYNC_FOR) { @@ -1298,7 +1299,7 @@ #line 949 "Python/bytecodes.c" assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { - #line 1302 "Python/generated_cases.c.h" + #line 1303 "Python/generated_cases.c.h" Py_DECREF(awaitable); Py_DECREF(exc); #line 952 "Python/bytecodes.c" @@ -1308,7 +1309,7 @@ _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; } - #line 1312 "Python/generated_cases.c.h" + #line 1313 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -1324,7 +1325,7 @@ assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); - #line 1328 "Python/generated_cases.c.h" + #line 1329 "Python/generated_cases.c.h" Py_DECREF(sub_iter); Py_DECREF(last_sent_val); Py_DECREF(exc_value); @@ -1335,7 +1336,7 @@ _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); goto exception_unwind; } - #line 1339 "Python/generated_cases.c.h" + #line 1340 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; stack_pointer[-2] = none; @@ -1346,7 +1347,7 @@ PyObject *value; #line 975 "Python/bytecodes.c" value = Py_NewRef(PyExc_AssertionError); - #line 1350 "Python/generated_cases.c.h" + #line 1351 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1376,7 +1377,7 @@ if (true) goto error; } } - #line 1380 "Python/generated_cases.c.h" + #line 1381 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = bc; DISPATCH(); @@ -1391,7 +1392,7 @@ if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - #line 1395 "Python/generated_cases.c.h" + #line 1396 "Python/generated_cases.c.h" Py_DECREF(v); #line 1010 "Python/bytecodes.c" if (true) goto pop_1_error; @@ -1400,11 +1401,11 @@ err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); - #line 1404 "Python/generated_cases.c.h" + #line 1405 "Python/generated_cases.c.h" Py_DECREF(v); #line 1017 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1408 "Python/generated_cases.c.h" + #line 1409 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1427,7 +1428,7 @@ name); goto error; } - #line 1431 "Python/generated_cases.c.h" + #line 1432 "Python/generated_cases.c.h" DISPATCH(); } @@ -1448,11 +1449,11 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = unpack_iterable(tstate, seq, oparg, -1, top); - #line 1452 "Python/generated_cases.c.h" + #line 1453 "Python/generated_cases.c.h" Py_DECREF(seq); #line 1060 "Python/bytecodes.c" if (res == 0) goto pop_1_error; - #line 1456 "Python/generated_cases.c.h" + #line 1457 "Python/generated_cases.c.h" STACK_SHRINK(1); STACK_GROW(oparg); next_instr += 1; @@ -1469,7 +1470,7 @@ STAT_INC(UNPACK_SEQUENCE, hit); values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - #line 1473 "Python/generated_cases.c.h" + #line 1474 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1488,7 +1489,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - #line 1492 "Python/generated_cases.c.h" + #line 1493 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1507,7 +1508,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - #line 1511 "Python/generated_cases.c.h" + #line 1512 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1521,11 +1522,11 @@ int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - #line 1525 "Python/generated_cases.c.h" + #line 1526 "Python/generated_cases.c.h" Py_DECREF(seq); #line 1100 "Python/bytecodes.c" if (res == 0) goto pop_1_error; - #line 1529 "Python/generated_cases.c.h" + #line 1530 "Python/generated_cases.c.h" STACK_GROW((oparg & 0xFF) + (oparg >> 8)); DISPATCH(); } @@ -1552,12 +1553,12 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, v); - #line 1556 "Python/generated_cases.c.h" + #line 1557 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(owner); #line 1127 "Python/bytecodes.c" if (err) goto pop_2_error; - #line 1561 "Python/generated_cases.c.h" + #line 1562 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -1568,11 +1569,11 @@ #line 1131 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); - #line 1572 "Python/generated_cases.c.h" + #line 1573 "Python/generated_cases.c.h" Py_DECREF(owner); #line 1134 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1576 "Python/generated_cases.c.h" + #line 1577 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1582,11 +1583,11 @@ #line 1138 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); - #line 1586 "Python/generated_cases.c.h" + #line 1587 "Python/generated_cases.c.h" Py_DECREF(v); #line 1141 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1590 "Python/generated_cases.c.h" + #line 1591 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1604,7 +1605,7 @@ } goto error; } - #line 1608 "Python/generated_cases.c.h" + #line 1609 "Python/generated_cases.c.h" DISPATCH(); } @@ -1669,7 +1670,7 @@ } } } - #line 1673 "Python/generated_cases.c.h" + #line 1674 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = v; DISPATCH(); @@ -1732,7 +1733,7 @@ } } null = NULL; - #line 1736 "Python/generated_cases.c.h" + #line 1737 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = v; @@ -1757,7 +1758,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - #line 1761 "Python/generated_cases.c.h" + #line 1762 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1787,7 +1788,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - #line 1791 "Python/generated_cases.c.h" + #line 1792 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1801,7 +1802,7 @@ PyObject *v = GETLOCAL(oparg); if (v == NULL) goto unbound_local_error; SETLOCAL(oparg, NULL); - #line 1805 "Python/generated_cases.c.h" + #line 1806 "Python/generated_cases.c.h" DISPATCH(); } @@ -1815,7 +1816,7 @@ goto resume_with_error; } SETLOCAL(oparg, cell); - #line 1819 "Python/generated_cases.c.h" + #line 1820 "Python/generated_cases.c.h" DISPATCH(); } @@ -1831,7 +1832,7 @@ } PyCell_SET(cell, NULL); Py_DECREF(oldobj); - #line 1835 "Python/generated_cases.c.h" + #line 1836 "Python/generated_cases.c.h" DISPATCH(); } @@ -1869,7 +1870,7 @@ } Py_INCREF(value); } - #line 1873 "Python/generated_cases.c.h" + #line 1874 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1885,7 +1886,7 @@ if (true) goto error; } Py_INCREF(value); - #line 1889 "Python/generated_cases.c.h" + #line 1890 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1898,7 +1899,7 @@ PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); - #line 1902 "Python/generated_cases.c.h" + #line 1903 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1915,7 +1916,7 @@ PyObject *o = PyTuple_GET_ITEM(closure, i); frame->localsplus[offset + i] = Py_NewRef(o); } - #line 1919 "Python/generated_cases.c.h" + #line 1920 "Python/generated_cases.c.h" DISPATCH(); } @@ -1924,13 +1925,13 @@ PyObject *str; #line 1404 "Python/bytecodes.c" str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - #line 1928 "Python/generated_cases.c.h" + #line 1929 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } #line 1406 "Python/bytecodes.c" if (str == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1934 "Python/generated_cases.c.h" + #line 1935 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = str; @@ -1943,7 +1944,7 @@ #line 1410 "Python/bytecodes.c" tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1947 "Python/generated_cases.c.h" + #line 1948 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = tup; @@ -1956,7 +1957,7 @@ #line 1415 "Python/bytecodes.c" list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1960 "Python/generated_cases.c.h" + #line 1961 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = list; @@ -1977,13 +1978,13 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - #line 1981 "Python/generated_cases.c.h" + #line 1982 "Python/generated_cases.c.h" Py_DECREF(iterable); #line 1431 "Python/bytecodes.c" if (true) goto pop_1_error; } Py_DECREF(none_val); - #line 1987 "Python/generated_cases.c.h" + #line 1988 "Python/generated_cases.c.h" Py_DECREF(iterable); STACK_SHRINK(1); DISPATCH(); @@ -1994,11 +1995,11 @@ PyObject *set = stack_pointer[-(2 + (oparg-1))]; #line 1438 "Python/bytecodes.c" int err = _PySet_Update(set, iterable); - #line 1998 "Python/generated_cases.c.h" + #line 1999 "Python/generated_cases.c.h" Py_DECREF(iterable); #line 1440 "Python/bytecodes.c" if (err < 0) goto pop_1_error; - #line 2002 "Python/generated_cases.c.h" + #line 2003 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -2021,7 +2022,7 @@ Py_DECREF(set); if (true) { STACK_SHRINK(oparg); goto error; } } - #line 2025 "Python/generated_cases.c.h" + #line 2026 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = set; @@ -2039,13 +2040,13 @@ if (map == NULL) goto error; - #line 2043 "Python/generated_cases.c.h" + #line 2044 "Python/generated_cases.c.h" for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } #line 1469 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } - #line 2049 "Python/generated_cases.c.h" + #line 2050 "Python/generated_cases.c.h" STACK_SHRINK(oparg*2); STACK_GROW(1); stack_pointer[-1] = map; @@ -2093,7 +2094,7 @@ Py_DECREF(ann_dict); } } - #line 2097 "Python/generated_cases.c.h" + #line 2098 "Python/generated_cases.c.h" DISPATCH(); } @@ -2111,14 +2112,14 @@ map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); - #line 2115 "Python/generated_cases.c.h" + #line 2116 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); #line 1525 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } - #line 2122 "Python/generated_cases.c.h" + #line 2123 "Python/generated_cases.c.h" STACK_SHRINK(oparg); stack_pointer[-1] = map; DISPATCH(); @@ -2134,12 +2135,12 @@ "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } - #line 2138 "Python/generated_cases.c.h" + #line 2139 "Python/generated_cases.c.h" Py_DECREF(update); #line 1537 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2143 "Python/generated_cases.c.h" + #line 2144 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); DISPATCH(); @@ -2152,12 +2153,12 @@ if (_PyDict_MergeEx(dict, update, 2) < 0) { format_kwargs_error(tstate, PEEK(3 + oparg), update); - #line 2156 "Python/generated_cases.c.h" + #line 2157 "Python/generated_cases.c.h" Py_DECREF(update); #line 1548 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2161 "Python/generated_cases.c.h" + #line 2162 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); PREDICT(CALL_FUNCTION_EX); @@ -2173,7 +2174,7 @@ /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - #line 2177 "Python/generated_cases.c.h" + #line 2178 "Python/generated_cases.c.h" STACK_SHRINK(2); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -2205,7 +2206,7 @@ // handle any case whose performance we care about PyObject *stack[] = {class, self}; PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - #line 2209 "Python/generated_cases.c.h" + #line 2210 "Python/generated_cases.c.h" Py_DECREF(global_super); Py_DECREF(class); Py_DECREF(self); @@ -2214,7 +2215,7 @@ res = PyObject_GetAttr(super, name); Py_DECREF(super); if (res == NULL) goto pop_3_error; - #line 2218 "Python/generated_cases.c.h" + #line 2219 "Python/generated_cases.c.h" STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2243,7 +2244,7 @@ Py_INCREF(res2); Py_DECREF(global_super); Py_DECREF(class); - #line 2247 "Python/generated_cases.c.h" + #line 2248 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; stack_pointer[-2] = res2; @@ -2291,7 +2292,7 @@ NULL | meth | arg1 | ... | argN */ - #line 2295 "Python/generated_cases.c.h" + #line 2296 "Python/generated_cases.c.h" Py_DECREF(owner); #line 1655 "Python/bytecodes.c" if (meth == NULL) goto pop_1_error; @@ -2302,12 +2303,12 @@ else { /* Classic, pushes one value. */ res = PyObject_GetAttr(owner, name); - #line 2306 "Python/generated_cases.c.h" + #line 2307 "Python/generated_cases.c.h" Py_DECREF(owner); #line 1664 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; } - #line 2311 "Python/generated_cases.c.h" + #line 2312 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -2334,7 +2335,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2338 "Python/generated_cases.c.h" + #line 2339 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2362,7 +2363,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2366 "Python/generated_cases.c.h" + #line 2367 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2404,7 +2405,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2408 "Python/generated_cases.c.h" + #line 2409 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2429,7 +2430,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2433 "Python/generated_cases.c.h" + #line 2434 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2456,7 +2457,7 @@ res = descr; assert(res != NULL); Py_INCREF(res); - #line 2460 "Python/generated_cases.c.h" + #line 2461 "Python/generated_cases.c.h" Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2494,7 +2495,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 2498 "Python/generated_cases.c.h" + #line 2499 "Python/generated_cases.c.h" } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { @@ -2528,7 +2529,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 2532 "Python/generated_cases.c.h" + #line 2533 "Python/generated_cases.c.h" } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -2554,7 +2555,7 @@ Py_DECREF(old_value); } Py_DECREF(owner); - #line 2558 "Python/generated_cases.c.h" + #line 2559 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2604,7 +2605,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - #line 2608 "Python/generated_cases.c.h" + #line 2609 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2625,7 +2626,7 @@ *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - #line 2629 "Python/generated_cases.c.h" + #line 2630 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2650,12 +2651,12 @@ #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); - #line 2654 "Python/generated_cases.c.h" + #line 2655 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); #line 1906 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2659 "Python/generated_cases.c.h" + #line 2660 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2678,7 +2679,7 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2682 "Python/generated_cases.c.h" + #line 2683 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2705,7 +2706,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2709 "Python/generated_cases.c.h" + #line 2710 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2729,7 +2730,7 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2733 "Python/generated_cases.c.h" + #line 2734 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2742,12 +2743,12 @@ PyObject *b; #line 1959 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; - #line 2746 "Python/generated_cases.c.h" + #line 2747 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); #line 1961 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2751 "Python/generated_cases.c.h" + #line 2752 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2759,13 +2760,13 @@ PyObject *b; #line 1965 "Python/bytecodes.c" int res = PySequence_Contains(right, left); - #line 2763 "Python/generated_cases.c.h" + #line 2764 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); #line 1967 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = Py_NewRef((res^oparg) ? Py_True : Py_False); - #line 2769 "Python/generated_cases.c.h" + #line 2770 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2778,7 +2779,7 @@ PyObject *match; #line 1972 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { - #line 2782 "Python/generated_cases.c.h" + #line 2783 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); #line 1974 "Python/bytecodes.c" @@ -2789,7 +2790,7 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); - #line 2793 "Python/generated_cases.c.h" + #line 2794 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); #line 1982 "Python/bytecodes.c" @@ -2801,7 +2802,7 @@ if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } - #line 2805 "Python/generated_cases.c.h" + #line 2806 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -2814,18 +2815,18 @@ #line 1993 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - #line 2818 "Python/generated_cases.c.h" + #line 2819 "Python/generated_cases.c.h" Py_DECREF(right); #line 1996 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - #line 2825 "Python/generated_cases.c.h" + #line 2826 "Python/generated_cases.c.h" Py_DECREF(right); #line 2001 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2829 "Python/generated_cases.c.h" + #line 2830 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -2837,12 +2838,12 @@ #line 2005 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); - #line 2841 "Python/generated_cases.c.h" + #line 2842 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); #line 2008 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2846 "Python/generated_cases.c.h" + #line 2847 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -2855,7 +2856,7 @@ PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - #line 2859 "Python/generated_cases.c.h" + #line 2860 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -2864,7 +2865,7 @@ TARGET(JUMP_FORWARD) { #line 2018 "Python/bytecodes.c" JUMPBY(oparg); - #line 2868 "Python/generated_cases.c.h" + #line 2869 "Python/generated_cases.c.h" DISPATCH(); } @@ -2873,7 +2874,7 @@ #line 2022 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); - #line 2877 "Python/generated_cases.c.h" + #line 2878 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2891,7 +2892,7 @@ } else { int err = PyObject_IsTrue(cond); - #line 2895 "Python/generated_cases.c.h" + #line 2896 "Python/generated_cases.c.h" Py_DECREF(cond); #line 2038 "Python/bytecodes.c" if (err == 0) { @@ -2901,7 +2902,7 @@ if (err < 0) goto pop_1_error; } } - #line 2905 "Python/generated_cases.c.h" + #line 2906 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -2918,7 +2919,7 @@ } else { int err = PyObject_IsTrue(cond); - #line 2922 "Python/generated_cases.c.h" + #line 2923 "Python/generated_cases.c.h" Py_DECREF(cond); #line 2058 "Python/bytecodes.c" if (err > 0) { @@ -2928,7 +2929,7 @@ if (err < 0) goto pop_1_error; } } - #line 2932 "Python/generated_cases.c.h" + #line 2933 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -2937,7 +2938,7 @@ PyObject *value = stack_pointer[-1]; #line 2068 "Python/bytecodes.c" if (!Py_IsNone(value)) { - #line 2941 "Python/generated_cases.c.h" + #line 2942 "Python/generated_cases.c.h" Py_DECREF(value); #line 2070 "Python/bytecodes.c" JUMPBY(oparg); @@ -2945,7 +2946,7 @@ else { _Py_DECREF_NO_DEALLOC(value); } - #line 2949 "Python/generated_cases.c.h" + #line 2950 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -2958,11 +2959,11 @@ JUMPBY(oparg); } else { - #line 2962 "Python/generated_cases.c.h" + #line 2963 "Python/generated_cases.c.h" Py_DECREF(value); #line 2084 "Python/bytecodes.c" } - #line 2966 "Python/generated_cases.c.h" + #line 2967 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -2975,7 +2976,7 @@ * (see bpo-30039). */ JUMPBY(-oparg); - #line 2979 "Python/generated_cases.c.h" + #line 2980 "Python/generated_cases.c.h" DISPATCH(); } @@ -2988,7 +2989,7 @@ if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 2992 "Python/generated_cases.c.h" + #line 2993 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -3004,7 +3005,7 @@ // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 3008 "Python/generated_cases.c.h" + #line 3009 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); @@ -3016,7 +3017,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_NewRef(Py_None); // Failure! } - #line 3020 "Python/generated_cases.c.h" + #line 3021 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -3028,7 +3029,7 @@ #line 2120 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = Py_NewRef(match ? Py_True : Py_False); - #line 3032 "Python/generated_cases.c.h" + #line 3033 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -3041,7 +3042,7 @@ #line 2126 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = Py_NewRef(match ? Py_True : Py_False); - #line 3045 "Python/generated_cases.c.h" + #line 3046 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -3056,7 +3057,7 @@ // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 3060 "Python/generated_cases.c.h" + #line 3061 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -3068,11 +3069,11 @@ #line 2138 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 3072 "Python/generated_cases.c.h" + #line 3073 "Python/generated_cases.c.h" Py_DECREF(iterable); #line 2141 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 3076 "Python/generated_cases.c.h" + #line 3077 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -3103,11 +3104,11 @@ if (iter == NULL) { goto error; } - #line 3107 "Python/generated_cases.c.h" + #line 3108 "Python/generated_cases.c.h" Py_DECREF(iterable); #line 2168 "Python/bytecodes.c" } - #line 3111 "Python/generated_cases.c.h" + #line 3112 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -3149,7 +3150,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator - #line 3153 "Python/generated_cases.c.h" + #line 3154 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3183,7 +3184,7 @@ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - #line 3187 "Python/generated_cases.c.h" + #line 3188 "Python/generated_cases.c.h" DISPATCH(); } @@ -3210,7 +3211,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator - #line 3214 "Python/generated_cases.c.h" + #line 3215 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3240,7 +3241,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator - #line 3244 "Python/generated_cases.c.h" + #line 3245 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3268,7 +3269,7 @@ if (next == NULL) { goto error; } - #line 3272 "Python/generated_cases.c.h" + #line 3273 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3292,7 +3293,7 @@ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); - #line 3296 "Python/generated_cases.c.h" + #line 3297 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { @@ -3322,7 +3323,7 @@ Py_DECREF(enter); goto error; } - #line 3326 "Python/generated_cases.c.h" + #line 3327 "Python/generated_cases.c.h" Py_DECREF(mgr); #line 2352 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); @@ -3331,7 +3332,7 @@ Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3335 "Python/generated_cases.c.h" + #line 3336 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3369,7 +3370,7 @@ Py_DECREF(enter); goto error; } - #line 3373 "Python/generated_cases.c.h" + #line 3374 "Python/generated_cases.c.h" Py_DECREF(mgr); #line 2388 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); @@ -3378,7 +3379,7 @@ Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3382 "Python/generated_cases.c.h" + #line 3383 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3411,7 +3412,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 3415 "Python/generated_cases.c.h" + #line 3416 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3430,7 +3431,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 3434 "Python/generated_cases.c.h" + #line 3435 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3461,7 +3462,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); - #line 3465 "Python/generated_cases.c.h" + #line 3466 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3485,7 +3486,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3489 "Python/generated_cases.c.h" + #line 3490 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3513,7 +3514,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3517 "Python/generated_cases.c.h" + #line 3518 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3526,7 +3527,7 @@ assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); - #line 3530 "Python/generated_cases.c.h" + #line 3531 "Python/generated_cases.c.h" DISPATCH(); } @@ -3544,7 +3545,7 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(CALL); - #line 3548 "Python/generated_cases.c.h" + #line 3549 "Python/generated_cases.c.h" } TARGET(CALL) { @@ -3636,7 +3637,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3640 "Python/generated_cases.c.h" + #line 3641 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3658,7 +3659,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3662 "Python/generated_cases.c.h" + #line 3663 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3693,7 +3694,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 3697 "Python/generated_cases.c.h" + #line 3698 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3737,7 +3738,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 3741 "Python/generated_cases.c.h" + #line 3742 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3755,7 +3756,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 3759 "Python/generated_cases.c.h" + #line 3760 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3779,7 +3780,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3783 "Python/generated_cases.c.h" + #line 3784 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3804,7 +3805,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3808 "Python/generated_cases.c.h" + #line 3809 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3840,7 +3841,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3844 "Python/generated_cases.c.h" + #line 3845 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3882,7 +3883,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3886 "Python/generated_cases.c.h" + #line 3887 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3928,7 +3929,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 3932 "Python/generated_cases.c.h" + #line 3933 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3974,7 +3975,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3978 "Python/generated_cases.c.h" + #line 3979 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4013,7 +4014,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4017 "Python/generated_cases.c.h" + #line 4018 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4053,7 +4054,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4057 "Python/generated_cases.c.h" + #line 4058 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4083,7 +4084,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 4087 "Python/generated_cases.c.h" + #line 4088 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { @@ -4121,7 +4122,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4125 "Python/generated_cases.c.h" + #line 4126 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4163,7 +4164,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4167 "Python/generated_cases.c.h" + #line 4168 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4205,7 +4206,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4209 "Python/generated_cases.c.h" + #line 4210 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4246,7 +4247,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4250 "Python/generated_cases.c.h" + #line 4251 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4258,7 +4259,7 @@ TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { #line 3068 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4262 "Python/generated_cases.c.h" + #line 4263 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4329,14 +4330,14 @@ } result = PyObject_Call(func, callargs, kwargs); } - #line 4333 "Python/generated_cases.c.h" + #line 4334 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); #line 3134 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4340 "Python/generated_cases.c.h" + #line 4341 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4380,7 +4381,7 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4384 "Python/generated_cases.c.h" + #line 4385 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); @@ -4408,7 +4409,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4412 "Python/generated_cases.c.h" + #line 4413 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4418,13 +4419,13 @@ PyObject *slice; #line 3198 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4422 "Python/generated_cases.c.h" + #line 4423 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); #line 3200 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4428 "Python/generated_cases.c.h" + #line 4429 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4470,7 +4471,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4474 "Python/generated_cases.c.h" + #line 4475 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4482,7 +4483,7 @@ #line 3241 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4486 "Python/generated_cases.c.h" + #line 4487 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4509,12 +4510,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4513 "Python/generated_cases.c.h" + #line 4514 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); #line 3261 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4518 "Python/generated_cases.c.h" + #line 4519 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4526,7 +4527,7 @@ PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; #line 3266 "Python/bytecodes.c" assert(oparg >= 2); - #line 4530 "Python/generated_cases.c.h" + #line 4531 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); @@ -4553,7 +4554,7 @@ } opcode = original_opcode; DISPATCH_GOTO(); - #line 4557 "Python/generated_cases.c.h" + #line 4558 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_INSTRUCTION) { @@ -4569,20 +4570,20 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4573 "Python/generated_cases.c.h" + #line 4574 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { #line 3306 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4579 "Python/generated_cases.c.h" + #line 4580 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { #line 3310 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4586 "Python/generated_cases.c.h" + #line 4587 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -4597,7 +4598,7 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4601 "Python/generated_cases.c.h" + #line 4602 "Python/generated_cases.c.h" DISPATCH(); } @@ -4611,7 +4612,7 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4615 "Python/generated_cases.c.h" + #line 4616 "Python/generated_cases.c.h" DISPATCH(); } @@ -4629,7 +4630,7 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4633 "Python/generated_cases.c.h" + #line 4634 "Python/generated_cases.c.h" DISPATCH(); } @@ -4647,7 +4648,7 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4651 "Python/generated_cases.c.h" + #line 4652 "Python/generated_cases.c.h" DISPATCH(); } @@ -4658,19 +4659,19 @@ oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4662 "Python/generated_cases.c.h" + #line 4663 "Python/generated_cases.c.h" } TARGET(CACHE) { #line 3375 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); - #line 4669 "Python/generated_cases.c.h" + #line 4670 "Python/generated_cases.c.h" } TARGET(RESERVED) { #line 3380 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); - #line 4676 "Python/generated_cases.c.h" + #line 4677 "Python/generated_cases.c.h" } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index c1a6ed4c18ab0a..670290e2f3f9f9 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -512,7 +512,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case SEND: return 2; case SEND_GEN: - return 1; + return 2; case INSTRUMENTED_YIELD_VALUE: return 1; case YIELD_VALUE: From ddc0e70a326180d90a45dfd8abeded9c9f911217 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 11 May 2023 15:01:05 +0200 Subject: [PATCH 155/204] gh-101819: Fix inverted debug preprocessor check in winconsoleio.c (#104388) --- Modules/_io/winconsoleio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index f341cb238ce85e..d65e247737a071 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -267,7 +267,7 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, int fd_is_own = 0; HANDLE handle = NULL; -#ifdef NDEBUG +#ifndef NDEBUG _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); assert(PyObject_TypeCheck(self, state->PyWindowsConsoleIO_Type)); #endif From 77262458fe3fe9f3f0266bbf578675a906b1e353 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Thu, 11 May 2023 08:08:13 -0600 Subject: [PATCH 156/204] gh-87729: improve hit rate of LOAD_SUPER_ATTR specialization (#104270) --- Include/internal/pycore_code.h | 7 +- Include/internal/pycore_opcode.h | 24 +- Include/internal/pycore_typeobject.h | 2 - Include/opcode.h | 55 +-- Lib/importlib/_bootstrap_external.py | 3 +- Lib/opcode.py | 4 +- Objects/typeobject.c | 12 - Python/bytecodes.c | 40 ++- Python/generated_cases.c.h | 507 ++++++++++++++------------- Python/opcode_metadata.h | 11 +- Python/opcode_targets.h | 20 +- Python/specialize.c | 33 +- Tools/scripts/summarize_stats.py | 8 +- 13 files changed, 374 insertions(+), 352 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index c1f017fdb753c5..75a23f3f5af560 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -53,9 +53,6 @@ typedef struct { typedef struct { uint16_t counter; - uint16_t class_version[2]; - uint16_t self_type_version[2]; - uint16_t method[4]; } _PySuperAttrCache; #define INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR CACHE_ENTRIES(_PySuperAttrCache) @@ -227,8 +224,8 @@ extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); /* Specialization functions */ -extern void _Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, PyObject *self, - _Py_CODEUNIT *instr, PyObject *name, int load_method); +extern void _Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, + _Py_CODEUNIT *instr, int load_method); extern void _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name); extern void _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 52ee70a7bfdc54..e823e1bfb5d452 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -42,7 +42,7 @@ const uint8_t _PyOpcode_Caches[256] = { [LOAD_GLOBAL] = 4, [BINARY_OP] = 1, [SEND] = 1, - [LOAD_SUPER_ATTR] = 9, + [LOAD_SUPER_ATTR] = 1, [CALL] = 3, }; @@ -182,6 +182,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [LOAD_GLOBAL_MODULE] = LOAD_GLOBAL, [LOAD_NAME] = LOAD_NAME, [LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR, + [LOAD_SUPER_ATTR_ATTR] = LOAD_SUPER_ATTR, [LOAD_SUPER_ATTR_METHOD] = LOAD_SUPER_ATTR, [MAKE_CELL] = MAKE_CELL, [MAKE_FUNCTION] = MAKE_FUNCTION, @@ -307,29 +308,29 @@ static const char *const _PyOpcode_OpName[267] = { [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [FOR_ITER_GEN] = "FOR_ITER_GEN", + [LOAD_SUPER_ATTR_ATTR] = "LOAD_SUPER_ATTR_ATTR", [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", - [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [GET_ITER] = "GET_ITER", [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", + [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", - [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [RETURN_GENERATOR] = "RETURN_GENERATOR", + [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", - [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [RETURN_VALUE] = "RETURN_VALUE", - [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", + [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", + [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [POP_EXCEPT] = "POP_EXCEPT", [STORE_NAME] = "STORE_NAME", [DELETE_NAME] = "DELETE_NAME", @@ -352,9 +353,9 @@ static const char *const _PyOpcode_OpName[267] = { [IMPORT_NAME] = "IMPORT_NAME", [IMPORT_FROM] = "IMPORT_FROM", [JUMP_FORWARD] = "JUMP_FORWARD", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", - [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -394,22 +395,22 @@ static const char *const _PyOpcode_OpName[267] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", + [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", - [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", + [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", - [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", + [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [SEND_GEN] = "SEND_GEN", - [168] = "<168>", [169] = "<169>", [170] = "<170>", [CALL] = "CALL", @@ -512,7 +513,6 @@ static const char *const _PyOpcode_OpName[267] = { #endif #define EXTRA_CASES \ - case 168: \ case 169: \ case 170: \ case 175: \ diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index f42f8f62de2c1d..c7d00002fbc63c 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -142,8 +142,6 @@ PyAPI_DATA(PyTypeObject) _PyBufferWrapper_Type; PyObject * _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, PyObject *name, int *meth_found); -PyObject * -_PySuper_LookupDescr(PyTypeObject *su_type, PyObject *su_obj, PyObject *name); #ifdef __cplusplus } diff --git a/Include/opcode.h b/Include/opcode.h index f6f4af8c793d8d..c3b6e5dce49ad3 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -185,33 +185,34 @@ extern "C" { #define FOR_ITER_TUPLE 63 #define FOR_ITER_RANGE 64 #define FOR_ITER_GEN 65 -#define LOAD_SUPER_ATTR_METHOD 66 -#define LOAD_ATTR_CLASS 67 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 70 -#define LOAD_ATTR_INSTANCE_VALUE 72 -#define LOAD_ATTR_MODULE 73 -#define LOAD_ATTR_PROPERTY 76 -#define LOAD_ATTR_SLOT 77 -#define LOAD_ATTR_WITH_HINT 78 -#define LOAD_ATTR_METHOD_LAZY_DICT 79 -#define LOAD_ATTR_METHOD_NO_DICT 80 -#define LOAD_ATTR_METHOD_WITH_VALUES 81 -#define LOAD_CONST__LOAD_FAST 82 -#define LOAD_FAST__LOAD_CONST 84 -#define LOAD_FAST__LOAD_FAST 86 -#define LOAD_GLOBAL_BUILTIN 87 -#define LOAD_GLOBAL_MODULE 88 -#define STORE_ATTR_INSTANCE_VALUE 111 -#define STORE_ATTR_SLOT 112 -#define STORE_ATTR_WITH_HINT 113 -#define STORE_FAST__LOAD_FAST 153 -#define STORE_FAST__STORE_FAST 154 -#define STORE_SUBSCR_DICT 158 -#define STORE_SUBSCR_LIST_INT 159 -#define UNPACK_SEQUENCE_LIST 160 -#define UNPACK_SEQUENCE_TUPLE 161 -#define UNPACK_SEQUENCE_TWO_TUPLE 166 -#define SEND_GEN 167 +#define LOAD_SUPER_ATTR_ATTR 66 +#define LOAD_SUPER_ATTR_METHOD 67 +#define LOAD_ATTR_CLASS 70 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 72 +#define LOAD_ATTR_INSTANCE_VALUE 73 +#define LOAD_ATTR_MODULE 76 +#define LOAD_ATTR_PROPERTY 77 +#define LOAD_ATTR_SLOT 78 +#define LOAD_ATTR_WITH_HINT 79 +#define LOAD_ATTR_METHOD_LAZY_DICT 80 +#define LOAD_ATTR_METHOD_NO_DICT 81 +#define LOAD_ATTR_METHOD_WITH_VALUES 82 +#define LOAD_CONST__LOAD_FAST 84 +#define LOAD_FAST__LOAD_CONST 86 +#define LOAD_FAST__LOAD_FAST 87 +#define LOAD_GLOBAL_BUILTIN 88 +#define LOAD_GLOBAL_MODULE 111 +#define STORE_ATTR_INSTANCE_VALUE 112 +#define STORE_ATTR_SLOT 113 +#define STORE_ATTR_WITH_HINT 153 +#define STORE_FAST__LOAD_FAST 154 +#define STORE_FAST__STORE_FAST 158 +#define STORE_SUBSCR_DICT 159 +#define STORE_SUBSCR_LIST_INT 160 +#define UNPACK_SEQUENCE_LIST 161 +#define UNPACK_SEQUENCE_TUPLE 166 +#define UNPACK_SEQUENCE_TWO_TUPLE 167 +#define SEND_GEN 168 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ || ((op) == JUMP) \ diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index d4a1593db2c874..3ca5a1aea28af7 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -443,6 +443,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12b1 3527 (Add LOAD_SUPER_ATTR) # Python 3.12b1 3528 (Add LOAD_SUPER_ATTR_METHOD specialization) # Python 3.12b1 3529 (Inline list/dict/set comprehensions) +# Python 3.12b1 3530 (Shrink the LOAD_SUPER_ATTR caches) # Python 3.13 will start with 3550 @@ -459,7 +460,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3529).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3530).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index d6d478aa1f479d..155466b1ab85f1 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -373,6 +373,7 @@ def pseudo_op(name, op, real_ops): "FOR_ITER_GEN", ], "LOAD_SUPER_ATTR": [ + "LOAD_SUPER_ATTR_ATTR", "LOAD_SUPER_ATTR_METHOD", ], "LOAD_ATTR": [ @@ -450,9 +451,6 @@ def pseudo_op(name, op, real_ops): }, "LOAD_SUPER_ATTR": { "counter": 1, - "class_version": 2, - "self_type_version": 2, - "method": 4, }, "LOAD_ATTR": { "counter": 1, diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 98fac276a873e1..f1745c91967358 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -10245,18 +10245,6 @@ _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, PyObject *name, int *me return res; } -PyObject * -_PySuper_LookupDescr(PyTypeObject *su_type, PyObject *su_obj, PyObject *name) -{ - PyTypeObject *su_obj_type = supercheck(su_type, su_obj); - if (su_obj_type == NULL) { - return NULL; - } - PyObject *res = _super_lookup_descr(su_type, su_obj_type, name); - Py_DECREF(su_obj_type); - return res; -} - static PyObject * super_descr_get(PyObject *self, PyObject *obj, PyObject *type) { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 1c09dc6be3d687..118a0b5ed4393e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1562,17 +1562,18 @@ dummy_func( family(load_super_attr, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR) = { LOAD_SUPER_ATTR, + LOAD_SUPER_ATTR_ATTR, LOAD_SUPER_ATTR_METHOD, }; - inst(LOAD_SUPER_ATTR, (unused/9, global_super, class, self -- res2 if (oparg & 1), res)) { + inst(LOAD_SUPER_ATTR, (unused/1, global_super, class, self -- res2 if (oparg & 1), res)) { PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 2); int load_method = oparg & 1; #if ENABLE_SPECIALIZATION _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { next_instr--; - _Py_Specialize_LoadSuperAttr(global_super, class, self, next_instr, name, load_method); + _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_SUPER_ATTR, deferred); @@ -1590,17 +1591,38 @@ dummy_func( ERROR_IF(res == NULL, error); } - inst(LOAD_SUPER_ATTR_METHOD, (unused/1, class_version/2, self_type_version/2, method/4, global_super, class, self -- res2, res)) { + inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super, class, self -- res2 if (oparg & 1), res)) { + assert(!(oparg & 1)); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); - DEOPT_IF(((PyTypeObject *)class)->tp_version_tag != class_version, LOAD_SUPER_ATTR); - PyTypeObject *self_type = Py_TYPE(self); - DEOPT_IF(self_type->tp_version_tag != self_type_version, LOAD_SUPER_ATTR); - res2 = method; - res = self; // transfer ownership - Py_INCREF(res2); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 2); + res = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + ERROR_IF(res == NULL, error); + DECREF_INPUTS(); + } + + inst(LOAD_SUPER_ATTR_METHOD, (unused/1, global_super, class, self -- res2, res)) { + assert(oparg & 1); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 2); + int method_found = 0; + res2 = _PySuper_Lookup((PyTypeObject *)class, self, name, &method_found); Py_DECREF(global_super); Py_DECREF(class); + if (res2 == NULL) { + Py_DECREF(self); + ERROR_IF(true, error); + } + if (method_found) { + res = self; // transfer ownership + } else { + Py_DECREF(self); + res = res2; + res2 = NULL; + } } family(load_attr, INLINE_CACHE_ENTRIES_LOAD_ATTR) = { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a86b895dd43d82..373c65e6ce84fc 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2182,20 +2182,20 @@ TARGET(LOAD_SUPER_ATTR) { PREDICTED(LOAD_SUPER_ATTR); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); PyObject *self = stack_pointer[-1]; PyObject *class = stack_pointer[-2]; PyObject *global_super = stack_pointer[-3]; PyObject *res2 = NULL; PyObject *res; - #line 1569 "Python/bytecodes.c" + #line 1570 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 2); int load_method = oparg & 1; #if ENABLE_SPECIALIZATION _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { next_instr--; - _Py_Specialize_LoadSuperAttr(global_super, class, self, next_instr, name, load_method); + _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_SUPER_ATTR, deferred); @@ -2210,7 +2210,7 @@ Py_DECREF(global_super); Py_DECREF(class); Py_DECREF(self); - #line 1587 "Python/bytecodes.c" + #line 1588 "Python/bytecodes.c" if (super == NULL) goto pop_3_error; res = PyObject_GetAttr(super, name); Py_DECREF(super); @@ -2220,7 +2220,33 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } - next_instr += 9; + next_instr += 1; + DISPATCH(); + } + + TARGET(LOAD_SUPER_ATTR_ATTR) { + PyObject *self = stack_pointer[-1]; + PyObject *class = stack_pointer[-2]; + PyObject *global_super = stack_pointer[-3]; + PyObject *res2 = NULL; + PyObject *res; + #line 1595 "Python/bytecodes.c" + assert(!(oparg & 1)); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 2); + res = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + if (res == NULL) goto pop_3_error; + #line 2242 "Python/generated_cases.c.h" + Py_DECREF(global_super); + Py_DECREF(class); + Py_DECREF(self); + STACK_SHRINK(2); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1] = res; + if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } + next_instr += 1; DISPATCH(); } @@ -2230,25 +2256,32 @@ PyObject *global_super = stack_pointer[-3]; PyObject *res2; PyObject *res; - uint32_t class_version = read_u32(&next_instr[1].cache); - uint32_t self_type_version = read_u32(&next_instr[3].cache); - PyObject *method = read_obj(&next_instr[5].cache); - #line 1594 "Python/bytecodes.c" + #line 1606 "Python/bytecodes.c" + assert(oparg & 1); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); - DEOPT_IF(((PyTypeObject *)class)->tp_version_tag != class_version, LOAD_SUPER_ATTR); - PyTypeObject *self_type = Py_TYPE(self); - DEOPT_IF(self_type->tp_version_tag != self_type_version, LOAD_SUPER_ATTR); - res2 = method; - res = self; // transfer ownership - Py_INCREF(res2); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 2); + int method_found = 0; + res2 = _PySuper_Lookup((PyTypeObject *)class, self, name, &method_found); Py_DECREF(global_super); Py_DECREF(class); - #line 2248 "Python/generated_cases.c.h" + if (res2 == NULL) { + Py_DECREF(self); + if (true) goto pop_3_error; + } + if (method_found) { + res = self; // transfer ownership + } else { + Py_DECREF(self); + res = res2; + res2 = NULL; + } + #line 2281 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; stack_pointer[-2] = res2; - next_instr += 9; + next_instr += 1; DISPATCH(); } @@ -2258,7 +2291,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; - #line 1621 "Python/bytecodes.c" + #line 1643 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2292,9 +2325,9 @@ NULL | meth | arg1 | ... | argN */ - #line 2296 "Python/generated_cases.c.h" + #line 2329 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1655 "Python/bytecodes.c" + #line 1677 "Python/bytecodes.c" if (meth == NULL) goto pop_1_error; res2 = NULL; res = meth; @@ -2303,12 +2336,12 @@ else { /* Classic, pushes one value. */ res = PyObject_GetAttr(owner, name); - #line 2307 "Python/generated_cases.c.h" + #line 2340 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1664 "Python/bytecodes.c" + #line 1686 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; } - #line 2312 "Python/generated_cases.c.h" + #line 2345 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -2322,7 +2355,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1669 "Python/bytecodes.c" + #line 1691 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2335,7 +2368,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2339 "Python/generated_cases.c.h" + #line 2372 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2350,7 +2383,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1685 "Python/bytecodes.c" + #line 1707 "Python/bytecodes.c" DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -2363,7 +2396,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2367 "Python/generated_cases.c.h" + #line 2400 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2378,7 +2411,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1701 "Python/bytecodes.c" + #line 1723 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2405,7 +2438,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2409 "Python/generated_cases.c.h" + #line 2442 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2420,7 +2453,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1731 "Python/bytecodes.c" + #line 1753 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2430,7 +2463,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2434 "Python/generated_cases.c.h" + #line 2467 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2445,7 +2478,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 1744 "Python/bytecodes.c" + #line 1766 "Python/bytecodes.c" DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, @@ -2457,7 +2490,7 @@ res = descr; assert(res != NULL); Py_INCREF(res); - #line 2461 "Python/generated_cases.c.h" + #line 2494 "Python/generated_cases.c.h" Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2471,7 +2504,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *fget = read_obj(&next_instr[5].cache); - #line 1759 "Python/bytecodes.c" + #line 1781 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -2495,7 +2528,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 2499 "Python/generated_cases.c.h" + #line 2532 "Python/generated_cases.c.h" } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { @@ -2503,7 +2536,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *getattribute = read_obj(&next_instr[5].cache); - #line 1785 "Python/bytecodes.c" + #line 1807 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -2529,7 +2562,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 2533 "Python/generated_cases.c.h" + #line 2566 "Python/generated_cases.c.h" } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -2537,7 +2570,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1813 "Python/bytecodes.c" + #line 1835 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2555,7 +2588,7 @@ Py_DECREF(old_value); } Py_DECREF(owner); - #line 2559 "Python/generated_cases.c.h" + #line 2592 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2566,7 +2599,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t hint = read_u16(&next_instr[3].cache); - #line 1833 "Python/bytecodes.c" + #line 1855 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2605,7 +2638,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - #line 2609 "Python/generated_cases.c.h" + #line 2642 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2616,7 +2649,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1874 "Python/bytecodes.c" + #line 1896 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2626,7 +2659,7 @@ *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - #line 2630 "Python/generated_cases.c.h" + #line 2663 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2638,7 +2671,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1893 "Python/bytecodes.c" + #line 1915 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2651,12 +2684,12 @@ #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); - #line 2655 "Python/generated_cases.c.h" + #line 2688 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1906 "Python/bytecodes.c" + #line 1928 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2660 "Python/generated_cases.c.h" + #line 2693 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2667,7 +2700,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1910 "Python/bytecodes.c" + #line 1932 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2679,7 +2712,7 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2683 "Python/generated_cases.c.h" + #line 2716 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2690,7 +2723,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1925 "Python/bytecodes.c" + #line 1947 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -2706,7 +2739,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2710 "Python/generated_cases.c.h" + #line 2743 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2717,7 +2750,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1944 "Python/bytecodes.c" + #line 1966 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2730,7 +2763,7 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2734 "Python/generated_cases.c.h" + #line 2767 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2741,14 +2774,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1959 "Python/bytecodes.c" + #line 1981 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; - #line 2747 "Python/generated_cases.c.h" + #line 2780 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1961 "Python/bytecodes.c" + #line 1983 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2752 "Python/generated_cases.c.h" + #line 2785 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2758,15 +2791,15 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1965 "Python/bytecodes.c" + #line 1987 "Python/bytecodes.c" int res = PySequence_Contains(right, left); - #line 2764 "Python/generated_cases.c.h" + #line 2797 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1967 "Python/bytecodes.c" + #line 1989 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = Py_NewRef((res^oparg) ? Py_True : Py_False); - #line 2770 "Python/generated_cases.c.h" + #line 2803 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2777,12 +2810,12 @@ PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; - #line 1972 "Python/bytecodes.c" + #line 1994 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { - #line 2783 "Python/generated_cases.c.h" + #line 2816 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1974 "Python/bytecodes.c" + #line 1996 "Python/bytecodes.c" if (true) goto pop_2_error; } @@ -2790,10 +2823,10 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); - #line 2794 "Python/generated_cases.c.h" + #line 2827 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1982 "Python/bytecodes.c" + #line 2004 "Python/bytecodes.c" if (res < 0) goto pop_2_error; assert((match == NULL) == (rest == NULL)); @@ -2802,7 +2835,7 @@ if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } - #line 2806 "Python/generated_cases.c.h" + #line 2839 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -2812,21 +2845,21 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1993 "Python/bytecodes.c" + #line 2015 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - #line 2819 "Python/generated_cases.c.h" + #line 2852 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1996 "Python/bytecodes.c" + #line 2018 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - #line 2826 "Python/generated_cases.c.h" + #line 2859 "Python/generated_cases.c.h" Py_DECREF(right); - #line 2001 "Python/bytecodes.c" + #line 2023 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2830 "Python/generated_cases.c.h" + #line 2863 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -2835,15 +2868,15 @@ PyObject *fromlist = stack_pointer[-1]; PyObject *level = stack_pointer[-2]; PyObject *res; - #line 2005 "Python/bytecodes.c" + #line 2027 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); - #line 2842 "Python/generated_cases.c.h" + #line 2875 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); - #line 2008 "Python/bytecodes.c" + #line 2030 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2847 "Python/generated_cases.c.h" + #line 2880 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -2852,29 +2885,29 @@ TARGET(IMPORT_FROM) { PyObject *from = stack_pointer[-1]; PyObject *res; - #line 2012 "Python/bytecodes.c" + #line 2034 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - #line 2860 "Python/generated_cases.c.h" + #line 2893 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); } TARGET(JUMP_FORWARD) { - #line 2018 "Python/bytecodes.c" + #line 2040 "Python/bytecodes.c" JUMPBY(oparg); - #line 2869 "Python/generated_cases.c.h" + #line 2902 "Python/generated_cases.c.h" DISPATCH(); } TARGET(JUMP_BACKWARD) { PREDICTED(JUMP_BACKWARD); - #line 2022 "Python/bytecodes.c" + #line 2044 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); - #line 2878 "Python/generated_cases.c.h" + #line 2911 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2882,7 +2915,7 @@ TARGET(POP_JUMP_IF_FALSE) { PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = stack_pointer[-1]; - #line 2028 "Python/bytecodes.c" + #line 2050 "Python/bytecodes.c" if (Py_IsTrue(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2892,9 +2925,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2896 "Python/generated_cases.c.h" + #line 2929 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 2038 "Python/bytecodes.c" + #line 2060 "Python/bytecodes.c" if (err == 0) { JUMPBY(oparg); } @@ -2902,14 +2935,14 @@ if (err < 0) goto pop_1_error; } } - #line 2906 "Python/generated_cases.c.h" + #line 2939 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = stack_pointer[-1]; - #line 2048 "Python/bytecodes.c" + #line 2070 "Python/bytecodes.c" if (Py_IsFalse(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2919,9 +2952,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2923 "Python/generated_cases.c.h" + #line 2956 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 2058 "Python/bytecodes.c" + #line 2080 "Python/bytecodes.c" if (err > 0) { JUMPBY(oparg); } @@ -2929,67 +2962,67 @@ if (err < 0) goto pop_1_error; } } - #line 2933 "Python/generated_cases.c.h" + #line 2966 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = stack_pointer[-1]; - #line 2068 "Python/bytecodes.c" + #line 2090 "Python/bytecodes.c" if (!Py_IsNone(value)) { - #line 2942 "Python/generated_cases.c.h" + #line 2975 "Python/generated_cases.c.h" Py_DECREF(value); - #line 2070 "Python/bytecodes.c" + #line 2092 "Python/bytecodes.c" JUMPBY(oparg); } else { _Py_DECREF_NO_DEALLOC(value); } - #line 2950 "Python/generated_cases.c.h" + #line 2983 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = stack_pointer[-1]; - #line 2078 "Python/bytecodes.c" + #line 2100 "Python/bytecodes.c" if (Py_IsNone(value)) { _Py_DECREF_NO_DEALLOC(value); JUMPBY(oparg); } else { - #line 2963 "Python/generated_cases.c.h" + #line 2996 "Python/generated_cases.c.h" Py_DECREF(value); - #line 2084 "Python/bytecodes.c" + #line 2106 "Python/bytecodes.c" } - #line 2967 "Python/generated_cases.c.h" + #line 3000 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - #line 2088 "Python/bytecodes.c" + #line 2110 "Python/bytecodes.c" /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ JUMPBY(-oparg); - #line 2980 "Python/generated_cases.c.h" + #line 3013 "Python/generated_cases.c.h" DISPATCH(); } TARGET(GET_LEN) { PyObject *obj = stack_pointer[-1]; PyObject *len_o; - #line 2097 "Python/bytecodes.c" + #line 2119 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 2993 "Python/generated_cases.c.h" + #line 3026 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -3000,16 +3033,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; - #line 2105 "Python/bytecodes.c" + #line 2127 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 3009 "Python/generated_cases.c.h" + #line 3042 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); - #line 2110 "Python/bytecodes.c" + #line 2132 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -3017,7 +3050,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_NewRef(Py_None); // Failure! } - #line 3021 "Python/generated_cases.c.h" + #line 3054 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -3026,10 +3059,10 @@ TARGET(MATCH_MAPPING) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2120 "Python/bytecodes.c" + #line 2142 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = Py_NewRef(match ? Py_True : Py_False); - #line 3033 "Python/generated_cases.c.h" + #line 3066 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -3039,10 +3072,10 @@ TARGET(MATCH_SEQUENCE) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2126 "Python/bytecodes.c" + #line 2148 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = Py_NewRef(match ? Py_True : Py_False); - #line 3046 "Python/generated_cases.c.h" + #line 3079 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -3053,11 +3086,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; - #line 2132 "Python/bytecodes.c" + #line 2154 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 3061 "Python/generated_cases.c.h" + #line 3094 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -3066,14 +3099,14 @@ TARGET(GET_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2138 "Python/bytecodes.c" + #line 2160 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 3073 "Python/generated_cases.c.h" + #line 3106 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2141 "Python/bytecodes.c" + #line 2163 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 3077 "Python/generated_cases.c.h" + #line 3110 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -3081,7 +3114,7 @@ TARGET(GET_YIELD_FROM_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2145 "Python/bytecodes.c" + #line 2167 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -3104,11 +3137,11 @@ if (iter == NULL) { goto error; } - #line 3108 "Python/generated_cases.c.h" + #line 3141 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2168 "Python/bytecodes.c" + #line 2190 "Python/bytecodes.c" } - #line 3112 "Python/generated_cases.c.h" + #line 3145 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -3119,7 +3152,7 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2187 "Python/bytecodes.c" + #line 2209 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -3150,7 +3183,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator - #line 3154 "Python/generated_cases.c.h" + #line 3187 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3158,7 +3191,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - #line 2220 "Python/bytecodes.c" + #line 2242 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -3184,14 +3217,14 @@ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - #line 3188 "Python/generated_cases.c.h" + #line 3221 "Python/generated_cases.c.h" DISPATCH(); } TARGET(FOR_ITER_LIST) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2248 "Python/bytecodes.c" + #line 2270 "Python/bytecodes.c" DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; STAT_INC(FOR_ITER, hit); @@ -3211,7 +3244,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator - #line 3215 "Python/generated_cases.c.h" + #line 3248 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3221,7 +3254,7 @@ TARGET(FOR_ITER_TUPLE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2270 "Python/bytecodes.c" + #line 2292 "Python/bytecodes.c" _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3241,7 +3274,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator - #line 3245 "Python/generated_cases.c.h" + #line 3278 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3251,7 +3284,7 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2292 "Python/bytecodes.c" + #line 2314 "Python/bytecodes.c" _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3269,7 +3302,7 @@ if (next == NULL) { goto error; } - #line 3273 "Python/generated_cases.c.h" + #line 3306 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3278,7 +3311,7 @@ TARGET(FOR_ITER_GEN) { PyObject *iter = stack_pointer[-1]; - #line 2312 "Python/bytecodes.c" + #line 2334 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -3293,14 +3326,14 @@ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); - #line 3297 "Python/generated_cases.c.h" + #line 3330 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2329 "Python/bytecodes.c" + #line 2351 "Python/bytecodes.c" PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -3323,16 +3356,16 @@ Py_DECREF(enter); goto error; } - #line 3327 "Python/generated_cases.c.h" + #line 3360 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2352 "Python/bytecodes.c" + #line 2374 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3336 "Python/generated_cases.c.h" + #line 3369 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3344,7 +3377,7 @@ PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2362 "Python/bytecodes.c" + #line 2384 "Python/bytecodes.c" /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -3370,16 +3403,16 @@ Py_DECREF(enter); goto error; } - #line 3374 "Python/generated_cases.c.h" + #line 3407 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2388 "Python/bytecodes.c" + #line 2410 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3383 "Python/generated_cases.c.h" + #line 3416 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3391,7 +3424,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; - #line 2397 "Python/bytecodes.c" + #line 2419 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3412,7 +3445,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 3416 "Python/generated_cases.c.h" + #line 3449 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3421,7 +3454,7 @@ TARGET(PUSH_EXC_INFO) { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; - #line 2420 "Python/bytecodes.c" + #line 2442 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3431,7 +3464,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 3435 "Python/generated_cases.c.h" + #line 3468 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3445,7 +3478,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2432 "Python/bytecodes.c" + #line 2454 "Python/bytecodes.c" /* Cached method object */ PyTypeObject *self_cls = Py_TYPE(self); assert(type_version != 0); @@ -3462,7 +3495,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); - #line 3466 "Python/generated_cases.c.h" + #line 3499 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3476,7 +3509,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2451 "Python/bytecodes.c" + #line 2473 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); @@ -3486,7 +3519,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3490 "Python/generated_cases.c.h" + #line 3523 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3500,7 +3533,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2463 "Python/bytecodes.c" + #line 2485 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; @@ -3514,7 +3547,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3518 "Python/generated_cases.c.h" + #line 3551 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3523,16 +3556,16 @@ } TARGET(KW_NAMES) { - #line 2479 "Python/bytecodes.c" + #line 2501 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); - #line 3531 "Python/generated_cases.c.h" + #line 3564 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_CALL) { - #line 2485 "Python/bytecodes.c" + #line 2507 "Python/bytecodes.c" int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); @@ -3545,7 +3578,7 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(CALL); - #line 3549 "Python/generated_cases.c.h" + #line 3582 "Python/generated_cases.c.h" } TARGET(CALL) { @@ -3555,7 +3588,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2530 "Python/bytecodes.c" + #line 2552 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3637,7 +3670,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3641 "Python/generated_cases.c.h" + #line 3674 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3649,7 +3682,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2618 "Python/bytecodes.c" + #line 2640 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3659,7 +3692,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3663 "Python/generated_cases.c.h" + #line 3696 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3668,7 +3701,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2630 "Python/bytecodes.c" + #line 2652 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3694,7 +3727,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 3698 "Python/generated_cases.c.h" + #line 3731 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3702,7 +3735,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2658 "Python/bytecodes.c" + #line 2680 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3738,7 +3771,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 3742 "Python/generated_cases.c.h" + #line 3775 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3746,7 +3779,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2696 "Python/bytecodes.c" + #line 2718 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3756,7 +3789,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 3760 "Python/generated_cases.c.h" + #line 3793 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3769,7 +3802,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2708 "Python/bytecodes.c" + #line 2730 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3780,7 +3813,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3784 "Python/generated_cases.c.h" + #line 3817 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3794,7 +3827,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2722 "Python/bytecodes.c" + #line 2744 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3805,7 +3838,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3809 "Python/generated_cases.c.h" + #line 3842 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3819,7 +3852,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2736 "Python/bytecodes.c" + #line 2758 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3841,7 +3874,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3845 "Python/generated_cases.c.h" + #line 3878 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3855,7 +3888,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2761 "Python/bytecodes.c" + #line 2783 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3883,7 +3916,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3887 "Python/generated_cases.c.h" + #line 3920 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3897,7 +3930,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2792 "Python/bytecodes.c" + #line 2814 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3929,7 +3962,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 3933 "Python/generated_cases.c.h" + #line 3966 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3943,7 +3976,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2827 "Python/bytecodes.c" + #line 2849 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -3975,7 +4008,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3979 "Python/generated_cases.c.h" + #line 4012 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3989,7 +4022,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2862 "Python/bytecodes.c" + #line 2884 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -4014,7 +4047,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4018 "Python/generated_cases.c.h" + #line 4051 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4027,7 +4060,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2889 "Python/bytecodes.c" + #line 2911 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -4054,7 +4087,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4058 "Python/generated_cases.c.h" + #line 4091 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4066,7 +4099,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2919 "Python/bytecodes.c" + #line 2941 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); assert(method != NULL); @@ -4084,14 +4117,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 4088 "Python/generated_cases.c.h" + #line 4121 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2939 "Python/bytecodes.c" + #line 2961 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4122,7 +4155,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4126 "Python/generated_cases.c.h" + #line 4159 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4135,7 +4168,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2973 "Python/bytecodes.c" + #line 2995 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4164,7 +4197,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4168 "Python/generated_cases.c.h" + #line 4201 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4177,7 +4210,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3005 "Python/bytecodes.c" + #line 3027 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4206,7 +4239,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4210 "Python/generated_cases.c.h" + #line 4243 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4219,7 +4252,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3037 "Python/bytecodes.c" + #line 3059 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4247,7 +4280,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4251 "Python/generated_cases.c.h" + #line 4284 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4257,9 +4290,9 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #line 3068 "Python/bytecodes.c" + #line 3090 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4263 "Python/generated_cases.c.h" + #line 4296 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4268,7 +4301,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 3072 "Python/bytecodes.c" + #line 3094 "Python/bytecodes.c" // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4330,14 +4363,14 @@ } result = PyObject_Call(func, callargs, kwargs); } - #line 4334 "Python/generated_cases.c.h" + #line 4367 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3134 "Python/bytecodes.c" + #line 3156 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4341 "Python/generated_cases.c.h" + #line 4374 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4352,7 +4385,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 3144 "Python/bytecodes.c" + #line 3166 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4381,14 +4414,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4385 "Python/generated_cases.c.h" + #line 4418 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3175 "Python/bytecodes.c" + #line 3197 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4409,7 +4442,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4413 "Python/generated_cases.c.h" + #line 4446 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4417,15 +4450,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3198 "Python/bytecodes.c" + #line 3220 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4423 "Python/generated_cases.c.h" + #line 4456 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3200 "Python/bytecodes.c" + #line 3222 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4429 "Python/generated_cases.c.h" + #line 4462 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4436,7 +4469,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 3204 "Python/bytecodes.c" + #line 3226 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4471,7 +4504,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4475 "Python/generated_cases.c.h" + #line 4508 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4480,10 +4513,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3241 "Python/bytecodes.c" + #line 3263 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4487 "Python/generated_cases.c.h" + #line 4520 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4495,7 +4528,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3246 "Python/bytecodes.c" + #line 3268 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4510,12 +4543,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4514 "Python/generated_cases.c.h" + #line 4547 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3261 "Python/bytecodes.c" + #line 3283 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4519 "Python/generated_cases.c.h" + #line 4552 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4525,16 +4558,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3266 "Python/bytecodes.c" + #line 3288 "Python/bytecodes.c" assert(oparg >= 2); - #line 4531 "Python/generated_cases.c.h" + #line 4564 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_LINE) { - #line 3270 "Python/bytecodes.c" + #line 3292 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( @@ -4554,11 +4587,11 @@ } opcode = original_opcode; DISPATCH_GOTO(); - #line 4558 "Python/generated_cases.c.h" + #line 4591 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3292 "Python/bytecodes.c" + #line 3314 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4570,26 +4603,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4574 "Python/generated_cases.c.h" + #line 4607 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3306 "Python/bytecodes.c" + #line 3328 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4580 "Python/generated_cases.c.h" + #line 4613 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3310 "Python/bytecodes.c" + #line 3332 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4587 "Python/generated_cases.c.h" + #line 4620 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3315 "Python/bytecodes.c" + #line 3337 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4598,12 +4631,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4602 "Python/generated_cases.c.h" + #line 4635 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3326 "Python/bytecodes.c" + #line 3348 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4612,12 +4645,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4616 "Python/generated_cases.c.h" + #line 4649 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3337 "Python/bytecodes.c" + #line 3359 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4630,12 +4663,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4634 "Python/generated_cases.c.h" + #line 4667 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3352 "Python/bytecodes.c" + #line 3374 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4648,30 +4681,30 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4652 "Python/generated_cases.c.h" + #line 4685 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3367 "Python/bytecodes.c" + #line 3389 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4663 "Python/generated_cases.c.h" + #line 4696 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3375 "Python/bytecodes.c" + #line 3397 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); - #line 4670 "Python/generated_cases.c.h" + #line 4703 "Python/generated_cases.c.h" } TARGET(RESERVED) { - #line 3380 "Python/bytecodes.c" + #line 3402 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); - #line 4677 "Python/generated_cases.c.h" + #line 4710 "Python/generated_cases.c.h" } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 670290e2f3f9f9..daf3a3888f0dbd 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -209,6 +209,8 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 2; case LOAD_SUPER_ATTR: return 3; + case LOAD_SUPER_ATTR_ATTR: + return 3; case LOAD_SUPER_ATTR_METHOD: return 3; case LOAD_ATTR: @@ -599,6 +601,8 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case LOAD_SUPER_ATTR: return ((oparg & 1) ? 1 : 0) + 1; + case LOAD_SUPER_ATTR_ATTR: + return ((oparg & 1) ? 1 : 0) + 1; case LOAD_SUPER_ATTR_METHOD: return 2; case LOAD_ATTR: @@ -783,7 +787,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { } #endif -enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC00, INSTR_FMT_IBC000, INSTR_FMT_IBC00000000, INSTR_FMT_IBIB, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC000, INSTR_FMT_IXC00000000 }; +enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC00, INSTR_FMT_IBC000, INSTR_FMT_IBC00000000, INSTR_FMT_IBIB, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC000 }; struct opcode_metadata { bool valid_entry; enum InstructionFormat instr_format; @@ -892,8 +896,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [DICT_UPDATE] = { true, INSTR_FMT_IB }, [DICT_MERGE] = { true, INSTR_FMT_IB }, [MAP_ADD] = { true, INSTR_FMT_IB }, - [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC00000000 }, - [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IXC00000000 }, + [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC }, + [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC }, + [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC }, [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000 }, [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000 }, [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000 }, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 00f15ff98da418..f2742f4ad1f20e 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -65,29 +65,29 @@ static void *opcode_targets[256] = { &&TARGET_FOR_ITER_TUPLE, &&TARGET_FOR_ITER_RANGE, &&TARGET_FOR_ITER_GEN, + &&TARGET_LOAD_SUPER_ATTR_ATTR, &&TARGET_LOAD_SUPER_ATTR_METHOD, - &&TARGET_LOAD_ATTR_CLASS, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, - &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, + &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_BUILD_CLASS, + &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, - &&TARGET_LOAD_ATTR_MODULE, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_LOAD_ATTR_MODULE, &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_LOAD_ATTR_METHOD_NO_DICT, &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, - &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_RETURN_VALUE, - &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_SETUP_ANNOTATIONS, + &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_LOAD_GLOBAL_BUILTIN, - &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_POP_EXCEPT, &&TARGET_STORE_NAME, &&TARGET_DELETE_NAME, @@ -110,9 +110,9 @@ static void *opcode_targets[256] = { &&TARGET_IMPORT_NAME, &&TARGET_IMPORT_FROM, &&TARGET_JUMP_FORWARD, + &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, - &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -152,24 +152,24 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, + &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_FAST__LOAD_FAST, - &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, + &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_UNPACK_SEQUENCE_LIST, - &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, + &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&TARGET_SEND_GEN, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_CALL, &&TARGET_KW_NAMES, &&TARGET_CALL_INTRINSIC_1, diff --git a/Python/specialize.c b/Python/specialize.c index 45f1cb412db241..5071d8ef9a495d 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -323,11 +323,8 @@ _PyCode_Quicken(PyCodeObject *code) /* Super */ -#define SPEC_FAIL_SUPER_NOT_LOAD_METHOD 9 -#define SPEC_FAIL_SUPER_BAD_CLASS 10 -#define SPEC_FAIL_SUPER_SHADOWED 11 -#define SPEC_FAIL_SUPER_NOT_METHOD 12 -#define SPEC_FAIL_SUPER_ERROR_OR_NOT_FOUND 13 +#define SPEC_FAIL_SUPER_BAD_CLASS 9 +#define SPEC_FAIL_SUPER_SHADOWED 10 /* Attributes */ @@ -516,15 +513,10 @@ specialize_module_load_attr( /* Attribute specialization */ void -_Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, PyObject *self, - _Py_CODEUNIT *instr, PyObject *name, int load_method) { +_Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, _Py_CODEUNIT *instr, int load_method) { assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[LOAD_SUPER_ATTR] == INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); _PySuperAttrCache *cache = (_PySuperAttrCache *)(instr + 1); - if (!load_method) { - SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_NOT_LOAD_METHOD); - goto fail; - } if (global_super != (PyObject *)&PySuper_Type) { SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_SHADOWED); goto fail; @@ -533,23 +525,8 @@ _Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, PyObject *se SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_BAD_CLASS); goto fail; } - PyTypeObject *tp = (PyTypeObject *)cls; - PyObject *res = _PySuper_LookupDescr(tp, self, name); - if (res == NULL) { - SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_ERROR_OR_NOT_FOUND); - PyErr_Clear(); - goto fail; - } - if (_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)) { - write_u32(cache->class_version, tp->tp_version_tag); - write_u32(cache->self_type_version, Py_TYPE(self)->tp_version_tag); - write_obj(cache->method, res); // borrowed - instr->op.code = LOAD_SUPER_ATTR_METHOD; - Py_DECREF(res); - goto success; - } - Py_DECREF(res); - SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_NOT_METHOD); + instr->op.code = load_method ? LOAD_SUPER_ATTR_METHOD : LOAD_SUPER_ATTR_ATTR; + goto success; fail: STAT_INC(LOAD_SUPER_ATTR, failure); diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 4f25ba36d86689..9c881897c2de1d 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -226,11 +226,13 @@ def pretty(defname): def kind_to_text(kind, defines, opname): if kind <= 8: return pretty(defines[kind][0]) - if opname.endswith("ATTR"): + if opname == "LOAD_SUPER_ATTR": + opname = "SUPER" + elif opname.endswith("ATTR"): opname = "ATTR" - if opname in ("FOR_ITER", "SEND"): + elif opname in ("FOR_ITER", "SEND"): opname = "ITER" - if opname.endswith("SUBSCR"): + elif opname.endswith("SUBSCR"): opname = "SUBSCR" for name in defines[kind]: if name.startswith(opname): From 27419a71b5aa18baf24f4e640c5a6e8df9338928 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Thu, 11 May 2023 17:44:39 +0300 Subject: [PATCH 157/204] gh-104057: Fix direct invocation of test_support (GH-104069) --- Lib/test/test_support.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 7738ca5e9b433d..85d692f3097494 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -30,7 +30,7 @@ def setUpClass(cls): "test.support.warnings_helper", like=".*used in test_support.*" ) cls._test_support_token = support.ignore_deprecations_from( - "test.test_support", like=".*You should NOT be seeing this.*" + __name__, like=".*You should NOT be seeing this.*" ) assert len(warnings.filters) == orig_filter_len + 2 From 0449ffe3a4ddf03367a5ee3d943c89f442b7b407 Mon Sep 17 00:00:00 2001 From: James Gerity Date: Thu, 11 May 2023 13:12:02 -0400 Subject: [PATCH 158/204] gh-104301: Allow leading whitespace in disambiguated pdb statements (#104342) --- Doc/library/pdb.rst | 14 ++++++++--- Lib/pdb.py | 11 +++++---- Lib/pydoc_data/topics.py | 13 +++++++---- Lib/test/test_pdb.py | 23 +++++++++++++++++++ ...-05-09-18-46-24.gh-issue-104301.gNnbId.rst | 1 + 5 files changed, 50 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-09-18-46-24.gh-issue-104301.gNnbId.rst diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 74bffef5562ae1..ef52370bff8058 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -602,9 +602,17 @@ can be overridden by the local file. Execute the (one-line) *statement* in the context of the current stack frame. The exclamation point can be omitted unless the first word of the statement - resembles a debugger command. To set a global variable, you can prefix the - assignment command with a :keyword:`global` statement on the same line, - e.g.:: + resembles a debugger command, e.g.: + + .. code-block:: none + + (Pdb) ! n=42 + (Pdb) + + To set a global variable, you can prefix the assignment command with a + :keyword:`global` statement on the same line, e.g.: + + .. code-block:: none (Pdb) global list_options; list_options = ['-l'] (Pdb) diff --git a/Lib/pdb.py b/Lib/pdb.py index b3dc5a455e56b9..6b6feac1ddead1 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -440,7 +440,7 @@ def displayhook(self, obj): self.message(repr(obj)) def default(self, line): - if line[:1] == '!': line = line[1:] + if line[:1] == '!': line = line[1:].strip() locals = self.curframe_locals globals = self.curframe.f_globals try: @@ -1642,9 +1642,12 @@ def help_exec(self): Execute the (one-line) statement in the context of the current stack frame. The exclamation point can be omitted unless the - first word of the statement resembles a debugger command. To - assign to a global variable you must always prefix the command - with a 'global' command, e.g.: + first word of the statement resembles a debugger command, e.g.: + (Pdb) ! n=42 + (Pdb) + + To assign to a global variable you must always prefix the command with + a 'global' command, e.g.: (Pdb) global list_options; list_options = ['-l'] (Pdb) """ diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 1babb5ce9476c9..3aaaee67fa35a6 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -5283,11 +5283,14 @@ 'current\n' ' stack frame. The exclamation point can be omitted unless the ' 'first\n' - ' word of the statement resembles a debugger command. To set ' - 'a\n' - ' global variable, you can prefix the assignment command with ' - 'a\n' - ' "global" statement on the same line, e.g.:\n' + ' word of the statement resembles a debugger command, e.g.:' + '\n' + ' (Pdb) ! n=42\n' + ' (Pdb)\n' + '\n' + ' To set a global variable, you can prefix the assignment command ' + ' with \n' + ' a "global" statement on the same line, e.g.:\n' '\n' " (Pdb) global list_options; list_options = ['-l']\n" ' (Pdb)\n' diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 482c92dbf1f6a0..037673dd0ea802 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -1798,6 +1798,29 @@ def test_pdb_issue_gh_101517(): (Pdb) continue """ +def test_pdb_ambiguous_statements(): + """See GH-104301 + + Make sure that ambiguous statements prefixed by '!' are properly disambiguated + + >>> with PdbTestInput([ + ... '! n = 42', # disambiguated statement: reassign the name n + ... 'n', # advance the debugger into the print() + ... 'continue' + ... ]): + ... n = -1 + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... print(f"The value of n is {n}") + > (8)() + -> print(f"The value of n is {n}") + (Pdb) ! n = 42 + (Pdb) n + The value of n is 42 + > (1)() + -> with PdbTestInput([ + (Pdb) continue + """ + @support.requires_subprocess() class PdbTestCase(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2023-05-09-18-46-24.gh-issue-104301.gNnbId.rst b/Misc/NEWS.d/next/Library/2023-05-09-18-46-24.gh-issue-104301.gNnbId.rst new file mode 100644 index 00000000000000..a44ad765e62458 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-09-18-46-24.gh-issue-104301.gNnbId.rst @@ -0,0 +1 @@ +Allow leading whitespace in disambiguated statements in :mod:`pdb`. From 7d7dd4cd70ed997ed7c3cda867c4e7b1ab02b205 Mon Sep 17 00:00:00 2001 From: Jonathan Protzenko Date: Thu, 11 May 2023 14:23:52 -0700 Subject: [PATCH 159/204] gh-99108: Refresh HACL* from upstream (#104401) Refresh HACL* from upstream and add a SHA3 test hashing over 4GiB of data. --- Lib/test/test_hashlib.py | 9 +++++++++ Modules/_hacl/Hacl_Hash_SHA3.c | 2 +- Modules/_hacl/refresh.sh | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 08cb5eb0c2bbab..73d758a3631b3a 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -359,6 +359,15 @@ def test_sha256_update_over_4gb(self): h.update(b"hello world") self.assertEqual(h.hexdigest(), "a5364f7a52ebe2e25f1838a4ca715a893b6fd7a23f2a0d9e9762120da8b1bf53") + @requires_resource('cpu') + def test_sha3_256_update_over_4gb(self): + zero_1mb = b"\0" * 1024 * 1024 + h = hashlib.sha3_256() + for i in range(0, 4096): + h.update(zero_1mb) + h.update(b"hello world") + self.assertEqual(h.hexdigest(), "e2d4535e3b613135c14f2fe4e026d7ad8d569db44901740beffa30d430acb038") + def check(self, name, data, hexdigest, shake=False, **kwargs): length = len(hexdigest)//2 hexdigest = hexdigest.lower() diff --git a/Modules/_hacl/Hacl_Hash_SHA3.c b/Modules/_hacl/Hacl_Hash_SHA3.c index 100afe7c2c6d1f..58eb436881d45c 100644 --- a/Modules/_hacl/Hacl_Hash_SHA3.c +++ b/Modules/_hacl/Hacl_Hash_SHA3.c @@ -244,7 +244,7 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint Hacl_Streaming_Keccak_hash_buf block_state = s.block_state; uint64_t total_len = s.total_len; Spec_Hash_Definitions_hash_alg i = block_state.fst; - if ((uint64_t)len > (uint64_t)0xffffffffU - total_len) + if ((uint64_t)len > (uint64_t)0xffffffffffffffffU - total_len) { return (uint32_t)1U; } diff --git a/Modules/_hacl/refresh.sh b/Modules/_hacl/refresh.sh index 220ebbe5561341..d2ba05f30d860f 100755 --- a/Modules/_hacl/refresh.sh +++ b/Modules/_hacl/refresh.sh @@ -22,7 +22,7 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. -expected_hacl_star_rev=363eae2c2eb60e46f182ddd4bd1cd3f1d00b35c9 +expected_hacl_star_rev=b6903a3e6458000730c3d83174d4b08d6d3e2ece hacl_dir="$(realpath "$1")" cd "$(dirname "$0")" From 434db68ee31514ddc4aa93f8dfc2eb874d3669c5 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Fri, 12 May 2023 07:45:55 +0900 Subject: [PATCH 160/204] gh-104396: uuid.py to skip platform check for emscripten and wasi (gh-104397) --- Lib/uuid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index 697f3b45597023..470bc0d68597ab 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -53,7 +53,7 @@ __author__ = 'Ka-Ping Yee ' # The recognized platforms - known behaviors -if sys.platform in ('win32', 'darwin'): +if sys.platform in ('win32', 'darwin', 'emscripten', 'wasi'): _AIX = _LINUX = False else: import platform From 37a5d256b97bc9d2a0ff445997fec851e328ebad Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 11 May 2023 16:41:26 -0700 Subject: [PATCH 161/204] gh-104392: Remove _paramspec_tvars from typing (#104393) This does nothing. --- Lib/typing.py | 19 ++++++------------- ...-05-11-07-50-00.gh-issue-104392.YSllzt.rst | 2 ++ 2 files changed, 8 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-11-07-50-00.gh-issue-104392.YSllzt.rst diff --git a/Lib/typing.py b/Lib/typing.py index 4f8cba88632d6a..62c7dd31a6290a 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -775,8 +775,7 @@ def Concatenate(self, parameters): "ParamSpec variable or ellipsis.") msg = "Concatenate[arg, ...]: each arg must be a type." parameters = (*(_type_check(p, msg) for p in parameters[:-1]), parameters[-1]) - return _ConcatenateGenericAlias(self, parameters, - _paramspec_tvars=True) + return _ConcatenateGenericAlias(self, parameters) @_SpecialForm @@ -1307,8 +1306,7 @@ def __getattr__(self, attr): raise AttributeError(attr) def __setattr__(self, attr, val): - if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams', - '_paramspec_tvars'}: + if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams'}: super().__setattr__(attr, val) else: setattr(self.__origin__, attr, val) @@ -1362,15 +1360,13 @@ class _GenericAlias(_BaseGenericAlias, _root=True): # ClassVar[float] # TypeVar[bool] - def __init__(self, origin, args, *, inst=True, name=None, - _paramspec_tvars=False): + def __init__(self, origin, args, *, inst=True, name=None): super().__init__(origin, inst=inst, name=name) if not isinstance(args, tuple): args = (args,) self.__args__ = tuple(... if a is _TypingEllipsis else a for a in args) self.__parameters__ = _collect_parameters(args) - self._paramspec_tvars = _paramspec_tvars if not name: self.__module__ = origin.__module__ @@ -1513,8 +1509,7 @@ def _make_substitution(self, args, new_arg_by_param): return new_args def copy_with(self, args): - return self.__class__(self.__origin__, args, name=self._name, inst=self._inst, - _paramspec_tvars=self._paramspec_tvars) + return self.__class__(self.__origin__, args, name=self._name, inst=self._inst) def __repr__(self): if self._name: @@ -1624,8 +1619,7 @@ def __reduce__(self): class _CallableType(_SpecialGenericAlias, _root=True): def copy_with(self, params): return _CallableGenericAlias(self.__origin__, params, - name=self._name, inst=self._inst, - _paramspec_tvars=True) + name=self._name, inst=self._inst) def __getitem__(self, params): if not isinstance(params, tuple) or len(params) != 2: @@ -1869,8 +1863,7 @@ def __class_getitem__(cls, params): new_args.append(new_arg) params = tuple(new_args) - return _GenericAlias(cls, params, - _paramspec_tvars=True) + return _GenericAlias(cls, params) def __init_subclass__(cls, *args, **kwargs): super().__init_subclass__(*args, **kwargs) diff --git a/Misc/NEWS.d/next/Library/2023-05-11-07-50-00.gh-issue-104392.YSllzt.rst b/Misc/NEWS.d/next/Library/2023-05-11-07-50-00.gh-issue-104392.YSllzt.rst new file mode 100644 index 00000000000000..b441b2acd19f1b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-11-07-50-00.gh-issue-104392.YSllzt.rst @@ -0,0 +1,2 @@ +Remove undocumented and unused ``_paramspec_tvars`` attribute from some +classes in :mod:`typing`. From ac66cc17f21653b66321b50d0a1f792982fca21f Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Thu, 11 May 2023 17:48:21 -0600 Subject: [PATCH 162/204] gh-104377: fix cell in comprehension that is free in outer scope (#104394) --- Lib/test/test_listcomps.py | 59 +++++++++++++++++++++++++++++++++++--- Python/compile.c | 15 ++++++++-- 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 1cc202bb599ae6..b2a3b7ea3e49b6 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -117,15 +117,15 @@ def get_output(moddict, name): newcode = code def get_output(moddict, name): return moddict[name] - ns = ns or {} + newns = ns.copy() if ns else {} try: - exec(newcode, ns) + exec(newcode, newns) except raises as e: # We care about e.g. NameError vs UnboundLocalError self.assertIs(type(e), raises) else: for k, v in (outputs or {}).items(): - self.assertEqual(get_output(ns, k), v) + self.assertEqual(get_output(newns, k), v) def test_lambdas_with_iteration_var_as_default(self): code = """ @@ -180,6 +180,26 @@ def test_closure_can_jump_over_comp_scope(self): z = [x() for x in items] """ outputs = {"z": [2, 2, 2, 2, 2]} + self._check_in_scopes(code, outputs, scopes=["module", "function"]) + + def test_cell_inner_free_outer(self): + code = """ + def f(): + return [lambda: x for x in (x, [1])[1]] + x = ... + y = [fn() for fn in f()] + """ + outputs = {"y": [1]} + self._check_in_scopes(code, outputs, scopes=["module", "function"]) + + def test_free_inner_cell_outer(self): + code = """ + g = 2 + def f(): + return g + y = [g for x in [1]] + """ + outputs = {"y": [2]} self._check_in_scopes(code, outputs) def test_inner_cell_shadows_outer_redefined(self): @@ -203,6 +223,37 @@ def inner(): outputs = {"x": -1} self._check_in_scopes(code, outputs, ns={"g": -1}) + def test_explicit_global(self): + code = """ + global g + x = g + g = 2 + items = [g for g in [1]] + y = g + """ + outputs = {"x": 1, "y": 2, "items": [1]} + self._check_in_scopes(code, outputs, ns={"g": 1}) + + def test_explicit_global_2(self): + code = """ + global g + x = g + g = 2 + items = [g for x in [1]] + y = g + """ + outputs = {"x": 1, "y": 2, "items": [2]} + self._check_in_scopes(code, outputs, ns={"g": 1}) + + def test_explicit_global_3(self): + code = """ + global g + fns = [lambda: g for g in [2]] + items = [fn() for fn in fns] + """ + outputs = {"items": [2]} + self._check_in_scopes(code, outputs, ns={"g": 1}) + def test_assignment_expression(self): code = """ x = -1 @@ -250,7 +301,7 @@ def g(): g() """ outputs = {"x": 1} - self._check_in_scopes(code, outputs) + self._check_in_scopes(code, outputs, scopes=["module", "function"]) def test_introspecting_frame_locals(self): code = """ diff --git a/Python/compile.c b/Python/compile.c index 941c6e9d4fdbb7..f8d0197e9f0682 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -5028,14 +5028,19 @@ push_inlined_comprehension_state(struct compiler *c, location loc, long scope = (symbol >> SCOPE_OFFSET) & SCOPE_MASK; PyObject *outv = PyDict_GetItemWithError(c->u->u_ste->ste_symbols, k); if (outv == NULL) { + assert(PyErr_Occurred()); return ERROR; } assert(PyLong_Check(outv)); long outsc = (PyLong_AS_LONG(outv) >> SCOPE_OFFSET) & SCOPE_MASK; - if (scope != outsc) { + if (scope != outsc && !(scope == CELL && outsc == FREE)) { // If a name has different scope inside than outside the // comprehension, we need to temporarily handle it with the - // right scope while compiling the comprehension. + // right scope while compiling the comprehension. (If it's free + // in outer scope and cell in inner scope, we can't treat it as + // both cell and free in the same function, but treating it as + // free throughout is fine; it's *_DEREF either way.) + if (state->temp_symbols == NULL) { state->temp_symbols = PyDict_New(); if (state->temp_symbols == NULL) { @@ -5071,7 +5076,11 @@ push_inlined_comprehension_state(struct compiler *c, location loc, // comprehension and restore the original one after ADDOP_NAME(c, loc, LOAD_FAST_AND_CLEAR, k, varnames); if (scope == CELL) { - ADDOP_NAME(c, loc, MAKE_CELL, k, cellvars); + if (outsc == FREE) { + ADDOP_NAME(c, loc, MAKE_CELL, k, freevars); + } else { + ADDOP_NAME(c, loc, MAKE_CELL, k, cellvars); + } } if (PyList_Append(state->pushed_locals, k) < 0) { return ERROR; From a0a98ddb31591357bead4694b21717cb4034924f Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 11 May 2023 22:22:40 -0700 Subject: [PATCH 163/204] gh-104371: Fix calls to `__release_buffer__` while an exception is active (#104378) Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> --- Lib/test/test_buffer.py | 13 +++++++++++++ Objects/typeobject.c | 14 ++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 2c65ae8114818f..94fc9d4436b717 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4749,6 +4749,19 @@ def __buffer__(self, flags): c.clear() self.assertIs(c.buffer, None) + def test_release_buffer_with_exception_set(self): + class A: + def __buffer__(self, flags): + return memoryview(bytes(8)) + def __release_buffer__(self, view): + pass + + b = bytearray(8) + with memoryview(b): + # now b.extend will raise an exception due to exports + with self.assertRaises(BufferError): + b.extend(A()) + if __name__ == "__main__": unittest.main() diff --git a/Objects/typeobject.c b/Objects/typeobject.c index f1745c91967358..f40e197f8c238a 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -9117,6 +9117,12 @@ releasebuffer_maybe_call_super(PyObject *self, Py_buffer *buffer) static void releasebuffer_call_python(PyObject *self, Py_buffer *buffer) { + // bf_releasebuffer may be called while an exception is already active. + // We have no way to report additional errors up the stack, because + // this slot returns void, so we simply stash away the active exception + // and restore it after the call to Python returns. + PyObject *exc = PyErr_GetRaisedException(); + PyObject *mv; bool is_buffer_wrapper = Py_TYPE(buffer->obj) == &_PyBufferWrapper_Type; if (is_buffer_wrapper) { @@ -9124,7 +9130,7 @@ releasebuffer_call_python(PyObject *self, Py_buffer *buffer) // __release_buffer__() that __buffer__() returned. PyBufferWrapper *bw = (PyBufferWrapper *)buffer->obj; if (bw->mv == NULL) { - return; + goto end; } mv = Py_NewRef(bw->mv); } @@ -9134,7 +9140,7 @@ releasebuffer_call_python(PyObject *self, Py_buffer *buffer) mv = PyMemoryView_FromBuffer(buffer); if (mv == NULL) { PyErr_WriteUnraisable(self); - return; + goto end; } // Set the memoryview to restricted mode, which forbids // users from saving any reference to the underlying buffer @@ -9155,6 +9161,10 @@ releasebuffer_call_python(PyObject *self, Py_buffer *buffer) PyObject_CallMethodNoArgs(mv, &_Py_ID(release)); } Py_DECREF(mv); +end: + assert(!PyErr_Occurred()); + + PyErr_SetRaisedException(exc); } /* From f0f5bb32043e1223d8c413e763cd93061d4f9fac Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 12 May 2023 07:01:31 +0100 Subject: [PATCH 164/204] gh-91896: Improve visibility of `ByteString` deprecation warnings (#104294) --- Lib/collections/abc.py | 9 +++++++++ Lib/test/libregrtest/refleak.py | 16 ++++++++++------ Lib/test/test_collections.py | 10 +++++++++- Lib/test/test_typing.py | 30 ++++++++++++++++++++++++++---- Lib/typing.py | 32 +++++++++++++++++++++++++++++++- 5 files changed, 85 insertions(+), 12 deletions(-) diff --git a/Lib/collections/abc.py b/Lib/collections/abc.py index 86ca8b8a8414b3..60b1eb60fa6ae1 100644 --- a/Lib/collections/abc.py +++ b/Lib/collections/abc.py @@ -1,3 +1,12 @@ from _collections_abc import * from _collections_abc import __all__ from _collections_abc import _CallableGenericAlias + +_deprecated_ByteString = globals().pop("ByteString") + +def __getattr__(attr): + if attr == "ByteString": + import warnings + warnings._deprecated("collections.abc.ByteString", remove=(3, 14)) + return _deprecated_ByteString + raise AttributeError(f"module 'collections.abc' has no attribute {attr!r}") diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 2de8c6cfbc61a1..776a9e9b587d32 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -48,11 +48,13 @@ def dash_R(ns, test_name, test_func): else: zdc = zipimport._zip_directory_cache.copy() abcs = {} - for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: - if not isabstract(abc): - continue - for obj in abc.__subclasses__() + [abc]: - abcs[obj] = _get_dump(obj)[0] + # catch and ignore collections.abc.ByteString deprecation + with warnings.catch_warnings(action='ignore', category=DeprecationWarning): + for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: + if not isabstract(abc): + continue + for obj in abc.__subclasses__() + [abc]: + abcs[obj] = _get_dump(obj)[0] # bpo-31217: Integer pool to get a single integer object for the same # value. The pool is used to prevent false alarm when checking for memory @@ -173,7 +175,9 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): zipimport._zip_directory_cache.update(zdc) # Clear ABC registries, restoring previously saved ABC registries. - abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] + # ignore deprecation warning for collections.abc.ByteString + with warnings.catch_warnings(action='ignore', category=DeprecationWarning): + abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] abs_classes = filter(isabstract, abs_classes) for abc in abs_classes: for obj in abc.__subclasses__() + [abc]: diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index bb8b352518ef3e..f0736b8081feac 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -11,6 +11,7 @@ import string import sys from test import support +from test.support.import_helper import import_fresh_module import types import unittest @@ -25,7 +26,7 @@ from collections.abc import Set, MutableSet from collections.abc import Mapping, MutableMapping, KeysView, ItemsView, ValuesView from collections.abc import Sequence, MutableSequence -from collections.abc import ByteString, Buffer +from collections.abc import Buffer class TestUserObjects(unittest.TestCase): @@ -1939,6 +1940,8 @@ def assert_index_same(seq1, seq2, index_args): nativeseq, seqseq, (letter, start, stop)) def test_ByteString(self): + with self.assertWarns(DeprecationWarning): + from collections.abc import ByteString for sample in [bytes, bytearray]: with self.assertWarns(DeprecationWarning): self.assertIsInstance(sample(), ByteString) @@ -1960,6 +1963,11 @@ class X(ByteString): pass # No metaclass conflict class Z(ByteString, Awaitable): pass + def test_ByteString_attribute_access(self): + collections_abc = import_fresh_module("collections.abc") + with self.assertWarns(DeprecationWarning): + collections_abc.ByteString + def test_Buffer(self): for sample in [bytes, bytearray, memoryview]: self.assertIsInstance(sample(b"x"), Buffer) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index f162e587810ac0..3422dc1ed3f5f8 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -8,6 +8,7 @@ import re import sys import warnings +from test.support.import_helper import import_fresh_module from unittest import TestCase, main, skipUnless, skip from unittest.mock import patch from copy import copy, deepcopy @@ -3908,7 +3909,14 @@ class MyChain(typing.ChainMap[str, T]): ... self.assertEqual(MyChain[int]().__orig_class__, MyChain[int]) def test_all_repr_eq_any(self): - objs = (getattr(typing, el) for el in typing.__all__) + typing = import_fresh_module("typing") + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('always', '', DeprecationWarning) + objs = [getattr(typing, el) for el in typing.__all__] + self.assertEqual( + [str(w.message) for w in wlog], + ["'typing.ByteString' is deprecated and slated for removal in Python 3.14"] + ) for obj in objs: self.assertNotEqual(repr(obj), '') self.assertEqual(obj, obj) @@ -5996,8 +6004,16 @@ def test_mutablesequence(self): self.assertNotIsInstance((), typing.MutableSequence) def test_bytestring(self): - self.assertIsInstance(b'', typing.ByteString) - self.assertIsInstance(bytearray(b''), typing.ByteString) + with self.assertWarns(DeprecationWarning): + from typing import ByteString + with self.assertWarns(DeprecationWarning): + self.assertIsInstance(b'', ByteString) + with self.assertWarns(DeprecationWarning): + self.assertIsInstance(bytearray(b''), ByteString) + with self.assertWarns(DeprecationWarning): + class Foo(ByteString): ... + with self.assertWarns(DeprecationWarning): + class Bar(ByteString, typing.Awaitable): ... def test_list(self): self.assertIsSubclass(list, typing.List) @@ -8293,6 +8309,10 @@ def test_no_isinstance(self): class SpecialAttrsTests(BaseTestCase): def test_special_attrs(self): + with warnings.catch_warnings( + action='ignore', category=DeprecationWarning + ): + typing_ByteString = typing.ByteString cls_to_check = { # ABC classes typing.AbstractSet: 'AbstractSet', @@ -8301,7 +8321,7 @@ def test_special_attrs(self): typing.AsyncIterable: 'AsyncIterable', typing.AsyncIterator: 'AsyncIterator', typing.Awaitable: 'Awaitable', - typing.ByteString: 'ByteString', + typing_ByteString: 'ByteString', typing.Callable: 'Callable', typing.ChainMap: 'ChainMap', typing.Collection: 'Collection', @@ -8626,6 +8646,8 @@ def test_all_exported_names(self): getattr(v, '__module__', None) == typing.__name__ ) } + # Deprecated; added dynamically via module __getattr__ + computed_all.add("ByteString") self.assertSetEqual(computed_all, actual_all) diff --git a/Lib/typing.py b/Lib/typing.py index 62c7dd31a6290a..bf7bd241972a6b 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1599,6 +1599,22 @@ def __or__(self, right): def __ror__(self, left): return Union[left, self] + +class _DeprecatedGenericAlias(_SpecialGenericAlias, _root=True): + def __init__( + self, origin, nparams, *, removal_version, inst=True, name=None + ): + super().__init__(origin, nparams, inst=inst, name=name) + self._removal_version = removal_version + + def __instancecheck__(self, inst): + import warnings + warnings._deprecated( + f"{self.__module__}.{self._name}", remove=self._removal_version + ) + return super().__instancecheck__(inst) + + class _CallableGenericAlias(_NotIterable, _GenericAlias, _root=True): def __repr__(self): assert self._name == 'Callable' @@ -2756,7 +2772,6 @@ class Other(Leaf): # Error reported by type checker MutableMapping = _alias(collections.abc.MutableMapping, 2) Sequence = _alias(collections.abc.Sequence, 1) MutableSequence = _alias(collections.abc.MutableSequence, 1) -ByteString = _alias(collections.abc.ByteString, 0) # Not generic # Tuple accepts variable number of parameters. Tuple = _TupleType(tuple, -1, inst=False, name='Tuple') Tuple.__doc__ = \ @@ -3556,3 +3571,18 @@ def method(self) -> None: # read-only property, TypeError if it's a builtin class. pass return method + + +def __getattr__(attr): + if attr == "ByteString": + import warnings + warnings._deprecated("typing.ByteString", remove=(3, 14)) + with warnings.catch_warnings( + action="ignore", category=DeprecationWarning + ): + # Not generic + ByteString = globals()["ByteString"] = _DeprecatedGenericAlias( + collections.abc.ByteString, 0, removal_version=(3, 14) + ) + return ByteString + raise AttributeError(f"module 'typing' has no attribute {attr!r}") From 3c2992e58b033a6c8adcb52b53c42a96002e7034 Mon Sep 17 00:00:00 2001 From: Alex Henrie Date: Fri, 12 May 2023 00:48:42 -0600 Subject: [PATCH 165/204] gh-87526: Remove dead initialization from _zoneinfo parse_abbr() (#24700) --- Modules/_zoneinfo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index 3b2d282d65cab9..c8c791b6d7c0d8 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -1709,11 +1709,11 @@ static Py_ssize_t parse_abbr(const char *const p, PyObject **abbr) { const char *ptr = p; - char buff = *ptr; const char *str_start; const char *str_end; if (*ptr == '<') { + char buff; ptr++; str_start = ptr; while ((buff = *ptr) != '>') { From a781484c8e9834538e5ee7b9e2e6bec7b679e033 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 12 May 2023 00:11:27 -0700 Subject: [PATCH 166/204] Fix refleak in `super_descr_get` (#104408) --- Objects/typeobject.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index f40e197f8c238a..a1ad5021c4150a 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -10277,8 +10277,10 @@ super_descr_get(PyObject *self, PyObject *obj, PyObject *type) return NULL; newobj = (superobject *)PySuper_Type.tp_new(&PySuper_Type, NULL, NULL); - if (newobj == NULL) + if (newobj == NULL) { + Py_DECREF(obj_type); return NULL; + } newobj->type = (PyTypeObject*)Py_NewRef(su->type); newobj->obj = Py_NewRef(obj); newobj->obj_type = obj_type; From 718b13277217e90232da5edf7ab3267e59189698 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 12 May 2023 00:30:14 -0700 Subject: [PATCH 167/204] gh-104413: Fix refleak when super attribute throws AttributeError (#104414) --- Python/bytecodes.c | 2 +- Python/generated_cases.c.h | 228 +++++++++++++++++++------------------ 2 files changed, 116 insertions(+), 114 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 118a0b5ed4393e..eee9147dd86603 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1598,8 +1598,8 @@ dummy_func( STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 2); res = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - ERROR_IF(res == NULL, error); DECREF_INPUTS(); + ERROR_IF(res == NULL, error); } inst(LOAD_SUPER_ATTR_METHOD, (unused/1, global_super, class, self -- res2, res)) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 373c65e6ce84fc..2ea15e98cb028d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2237,11 +2237,13 @@ STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 2); res = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - if (res == NULL) goto pop_3_error; - #line 2242 "Python/generated_cases.c.h" + #line 2241 "Python/generated_cases.c.h" Py_DECREF(global_super); Py_DECREF(class); Py_DECREF(self); + #line 1602 "Python/bytecodes.c" + if (res == NULL) goto pop_3_error; + #line 2247 "Python/generated_cases.c.h" STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2277,7 +2279,7 @@ res = res2; res2 = NULL; } - #line 2281 "Python/generated_cases.c.h" + #line 2283 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; stack_pointer[-2] = res2; @@ -2325,7 +2327,7 @@ NULL | meth | arg1 | ... | argN */ - #line 2329 "Python/generated_cases.c.h" + #line 2331 "Python/generated_cases.c.h" Py_DECREF(owner); #line 1677 "Python/bytecodes.c" if (meth == NULL) goto pop_1_error; @@ -2336,12 +2338,12 @@ else { /* Classic, pushes one value. */ res = PyObject_GetAttr(owner, name); - #line 2340 "Python/generated_cases.c.h" + #line 2342 "Python/generated_cases.c.h" Py_DECREF(owner); #line 1686 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; } - #line 2345 "Python/generated_cases.c.h" + #line 2347 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -2368,7 +2370,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2372 "Python/generated_cases.c.h" + #line 2374 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2396,7 +2398,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2400 "Python/generated_cases.c.h" + #line 2402 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2438,7 +2440,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2442 "Python/generated_cases.c.h" + #line 2444 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2463,7 +2465,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2467 "Python/generated_cases.c.h" + #line 2469 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2490,7 +2492,7 @@ res = descr; assert(res != NULL); Py_INCREF(res); - #line 2494 "Python/generated_cases.c.h" + #line 2496 "Python/generated_cases.c.h" Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2528,7 +2530,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 2532 "Python/generated_cases.c.h" + #line 2534 "Python/generated_cases.c.h" } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { @@ -2562,7 +2564,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 2566 "Python/generated_cases.c.h" + #line 2568 "Python/generated_cases.c.h" } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -2588,7 +2590,7 @@ Py_DECREF(old_value); } Py_DECREF(owner); - #line 2592 "Python/generated_cases.c.h" + #line 2594 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2638,7 +2640,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - #line 2642 "Python/generated_cases.c.h" + #line 2644 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2659,7 +2661,7 @@ *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - #line 2663 "Python/generated_cases.c.h" + #line 2665 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2684,12 +2686,12 @@ #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); - #line 2688 "Python/generated_cases.c.h" + #line 2690 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); #line 1928 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2693 "Python/generated_cases.c.h" + #line 2695 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2712,7 +2714,7 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2716 "Python/generated_cases.c.h" + #line 2718 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2739,7 +2741,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2743 "Python/generated_cases.c.h" + #line 2745 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2763,7 +2765,7 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2767 "Python/generated_cases.c.h" + #line 2769 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2776,12 +2778,12 @@ PyObject *b; #line 1981 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; - #line 2780 "Python/generated_cases.c.h" + #line 2782 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); #line 1983 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2785 "Python/generated_cases.c.h" + #line 2787 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2793,13 +2795,13 @@ PyObject *b; #line 1987 "Python/bytecodes.c" int res = PySequence_Contains(right, left); - #line 2797 "Python/generated_cases.c.h" + #line 2799 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); #line 1989 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = Py_NewRef((res^oparg) ? Py_True : Py_False); - #line 2803 "Python/generated_cases.c.h" + #line 2805 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2812,7 +2814,7 @@ PyObject *match; #line 1994 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { - #line 2816 "Python/generated_cases.c.h" + #line 2818 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); #line 1996 "Python/bytecodes.c" @@ -2823,7 +2825,7 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); - #line 2827 "Python/generated_cases.c.h" + #line 2829 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); #line 2004 "Python/bytecodes.c" @@ -2835,7 +2837,7 @@ if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } - #line 2839 "Python/generated_cases.c.h" + #line 2841 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -2848,18 +2850,18 @@ #line 2015 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - #line 2852 "Python/generated_cases.c.h" + #line 2854 "Python/generated_cases.c.h" Py_DECREF(right); #line 2018 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - #line 2859 "Python/generated_cases.c.h" + #line 2861 "Python/generated_cases.c.h" Py_DECREF(right); #line 2023 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2863 "Python/generated_cases.c.h" + #line 2865 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -2871,12 +2873,12 @@ #line 2027 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); - #line 2875 "Python/generated_cases.c.h" + #line 2877 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); #line 2030 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2880 "Python/generated_cases.c.h" + #line 2882 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -2889,7 +2891,7 @@ PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - #line 2893 "Python/generated_cases.c.h" + #line 2895 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -2898,7 +2900,7 @@ TARGET(JUMP_FORWARD) { #line 2040 "Python/bytecodes.c" JUMPBY(oparg); - #line 2902 "Python/generated_cases.c.h" + #line 2904 "Python/generated_cases.c.h" DISPATCH(); } @@ -2907,7 +2909,7 @@ #line 2044 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); - #line 2911 "Python/generated_cases.c.h" + #line 2913 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2925,7 +2927,7 @@ } else { int err = PyObject_IsTrue(cond); - #line 2929 "Python/generated_cases.c.h" + #line 2931 "Python/generated_cases.c.h" Py_DECREF(cond); #line 2060 "Python/bytecodes.c" if (err == 0) { @@ -2935,7 +2937,7 @@ if (err < 0) goto pop_1_error; } } - #line 2939 "Python/generated_cases.c.h" + #line 2941 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -2952,7 +2954,7 @@ } else { int err = PyObject_IsTrue(cond); - #line 2956 "Python/generated_cases.c.h" + #line 2958 "Python/generated_cases.c.h" Py_DECREF(cond); #line 2080 "Python/bytecodes.c" if (err > 0) { @@ -2962,7 +2964,7 @@ if (err < 0) goto pop_1_error; } } - #line 2966 "Python/generated_cases.c.h" + #line 2968 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -2971,7 +2973,7 @@ PyObject *value = stack_pointer[-1]; #line 2090 "Python/bytecodes.c" if (!Py_IsNone(value)) { - #line 2975 "Python/generated_cases.c.h" + #line 2977 "Python/generated_cases.c.h" Py_DECREF(value); #line 2092 "Python/bytecodes.c" JUMPBY(oparg); @@ -2979,7 +2981,7 @@ else { _Py_DECREF_NO_DEALLOC(value); } - #line 2983 "Python/generated_cases.c.h" + #line 2985 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -2992,11 +2994,11 @@ JUMPBY(oparg); } else { - #line 2996 "Python/generated_cases.c.h" + #line 2998 "Python/generated_cases.c.h" Py_DECREF(value); #line 2106 "Python/bytecodes.c" } - #line 3000 "Python/generated_cases.c.h" + #line 3002 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -3009,7 +3011,7 @@ * (see bpo-30039). */ JUMPBY(-oparg); - #line 3013 "Python/generated_cases.c.h" + #line 3015 "Python/generated_cases.c.h" DISPATCH(); } @@ -3022,7 +3024,7 @@ if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 3026 "Python/generated_cases.c.h" + #line 3028 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -3038,7 +3040,7 @@ // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 3042 "Python/generated_cases.c.h" + #line 3044 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); @@ -3050,7 +3052,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_NewRef(Py_None); // Failure! } - #line 3054 "Python/generated_cases.c.h" + #line 3056 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -3062,7 +3064,7 @@ #line 2142 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = Py_NewRef(match ? Py_True : Py_False); - #line 3066 "Python/generated_cases.c.h" + #line 3068 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -3075,7 +3077,7 @@ #line 2148 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = Py_NewRef(match ? Py_True : Py_False); - #line 3079 "Python/generated_cases.c.h" + #line 3081 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -3090,7 +3092,7 @@ // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 3094 "Python/generated_cases.c.h" + #line 3096 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -3102,11 +3104,11 @@ #line 2160 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 3106 "Python/generated_cases.c.h" + #line 3108 "Python/generated_cases.c.h" Py_DECREF(iterable); #line 2163 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 3110 "Python/generated_cases.c.h" + #line 3112 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -3137,11 +3139,11 @@ if (iter == NULL) { goto error; } - #line 3141 "Python/generated_cases.c.h" + #line 3143 "Python/generated_cases.c.h" Py_DECREF(iterable); #line 2190 "Python/bytecodes.c" } - #line 3145 "Python/generated_cases.c.h" + #line 3147 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -3183,7 +3185,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator - #line 3187 "Python/generated_cases.c.h" + #line 3189 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3217,7 +3219,7 @@ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - #line 3221 "Python/generated_cases.c.h" + #line 3223 "Python/generated_cases.c.h" DISPATCH(); } @@ -3244,7 +3246,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator - #line 3248 "Python/generated_cases.c.h" + #line 3250 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3274,7 +3276,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator - #line 3278 "Python/generated_cases.c.h" + #line 3280 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3302,7 +3304,7 @@ if (next == NULL) { goto error; } - #line 3306 "Python/generated_cases.c.h" + #line 3308 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3326,7 +3328,7 @@ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); - #line 3330 "Python/generated_cases.c.h" + #line 3332 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { @@ -3356,7 +3358,7 @@ Py_DECREF(enter); goto error; } - #line 3360 "Python/generated_cases.c.h" + #line 3362 "Python/generated_cases.c.h" Py_DECREF(mgr); #line 2374 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); @@ -3365,7 +3367,7 @@ Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3369 "Python/generated_cases.c.h" + #line 3371 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3403,7 +3405,7 @@ Py_DECREF(enter); goto error; } - #line 3407 "Python/generated_cases.c.h" + #line 3409 "Python/generated_cases.c.h" Py_DECREF(mgr); #line 2410 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); @@ -3412,7 +3414,7 @@ Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3416 "Python/generated_cases.c.h" + #line 3418 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3445,7 +3447,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 3449 "Python/generated_cases.c.h" + #line 3451 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3464,7 +3466,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 3468 "Python/generated_cases.c.h" + #line 3470 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3495,7 +3497,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); - #line 3499 "Python/generated_cases.c.h" + #line 3501 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3519,7 +3521,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3523 "Python/generated_cases.c.h" + #line 3525 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3547,7 +3549,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3551 "Python/generated_cases.c.h" + #line 3553 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3560,7 +3562,7 @@ assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); - #line 3564 "Python/generated_cases.c.h" + #line 3566 "Python/generated_cases.c.h" DISPATCH(); } @@ -3578,7 +3580,7 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(CALL); - #line 3582 "Python/generated_cases.c.h" + #line 3584 "Python/generated_cases.c.h" } TARGET(CALL) { @@ -3670,7 +3672,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3674 "Python/generated_cases.c.h" + #line 3676 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3692,7 +3694,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3696 "Python/generated_cases.c.h" + #line 3698 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3727,7 +3729,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 3731 "Python/generated_cases.c.h" + #line 3733 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3771,7 +3773,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 3775 "Python/generated_cases.c.h" + #line 3777 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3789,7 +3791,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 3793 "Python/generated_cases.c.h" + #line 3795 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3813,7 +3815,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3817 "Python/generated_cases.c.h" + #line 3819 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3838,7 +3840,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3842 "Python/generated_cases.c.h" + #line 3844 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3874,7 +3876,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3878 "Python/generated_cases.c.h" + #line 3880 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3916,7 +3918,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3920 "Python/generated_cases.c.h" + #line 3922 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3962,7 +3964,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 3966 "Python/generated_cases.c.h" + #line 3968 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4008,7 +4010,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4012 "Python/generated_cases.c.h" + #line 4014 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4047,7 +4049,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4051 "Python/generated_cases.c.h" + #line 4053 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4087,7 +4089,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4091 "Python/generated_cases.c.h" + #line 4093 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4117,7 +4119,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 4121 "Python/generated_cases.c.h" + #line 4123 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { @@ -4155,7 +4157,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4159 "Python/generated_cases.c.h" + #line 4161 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4197,7 +4199,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4201 "Python/generated_cases.c.h" + #line 4203 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4239,7 +4241,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4243 "Python/generated_cases.c.h" + #line 4245 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4280,7 +4282,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4284 "Python/generated_cases.c.h" + #line 4286 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4292,7 +4294,7 @@ TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { #line 3090 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4296 "Python/generated_cases.c.h" + #line 4298 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4363,14 +4365,14 @@ } result = PyObject_Call(func, callargs, kwargs); } - #line 4367 "Python/generated_cases.c.h" + #line 4369 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); #line 3156 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4374 "Python/generated_cases.c.h" + #line 4376 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4414,7 +4416,7 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4418 "Python/generated_cases.c.h" + #line 4420 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); @@ -4442,7 +4444,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4446 "Python/generated_cases.c.h" + #line 4448 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4452,13 +4454,13 @@ PyObject *slice; #line 3220 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4456 "Python/generated_cases.c.h" + #line 4458 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); #line 3222 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4462 "Python/generated_cases.c.h" + #line 4464 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4504,7 +4506,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4508 "Python/generated_cases.c.h" + #line 4510 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4516,7 +4518,7 @@ #line 3263 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4520 "Python/generated_cases.c.h" + #line 4522 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4543,12 +4545,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4547 "Python/generated_cases.c.h" + #line 4549 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); #line 3283 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4552 "Python/generated_cases.c.h" + #line 4554 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4560,7 +4562,7 @@ PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; #line 3288 "Python/bytecodes.c" assert(oparg >= 2); - #line 4564 "Python/generated_cases.c.h" + #line 4566 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); @@ -4587,7 +4589,7 @@ } opcode = original_opcode; DISPATCH_GOTO(); - #line 4591 "Python/generated_cases.c.h" + #line 4593 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_INSTRUCTION) { @@ -4603,20 +4605,20 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4607 "Python/generated_cases.c.h" + #line 4609 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { #line 3328 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4613 "Python/generated_cases.c.h" + #line 4615 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { #line 3332 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4620 "Python/generated_cases.c.h" + #line 4622 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -4631,7 +4633,7 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4635 "Python/generated_cases.c.h" + #line 4637 "Python/generated_cases.c.h" DISPATCH(); } @@ -4645,7 +4647,7 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4649 "Python/generated_cases.c.h" + #line 4651 "Python/generated_cases.c.h" DISPATCH(); } @@ -4663,7 +4665,7 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4667 "Python/generated_cases.c.h" + #line 4669 "Python/generated_cases.c.h" DISPATCH(); } @@ -4681,7 +4683,7 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4685 "Python/generated_cases.c.h" + #line 4687 "Python/generated_cases.c.h" DISPATCH(); } @@ -4692,19 +4694,19 @@ oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4696 "Python/generated_cases.c.h" + #line 4698 "Python/generated_cases.c.h" } TARGET(CACHE) { #line 3397 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); - #line 4703 "Python/generated_cases.c.h" + #line 4705 "Python/generated_cases.c.h" } TARGET(RESERVED) { #line 3402 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); - #line 4710 "Python/generated_cases.c.h" + #line 4712 "Python/generated_cases.c.h" } From 15795b57d92ee6315b5c8263290944b16834b5f2 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 12 May 2023 09:30:26 +0200 Subject: [PATCH 168/204] gh-101819: Prepare _io._IOBase for module state (#104386) - Add PyIOBase_Type to _io module state - Pass defining class to _io._IOBase.fileno --- Modules/_io/_iomodule.c | 3 +++ Modules/_io/_iomodule.h | 1 + Modules/_io/clinic/iobase.c.h | 18 +++++++++++------- Modules/_io/iobase.c | 10 ++++++---- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index ee4eca70e2af8f..2457cb124036fe 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -582,6 +582,7 @@ iomodule_traverse(PyObject *mod, visitproc visit, void *arg) { return 0; Py_VISIT(state->unsupported_operation); + Py_VISIT(state->PyIOBase_Type); Py_VISIT(state->PyIncrementalNewlineDecoder_Type); Py_VISIT(state->PyRawIOBase_Type); Py_VISIT(state->PyBufferedIOBase_Type); @@ -609,6 +610,7 @@ iomodule_clear(PyObject *mod) { return 0; Py_CLEAR(state->unsupported_operation); + Py_CLEAR(state->PyIOBase_Type); Py_CLEAR(state->PyIncrementalNewlineDecoder_Type); Py_CLEAR(state->PyRawIOBase_Type); Py_CLEAR(state->PyBufferedIOBase_Type); @@ -751,6 +753,7 @@ PyInit__io(void) } // Base classes + state->PyIOBase_Type = (PyTypeObject *)Py_NewRef(&PyIOBase_Type); ADD_TYPE(m, state->PyIncrementalNewlineDecoder_Type, &nldecoder_spec, NULL); ADD_TYPE(m, state->PyBytesIOBuffer_Type, &bytesiobuf_spec, NULL); diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index 44d651338e6975..ae06fecc48b450 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -149,6 +149,7 @@ struct _io_state { PyObject *unsupported_operation; /* Types */ + PyTypeObject *PyIOBase_Type; PyTypeObject *PyIncrementalNewlineDecoder_Type; PyTypeObject *PyRawIOBase_Type; PyTypeObject *PyBufferedIOBase_Type; diff --git a/Modules/_io/clinic/iobase.c.h b/Modules/_io/clinic/iobase.c.h index 727398800bec01..7e6b3b5b78e8f6 100644 --- a/Modules/_io/clinic/iobase.c.h +++ b/Modules/_io/clinic/iobase.c.h @@ -231,20 +231,24 @@ PyDoc_STRVAR(_io__IOBase_fileno__doc__, "fileno($self, /)\n" "--\n" "\n" -"Returns underlying file descriptor if one exists.\n" +"Return underlying file descriptor if one exists.\n" "\n" -"OSError is raised if the IO object does not use a file descriptor."); +"Raise OSError if the IO object does not use a file descriptor."); #define _IO__IOBASE_FILENO_METHODDEF \ - {"fileno", (PyCFunction)_io__IOBase_fileno, METH_NOARGS, _io__IOBase_fileno__doc__}, + {"fileno", _PyCFunction_CAST(_io__IOBase_fileno), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__IOBase_fileno__doc__}, static PyObject * -_io__IOBase_fileno_impl(PyObject *self); +_io__IOBase_fileno_impl(PyObject *self, PyTypeObject *cls); static PyObject * -_io__IOBase_fileno(PyObject *self, PyObject *Py_UNUSED(ignored)) +_io__IOBase_fileno(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _io__IOBase_fileno_impl(self); + if (nargs) { + PyErr_SetString(PyExc_TypeError, "fileno() takes no arguments"); + return NULL; + } + return _io__IOBase_fileno_impl(self, cls); } PyDoc_STRVAR(_io__IOBase_isatty__doc__, @@ -416,4 +420,4 @@ _io__RawIOBase_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _io__RawIOBase_readall_impl(self); } -/*[clinic end generated code: output=b6d4845254da1da2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=63bc25a5bfcecaf0 input=a9049054013a1b77]*/ diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index a74e46cc8dc536..26f2a3155bda62 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -511,15 +511,17 @@ iobase_exit(PyObject *self, PyObject *args) /*[clinic input] _io._IOBase.fileno + cls: defining_class + / -Returns underlying file descriptor if one exists. +Return underlying file descriptor if one exists. -OSError is raised if the IO object does not use a file descriptor. +Raise OSError if the IO object does not use a file descriptor. [clinic start generated code]*/ static PyObject * -_io__IOBase_fileno_impl(PyObject *self) -/*[clinic end generated code: output=7cc0973f0f5f3b73 input=4e37028947dc1cc8]*/ +_io__IOBase_fileno_impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=7caaa32a6f4ada3d input=1927c8bea5c85099]*/ { _PyIO_State *state = IO_STATE(); return iobase_unsupported(state, "fileno"); From b2c1b4da1935639cb89fbbad0ce170a1182537bd Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 12 May 2023 10:34:00 +0200 Subject: [PATCH 169/204] gh-104389: Add 'unused' keyword to Argument Clinic C converters (#104390) Use the unused keyword param in the converter to explicitly mark an argument as unused: /*[clinic input] SomeBaseClass.stubmethod flag: bool(unused=True) [clinic start generated code]*/ --- Doc/howto/clinic.rst | 3 ++ Lib/test/test_clinic.py | 38 +++++++++++++++++++ ...-05-11-15-12-11.gh-issue-104389.EiOhB3.rst | 2 + Tools/clinic/clinic.py | 20 +++++++++- 4 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2023-05-11-15-12-11.gh-issue-104389.EiOhB3.rst diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 6ebc2d9b0a71a9..4620b4617e3450 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -775,6 +775,9 @@ All Argument Clinic converters accept the following arguments: because :pep:`8` mandates that the Python library may not use annotations. + ``unused`` + Wrap the argument with :c:macro:`Py_UNUSED` in the impl function signature. + In addition, some converters accept additional arguments. Here is a list of these arguments, along with their meanings: diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 6aaf4d1ed8d560..28d9f650926427 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -774,6 +774,44 @@ def test_legacy_converters(self): module, function = block.signatures self.assertIsInstance((function.parameters['path']).converter, clinic.str_converter) + def test_unused_param(self): + block = self.parse(""" + module foo + foo.func + fn: object + k: float + i: float(unused=True) + / + * + flag: bool(unused=True) = False + """) + sig = block.signatures[1] # Function index == 1 + params = sig.parameters + conv = lambda fn: params[fn].converter + dataset = ( + {"name": "fn", "unused": False}, + {"name": "k", "unused": False}, + {"name": "i", "unused": True}, + {"name": "flag", "unused": True}, + ) + for param in dataset: + name, unused = param.values() + with self.subTest(name=name, unused=unused): + p = conv(name) + # Verify that the unused flag is parsed correctly. + self.assertEqual(unused, p.unused) + + # Now, check that we'll produce correct code. + decl = p.simple_declaration(in_parser=False) + if unused: + self.assertIn("Py_UNUSED", decl) + else: + self.assertNotIn("Py_UNUSED", decl) + + # Make sure the Py_UNUSED macro is not used in the parser body. + parser_decl = p.simple_declaration(in_parser=True) + self.assertNotIn("Py_UNUSED", parser_decl) + def parse(self, text): c = FakeClinic() parser = DSLParser(c) diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-05-11-15-12-11.gh-issue-104389.EiOhB3.rst b/Misc/NEWS.d/next/Tools-Demos/2023-05-11-15-12-11.gh-issue-104389.EiOhB3.rst new file mode 100644 index 00000000000000..854e1cca967c42 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-05-11-15-12-11.gh-issue-104389.EiOhB3.rst @@ -0,0 +1,2 @@ +Argument Clinic C converters now accept the ``unused`` keyword, for wrapping +a parameter with :c:macro:`Py_UNUSED`. Patch by Erlend E. Aasland. diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 2ae8a02aa6d285..19c4cd299f0bbb 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -2599,6 +2599,9 @@ class CConverter(metaclass=CConverterAutoRegister): # Every non-abstract subclass should supply a valid value. c_ignored_default = 'NULL' + # If true, wrap with Py_UNUSED. + unused = False + # The C converter *function* to be used, if any. # (If this is not None, format_unit must be 'O&'.) converter = None @@ -2651,9 +2654,22 @@ class CConverter(metaclass=CConverterAutoRegister): signature_name = None # keep in sync with self_converter.__init__! - def __init__(self, name, py_name, function, default=unspecified, *, c_default=None, py_default=None, annotation=unspecified, **kwargs): + def __init__(self, + # Positional args: + name, + py_name, + function, + default=unspecified, + *, # Keyword only args: + c_default=None, + py_default=None, + annotation=unspecified, + unused=False, + **kwargs + ): self.name = ensure_legal_c_identifier(name) self.py_name = py_name + self.unused = unused if default is not unspecified: if self.default_type and not isinstance(default, (self.default_type, Unknown)): @@ -2800,6 +2816,8 @@ def simple_declaration(self, by_reference=False, *, in_parser=False): name = self.parser_name else: name = self.name + if self.unused: + name = f"Py_UNUSED({name})" prototype.append(name) return "".join(prototype) From 19ee53d52e8adf267dfd588c2142967734a3b65a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jurica=20Bradari=C4=87?= Date: Fri, 12 May 2023 11:26:07 +0200 Subject: [PATCH 170/204] gh-93649: Split gc- and allocation tests from _testcapimodule.c (GH-104403) --- Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/gc.c | 344 ++++++++++++++++++++++++++++++ Modules/_testcapi/parts.h | 1 + Modules/_testcapimodule.c | 329 +--------------------------- PCbuild/_testcapi.vcxproj | 1 + PCbuild/_testcapi.vcxproj.filters | 3 + 6 files changed, 353 insertions(+), 327 deletions(-) create mode 100644 Modules/_testcapi/gc.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 8e66576b5c5f00..95409d48c0da02 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -169,7 +169,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c # Some testing modules MUST be built as shared libraries. diff --git a/Modules/_testcapi/gc.c b/Modules/_testcapi/gc.c new file mode 100644 index 00000000000000..829200ad12cd3c --- /dev/null +++ b/Modules/_testcapi/gc.c @@ -0,0 +1,344 @@ +#include "parts.h" + +static PyObject* +test_gc_control(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + int orig_enabled = PyGC_IsEnabled(); + const char* msg = "ok"; + int old_state; + + old_state = PyGC_Enable(); + msg = "Enable(1)"; + if (old_state != orig_enabled) { + goto failed; + } + msg = "IsEnabled(1)"; + if (!PyGC_IsEnabled()) { + goto failed; + } + + old_state = PyGC_Disable(); + msg = "disable(2)"; + if (!old_state) { + goto failed; + } + msg = "IsEnabled(2)"; + if (PyGC_IsEnabled()) { + goto failed; + } + + old_state = PyGC_Enable(); + msg = "enable(3)"; + if (old_state) { + goto failed; + } + msg = "IsEnabled(3)"; + if (!PyGC_IsEnabled()) { + goto failed; + } + + if (!orig_enabled) { + old_state = PyGC_Disable(); + msg = "disable(4)"; + if (old_state) { + goto failed; + } + msg = "IsEnabled(4)"; + if (PyGC_IsEnabled()) { + goto failed; + } + } + + Py_RETURN_NONE; + +failed: + /* Try to clean up if we can. */ + if (orig_enabled) { + PyGC_Enable(); + } else { + PyGC_Disable(); + } + PyErr_Format(PyExc_ValueError, "GC control failed in %s", msg); + return NULL; +} + +static PyObject * +without_gc(PyObject *Py_UNUSED(self), PyObject *obj) +{ + PyTypeObject *tp = (PyTypeObject*)obj; + if (!PyType_Check(obj) || !PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { + return PyErr_Format(PyExc_TypeError, "heap type expected, got %R", obj); + } + if (PyType_IS_GC(tp)) { + // Don't try this at home, kids: + tp->tp_flags -= Py_TPFLAGS_HAVE_GC; + tp->tp_free = PyObject_Del; + tp->tp_traverse = NULL; + tp->tp_clear = NULL; + } + assert(!PyType_IS_GC(tp)); + return Py_NewRef(obj); +} + +static void +slot_tp_del(PyObject *self) +{ + PyObject *del, *res; + + /* Temporarily resurrect the object. */ + assert(Py_REFCNT(self) == 0); + Py_SET_REFCNT(self, 1); + + /* Save the current exception, if any. */ + PyObject *exc = PyErr_GetRaisedException(); + + PyObject *tp_del = PyUnicode_InternFromString("__tp_del__"); + if (tp_del == NULL) { + PyErr_WriteUnraisable(NULL); + PyErr_SetRaisedException(exc); + return; + } + /* Execute __del__ method, if any. */ + del = _PyType_Lookup(Py_TYPE(self), tp_del); + Py_DECREF(tp_del); + if (del != NULL) { + res = PyObject_CallOneArg(del, self); + if (res == NULL) + PyErr_WriteUnraisable(del); + else + Py_DECREF(res); + } + + /* Restore the saved exception. */ + PyErr_SetRaisedException(exc); + + /* Undo the temporary resurrection; can't use DECREF here, it would + * cause a recursive call. + */ + assert(Py_REFCNT(self) > 0); + Py_SET_REFCNT(self, Py_REFCNT(self) - 1); + if (Py_REFCNT(self) == 0) { + /* this is the normal path out */ + return; + } + + /* __del__ resurrected it! Make it look like the original Py_DECREF + * never happened. + */ + { + Py_ssize_t refcnt = Py_REFCNT(self); + _Py_NewReferenceNoTotal(self); + Py_SET_REFCNT(self, refcnt); + } + assert(!PyType_IS_GC(Py_TYPE(self)) || PyObject_GC_IsTracked(self)); +} + +static PyObject * +with_tp_del(PyObject *self, PyObject *args) +{ + PyObject *obj; + PyTypeObject *tp; + + if (!PyArg_ParseTuple(args, "O:with_tp_del", &obj)) + return NULL; + tp = (PyTypeObject *) obj; + if (!PyType_Check(obj) || !PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { + PyErr_Format(PyExc_TypeError, + "heap type expected, got %R", obj); + return NULL; + } + tp->tp_del = slot_tp_del; + return Py_NewRef(obj); +} + + +struct gc_visit_state_basic { + PyObject *target; + int found; +}; + +static int +gc_visit_callback_basic(PyObject *obj, void *arg) +{ + struct gc_visit_state_basic *state = (struct gc_visit_state_basic *)arg; + if (obj == state->target) { + state->found = 1; + return 0; + } + return 1; +} + +static PyObject * +test_gc_visit_objects_basic(PyObject *Py_UNUSED(self), + PyObject *Py_UNUSED(ignored)) +{ + PyObject *obj; + struct gc_visit_state_basic state; + + obj = PyList_New(0); + if (obj == NULL) { + return NULL; + } + state.target = obj; + state.found = 0; + + PyUnstable_GC_VisitObjects(gc_visit_callback_basic, &state); + Py_DECREF(obj); + if (!state.found) { + PyErr_SetString( + PyExc_AssertionError, + "test_gc_visit_objects_basic: Didn't find live list"); + return NULL; + } + Py_RETURN_NONE; +} + +static int +gc_visit_callback_exit_early(PyObject *obj, void *arg) + { + int *visited_i = (int *)arg; + (*visited_i)++; + if (*visited_i == 2) { + return 0; + } + return 1; +} + +static PyObject * +test_gc_visit_objects_exit_early(PyObject *Py_UNUSED(self), + PyObject *Py_UNUSED(ignored)) +{ + int visited_i = 0; + PyUnstable_GC_VisitObjects(gc_visit_callback_exit_early, &visited_i); + if (visited_i != 2) { + PyErr_SetString( + PyExc_AssertionError, + "test_gc_visit_objects_exit_early: did not exit when expected"); + } + Py_RETURN_NONE; +} + +typedef struct { + PyObject_HEAD +} ObjExtraData; + +static PyObject * +obj_extra_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + size_t extra_size = sizeof(PyObject *); + PyObject *obj = PyUnstable_Object_GC_NewWithExtraData(type, extra_size); + if (obj == NULL) { + return PyErr_NoMemory(); + } + PyObject_GC_Track(obj); + return obj; +} + +static PyObject ** +obj_extra_data_get_extra_storage(PyObject *self) +{ + return (PyObject **)((char *)self + Py_TYPE(self)->tp_basicsize); +} + +static PyObject * +obj_extra_data_get(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyObject **extra_storage = obj_extra_data_get_extra_storage(self); + PyObject *value = *extra_storage; + if (!value) { + Py_RETURN_NONE; + } + return Py_NewRef(value); +} + +static int +obj_extra_data_set(PyObject *self, PyObject *newval, void *Py_UNUSED(ignored)) +{ + PyObject **extra_storage = obj_extra_data_get_extra_storage(self); + Py_CLEAR(*extra_storage); + if (newval) { + *extra_storage = Py_NewRef(newval); + } + return 0; +} + +static PyGetSetDef obj_extra_data_getset[] = { + {"extra", (getter)obj_extra_data_get, (setter)obj_extra_data_set, NULL}, + {NULL} +}; + +static int +obj_extra_data_traverse(PyObject *self, visitproc visit, void *arg) +{ + PyObject **extra_storage = obj_extra_data_get_extra_storage(self); + PyObject *value = *extra_storage; + Py_VISIT(value); + return 0; +} + +static int +obj_extra_data_clear(PyObject *self) +{ + PyObject **extra_storage = obj_extra_data_get_extra_storage(self); + Py_CLEAR(*extra_storage); + return 0; +} + +static void +obj_extra_data_dealloc(PyObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); + obj_extra_data_clear(self); + tp->tp_free(self); + Py_DECREF(tp); +} + +static PyType_Slot ObjExtraData_Slots[] = { + {Py_tp_getset, obj_extra_data_getset}, + {Py_tp_dealloc, obj_extra_data_dealloc}, + {Py_tp_traverse, obj_extra_data_traverse}, + {Py_tp_clear, obj_extra_data_clear}, + {Py_tp_new, obj_extra_data_new}, + {Py_tp_free, PyObject_GC_Del}, + {0, NULL}, +}; + +static PyType_Spec ObjExtraData_TypeSpec = { + .name = "_testcapi.ObjExtraData", + .basicsize = sizeof(ObjExtraData), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .slots = ObjExtraData_Slots, +}; + +static PyMethodDef test_methods[] = { + {"test_gc_control", test_gc_control, METH_NOARGS}, + {"test_gc_visit_objects_basic", test_gc_visit_objects_basic, METH_NOARGS, NULL}, + {"test_gc_visit_objects_exit_early", test_gc_visit_objects_exit_early, METH_NOARGS, NULL}, + {"without_gc", without_gc, METH_O, NULL}, + {"with_tp_del", with_tp_del, METH_VARARGS, NULL}, + {NULL} +}; + +int _PyTestCapi_Init_GC(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + + PyObject *ObjExtraData_Type = PyType_FromModuleAndSpec( + mod, &ObjExtraData_TypeSpec, NULL); + if (ObjExtraData_Type == 0) { + return -1; + } + int ret = PyModule_AddType(mod, (PyTypeObject*)ObjExtraData_Type); + Py_DECREF(ObjExtraData_Type); + if (ret < 0) { + return ret; + } + + return 0; +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 663d4f2255de88..d1991ac6b464f2 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -41,6 +41,7 @@ int _PyTestCapi_Init_Code(PyObject *module); int _PyTestCapi_Init_Buffer(PyObject *module); int _PyTestCapi_Init_PyOS(PyObject *module); int _PyTestCapi_Init_Immortal(PyObject *module); +int _PyTestCapi_Init_GC(PyObject *mod); #ifdef LIMITED_API_AVAILABLE int _PyTestCapi_Init_VectorcallLimited(PyObject *module); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 79ab7d3f5555c2..c29d29c4791134 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -154,68 +154,6 @@ test_sizeof_c_types(PyObject *self, PyObject *Py_UNUSED(ignored)) #endif } -static PyObject* -test_gc_control(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - int orig_enabled = PyGC_IsEnabled(); - const char* msg = "ok"; - int old_state; - - old_state = PyGC_Enable(); - msg = "Enable(1)"; - if (old_state != orig_enabled) { - goto failed; - } - msg = "IsEnabled(1)"; - if (!PyGC_IsEnabled()) { - goto failed; - } - - old_state = PyGC_Disable(); - msg = "disable(2)"; - if (!old_state) { - goto failed; - } - msg = "IsEnabled(2)"; - if (PyGC_IsEnabled()) { - goto failed; - } - - old_state = PyGC_Enable(); - msg = "enable(3)"; - if (old_state) { - goto failed; - } - msg = "IsEnabled(3)"; - if (!PyGC_IsEnabled()) { - goto failed; - } - - if (!orig_enabled) { - old_state = PyGC_Disable(); - msg = "disable(4)"; - if (old_state) { - goto failed; - } - msg = "IsEnabled(4)"; - if (PyGC_IsEnabled()) { - goto failed; - } - } - - Py_RETURN_NONE; - -failed: - /* Try to clean up if we can. */ - if (orig_enabled) { - PyGC_Enable(); - } else { - PyGC_Disable(); - } - PyErr_Format(TestError, "GC control failed in %s", msg); - return NULL; -} - static PyObject* test_list_api(PyObject *self, PyObject *Py_UNUSED(ignored)) { @@ -1627,95 +1565,6 @@ restore_crossinterp_data(PyObject *self, PyObject *args) return _PyCrossInterpreterData_NewObject(data); } -static void -slot_tp_del(PyObject *self) -{ - PyObject *del, *res; - - /* Temporarily resurrect the object. */ - assert(Py_REFCNT(self) == 0); - Py_SET_REFCNT(self, 1); - - /* Save the current exception, if any. */ - PyObject *exc = PyErr_GetRaisedException(); - - PyObject *tp_del = PyUnicode_InternFromString("__tp_del__"); - if (tp_del == NULL) { - PyErr_WriteUnraisable(NULL); - PyErr_SetRaisedException(exc); - return; - } - /* Execute __del__ method, if any. */ - del = _PyType_Lookup(Py_TYPE(self), tp_del); - Py_DECREF(tp_del); - if (del != NULL) { - res = PyObject_CallOneArg(del, self); - if (res == NULL) - PyErr_WriteUnraisable(del); - else - Py_DECREF(res); - } - - /* Restore the saved exception. */ - PyErr_SetRaisedException(exc); - - /* Undo the temporary resurrection; can't use DECREF here, it would - * cause a recursive call. - */ - assert(Py_REFCNT(self) > 0); - Py_SET_REFCNT(self, Py_REFCNT(self) - 1); - if (Py_REFCNT(self) == 0) { - /* this is the normal path out */ - return; - } - - /* __del__ resurrected it! Make it look like the original Py_DECREF - * never happened. - */ - { - Py_ssize_t refcnt = Py_REFCNT(self); - _Py_NewReferenceNoTotal(self); - Py_SET_REFCNT(self, refcnt); - } - assert(!PyType_IS_GC(Py_TYPE(self)) || PyObject_GC_IsTracked(self)); -} - -static PyObject * -with_tp_del(PyObject *self, PyObject *args) -{ - PyObject *obj; - PyTypeObject *tp; - - if (!PyArg_ParseTuple(args, "O:with_tp_del", &obj)) - return NULL; - tp = (PyTypeObject *) obj; - if (!PyType_Check(obj) || !PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { - PyErr_Format(PyExc_TypeError, - "heap type expected, got %R", obj); - return NULL; - } - tp->tp_del = slot_tp_del; - return Py_NewRef(obj); -} - -static PyObject * -without_gc(PyObject *Py_UNUSED(self), PyObject *obj) -{ - PyTypeObject *tp = (PyTypeObject*)obj; - if (!PyType_Check(obj) || !PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { - return PyErr_Format(PyExc_TypeError, "heap type expected, got %R", obj); - } - if (PyType_IS_GC(tp)) { - // Don't try this at home, kids: - tp->tp_flags -= Py_TPFLAGS_HAVE_GC; - tp->tp_free = PyObject_Del; - tp->tp_traverse = NULL; - tp->tp_clear = NULL; - } - assert(!PyType_IS_GC(tp)); - return Py_NewRef(obj); -} - static PyMethodDef ml; static PyObject * @@ -3342,165 +3191,6 @@ function_set_kw_defaults(PyObject *self, PyObject *args) Py_RETURN_NONE; } -struct gc_visit_state_basic { - PyObject *target; - int found; -}; - -static int -gc_visit_callback_basic(PyObject *obj, void *arg) -{ - struct gc_visit_state_basic *state = (struct gc_visit_state_basic *)arg; - if (obj == state->target) { - state->found = 1; - return 0; - } - return 1; -} - -static PyObject * -test_gc_visit_objects_basic(PyObject *Py_UNUSED(self), - PyObject *Py_UNUSED(ignored)) -{ - PyObject *obj; - struct gc_visit_state_basic state; - - obj = PyList_New(0); - if (obj == NULL) { - return NULL; - } - state.target = obj; - state.found = 0; - - PyUnstable_GC_VisitObjects(gc_visit_callback_basic, &state); - Py_DECREF(obj); - if (!state.found) { - PyErr_SetString( - PyExc_AssertionError, - "test_gc_visit_objects_basic: Didn't find live list"); - return NULL; - } - Py_RETURN_NONE; -} - -static int -gc_visit_callback_exit_early(PyObject *obj, void *arg) - { - int *visited_i = (int *)arg; - (*visited_i)++; - if (*visited_i == 2) { - return 0; - } - return 1; -} - -static PyObject * -test_gc_visit_objects_exit_early(PyObject *Py_UNUSED(self), - PyObject *Py_UNUSED(ignored)) -{ - int visited_i = 0; - PyUnstable_GC_VisitObjects(gc_visit_callback_exit_early, &visited_i); - if (visited_i != 2) { - PyErr_SetString( - PyExc_AssertionError, - "test_gc_visit_objects_exit_early: did not exit when expected"); - } - Py_RETURN_NONE; -} - -typedef struct { - PyObject_HEAD -} ObjExtraData; - -static PyObject * -obj_extra_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - size_t extra_size = sizeof(PyObject *); - PyObject *obj = PyUnstable_Object_GC_NewWithExtraData(type, extra_size); - if (obj == NULL) { - return PyErr_NoMemory(); - } - PyObject_GC_Track(obj); - return obj; -} - -static PyObject ** -obj_extra_data_get_extra_storage(PyObject *self) -{ - return (PyObject **)((char *)self + Py_TYPE(self)->tp_basicsize); -} - -static PyObject * -obj_extra_data_get(PyObject *self, void *Py_UNUSED(ignored)) -{ - PyObject **extra_storage = obj_extra_data_get_extra_storage(self); - PyObject *value = *extra_storage; - if (!value) { - Py_RETURN_NONE; - } - return Py_NewRef(value); -} - -static int -obj_extra_data_set(PyObject *self, PyObject *newval, void *Py_UNUSED(ignored)) -{ - PyObject **extra_storage = obj_extra_data_get_extra_storage(self); - Py_CLEAR(*extra_storage); - if (newval) { - *extra_storage = Py_NewRef(newval); - } - return 0; -} - -static PyGetSetDef obj_extra_data_getset[] = { - {"extra", (getter)obj_extra_data_get, (setter)obj_extra_data_set, NULL}, - {NULL} -}; - -static int -obj_extra_data_traverse(PyObject *self, visitproc visit, void *arg) -{ - PyObject **extra_storage = obj_extra_data_get_extra_storage(self); - PyObject *value = *extra_storage; - Py_VISIT(value); - return 0; -} - -static int -obj_extra_data_clear(PyObject *self) -{ - PyObject **extra_storage = obj_extra_data_get_extra_storage(self); - Py_CLEAR(*extra_storage); - return 0; -} - -static void -obj_extra_data_dealloc(PyObject *self) -{ - PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_UnTrack(self); - obj_extra_data_clear(self); - tp->tp_free(self); - Py_DECREF(tp); -} - -static PyType_Slot ObjExtraData_Slots[] = { - {Py_tp_getset, obj_extra_data_getset}, - {Py_tp_dealloc, obj_extra_data_dealloc}, - {Py_tp_traverse, obj_extra_data_traverse}, - {Py_tp_clear, obj_extra_data_clear}, - {Py_tp_new, obj_extra_data_new}, - {Py_tp_free, PyObject_GC_Del}, - {0, NULL}, -}; - -static PyType_Spec ObjExtraData_TypeSpec = { - .name = "_testcapi.ObjExtraData", - .basicsize = sizeof(ObjExtraData), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - .slots = ObjExtraData_Slots, -}; - struct atexit_data { int called; }; @@ -3538,7 +3228,6 @@ static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, {"test_config", test_config, METH_NOARGS}, {"test_sizeof_c_types", test_sizeof_c_types, METH_NOARGS}, - {"test_gc_control", test_gc_control, METH_NOARGS}, {"test_list_api", test_list_api, METH_NOARGS}, {"test_dict_iteration", test_dict_iteration, METH_NOARGS}, {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, @@ -3590,7 +3279,6 @@ static PyMethodDef TestMethods[] = { METH_VARARGS | METH_KEYWORDS}, {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, {"restore_crossinterp_data", restore_crossinterp_data, METH_VARARGS}, - {"with_tp_del", with_tp_del, METH_VARARGS}, {"create_cfunction", create_cfunction, METH_NOARGS}, {"call_in_temporary_c_thread", call_in_temporary_c_thread, METH_VARARGS, PyDoc_STR("set_error_class(error_class) -> None")}, @@ -3641,7 +3329,6 @@ static PyMethodDef TestMethods[] = { {"meth_fastcall", _PyCFunction_CAST(meth_fastcall), METH_FASTCALL}, {"meth_fastcall_keywords", _PyCFunction_CAST(meth_fastcall_keywords), METH_FASTCALL|METH_KEYWORDS}, {"pynumber_tobase", pynumber_tobase, METH_VARARGS}, - {"without_gc", without_gc, METH_O}, {"test_set_type_size", test_set_type_size, METH_NOARGS}, {"test_py_clear", test_py_clear, METH_NOARGS}, {"test_py_setref", test_py_setref, METH_NOARGS}, @@ -3675,8 +3362,6 @@ static PyMethodDef TestMethods[] = { {"function_set_defaults", function_set_defaults, METH_VARARGS, NULL}, {"function_get_kw_defaults", function_get_kw_defaults, METH_O, NULL}, {"function_set_kw_defaults", function_set_kw_defaults, METH_VARARGS, NULL}, - {"test_gc_visit_objects_basic", test_gc_visit_objects_basic, METH_NOARGS, NULL}, - {"test_gc_visit_objects_exit_early", test_gc_visit_objects_exit_early, METH_NOARGS, NULL}, {"test_atexit", test_atexit, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; @@ -4223,17 +3908,6 @@ PyInit__testcapi(void) Py_INCREF(&MethStatic_Type); PyModule_AddObject(m, "MethStatic", (PyObject *)&MethStatic_Type); - PyObject *ObjExtraData_Type = PyType_FromModuleAndSpec( - m, &ObjExtraData_TypeSpec, NULL); - if (ObjExtraData_Type == 0) { - return NULL; - } - int ret = PyModule_AddType(m, (PyTypeObject*)ObjExtraData_Type); - Py_DECREF(ObjExtraData_Type); - if (ret < 0) { - return NULL; - } - PyModule_AddObject(m, "CHAR_MAX", PyLong_FromLong(CHAR_MAX)); PyModule_AddObject(m, "CHAR_MIN", PyLong_FromLong(CHAR_MIN)); PyModule_AddObject(m, "UCHAR_MAX", PyLong_FromLong(UCHAR_MAX)); @@ -4327,6 +4001,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Immortal(m) < 0) { return NULL; } + if (_PyTestCapi_Init_GC(m) < 0) { + return NULL; + } #ifndef LIMITED_API_AVAILABLE PyModule_AddObjectRef(m, "LIMITED_API_AVAILABLE", Py_False); diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 350f97f8ff41aa..3db9426d1a25ff 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -113,6 +113,7 @@ + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index af80f1eebb3c4d..8df4874659fa1c 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -66,6 +66,9 @@ Source Files + + Source Files + From 45f5aa8fc73acf516071d52ef8213532f0381316 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 12 May 2023 12:21:20 +0100 Subject: [PATCH 171/204] GH-103082: Filter LINE events in VM, to simplify tool implementation. (GH-104387) When monitoring LINE events, instrument all instructions that can have a predecessor on a different line. Then check that the a new line has been hit in the instrumentation code. This brings the behavior closer to that of 3.11, simplifying implementation and porting of tools. --- Include/internal/pycore_frame.h | 1 - Include/internal/pycore_instruments.h | 5 +- Lib/test/test_monitoring.py | 63 +++++++++- Lib/test/test_pdb.py | 5 +- Lib/test/test_sys.py | 2 +- Lib/test/test_sys_settrace.py | 1 - ...-05-10-20-52-29.gh-issue-103082.y3LG5Q.rst | 5 + Objects/frameobject.c | 3 - Python/bytecodes.c | 22 ---- Python/ceval.c | 35 ++++++ Python/ceval_macros.h | 5 +- Python/generated_cases.c.h | 64 +++------- Python/instrumentation.c | 119 +++++++++++++++--- Python/legacy_tracing.c | 73 +++-------- Python/opcode_metadata.h | 5 - Tools/c-analyzer/cpython/globals-to-fix.tsv | 2 +- 16 files changed, 252 insertions(+), 158 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-10-20-52-29.gh-issue-103082.y3LG5Q.rst diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 3d3cbbff7aae81..a72e03f1438fc8 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -19,7 +19,6 @@ struct _frame { struct _PyInterpreterFrame *f_frame; /* points to the frame data */ PyObject *f_trace; /* Trace function */ int f_lineno; /* Current line number. Only valid if non-zero */ - int f_last_traced_line; /* The last line traced for this frame */ char f_trace_lines; /* Emit per-line trace events? */ char f_trace_opcodes; /* Emit per-opcode trace events? */ char f_fast_as_locals; /* Have the fast locals of this frame been converted to a dict? */ diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index e94d8755546efd..9fb3952227af18 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -69,13 +69,13 @@ _Py_call_instrumentation(PyThreadState *tstate, int event, extern int _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, - _Py_CODEUNIT *instr); + _Py_CODEUNIT *instr, _Py_CODEUNIT *prev); extern int _Py_call_instrumentation_instruction( PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr); -int +_Py_CODEUNIT * _Py_call_instrumentation_jump( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target); @@ -100,6 +100,7 @@ extern int _Py_Instrumentation_GetLine(PyCodeObject *code, int index); extern PyObject _PyInstrumentation_MISSING; +extern PyObject _PyInstrumentation_DISABLE; #ifdef __cplusplus } diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index a493bb54d70d38..06e54fa2965f0a 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -524,7 +524,7 @@ def test_lines_loop(self): sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) start = LineMonitoringTest.test_lines_loop.__code__.co_firstlineno - self.assertEqual(events, [start+7, 21, 22, 22, 21, start+8]) + self.assertEqual(events, [start+7, 21, 22, 21, 22, 21, start+8]) finally: sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) @@ -1050,6 +1050,8 @@ def func3(): def line_from_offset(code, offset): for start, end, line in code.co_lines(): if start <= offset < end: + if line is None: + return f"[offset={offset}]" return line - code.co_firstlineno return -1 @@ -1072,9 +1074,20 @@ class BranchRecorder(JumpRecorder): event_type = E.BRANCH name = "branch" +class ReturnRecorder: + + event_type = E.PY_RETURN + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset, val): + self.events.append(("return", val)) + JUMP_AND_BRANCH_RECORDERS = JumpRecorder, BranchRecorder JUMP_BRANCH_AND_LINE_RECORDERS = JumpRecorder, BranchRecorder, LineRecorder +FLOW_AND_LINE_RECORDERS = JumpRecorder, BranchRecorder, LineRecorder, ExceptionRecorder, ReturnRecorder class TestBranchAndJumpEvents(CheckEvents): maxDiff = None @@ -1098,7 +1111,6 @@ def func(): ('jump', 'func', 4, 2), ('branch', 'func', 2, 2)]) - self.check_events(func, recorders = JUMP_BRANCH_AND_LINE_RECORDERS, expected = [ ('line', 'check_events', 10), ('line', 'func', 1), @@ -1108,15 +1120,62 @@ def func(): ('branch', 'func', 3, 6), ('line', 'func', 6), ('jump', 'func', 6, 2), + ('line', 'func', 2), ('branch', 'func', 2, 2), ('line', 'func', 3), ('branch', 'func', 3, 4), ('line', 'func', 4), ('jump', 'func', 4, 2), + ('line', 'func', 2), ('branch', 'func', 2, 2), + ('line', 'check_events', 11)]) + + def test_except_star(self): + + class Foo: + def meth(self): + pass + + def func(): + try: + try: + raise KeyError + except* Exception as e: + f = Foo(); f.meth() + except KeyError: + pass + + + self.check_events(func, recorders = JUMP_BRANCH_AND_LINE_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('line', 'func', 1), ('line', 'func', 2), + ('line', 'func', 3), + ('line', 'func', 4), + ('branch', 'func', 4, 4), + ('line', 'func', 5), + ('line', 'meth', 1), + ('jump', 'func', 5, 5), + ('jump', 'func', 5, '[offset=114]'), + ('branch', 'func', '[offset=120]', '[offset=122]'), ('line', 'check_events', 11)]) + self.check_events(func, recorders = FLOW_AND_LINE_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('line', 'func', 1), + ('line', 'func', 2), + ('line', 'func', 3), + ('raise', KeyError), + ('line', 'func', 4), + ('branch', 'func', 4, 4), + ('line', 'func', 5), + ('line', 'meth', 1), + ('return', None), + ('jump', 'func', 5, 5), + ('jump', 'func', 5, '[offset=114]'), + ('branch', 'func', '[offset=120]', '[offset=122]'), + ('return', None), + ('line', 'check_events', 11)]) class TestSetGetEvents(MonitoringTestBase, unittest.TestCase): diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 037673dd0ea802..83c7cdff87fd34 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -1793,8 +1793,9 @@ def test_pdb_issue_gh_101517(): ... 'continue' ... ]): ... test_function() - > (5)test_function() - -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + --Return-- + > (None)test_function()->None + -> Warning: lineno is None (Pdb) continue """ diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 611cd27ecf1240..e1db450bf8a2ef 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1446,7 +1446,7 @@ class C(object): pass def func(): return sys._getframe() x = func() - check(x, size('3Pii3c7P2ic??2P')) + check(x, size('3Pi3c7P2ic??2P')) # function def func(): pass check(func, size('14Pi')) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 980321e169b9e5..4411603af18cd3 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2867,6 +2867,5 @@ def func(arg = 1): sys.settrace(None) - if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-10-20-52-29.gh-issue-103082.y3LG5Q.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-10-20-52-29.gh-issue-103082.y3LG5Q.rst new file mode 100644 index 00000000000000..40eee64dfa717d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-10-20-52-29.gh-issue-103082.y3LG5Q.rst @@ -0,0 +1,5 @@ +Change behavior of ``sys.monitoring.events.LINE`` events in +``sys.monitoring``: Line events now occur when a new line is reached +dynamically, instead of using a static approximation, as before. This makes +the behavior very similar to that of "line" events in ``sys.settrace``. This +should ease porting of tools from 3.11 to 3.12. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index d9aaea7831a380..2c90a6b71311ca 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -831,7 +831,6 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore start_stack = pop_value(start_stack); } /* Finally set the new lasti and return OK. */ - f->f_last_traced_line = new_lineno; f->f_lineno = 0; f->f_frame->prev_instr = _PyCode_CODE(f->f_frame->f_code) + best_addr; return 0; @@ -854,7 +853,6 @@ frame_settrace(PyFrameObject *f, PyObject* v, void *closure) } if (v != f->f_trace) { Py_XSETREF(f->f_trace, Py_XNewRef(v)); - f->f_last_traced_line = -1; } return 0; } @@ -1056,7 +1054,6 @@ _PyFrame_New_NoTrack(PyCodeObject *code) f->f_trace_opcodes = 0; f->f_fast_as_locals = 0; f->f_lineno = 0; - f->f_last_traced_line = -1; return f; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index eee9147dd86603..99935a3d79075c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3288,28 +3288,6 @@ dummy_func( assert(oparg >= 2); } - inst(INSTRUMENTED_LINE, ( -- )) { - _Py_CODEUNIT *here = next_instr-1; - _PyFrame_SetStackPointer(frame, stack_pointer); - int original_opcode = _Py_call_instrumentation_line( - tstate, frame, here); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = here+1; - goto error; - } - next_instr = frame->prev_instr; - if (next_instr != here) { - DISPATCH(); - } - if (_PyOpcode_Caches[original_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - INCREMENT_ADAPTIVE_COUNTER(cache->counter); - } - opcode = original_opcode; - DISPATCH_GOTO(); - } - inst(INSTRUMENTED_INSTRUCTION, ( -- )) { int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); diff --git a/Python/ceval.c b/Python/ceval.c index 56a3b123f46331..e8534ec6598e2b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -775,6 +775,41 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #include "generated_cases.c.h" + /* INSTRUMENTED_LINE has to be here, rather than in bytecodes.c, + * because it needs to capture frame->prev_instr before it is updated, + * as happens in the standard instruction prologue. + */ +#if USE_COMPUTED_GOTOS + TARGET_INSTRUMENTED_LINE: +#else + case INSTRUMENTED_LINE: +#endif + { + _Py_CODEUNIT *prev = frame->prev_instr; + _Py_CODEUNIT *here = frame->prev_instr = next_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + int original_opcode = _Py_call_instrumentation_line( + tstate, frame, here, prev); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = here+1; + goto error; + } + next_instr = frame->prev_instr; + if (next_instr != here) { + DISPATCH(); + } + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + /* Prevent the underlying instruction from specializing + * and overwriting the instrumentation. */ + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + } + opcode = original_opcode; + DISPATCH_GOTO(); + } + + #if USE_COMPUTED_GOTOS _unknown_opcode: #else diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 485771ac65a767..f5515d0d32dedf 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -334,11 +334,10 @@ do { \ #define INSTRUMENTED_JUMP(src, dest, event) \ do { \ _PyFrame_SetStackPointer(frame, stack_pointer); \ - int err = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ + next_instr = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ stack_pointer = _PyFrame_GetStackPointer(frame); \ - if (err) { \ + if (next_instr == NULL) { \ next_instr = (dest)+1; \ goto error; \ } \ - next_instr = frame->prev_instr; \ } while (0); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2ea15e98cb028d..0ded2f9dcdc2e4 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4568,32 +4568,8 @@ DISPATCH(); } - TARGET(INSTRUMENTED_LINE) { - #line 3292 "Python/bytecodes.c" - _Py_CODEUNIT *here = next_instr-1; - _PyFrame_SetStackPointer(frame, stack_pointer); - int original_opcode = _Py_call_instrumentation_line( - tstate, frame, here); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = here+1; - goto error; - } - next_instr = frame->prev_instr; - if (next_instr != here) { - DISPATCH(); - } - if (_PyOpcode_Caches[original_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - INCREMENT_ADAPTIVE_COUNTER(cache->counter); - } - opcode = original_opcode; - DISPATCH_GOTO(); - #line 4593 "Python/generated_cases.c.h" - } - TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3314 "Python/bytecodes.c" + #line 3292 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4605,26 +4581,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4609 "Python/generated_cases.c.h" + #line 4585 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3328 "Python/bytecodes.c" + #line 3306 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4615 "Python/generated_cases.c.h" + #line 4591 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3332 "Python/bytecodes.c" + #line 3310 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4622 "Python/generated_cases.c.h" + #line 4598 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3337 "Python/bytecodes.c" + #line 3315 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4633,12 +4609,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4637 "Python/generated_cases.c.h" + #line 4613 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3348 "Python/bytecodes.c" + #line 3326 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4647,12 +4623,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4651 "Python/generated_cases.c.h" + #line 4627 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3359 "Python/bytecodes.c" + #line 3337 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4665,12 +4641,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4669 "Python/generated_cases.c.h" + #line 4645 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3374 "Python/bytecodes.c" + #line 3352 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4683,30 +4659,30 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4687 "Python/generated_cases.c.h" + #line 4663 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3389 "Python/bytecodes.c" + #line 3367 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4698 "Python/generated_cases.c.h" + #line 4674 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3397 "Python/bytecodes.c" + #line 3375 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); - #line 4705 "Python/generated_cases.c.h" + #line 4681 "Python/generated_cases.c.h" } TARGET(RESERVED) { - #line 3402 "Python/bytecodes.c" + #line 3380 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); - #line 4712 "Python/generated_cases.c.h" + #line 4688 "Python/generated_cases.c.h" } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index a1423240609699..9152744d7c2c1c 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -14,7 +14,7 @@ /* Uncomment this to dump debugging output when assertions fail */ // #define INSTRUMENT_DEBUG 1 -static PyObject DISABLE = +PyObject _PyInstrumentation_DISABLE = { .ob_refcnt = _Py_IMMORTAL_REFCNT, .ob_type = &PyBaseObject_Type @@ -859,7 +859,7 @@ call_one_instrument( return -1; } Py_DECREF(res); - return (res == &DISABLE); + return (res == &_PyInstrumentation_DISABLE); } static const int8_t MOST_SIGNIFICANT_BITS[16] = { @@ -1002,7 +1002,7 @@ _Py_call_instrumentation_2args( return call_instrumentation_vector(tstate, event, frame, instr, 4, args); } -int +_Py_CODEUNIT * _Py_call_instrumentation_jump( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target) @@ -1010,17 +1010,27 @@ _Py_call_instrumentation_jump( assert(event == PY_MONITORING_EVENT_JUMP || event == PY_MONITORING_EVENT_BRANCH); assert(frame->prev_instr == instr); + /* Event should occur after the jump */ frame->prev_instr = target; PyCodeObject *code = frame->f_code; int to = (int)(target - _PyCode_CODE(code)); PyObject *to_obj = PyLong_FromLong(to * (int)sizeof(_Py_CODEUNIT)); if (to_obj == NULL) { - return -1; + return NULL; } PyObject *args[4] = { NULL, NULL, NULL, to_obj }; int err = call_instrumentation_vector(tstate, event, frame, instr, 3, args); Py_DECREF(to_obj); - return err; + if (err) { + return NULL; + } + if (frame->prev_instr != target) { + /* The callback has caused a jump (by setting the line number) */ + return frame->prev_instr; + } + /* Reset prev_instr for INSTRUMENTED_LINE */ + frame->prev_instr = instr; + return target; } static void @@ -1076,13 +1086,14 @@ _Py_Instrumentation_GetLine(PyCodeObject *code, int index) } int -_Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr) +_Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev) { frame->prev_instr = instr; PyCodeObject *code = frame->f_code; assert(is_version_up_to_date(code, tstate->interp)); assert(instrumentation_cross_checks(tstate->interp, code)); int i = (int)(instr - _PyCode_CODE(code)); + _PyCoMonitoringData *monitoring = code->_co_monitoring; _PyCoLineInstrumentationData *line_data = &monitoring->lines[i]; uint8_t original_opcode = line_data->original_opcode; @@ -1092,6 +1103,18 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, PyInterpreterState *interp = tstate->interp; int8_t line_delta = line_data->line_delta; int line = compute_line(code, i, line_delta); + assert(line >= 0); + int prev_index = (int)(prev - _PyCode_CODE(code)); + int prev_line = _Py_Instrumentation_GetLine(code, prev_index); + if (prev_line == line) { + int prev_opcode = _PyCode_CODE(code)[prev_index].op.code; + /* RESUME and INSTRUMENTED_RESUME are needed for the operation of + * instrumentation, so must never be hidden by an INSTRUMENTED_LINE. + */ + if (prev_opcode != RESUME && prev_opcode != INSTRUMENTED_RESUME) { + goto done; + } + } uint8_t tools = code->_co_monitoring->line_tools != NULL ? code->_co_monitoring->line_tools[i] : (interp->monitors.tools[PY_MONITORING_EVENT_LINE] | @@ -1275,29 +1298,91 @@ initialize_lines(PyCodeObject *code) line_data[i].original_opcode = 0; break; default: + /* Set original_opcode to the opcode iff the instruction + * starts a line, and thus should be instrumented. + * This saves having to perform this check every time the + * we turn instrumentation on or off, and serves as a sanity + * check when debugging. + */ if (line != current_line && line >= 0) { line_data[i].original_opcode = opcode; } else { line_data[i].original_opcode = 0; } - if (line >= 0) { - current_line = line; - } + current_line = line; } for (int j = 1; j < length; j++) { line_data[i+j].original_opcode = 0; line_data[i+j].line_delta = NO_LINE; } + i += length; + } + for (int i = code->_co_firsttraceable; i < code_len; ) { + int opcode = _Py_GetBaseOpcode(code, i); + int oparg = 0; + while (opcode == EXTENDED_ARG) { + oparg = (oparg << 8) | _PyCode_CODE(code)[i].op.arg; + i++; + opcode = _Py_GetBaseOpcode(code, i); + } + oparg = (oparg << 8) | _PyCode_CODE(code)[i].op.arg; + i += instruction_length(code, i); + int target = -1; switch (opcode) { - case RETURN_VALUE: - case RAISE_VARARGS: - case RERAISE: - /* Blocks of code after these terminators - * should be treated as different lines */ - current_line = -1; + case POP_JUMP_IF_FALSE: + case POP_JUMP_IF_TRUE: + case POP_JUMP_IF_NONE: + case POP_JUMP_IF_NOT_NONE: + case JUMP_FORWARD: + { + target = i + oparg; + break; + } + case FOR_ITER: + case SEND: + { + /* Skip over END_FOR/END_SEND */ + target = i + oparg + 1; + break; + } + case JUMP_BACKWARD: + case JUMP_BACKWARD_NO_INTERRUPT: + { + target = i - oparg; + break; + } + default: + continue; + } + assert(target >= 0); + if (line_data[target].line_delta != NO_LINE) { + line_data[target].original_opcode = _Py_GetBaseOpcode(code, target); + } + } + /* Scan exception table */ + unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code->co_exceptiontable); + unsigned char *end = start + PyBytes_GET_SIZE(code->co_exceptiontable); + unsigned char *scan = start; + while (scan < end) { + int start_offset, size, handler; + scan = parse_varint(scan, &start_offset); + assert(start_offset >= 0 && start_offset < code_len); + scan = parse_varint(scan, &size); + assert(size >= 0 && start_offset+size <= code_len); + scan = parse_varint(scan, &handler); + assert(handler >= 0 && handler < code_len); + int depth_and_lasti; + scan = parse_varint(scan, &depth_and_lasti); + int original_opcode = _Py_GetBaseOpcode(code, handler); + /* Skip if not the start of a line. + * END_ASYNC_FOR is a bit special as it marks the end of + * an `async for` loop, which should not generate its own + * line event. */ + if (line_data[handler].line_delta != NO_LINE && + original_opcode != END_ASYNC_FOR) { + line_data[handler].original_opcode = original_opcode; } - i += length; } } @@ -2010,7 +2095,7 @@ PyObject *_Py_CreateMonitoringObject(void) if (mod == NULL) { return NULL; } - if (PyObject_SetAttrString(mod, "DISABLE", &DISABLE)) { + if (PyObject_SetAttrString(mod, "DISABLE", &_PyInstrumentation_DISABLE)) { goto error; } if (PyObject_SetAttrString(mod, "MISSING", &_PyInstrumentation_MISSING)) { diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index e509e63a087a52..5143b79b0864d8 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -4,6 +4,7 @@ #include #include "Python.h" +#include "opcode.h" #include "pycore_ceval.h" #include "pycore_object.h" #include "pycore_sysmodule.h" @@ -213,7 +214,6 @@ trace_line( if (line < 0) { Py_RETURN_NONE; } - frame ->f_last_traced_line = line; Py_INCREF(frame); frame->f_lineno = line; int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, Py_None); @@ -245,14 +245,12 @@ sys_trace_line_func( return NULL; } assert(args[0] == (PyObject *)frame->f_frame->f_code); - if (frame ->f_last_traced_line == line) { - /* Already traced this line */ - Py_RETURN_NONE; - } return trace_line(tstate, self, frame, line); } - +/* sys.settrace generates line events for all backward + * edges, even if on the same line. + * Handle that case here */ static PyObject * sys_trace_jump_func( _PyLegacyEventHandler *self, PyObject *const *args, @@ -268,61 +266,33 @@ sys_trace_jump_func( assert(from >= 0); int to = _PyLong_AsInt(args[2])/sizeof(_Py_CODEUNIT); assert(to >= 0); - PyFrameObject *frame = PyEval_GetFrame(); - if (frame == NULL) { - PyErr_SetString(PyExc_SystemError, - "Missing frame when calling trace function."); - return NULL; - } - if (!frame->f_trace_lines) { - Py_RETURN_NONE; + if (to > from) { + /* Forward jump */ + return &_PyInstrumentation_DISABLE; } PyCodeObject *code = (PyCodeObject *)args[0]; assert(PyCode_Check(code)); - assert(code == frame->f_frame->f_code); /* We can call _Py_Instrumentation_GetLine because we always set * line events for tracing */ int to_line = _Py_Instrumentation_GetLine(code, to); - /* Backward jump: Always generate event - * Forward jump: Only generate event if jumping to different line. */ - if (to > from && frame->f_last_traced_line == to_line) { - /* Already traced this line */ - Py_RETURN_NONE; + int from_line = _Py_Instrumentation_GetLine(code, from); + if (to_line != from_line) { + /* Will be handled by target INSTRUMENTED_LINE */ + return &_PyInstrumentation_DISABLE; } - return trace_line(tstate, self, frame, to_line); -} - -/* We don't care about the exception here, - * we just treat it as a possible new line - */ -static PyObject * -sys_trace_exception_handled( - _PyLegacyEventHandler *self, PyObject *const *args, - size_t nargsf, PyObject *kwnames -) { - assert(kwnames == NULL); - PyThreadState *tstate = _PyThreadState_GET(); - if (tstate->c_tracefunc == NULL) { - Py_RETURN_NONE; - } - assert(PyVectorcall_NARGS(nargsf) == 3); PyFrameObject *frame = PyEval_GetFrame(); - PyCodeObject *code = (PyCodeObject *)args[0]; - assert(PyCode_Check(code)); + if (frame == NULL) { + PyErr_SetString(PyExc_SystemError, + "Missing frame when calling trace function."); + return NULL; + } assert(code == frame->f_frame->f_code); - assert(PyLong_Check(args[1])); - int offset = _PyLong_AsInt(args[1])/sizeof(_Py_CODEUNIT); - /* We can call _Py_Instrumentation_GetLine because we always set - * line events for tracing */ - int line = _Py_Instrumentation_GetLine(code, offset); - if (frame->f_last_traced_line == line) { - /* Already traced this line */ + if (!frame->f_trace_lines) { Py_RETURN_NONE; } - return trace_line(tstate, self, frame, line); + return trace_line(tstate, self, frame, to_line); } - PyTypeObject _PyLegacyEventHandler_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "sys.legacy_event_handler", @@ -487,7 +457,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) } if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, (vectorcallfunc)sys_trace_jump_func, PyTrace_LINE, - PY_MONITORING_EVENT_JUMP, PY_MONITORING_EVENT_BRANCH)) { + PY_MONITORING_EVENT_JUMP, -1)) { return -1; } if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, @@ -495,11 +465,6 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) PY_MONITORING_EVENT_INSTRUCTION, -1)) { return -1; } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - (vectorcallfunc)sys_trace_exception_handled, PyTrace_LINE, - PY_MONITORING_EVENT_EXCEPTION_HANDLED, -1)) { - return -1; - } } int delta = (func != NULL) - (tstate->c_tracefunc != NULL); diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index daf3a3888f0dbd..ae68e045a6119f 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -367,8 +367,6 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 2; case SWAP: return (oparg-2) + 2; - case INSTRUMENTED_LINE: - return 0; case INSTRUMENTED_INSTRUCTION: return 0; case INSTRUMENTED_JUMP_FORWARD: @@ -759,8 +757,6 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case SWAP: return (oparg-2) + 2; - case INSTRUMENTED_LINE: - return 0; case INSTRUMENTED_INSTRUCTION: return 0; case INSTRUMENTED_JUMP_FORWARD: @@ -976,7 +972,6 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [COPY] = { true, INSTR_FMT_IB }, [BINARY_OP] = { true, INSTR_FMT_IBC }, [SWAP] = { true, INSTR_FMT_IB }, - [INSTRUMENTED_LINE] = { true, INSTR_FMT_IX }, [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IB }, diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 453f63ec3f1cfb..8afa92ef25d376 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -301,7 +301,7 @@ Objects/object.c - _Py_NotImplementedStruct - Objects/setobject.c - _dummy_struct - Objects/setobject.c - _PySet_Dummy - Objects/sliceobject.c - _Py_EllipsisObject - -Python/instrumentation.c - DISABLE - +Python/instrumentation.c - _PyInstrumentation_DISABLE - Python/instrumentation.c - _PyInstrumentation_MISSING - From d50c37d8adb2d2da9808089d959ca7d6791ac59f Mon Sep 17 00:00:00 2001 From: Paul Ganssle <1377457+pganssle@users.noreply.github.com> Date: Fri, 12 May 2023 08:35:53 -0400 Subject: [PATCH 172/204] GH-86275: Implementation of hypothesis stubs for property-based tests, with zoneinfo tests (#22863) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These are stubs to be used for adding hypothesis (https://hypothesis.readthedocs.io/en/latest/) tests to the standard library. When the tests are run in an environment where `hypothesis` and its various dependencies are not installed, the stubs will turn any tests with examples into simple parameterized tests and any tests without examples are skipped. It also adds hypothesis tests for the `zoneinfo` module, and a Github Actions workflow to run the hypothesis tests as a non-required CI job. The full hypothesis interface is not stubbed out — missing stubs can be added as necessary. Co-authored-by: Zac Hatfield-Dodds --- .github/workflows/build.yml | 96 +++++ Lib/test/libregrtest/save_env.py | 4 +- .../support/_hypothesis_stubs/__init__.py | 111 ++++++ .../support/_hypothesis_stubs/_helpers.py | 43 ++ .../support/_hypothesis_stubs/strategies.py | 91 +++++ Lib/test/support/hypothesis_helper.py | 4 + Lib/test/test_zoneinfo/__init__.py | 1 + .../test_zoneinfo/test_zoneinfo_property.py | 368 ++++++++++++++++++ ...3-04-25-12-19-37.gh-issue-86275.-RoLIt.rst | 2 + 9 files changed, 719 insertions(+), 1 deletion(-) create mode 100644 Lib/test/support/_hypothesis_stubs/__init__.py create mode 100644 Lib/test/support/_hypothesis_stubs/_helpers.py create mode 100644 Lib/test/support/_hypothesis_stubs/strategies.py create mode 100644 Lib/test/support/hypothesis_helper.py create mode 100644 Lib/test/test_zoneinfo/test_zoneinfo_property.py create mode 100644 Misc/NEWS.d/next/Tests/2023-04-25-12-19-37.gh-issue-86275.-RoLIt.rst diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index df0f107a541614..f0a2dd5694c078 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,6 +36,7 @@ jobs: timeout-minutes: 10 outputs: run_tests: ${{ steps.check.outputs.run_tests }} + run_hypothesis: ${{ steps.check.outputs.run_hypothesis }} steps: - uses: actions/checkout@v3 - name: Check for source changes @@ -61,6 +62,17 @@ jobs: git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true fi + # Check if we should run hypothesis tests + GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} + echo $GIT_BRANCH + if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then + echo "Branch too old for hypothesis tests" + echo "run_hypothesis=false" >> $GITHUB_OUTPUT + else + echo "Run hypothesis tests" + echo "run_hypothesis=true" >> $GITHUB_OUTPUT + fi + check_generated_files: name: 'Check if generated files are up to date' runs-on: ubuntu-latest @@ -291,6 +303,90 @@ jobs: - name: SSL tests run: ./python Lib/test/ssltests.py + test_hypothesis: + name: "Hypothesis Tests on Ubuntu" + runs-on: ubuntu-20.04 + timeout-minutes: 60 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' + env: + OPENSSL_VER: 1.1.1t + PYTHONSTRICTEXTENSIONBUILD: 1 + steps: + - uses: actions/checkout@v3 + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v3 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + - name: Setup directory envs for out-of-tree builds + run: | + echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV + echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + - name: Create directories for read-only out-of-tree builds + run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR + - name: Bind mount sources read-only + run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + - name: Configure CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: ../cpython-ro-srcdir/configure --with-pydebug --with-openssl=$OPENSSL_DIR + - name: Build CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make -j4 + - name: Display build info + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make pythoninfo + - name: Remount sources writable for tests + # some tests write to srcdir, lack of pyc files slows down testing + run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw + - name: Setup directory envs for out-of-tree builds + run: | + echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + - name: "Create hypothesis venv" + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: | + VENV_LOC=$(realpath -m .)/hypovenv + VENV_PYTHON=$VENV_LOC/bin/python + echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV + echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV + ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -U hypothesis + - name: "Run tests" + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: | + # Most of the excluded tests are slow test suites with no property tests + # + # (GH-104097) test_sysconfig is skipped because it has tests that are + # failing when executed from inside a virtual environment. + ${{ env.VENV_PYTHON }} -m test \ + -W \ + -x test_asyncio \ + -x test_multiprocessing_fork \ + -x test_multiprocessing_forkserver \ + -x test_multiprocessing_spawn \ + -x test_concurrent_futures \ + -x test_socket \ + -x test_subprocess \ + -x test_signal \ + -x test_sysconfig + build_asan: name: 'Address sanitizer' diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index cc5870ab2b833e..c7801b767c590c 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -257,8 +257,10 @@ def restore_sysconfig__INSTALL_SCHEMES(self, saved): sysconfig._INSTALL_SCHEMES.update(saved[2]) def get_files(self): + # XXX: Maybe add an allow-list here? return sorted(fn + ('/' if os.path.isdir(fn) else '') - for fn in os.listdir()) + for fn in os.listdir() + if not fn.startswith(".hypothesis")) def restore_files(self, saved_value): fn = os_helper.TESTFN if fn not in saved_value and (fn + '/') not in saved_value: diff --git a/Lib/test/support/_hypothesis_stubs/__init__.py b/Lib/test/support/_hypothesis_stubs/__init__.py new file mode 100644 index 00000000000000..6ba5bb814b92f7 --- /dev/null +++ b/Lib/test/support/_hypothesis_stubs/__init__.py @@ -0,0 +1,111 @@ +from enum import Enum +import functools +import unittest + +__all__ = [ + "given", + "example", + "assume", + "reject", + "register_random", + "strategies", + "HealthCheck", + "settings", + "Verbosity", +] + +from . import strategies + + +def given(*_args, **_kwargs): + def decorator(f): + if examples := getattr(f, "_examples", []): + + @functools.wraps(f) + def test_function(self): + for example_args, example_kwargs in examples: + with self.subTest(*example_args, **example_kwargs): + f(self, *example_args, **example_kwargs) + + else: + # If we have found no examples, we must skip the test. If @example + # is applied after @given, it will re-wrap the test to remove the + # skip decorator. + test_function = unittest.skip( + "Hypothesis required for property test with no " + + "specified examples" + )(f) + + test_function._given = True + return test_function + + return decorator + + +def example(*args, **kwargs): + if bool(args) == bool(kwargs): + raise ValueError("Must specify exactly one of *args or **kwargs") + + def decorator(f): + base_func = getattr(f, "__wrapped__", f) + if not hasattr(base_func, "_examples"): + base_func._examples = [] + + base_func._examples.append((args, kwargs)) + + if getattr(f, "_given", False): + # If the given decorator is below all the example decorators, + # it would be erroneously skipped, so we need to re-wrap the new + # base function. + f = given()(base_func) + + return f + + return decorator + + +def assume(condition): + if not condition: + raise unittest.SkipTest("Unsatisfied assumption") + return True + + +def reject(): + assume(False) + + +def register_random(*args, **kwargs): + pass # pragma: no cover + + +def settings(*args, **kwargs): + return lambda f: f # pragma: nocover + + +class HealthCheck(Enum): + data_too_large = 1 + filter_too_much = 2 + too_slow = 3 + return_value = 5 + large_base_example = 7 + not_a_test_method = 8 + + @classmethod + def all(cls): + return list(cls) + + +class Verbosity(Enum): + quiet = 0 + normal = 1 + verbose = 2 + debug = 3 + + +class Phase(Enum): + explicit = 0 + reuse = 1 + generate = 2 + target = 3 + shrink = 4 + explain = 5 diff --git a/Lib/test/support/_hypothesis_stubs/_helpers.py b/Lib/test/support/_hypothesis_stubs/_helpers.py new file mode 100644 index 00000000000000..3f6244e4dbc0b1 --- /dev/null +++ b/Lib/test/support/_hypothesis_stubs/_helpers.py @@ -0,0 +1,43 @@ +# Stub out only the subset of the interface that we actually use in our tests. +class StubClass: + def __init__(self, *args, **kwargs): + self.__stub_args = args + self.__stub_kwargs = kwargs + self.__repr = None + + def _with_repr(self, new_repr): + new_obj = self.__class__(*self.__stub_args, **self.__stub_kwargs) + new_obj.__repr = new_repr + return new_obj + + def __repr__(self): + if self.__repr is not None: + return self.__repr + + argstr = ", ".join(self.__stub_args) + kwargstr = ", ".join(f"{kw}={val}" for kw, val in self.__stub_kwargs.items()) + + in_parens = argstr + if kwargstr: + in_parens += ", " + kwargstr + + return f"{self.__class__.__qualname__}({in_parens})" + + +def stub_factory(klass, name, *, with_repr=None, _seen={}): + if (klass, name) not in _seen: + + class Stub(klass): + def __init__(self, *args, **kwargs): + super().__init__() + self.__stub_args = args + self.__stub_kwargs = kwargs + + Stub.__name__ = name + Stub.__qualname__ = name + if with_repr is not None: + Stub._repr = None + + _seen.setdefault((klass, name, with_repr), Stub) + + return _seen[(klass, name, with_repr)] diff --git a/Lib/test/support/_hypothesis_stubs/strategies.py b/Lib/test/support/_hypothesis_stubs/strategies.py new file mode 100644 index 00000000000000..d2b885d41e16d0 --- /dev/null +++ b/Lib/test/support/_hypothesis_stubs/strategies.py @@ -0,0 +1,91 @@ +import functools + +from ._helpers import StubClass, stub_factory + + +class StubStrategy(StubClass): + def __make_trailing_repr(self, transformation_name, func): + func_name = func.__name__ or repr(func) + return f"{self!r}.{transformation_name}({func_name})" + + def map(self, pack): + return self._with_repr(self.__make_trailing_repr("map", pack)) + + def flatmap(self, expand): + return self._with_repr(self.__make_trailing_repr("flatmap", expand)) + + def filter(self, condition): + return self._with_repr(self.__make_trailing_repr("filter", condition)) + + def __or__(self, other): + new_repr = f"one_of({self!r}, {other!r})" + return self._with_repr(new_repr) + + +_STRATEGIES = { + "binary", + "booleans", + "builds", + "characters", + "complex_numbers", + "composite", + "data", + "dates", + "datetimes", + "decimals", + "deferred", + "dictionaries", + "emails", + "fixed_dictionaries", + "floats", + "fractions", + "from_regex", + "from_type", + "frozensets", + "functions", + "integers", + "iterables", + "just", + "lists", + "none", + "nothing", + "one_of", + "permutations", + "random_module", + "randoms", + "recursive", + "register_type_strategy", + "runner", + "sampled_from", + "sets", + "shared", + "slices", + "timedeltas", + "times", + "text", + "tuples", + "uuids", +} + +__all__ = sorted(_STRATEGIES) + + +def composite(f): + strategy = stub_factory(StubStrategy, f.__name__) + + @functools.wraps(f) + def inner(*args, **kwargs): + return strategy(*args, **kwargs) + + return inner + + +def __getattr__(name): + if name not in _STRATEGIES: + raise AttributeError(f"Unknown attribute {name}") + + return stub_factory(StubStrategy, f"hypothesis.strategies.{name}") + + +def __dir__(): + return __all__ diff --git a/Lib/test/support/hypothesis_helper.py b/Lib/test/support/hypothesis_helper.py new file mode 100644 index 00000000000000..76bd2490fe6e3d --- /dev/null +++ b/Lib/test/support/hypothesis_helper.py @@ -0,0 +1,4 @@ +try: + import hypothesis +except ImportError: + from . import _hypothesis_stubs as hypothesis diff --git a/Lib/test/test_zoneinfo/__init__.py b/Lib/test/test_zoneinfo/__init__.py index 98cc4412ae16c2..c3ea567103275d 100644 --- a/Lib/test/test_zoneinfo/__init__.py +++ b/Lib/test/test_zoneinfo/__init__.py @@ -1 +1,2 @@ from .test_zoneinfo import * +from .test_zoneinfo_property import * diff --git a/Lib/test/test_zoneinfo/test_zoneinfo_property.py b/Lib/test/test_zoneinfo/test_zoneinfo_property.py new file mode 100644 index 00000000000000..feaa77f3e7f0b9 --- /dev/null +++ b/Lib/test/test_zoneinfo/test_zoneinfo_property.py @@ -0,0 +1,368 @@ +import contextlib +import datetime +import os +import pickle +import unittest +import zoneinfo + +from test.support.hypothesis_helper import hypothesis + +import test.test_zoneinfo._support as test_support + +ZoneInfoTestBase = test_support.ZoneInfoTestBase + +py_zoneinfo, c_zoneinfo = test_support.get_modules() + +UTC = datetime.timezone.utc +MIN_UTC = datetime.datetime.min.replace(tzinfo=UTC) +MAX_UTC = datetime.datetime.max.replace(tzinfo=UTC) +ZERO = datetime.timedelta(0) + + +def _valid_keys(): + """Get available time zones, including posix/ and right/ directories.""" + from importlib import resources + + available_zones = sorted(zoneinfo.available_timezones()) + TZPATH = zoneinfo.TZPATH + + def valid_key(key): + for root in TZPATH: + key_file = os.path.join(root, key) + if os.path.exists(key_file): + return True + + components = key.split("/") + package_name = ".".join(["tzdata.zoneinfo"] + components[:-1]) + resource_name = components[-1] + + try: + return resources.files(package_name).joinpath(resource_name).is_file() + except ModuleNotFoundError: + return False + + # This relies on the fact that dictionaries maintain insertion order — for + # shrinking purposes, it is preferable to start with the standard version, + # then move to the posix/ version, then to the right/ version. + out_zones = {"": available_zones} + for prefix in ["posix", "right"]: + prefix_out = [] + for key in available_zones: + prefix_key = f"{prefix}/{key}" + if valid_key(prefix_key): + prefix_out.append(prefix_key) + + out_zones[prefix] = prefix_out + + output = [] + for keys in out_zones.values(): + output.extend(keys) + + return output + + +VALID_KEYS = _valid_keys() +if not VALID_KEYS: + raise unittest.SkipTest("No time zone data available") + + +def valid_keys(): + return hypothesis.strategies.sampled_from(VALID_KEYS) + + +KEY_EXAMPLES = [ + "Africa/Abidjan", + "Africa/Casablanca", + "America/Los_Angeles", + "America/Santiago", + "Asia/Tokyo", + "Australia/Sydney", + "Europe/Dublin", + "Europe/Lisbon", + "Europe/London", + "Pacific/Kiritimati", + "UTC", +] + + +def add_key_examples(f): + for key in KEY_EXAMPLES: + f = hypothesis.example(key)(f) + return f + + +class ZoneInfoTest(ZoneInfoTestBase): + module = py_zoneinfo + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_str(self, key): + zi = self.klass(key) + self.assertEqual(str(zi), key) + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_key(self, key): + zi = self.klass(key) + + self.assertEqual(zi.key, key) + + @hypothesis.given( + dt=hypothesis.strategies.one_of( + hypothesis.strategies.datetimes(), hypothesis.strategies.times() + ) + ) + @hypothesis.example(dt=datetime.datetime.min) + @hypothesis.example(dt=datetime.datetime.max) + @hypothesis.example(dt=datetime.datetime(1970, 1, 1)) + @hypothesis.example(dt=datetime.datetime(2039, 1, 1)) + @hypothesis.example(dt=datetime.time(0)) + @hypothesis.example(dt=datetime.time(12, 0)) + @hypothesis.example(dt=datetime.time(23, 59, 59, 999999)) + def test_utc(self, dt): + zi = self.klass("UTC") + dt_zi = dt.replace(tzinfo=zi) + + self.assertEqual(dt_zi.utcoffset(), ZERO) + self.assertEqual(dt_zi.dst(), ZERO) + self.assertEqual(dt_zi.tzname(), "UTC") + + +class CZoneInfoTest(ZoneInfoTest): + module = c_zoneinfo + + +class ZoneInfoPickleTest(ZoneInfoTestBase): + module = py_zoneinfo + + def setUp(self): + with contextlib.ExitStack() as stack: + stack.enter_context(test_support.set_zoneinfo_module(self.module)) + self.addCleanup(stack.pop_all().close) + + super().setUp() + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_pickle_unpickle_cache(self, key): + zi = self.klass(key) + pkl_str = pickle.dumps(zi) + zi_rt = pickle.loads(pkl_str) + + self.assertIs(zi, zi_rt) + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_pickle_unpickle_no_cache(self, key): + zi = self.klass.no_cache(key) + pkl_str = pickle.dumps(zi) + zi_rt = pickle.loads(pkl_str) + + self.assertIsNot(zi, zi_rt) + self.assertEqual(str(zi), str(zi_rt)) + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_pickle_unpickle_cache_multiple_rounds(self, key): + """Test that pickle/unpickle is idempotent.""" + zi_0 = self.klass(key) + pkl_str_0 = pickle.dumps(zi_0) + zi_1 = pickle.loads(pkl_str_0) + pkl_str_1 = pickle.dumps(zi_1) + zi_2 = pickle.loads(pkl_str_1) + pkl_str_2 = pickle.dumps(zi_2) + + self.assertEqual(pkl_str_0, pkl_str_1) + self.assertEqual(pkl_str_1, pkl_str_2) + + self.assertIs(zi_0, zi_1) + self.assertIs(zi_0, zi_2) + self.assertIs(zi_1, zi_2) + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_pickle_unpickle_no_cache_multiple_rounds(self, key): + """Test that pickle/unpickle is idempotent.""" + zi_cache = self.klass(key) + + zi_0 = self.klass.no_cache(key) + pkl_str_0 = pickle.dumps(zi_0) + zi_1 = pickle.loads(pkl_str_0) + pkl_str_1 = pickle.dumps(zi_1) + zi_2 = pickle.loads(pkl_str_1) + pkl_str_2 = pickle.dumps(zi_2) + + self.assertEqual(pkl_str_0, pkl_str_1) + self.assertEqual(pkl_str_1, pkl_str_2) + + self.assertIsNot(zi_0, zi_1) + self.assertIsNot(zi_0, zi_2) + self.assertIsNot(zi_1, zi_2) + + self.assertIsNot(zi_0, zi_cache) + self.assertIsNot(zi_1, zi_cache) + self.assertIsNot(zi_2, zi_cache) + + +class CZoneInfoPickleTest(ZoneInfoPickleTest): + module = c_zoneinfo + + +class ZoneInfoCacheTest(ZoneInfoTestBase): + module = py_zoneinfo + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_cache(self, key): + zi_0 = self.klass(key) + zi_1 = self.klass(key) + + self.assertIs(zi_0, zi_1) + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_no_cache(self, key): + zi_0 = self.klass.no_cache(key) + zi_1 = self.klass.no_cache(key) + + self.assertIsNot(zi_0, zi_1) + + +class CZoneInfoCacheTest(ZoneInfoCacheTest): + klass = c_zoneinfo.ZoneInfo + + +class PythonCConsistencyTest(unittest.TestCase): + """Tests that the C and Python versions do the same thing.""" + + def _is_ambiguous(self, dt): + return dt.replace(fold=not dt.fold).utcoffset() == dt.utcoffset() + + @hypothesis.given(dt=hypothesis.strategies.datetimes(), key=valid_keys()) + @hypothesis.example(dt=datetime.datetime.min, key="America/New_York") + @hypothesis.example(dt=datetime.datetime.max, key="America/New_York") + @hypothesis.example(dt=datetime.datetime(1970, 1, 1), key="America/New_York") + @hypothesis.example(dt=datetime.datetime(2020, 1, 1), key="Europe/Paris") + @hypothesis.example(dt=datetime.datetime(2020, 6, 1), key="Europe/Paris") + def test_same_str(self, dt, key): + py_dt = dt.replace(tzinfo=py_zoneinfo.ZoneInfo(key)) + c_dt = dt.replace(tzinfo=c_zoneinfo.ZoneInfo(key)) + + self.assertEqual(str(py_dt), str(c_dt)) + + @hypothesis.given(dt=hypothesis.strategies.datetimes(), key=valid_keys()) + @hypothesis.example(dt=datetime.datetime(1970, 1, 1), key="America/New_York") + @hypothesis.example(dt=datetime.datetime(2020, 2, 5), key="America/New_York") + @hypothesis.example(dt=datetime.datetime(2020, 8, 12), key="America/New_York") + @hypothesis.example(dt=datetime.datetime(2040, 1, 1), key="Africa/Casablanca") + @hypothesis.example(dt=datetime.datetime(1970, 1, 1), key="Europe/Paris") + @hypothesis.example(dt=datetime.datetime(2040, 1, 1), key="Europe/Paris") + @hypothesis.example(dt=datetime.datetime.min, key="Asia/Tokyo") + @hypothesis.example(dt=datetime.datetime.max, key="Asia/Tokyo") + def test_same_offsets_and_names(self, dt, key): + py_dt = dt.replace(tzinfo=py_zoneinfo.ZoneInfo(key)) + c_dt = dt.replace(tzinfo=c_zoneinfo.ZoneInfo(key)) + + self.assertEqual(py_dt.tzname(), c_dt.tzname()) + self.assertEqual(py_dt.utcoffset(), c_dt.utcoffset()) + self.assertEqual(py_dt.dst(), c_dt.dst()) + + @hypothesis.given( + dt=hypothesis.strategies.datetimes(timezones=hypothesis.strategies.just(UTC)), + key=valid_keys(), + ) + @hypothesis.example(dt=MIN_UTC, key="Asia/Tokyo") + @hypothesis.example(dt=MAX_UTC, key="Asia/Tokyo") + @hypothesis.example(dt=MIN_UTC, key="America/New_York") + @hypothesis.example(dt=MAX_UTC, key="America/New_York") + @hypothesis.example( + dt=datetime.datetime(2006, 10, 29, 5, 15, tzinfo=UTC), + key="America/New_York", + ) + def test_same_from_utc(self, dt, key): + py_zi = py_zoneinfo.ZoneInfo(key) + c_zi = c_zoneinfo.ZoneInfo(key) + + # Convert to UTC: This can overflow, but we just care about consistency + py_overflow_exc = None + c_overflow_exc = None + try: + py_dt = dt.astimezone(py_zi) + except OverflowError as e: + py_overflow_exc = e + + try: + c_dt = dt.astimezone(c_zi) + except OverflowError as e: + c_overflow_exc = e + + if (py_overflow_exc is not None) != (c_overflow_exc is not None): + raise py_overflow_exc or c_overflow_exc # pragma: nocover + + if py_overflow_exc is not None: + return # Consistently raises the same exception + + # PEP 495 says that an inter-zone comparison between ambiguous + # datetimes is always False. + if py_dt != c_dt: + self.assertEqual( + self._is_ambiguous(py_dt), + self._is_ambiguous(c_dt), + (py_dt, c_dt), + ) + + self.assertEqual(py_dt.tzname(), c_dt.tzname()) + self.assertEqual(py_dt.utcoffset(), c_dt.utcoffset()) + self.assertEqual(py_dt.dst(), c_dt.dst()) + + @hypothesis.given(dt=hypothesis.strategies.datetimes(), key=valid_keys()) + @hypothesis.example(dt=datetime.datetime.max, key="America/New_York") + @hypothesis.example(dt=datetime.datetime.min, key="America/New_York") + @hypothesis.example(dt=datetime.datetime.min, key="Asia/Tokyo") + @hypothesis.example(dt=datetime.datetime.max, key="Asia/Tokyo") + def test_same_to_utc(self, dt, key): + py_dt = dt.replace(tzinfo=py_zoneinfo.ZoneInfo(key)) + c_dt = dt.replace(tzinfo=c_zoneinfo.ZoneInfo(key)) + + # Convert from UTC: Overflow OK if it happens in both implementations + py_overflow_exc = None + c_overflow_exc = None + try: + py_utc = py_dt.astimezone(UTC) + except OverflowError as e: + py_overflow_exc = e + + try: + c_utc = c_dt.astimezone(UTC) + except OverflowError as e: + c_overflow_exc = e + + if (py_overflow_exc is not None) != (c_overflow_exc is not None): + raise py_overflow_exc or c_overflow_exc # pragma: nocover + + if py_overflow_exc is not None: + return # Consistently raises the same exception + + self.assertEqual(py_utc, c_utc) + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_cross_module_pickle(self, key): + py_zi = py_zoneinfo.ZoneInfo(key) + c_zi = c_zoneinfo.ZoneInfo(key) + + with test_support.set_zoneinfo_module(py_zoneinfo): + py_pkl = pickle.dumps(py_zi) + + with test_support.set_zoneinfo_module(c_zoneinfo): + c_pkl = pickle.dumps(c_zi) + + with test_support.set_zoneinfo_module(c_zoneinfo): + # Python → C + py_to_c_zi = pickle.loads(py_pkl) + self.assertIs(py_to_c_zi, c_zi) + + with test_support.set_zoneinfo_module(py_zoneinfo): + # C → Python + c_to_py_zi = pickle.loads(c_pkl) + self.assertIs(c_to_py_zi, py_zi) diff --git a/Misc/NEWS.d/next/Tests/2023-04-25-12-19-37.gh-issue-86275.-RoLIt.rst b/Misc/NEWS.d/next/Tests/2023-04-25-12-19-37.gh-issue-86275.-RoLIt.rst new file mode 100644 index 00000000000000..37ab74ffc441ed --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-04-25-12-19-37.gh-issue-86275.-RoLIt.rst @@ -0,0 +1,2 @@ +Added property-based tests to the :mod:`zoneinfo` tests, along with stubs +for the ``hypothesis`` interface. (Patch by Paul Ganssle) From 5b8cd5abe5924646b9ed90e7ba90085b56d5f634 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Fri, 12 May 2023 15:36:12 +0300 Subject: [PATCH 173/204] gh-104415: Fix refleak tests for `typing.ByteString` deprecation (#104416) --- Lib/typing.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Lib/typing.py b/Lib/typing.py index bf7bd241972a6b..513d4d96dd6e1d 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -3586,3 +3586,12 @@ def __getattr__(attr): ) return ByteString raise AttributeError(f"module 'typing' has no attribute {attr!r}") + + +def _remove_cached_ByteString_from_globals(): + try: + del globals()["ByteString"] + except KeyError: + pass + +_cleanups.append(_remove_cached_ByteString_from_globals) From a052be4c0d9707ebdb935b8314ed22e8d3d99fc0 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Fri, 12 May 2023 18:27:06 +0530 Subject: [PATCH 174/204] GH-104371: check return value of calling `mv.release` (#104417) --- Objects/typeobject.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index a1ad5021c4150a..c1525320a7cbd5 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -9158,7 +9158,13 @@ releasebuffer_call_python(PyObject *self, Py_buffer *buffer) Py_DECREF(ret); } if (!is_buffer_wrapper) { - PyObject_CallMethodNoArgs(mv, &_Py_ID(release)); + PyObject *res = PyObject_CallMethodNoArgs(mv, &_Py_ID(release)); + if (res == NULL) { + PyErr_WriteUnraisable(self); + } + else { + Py_DECREF(res); + } } Py_DECREF(mv); end: From ce4eecf989e23fa26c033de78f1ca8b035a979cb Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 12 May 2023 15:38:35 +0100 Subject: [PATCH 175/204] gh-91896: Fixup some docs issues following ByteString deprecation (#104422) Co-authored-by: Jelle Zijlstra --- Doc/library/collections.abc.rst | 5 ++++- Doc/whatsnew/3.12.rst | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 43a3286ba832cf..158f4851634652 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -14,7 +14,10 @@ .. testsetup:: * - from collections.abc import * + import warnings + # Ignore warning when ByteString is imported + with warnings.catch_warnings(action='ignore', category=DeprecationWarning): + from collections.abc import * import itertools __name__ = '' diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index eb13d4bf031c95..546c7147bb3b27 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -831,6 +831,9 @@ Pending Removal in Python 3.14 For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. (Contributed by Shantanu Jain in :gh:`91896`.) +* :class:`typing.ByteString`, deprecated since Python 3.9, now causes an + :exc:`DeprecationWarning` to be emitted when it is used or accessed. + * Creating immutable types (:data:`Py_TPFLAGS_IMMUTABLETYPE`) with mutable bases using the C API. From b7597dab2c9ef7b825e190c5e4b130203361df24 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 12 May 2023 08:51:46 -0700 Subject: [PATCH 176/204] gh-86275: Fix install of new _hypothesis_stubs directory (#104425) --- Makefile.pre.in | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile.pre.in b/Makefile.pre.in index 329466580b9cb0..52cafabd1ab999 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2060,6 +2060,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/sndhdrdata \ test/subprocessdata \ test/support \ + test/support/_hypothesis_stubs \ test/test_asyncio \ test/test_capi \ test/test_ctypes \ From 590d7a527d43e4a50fcf74839e516ff7b869434f Mon Sep 17 00:00:00 2001 From: Paul Ganssle <1377457+pganssle@users.noreply.github.com> Date: Fri, 12 May 2023 12:23:08 -0400 Subject: [PATCH 177/204] GH-86275: Run hypothesis tests in parallel (#104427) Run hypothesis tests in parallel --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f0a2dd5694c078..69b78e5567adb1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -377,6 +377,8 @@ jobs: # failing when executed from inside a virtual environment. ${{ env.VENV_PYTHON }} -m test \ -W \ + -o \ + -j4 \ -x test_asyncio \ -x test_multiprocessing_fork \ -x test_multiprocessing_forkserver \ From 25db95d224d18fb7b7f53165aeaa87632b0230f2 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Fri, 12 May 2023 21:25:45 +0300 Subject: [PATCH 178/204] gh-103857: Update deprecation stacktrace to point to calling line (#104431) --- .../Library/2023-05-12-19-29-28.gh-issue-103857.0IzSxr.rst | 1 + Modules/_datetimemodule.c | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-12-19-29-28.gh-issue-103857.0IzSxr.rst diff --git a/Misc/NEWS.d/next/Library/2023-05-12-19-29-28.gh-issue-103857.0IzSxr.rst b/Misc/NEWS.d/next/Library/2023-05-12-19-29-28.gh-issue-103857.0IzSxr.rst new file mode 100644 index 00000000000000..6e8162d9ecc286 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-12-19-29-28.gh-issue-103857.0IzSxr.rst @@ -0,0 +1 @@ +Update datetime deprecations' stracktrace to point to the calling line diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 8f86fc91966205..8b417bdd0fb5b6 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -5147,7 +5147,7 @@ datetime_utcnow(PyObject *cls, PyObject *dummy) if (PyErr_WarnEx(PyExc_DeprecationWarning, "datetime.utcnow() is deprecated and scheduled for removal in a " "future version. Use timezone-aware objects to represent datetimes " - "in UTC: datetime.now(datetime.UTC).", 2)) + "in UTC: datetime.now(datetime.UTC).", 1)) { return NULL; } @@ -5190,7 +5190,7 @@ datetime_utcfromtimestamp(PyObject *cls, PyObject *args) if (PyErr_WarnEx(PyExc_DeprecationWarning, "datetime.utcfromtimestamp() is deprecated and scheduled for removal " "in a future version. Use timezone-aware objects to represent " - "datetimes in UTC: datetime.now(datetime.UTC).", 2)) + "datetimes in UTC: datetime.now(datetime.UTC).", 1)) { return NULL; } From cf720acfcbd8c9c25a706a4b6df136465a803992 Mon Sep 17 00:00:00 2001 From: Ben Kallus <49924171+kenballus@users.noreply.github.com> Date: Fri, 12 May 2023 16:25:58 -0400 Subject: [PATCH 179/204] gh-103204: `http.server` - Enforce that HTTP version numbers must consist only of digits (#103205) Reject HTTP requests with invalid http/x.y version numbers: x or y being non-digits or too-long. --------- Co-authored-by: Oleg Iarygin Co-authored-by: Gregory P. Smith --- Lib/http/server.py | 4 ++++ Lib/test/test_httpservers.py | 21 +++++++++++++++++++ ...-04-02-23-05-22.gh-issue-103204.bbDmu0.rst | 3 +++ 3 files changed, 28 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-04-02-23-05-22.gh-issue-103204.bbDmu0.rst diff --git a/Lib/http/server.py b/Lib/http/server.py index a245ffb307860a..ca6240d9a921e6 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -300,6 +300,10 @@ def parse_request(self): # - Leading zeros MUST be ignored by recipients. if len(version_number) != 2: raise ValueError + if any(not component.isdigit() for component in version_number): + raise ValueError("non digit in http version") + if any(len(component) > 10 for component in version_number): + raise ValueError("unreasonable length http version") version_number = int(version_number[0]), int(version_number[1]) except (ValueError, IndexError): self.send_error( diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 0382b5ec448d57..cdd1bea754a020 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -164,6 +164,27 @@ def test_version_digits(self): res = self.con.getresponse() self.assertEqual(res.status, HTTPStatus.BAD_REQUEST) + def test_version_signs_and_underscores(self): + self.con._http_vsn_str = 'HTTP/-9_9_9.+9_9_9' + self.con.putrequest('GET', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, HTTPStatus.BAD_REQUEST) + + def test_major_version_number_too_long(self): + self.con._http_vsn_str = 'HTTP/909876543210.0' + self.con.putrequest('GET', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, HTTPStatus.BAD_REQUEST) + + def test_minor_version_number_too_long(self): + self.con._http_vsn_str = 'HTTP/1.909876543210' + self.con.putrequest('GET', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, HTTPStatus.BAD_REQUEST) + def test_version_none_get(self): self.con._http_vsn_str = '' self.con.putrequest('GET', '/') diff --git a/Misc/NEWS.d/next/Library/2023-04-02-23-05-22.gh-issue-103204.bbDmu0.rst b/Misc/NEWS.d/next/Library/2023-04-02-23-05-22.gh-issue-103204.bbDmu0.rst new file mode 100644 index 00000000000000..f8b3aa5151b622 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-02-23-05-22.gh-issue-103204.bbDmu0.rst @@ -0,0 +1,3 @@ +Fixes :mod:`http.server` accepting HTTP requests with HTTP version numbers +preceded by '+', or '-', or with digit-separating '_' characters. The length +of the version numbers is also constrained. From 79b17f2cf0e1a2688bd91df02744f461835c4253 Mon Sep 17 00:00:00 2001 From: Charles Machalow Date: Fri, 12 May 2023 13:33:23 -0700 Subject: [PATCH 180/204] gh-103333: Pickle the keyword attributes of AttributeError (#103352) * Pickle the `name` and `args` attributes of AttributeError when present. Co-authored-by: Gregory P. Smith Co-authored-by: Erlend E. Aasland --- Lib/test/test_exceptions.py | 61 +++++++++++-------- ...-04-09-04-30-02.gh-issue-103333.gKOetS.rst | 1 + Objects/exceptions.c | 44 ++++++++++++- 3 files changed, 78 insertions(+), 28 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-09-04-30-02.gh-issue-103333.gKOetS.rst diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 4ef7decfbc263e..f3554f1c4bb3f3 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -417,45 +417,45 @@ def testAttributes(self): # test that exception attributes are happy exceptionList = [ - (BaseException, (), {'args' : ()}), - (BaseException, (1, ), {'args' : (1,)}), - (BaseException, ('foo',), + (BaseException, (), {}, {'args' : ()}), + (BaseException, (1, ), {}, {'args' : (1,)}), + (BaseException, ('foo',), {}, {'args' : ('foo',)}), - (BaseException, ('foo', 1), + (BaseException, ('foo', 1), {}, {'args' : ('foo', 1)}), - (SystemExit, ('foo',), + (SystemExit, ('foo',), {}, {'args' : ('foo',), 'code' : 'foo'}), - (OSError, ('foo',), + (OSError, ('foo',), {}, {'args' : ('foo',), 'filename' : None, 'filename2' : None, 'errno' : None, 'strerror' : None}), - (OSError, ('foo', 'bar'), + (OSError, ('foo', 'bar'), {}, {'args' : ('foo', 'bar'), 'filename' : None, 'filename2' : None, 'errno' : 'foo', 'strerror' : 'bar'}), - (OSError, ('foo', 'bar', 'baz'), + (OSError, ('foo', 'bar', 'baz'), {}, {'args' : ('foo', 'bar'), 'filename' : 'baz', 'filename2' : None, 'errno' : 'foo', 'strerror' : 'bar'}), - (OSError, ('foo', 'bar', 'baz', None, 'quux'), + (OSError, ('foo', 'bar', 'baz', None, 'quux'), {}, {'args' : ('foo', 'bar'), 'filename' : 'baz', 'filename2': 'quux'}), - (OSError, ('errnoStr', 'strErrorStr', 'filenameStr'), + (OSError, ('errnoStr', 'strErrorStr', 'filenameStr'), {}, {'args' : ('errnoStr', 'strErrorStr'), 'strerror' : 'strErrorStr', 'errno' : 'errnoStr', 'filename' : 'filenameStr'}), - (OSError, (1, 'strErrorStr', 'filenameStr'), + (OSError, (1, 'strErrorStr', 'filenameStr'), {}, {'args' : (1, 'strErrorStr'), 'errno' : 1, 'strerror' : 'strErrorStr', 'filename' : 'filenameStr', 'filename2' : None}), - (SyntaxError, (), {'msg' : None, 'text' : None, + (SyntaxError, (), {}, {'msg' : None, 'text' : None, 'filename' : None, 'lineno' : None, 'offset' : None, 'end_offset': None, 'print_file_and_line' : None}), - (SyntaxError, ('msgStr',), + (SyntaxError, ('msgStr',), {}, {'args' : ('msgStr',), 'text' : None, 'print_file_and_line' : None, 'msg' : 'msgStr', 'filename' : None, 'lineno' : None, 'offset' : None, 'end_offset': None}), (SyntaxError, ('msgStr', ('filenameStr', 'linenoStr', 'offsetStr', - 'textStr', 'endLinenoStr', 'endOffsetStr')), + 'textStr', 'endLinenoStr', 'endOffsetStr')), {}, {'offset' : 'offsetStr', 'text' : 'textStr', 'args' : ('msgStr', ('filenameStr', 'linenoStr', 'offsetStr', 'textStr', @@ -465,7 +465,7 @@ def testAttributes(self): 'end_lineno': 'endLinenoStr', 'end_offset': 'endOffsetStr'}), (SyntaxError, ('msgStr', 'filenameStr', 'linenoStr', 'offsetStr', 'textStr', 'endLinenoStr', 'endOffsetStr', - 'print_file_and_lineStr'), + 'print_file_and_lineStr'), {}, {'text' : None, 'args' : ('msgStr', 'filenameStr', 'linenoStr', 'offsetStr', 'textStr', 'endLinenoStr', 'endOffsetStr', @@ -473,38 +473,40 @@ def testAttributes(self): 'print_file_and_line' : None, 'msg' : 'msgStr', 'filename' : None, 'lineno' : None, 'offset' : None, 'end_lineno': None, 'end_offset': None}), - (UnicodeError, (), {'args' : (),}), + (UnicodeError, (), {}, {'args' : (),}), (UnicodeEncodeError, ('ascii', 'a', 0, 1, - 'ordinal not in range'), + 'ordinal not in range'), {}, {'args' : ('ascii', 'a', 0, 1, 'ordinal not in range'), 'encoding' : 'ascii', 'object' : 'a', 'start' : 0, 'reason' : 'ordinal not in range'}), (UnicodeDecodeError, ('ascii', bytearray(b'\xff'), 0, 1, - 'ordinal not in range'), + 'ordinal not in range'), {}, {'args' : ('ascii', bytearray(b'\xff'), 0, 1, 'ordinal not in range'), 'encoding' : 'ascii', 'object' : b'\xff', 'start' : 0, 'reason' : 'ordinal not in range'}), (UnicodeDecodeError, ('ascii', b'\xff', 0, 1, - 'ordinal not in range'), + 'ordinal not in range'), {}, {'args' : ('ascii', b'\xff', 0, 1, 'ordinal not in range'), 'encoding' : 'ascii', 'object' : b'\xff', 'start' : 0, 'reason' : 'ordinal not in range'}), - (UnicodeTranslateError, ("\u3042", 0, 1, "ouch"), + (UnicodeTranslateError, ("\u3042", 0, 1, "ouch"), {}, {'args' : ('\u3042', 0, 1, 'ouch'), 'object' : '\u3042', 'reason' : 'ouch', 'start' : 0, 'end' : 1}), - (NaiveException, ('foo',), + (NaiveException, ('foo',), {}, {'args': ('foo',), 'x': 'foo'}), - (SlottedNaiveException, ('foo',), + (SlottedNaiveException, ('foo',), {}, {'args': ('foo',), 'x': 'foo'}), + (AttributeError, ('foo',), dict(name='name', obj='obj'), + dict(args=('foo',), name='name', obj='obj')), ] try: # More tests are in test_WindowsError exceptionList.append( - (WindowsError, (1, 'strErrorStr', 'filenameStr'), + (WindowsError, (1, 'strErrorStr', 'filenameStr'), {}, {'args' : (1, 'strErrorStr'), 'strerror' : 'strErrorStr', 'winerror' : None, 'errno' : 1, @@ -513,11 +515,11 @@ def testAttributes(self): except NameError: pass - for exc, args, expected in exceptionList: + for exc, args, kwargs, expected in exceptionList: try: - e = exc(*args) + e = exc(*args, **kwargs) except: - print("\nexc=%r, args=%r" % (exc, args), file=sys.stderr) + print(f"\nexc={exc!r}, args={args!r}", file=sys.stderr) # raise else: # Verify module name @@ -540,7 +542,12 @@ def testAttributes(self): new = p.loads(s) for checkArgName in expected: got = repr(getattr(new, checkArgName)) - want = repr(expected[checkArgName]) + if exc == AttributeError and checkArgName == 'obj': + # See GH-103352, we're not pickling + # obj at this point. So verify it's None. + want = repr(None) + else: + want = repr(expected[checkArgName]) self.assertEqual(got, want, 'pickled "%r", attribute "%s' % (e, checkArgName)) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-09-04-30-02.gh-issue-103333.gKOetS.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-09-04-30-02.gh-issue-103333.gKOetS.rst new file mode 100644 index 00000000000000..793f02c2afdcff --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-04-09-04-30-02.gh-issue-103333.gKOetS.rst @@ -0,0 +1 @@ +:exc:`AttributeError` now retains the ``name`` attribute when pickled and unpickled. diff --git a/Objects/exceptions.c b/Objects/exceptions.c index ba5ee291f08b0c..59c63f4aa44958 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -2287,6 +2287,46 @@ AttributeError_traverse(PyAttributeErrorObject *self, visitproc visit, void *arg return BaseException_traverse((PyBaseExceptionObject *)self, visit, arg); } +/* Pickling support */ +static PyObject * +AttributeError_getstate(PyAttributeErrorObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *dict = ((PyAttributeErrorObject *)self)->dict; + if (self->name || self->args) { + dict = dict ? PyDict_Copy(dict) : PyDict_New(); + if (dict == NULL) { + return NULL; + } + if (self->name && PyDict_SetItemString(dict, "name", self->name) < 0) { + Py_DECREF(dict); + return NULL; + } + /* We specifically are not pickling the obj attribute since there are many + cases where it is unlikely to be picklable. See GH-103352. + */ + if (self->args && PyDict_SetItemString(dict, "args", self->args) < 0) { + Py_DECREF(dict); + return NULL; + } + return dict; + } + else if (dict) { + return Py_NewRef(dict); + } + Py_RETURN_NONE; +} + +static PyObject * +AttributeError_reduce(PyAttributeErrorObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *state = AttributeError_getstate(self, NULL); + if (state == NULL) { + return NULL; + } + + return PyTuple_Pack(3, Py_TYPE(self), self->args, state); +} + static PyMemberDef AttributeError_members[] = { {"name", T_OBJECT, offsetof(PyAttributeErrorObject, name), 0, PyDoc_STR("attribute name")}, {"obj", T_OBJECT, offsetof(PyAttributeErrorObject, obj), 0, PyDoc_STR("object")}, @@ -2294,7 +2334,9 @@ static PyMemberDef AttributeError_members[] = { }; static PyMethodDef AttributeError_methods[] = { - {NULL} /* Sentinel */ + {"__getstate__", (PyCFunction)AttributeError_getstate, METH_NOARGS}, + {"__reduce__", (PyCFunction)AttributeError_reduce, METH_NOARGS }, + {NULL} }; ComplexExtendsException(PyExc_Exception, AttributeError, From a10b026f0fdceac42c4b928917894d77da996555 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 12 May 2023 14:26:54 -0700 Subject: [PATCH 181/204] GH-94841: Fix usage of Py_ALWAYS_INLINE (GH-104409) --- Objects/obmalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index de62aeb04461fa..090129fe8653f2 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -882,7 +882,7 @@ _Py_GetGlobalAllocatedBlocks(void) /* Return a pointer to a bottom tree node, return NULL if it doesn't exist or * it cannot be created */ -static Py_ALWAYS_INLINE arena_map_bot_t * +static inline Py_ALWAYS_INLINE arena_map_bot_t * arena_map_get(OMState *state, pymem_block *p, int create) { #ifdef USE_INTERIOR_NODES From 1eb950ca55b3e0b6524b3f03b0b519723916eca2 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 12 May 2023 15:23:13 -0700 Subject: [PATCH 182/204] GH-104405: Add missing PEP 523 checks (GH-104406) --- Lib/test/test_capi/test_misc.py | 88 +- ...-05-11-15-56-07.gh-issue-104405.tXV5fn.rst | 2 + Python/bytecodes.c | 8 +- Python/ceval_macros.h | 1 + Python/generated_cases.c.h | 806 +++++++++--------- Python/specialize.c | 22 + 6 files changed, 506 insertions(+), 421 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-11-15-56-07.gh-issue-104405.tXV5fn.rst diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 3fc2c07f933061..0ef0b5d8b65803 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -1748,28 +1748,80 @@ class Subclass(BaseException, self.module.StateAccessType): class Test_Pep523API(unittest.TestCase): - def do_test(self, func): - calls = [] + def do_test(self, func, names): + actual_calls = [] start = SUFFICIENT_TO_DEOPT_AND_SPECIALIZE count = start + SUFFICIENT_TO_DEOPT_AND_SPECIALIZE - for i in range(count): - if i == start: - _testinternalcapi.set_eval_frame_record(calls) - func() - _testinternalcapi.set_eval_frame_default() - self.assertEqual(len(calls), SUFFICIENT_TO_DEOPT_AND_SPECIALIZE) - for name in calls: - self.assertEqual(name, func.__name__) - - def test_pep523_with_specialization_simple(self): - def func1(): - pass - self.do_test(func1) + try: + for i in range(count): + if i == start: + _testinternalcapi.set_eval_frame_record(actual_calls) + func() + finally: + _testinternalcapi.set_eval_frame_default() + expected_calls = names * SUFFICIENT_TO_DEOPT_AND_SPECIALIZE + self.assertEqual(len(expected_calls), len(actual_calls)) + for expected, actual in zip(expected_calls, actual_calls, strict=True): + self.assertEqual(expected, actual) + + def test_inlined_binary_subscr(self): + class C: + def __getitem__(self, other): + return None + def func(): + C()[42] + names = ["func", "__getitem__"] + self.do_test(func, names) - def test_pep523_with_specialization_with_default(self): - def func2(x=None): + def test_inlined_call(self): + def inner(x=42): + pass + def func(): + inner() + inner(42) + names = ["func", "inner", "inner"] + self.do_test(func, names) + + def test_inlined_call_function_ex(self): + def inner(x): pass - self.do_test(func2) + def func(): + inner(*[42]) + names = ["func", "inner"] + self.do_test(func, names) + + def test_inlined_for_iter(self): + def gen(): + yield 42 + def func(): + for _ in gen(): + pass + names = ["func", "gen", "gen", "gen"] + self.do_test(func, names) + + def test_inlined_load_attr(self): + class C: + @property + def a(self): + return 42 + class D: + def __getattribute__(self, name): + return 42 + def func(): + C().a + D().a + names = ["func", "a", "__getattribute__"] + self.do_test(func, names) + + def test_inlined_send(self): + def inner(): + yield 42 + def outer(): + yield from inner() + def func(): + list(outer()) + names = ["func", "outer", "outer", "inner", "inner", "outer", "inner"] + self.do_test(func, names) if __name__ == "__main__": diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-11-15-56-07.gh-issue-104405.tXV5fn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-11-15-56-07.gh-issue-104405.tXV5fn.rst new file mode 100644 index 00000000000000..06ec5d7b0f0cc9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-11-15-56-07.gh-issue-104405.tXV5fn.rst @@ -0,0 +1,2 @@ +Fix an issue where some :term:`bytecode` instructions could ignore +:pep:`523` when "inlining" calls. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 99935a3d79075c..d84a078f110085 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -494,6 +494,7 @@ dummy_func( } inst(BINARY_SUBSCR_GETITEM, (unused/1, container, sub -- unused)) { + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; @@ -830,8 +831,9 @@ dummy_func( DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ assert(frame != &entry_frame); - if ((Py_TYPE(receiver) == &PyGen_Type || - Py_TYPE(receiver) == &PyCoro_Type) && ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) + if ((tstate->interp->eval_frame == NULL) && + (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && + ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) { PyGenObject *gen = (PyGenObject *)receiver; _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; @@ -867,6 +869,7 @@ dummy_func( } inst(SEND_GEN, (unused/1, receiver, v -- receiver, unused)) { + DEOPT_IF(tstate->interp->eval_frame, SEND); PyGenObject *gen = (PyGenObject *)receiver; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); @@ -2331,6 +2334,7 @@ dummy_func( } inst(FOR_ITER_GEN, (unused/1, iter -- iter, unused)) { + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index f5515d0d32dedf..e92ff0bd6fba6f 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -105,6 +105,7 @@ #define DISPATCH_INLINED(NEW_FRAME) \ do { \ + assert(tstate->interp->eval_frame == NULL); \ _PyFrame_SetStackPointer(frame, stack_pointer); \ frame->prev_instr = next_instr - 1; \ (NEW_FRAME)->previous = frame; \ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0ded2f9dcdc2e4..014a69ff7e9615 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -708,6 +708,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; #line 497 "Python/bytecodes.c" + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; @@ -729,15 +730,15 @@ JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 733 "Python/generated_cases.c.h" + #line 734 "Python/generated_cases.c.h" } TARGET(LIST_APPEND) { PyObject *v = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 521 "Python/bytecodes.c" + #line 522 "Python/bytecodes.c" if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; - #line 741 "Python/generated_cases.c.h" + #line 742 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -746,13 +747,13 @@ TARGET(SET_ADD) { PyObject *v = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 526 "Python/bytecodes.c" + #line 527 "Python/bytecodes.c" int err = PySet_Add(set, v); - #line 752 "Python/generated_cases.c.h" + #line 753 "Python/generated_cases.c.h" Py_DECREF(v); - #line 528 "Python/bytecodes.c" + #line 529 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 756 "Python/generated_cases.c.h" + #line 757 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -765,7 +766,7 @@ PyObject *container = stack_pointer[-2]; PyObject *v = stack_pointer[-3]; uint16_t counter = read_u16(&next_instr[0].cache); - #line 539 "Python/bytecodes.c" + #line 540 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr--; @@ -780,13 +781,13 @@ #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); - #line 784 "Python/generated_cases.c.h" + #line 785 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); - #line 554 "Python/bytecodes.c" + #line 555 "Python/bytecodes.c" if (err) goto pop_3_error; - #line 790 "Python/generated_cases.c.h" + #line 791 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -796,7 +797,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; - #line 558 "Python/bytecodes.c" + #line 559 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -813,7 +814,7 @@ Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - #line 817 "Python/generated_cases.c.h" + #line 818 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -823,13 +824,13 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; - #line 577 "Python/bytecodes.c" + #line 578 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); if (err) goto pop_3_error; - #line 833 "Python/generated_cases.c.h" + #line 834 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -838,15 +839,15 @@ TARGET(DELETE_SUBSCR) { PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; - #line 585 "Python/bytecodes.c" + #line 586 "Python/bytecodes.c" /* del container[sub] */ int err = PyObject_DelItem(container, sub); - #line 845 "Python/generated_cases.c.h" + #line 846 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); - #line 588 "Python/bytecodes.c" + #line 589 "Python/bytecodes.c" if (err) goto pop_2_error; - #line 850 "Python/generated_cases.c.h" + #line 851 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -854,14 +855,14 @@ TARGET(CALL_INTRINSIC_1) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 592 "Python/bytecodes.c" + #line 593 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg](tstate, value); - #line 861 "Python/generated_cases.c.h" + #line 862 "Python/generated_cases.c.h" Py_DECREF(value); - #line 595 "Python/bytecodes.c" + #line 596 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 865 "Python/generated_cases.c.h" + #line 866 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -870,15 +871,15 @@ PyObject *value1 = stack_pointer[-1]; PyObject *value2 = stack_pointer[-2]; PyObject *res; - #line 599 "Python/bytecodes.c" + #line 600 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_2); res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1); - #line 877 "Python/generated_cases.c.h" + #line 878 "Python/generated_cases.c.h" Py_DECREF(value2); Py_DECREF(value1); - #line 602 "Python/bytecodes.c" + #line 603 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 882 "Python/generated_cases.c.h" + #line 883 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -886,7 +887,7 @@ TARGET(RAISE_VARARGS) { PyObject **args = (stack_pointer - oparg); - #line 606 "Python/bytecodes.c" + #line 607 "Python/bytecodes.c" PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -904,12 +905,12 @@ break; } if (true) { STACK_SHRINK(oparg); goto error; } - #line 908 "Python/generated_cases.c.h" + #line 909 "Python/generated_cases.c.h" } TARGET(INTERPRETER_EXIT) { PyObject *retval = stack_pointer[-1]; - #line 626 "Python/bytecodes.c" + #line 627 "Python/bytecodes.c" assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); STACK_SHRINK(1); // Since we're not going to DISPATCH() @@ -920,12 +921,12 @@ assert(!_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallTstate(tstate); return retval; - #line 924 "Python/generated_cases.c.h" + #line 925 "Python/generated_cases.c.h" } TARGET(RETURN_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 639 "Python/bytecodes.c" + #line 640 "Python/bytecodes.c" STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -938,12 +939,12 @@ frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 942 "Python/generated_cases.c.h" + #line 943 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_RETURN_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 654 "Python/bytecodes.c" + #line 655 "Python/bytecodes.c" int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, next_instr-1, retval); @@ -960,11 +961,11 @@ frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 964 "Python/generated_cases.c.h" + #line 965 "Python/generated_cases.c.h" } TARGET(RETURN_CONST) { - #line 673 "Python/bytecodes.c" + #line 674 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(retval); assert(EMPTY()); @@ -978,11 +979,11 @@ frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 982 "Python/generated_cases.c.h" + #line 983 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_RETURN_CONST) { - #line 689 "Python/bytecodes.c" + #line 690 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, @@ -1000,13 +1001,13 @@ frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 1004 "Python/generated_cases.c.h" + #line 1005 "Python/generated_cases.c.h" } TARGET(GET_AITER) { PyObject *obj = stack_pointer[-1]; PyObject *iter; - #line 709 "Python/bytecodes.c" + #line 710 "Python/bytecodes.c" unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); @@ -1019,16 +1020,16 @@ "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - #line 1023 "Python/generated_cases.c.h" + #line 1024 "Python/generated_cases.c.h" Py_DECREF(obj); - #line 722 "Python/bytecodes.c" + #line 723 "Python/bytecodes.c" if (true) goto pop_1_error; } iter = (*getter)(obj); - #line 1030 "Python/generated_cases.c.h" + #line 1031 "Python/generated_cases.c.h" Py_DECREF(obj); - #line 727 "Python/bytecodes.c" + #line 728 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; if (Py_TYPE(iter)->tp_as_async == NULL || @@ -1041,7 +1042,7 @@ Py_DECREF(iter); if (true) goto pop_1_error; } - #line 1045 "Python/generated_cases.c.h" + #line 1046 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -1049,7 +1050,7 @@ TARGET(GET_ANEXT) { PyObject *aiter = stack_pointer[-1]; PyObject *awaitable; - #line 742 "Python/bytecodes.c" + #line 743 "Python/bytecodes.c" unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); @@ -1093,7 +1094,7 @@ } } - #line 1097 "Python/generated_cases.c.h" + #line 1098 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = awaitable; PREDICT(LOAD_CONST); @@ -1104,16 +1105,16 @@ PREDICTED(GET_AWAITABLE); PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 789 "Python/bytecodes.c" + #line 790 "Python/bytecodes.c" iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { format_awaitable_error(tstate, Py_TYPE(iterable), oparg); } - #line 1115 "Python/generated_cases.c.h" + #line 1116 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 796 "Python/bytecodes.c" + #line 797 "Python/bytecodes.c" if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); @@ -1131,7 +1132,7 @@ if (iter == NULL) goto pop_1_error; - #line 1135 "Python/generated_cases.c.h" + #line 1136 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -1143,7 +1144,7 @@ PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; PyObject *retval; - #line 822 "Python/bytecodes.c" + #line 823 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PySendCache *cache = (_PySendCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1155,8 +1156,9 @@ DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ assert(frame != &entry_frame); - if ((Py_TYPE(receiver) == &PyGen_Type || - Py_TYPE(receiver) == &PyCoro_Type) && ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) + if ((tstate->interp->eval_frame == NULL) && + (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && + ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) { PyGenObject *gen = (PyGenObject *)receiver; _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; @@ -1189,7 +1191,7 @@ } } Py_DECREF(v); - #line 1193 "Python/generated_cases.c.h" + #line 1195 "Python/generated_cases.c.h" stack_pointer[-1] = retval; next_instr += 1; DISPATCH(); @@ -1198,7 +1200,8 @@ TARGET(SEND_GEN) { PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 870 "Python/bytecodes.c" + #line 872 "Python/bytecodes.c" + DEOPT_IF(tstate->interp->eval_frame, SEND); PyGenObject *gen = (PyGenObject *)receiver; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); @@ -1213,12 +1216,12 @@ tstate->exc_info = &gen->gi_exc_state; JUMPBY(INLINE_CACHE_ENTRIES_SEND); DISPATCH_INLINED(gen_frame); - #line 1217 "Python/generated_cases.c.h" + #line 1220 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_YIELD_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 887 "Python/bytecodes.c" + #line 890 "Python/bytecodes.c" assert(frame != &entry_frame); PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; @@ -1235,12 +1238,12 @@ gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 1239 "Python/generated_cases.c.h" + #line 1242 "Python/generated_cases.c.h" } TARGET(YIELD_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 906 "Python/bytecodes.c" + #line 909 "Python/bytecodes.c" // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -1256,15 +1259,15 @@ gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 1260 "Python/generated_cases.c.h" + #line 1263 "Python/generated_cases.c.h" } TARGET(POP_EXCEPT) { PyObject *exc_value = stack_pointer[-1]; - #line 924 "Python/bytecodes.c" + #line 927 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); - #line 1268 "Python/generated_cases.c.h" + #line 1271 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1272,7 +1275,7 @@ TARGET(RERAISE) { PyObject *exc = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); - #line 929 "Python/bytecodes.c" + #line 932 "Python/bytecodes.c" assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1290,26 +1293,26 @@ Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; - #line 1294 "Python/generated_cases.c.h" + #line 1297 "Python/generated_cases.c.h" } TARGET(END_ASYNC_FOR) { PyObject *exc = stack_pointer[-1]; PyObject *awaitable = stack_pointer[-2]; - #line 949 "Python/bytecodes.c" + #line 952 "Python/bytecodes.c" assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { - #line 1303 "Python/generated_cases.c.h" + #line 1306 "Python/generated_cases.c.h" Py_DECREF(awaitable); Py_DECREF(exc); - #line 952 "Python/bytecodes.c" + #line 955 "Python/bytecodes.c" } else { Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; } - #line 1313 "Python/generated_cases.c.h" + #line 1316 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -1320,23 +1323,23 @@ PyObject *sub_iter = stack_pointer[-3]; PyObject *none; PyObject *value; - #line 961 "Python/bytecodes.c" + #line 964 "Python/bytecodes.c" assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); - #line 1329 "Python/generated_cases.c.h" + #line 1332 "Python/generated_cases.c.h" Py_DECREF(sub_iter); Py_DECREF(last_sent_val); Py_DECREF(exc_value); - #line 966 "Python/bytecodes.c" + #line 969 "Python/bytecodes.c" none = Py_NewRef(Py_None); } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); goto exception_unwind; } - #line 1340 "Python/generated_cases.c.h" + #line 1343 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; stack_pointer[-2] = none; @@ -1345,9 +1348,9 @@ TARGET(LOAD_ASSERTION_ERROR) { PyObject *value; - #line 975 "Python/bytecodes.c" + #line 978 "Python/bytecodes.c" value = Py_NewRef(PyExc_AssertionError); - #line 1351 "Python/generated_cases.c.h" + #line 1354 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1355,7 +1358,7 @@ TARGET(LOAD_BUILD_CLASS) { PyObject *bc; - #line 979 "Python/bytecodes.c" + #line 982 "Python/bytecodes.c" if (PyDict_CheckExact(BUILTINS())) { bc = _PyDict_GetItemWithError(BUILTINS(), &_Py_ID(__build_class__)); @@ -1377,7 +1380,7 @@ if (true) goto error; } } - #line 1381 "Python/generated_cases.c.h" + #line 1384 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = bc; DISPATCH(); @@ -1385,33 +1388,33 @@ TARGET(STORE_NAME) { PyObject *v = stack_pointer[-1]; - #line 1003 "Python/bytecodes.c" + #line 1006 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - #line 1396 "Python/generated_cases.c.h" + #line 1399 "Python/generated_cases.c.h" Py_DECREF(v); - #line 1010 "Python/bytecodes.c" + #line 1013 "Python/bytecodes.c" if (true) goto pop_1_error; } if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); - #line 1405 "Python/generated_cases.c.h" + #line 1408 "Python/generated_cases.c.h" Py_DECREF(v); - #line 1017 "Python/bytecodes.c" + #line 1020 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1409 "Python/generated_cases.c.h" + #line 1412 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_NAME) { - #line 1021 "Python/bytecodes.c" + #line 1024 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; @@ -1428,7 +1431,7 @@ name); goto error; } - #line 1432 "Python/generated_cases.c.h" + #line 1435 "Python/generated_cases.c.h" DISPATCH(); } @@ -1436,7 +1439,7 @@ PREDICTED(UNPACK_SEQUENCE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq = stack_pointer[-1]; - #line 1047 "Python/bytecodes.c" + #line 1050 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1449,11 +1452,11 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = unpack_iterable(tstate, seq, oparg, -1, top); - #line 1453 "Python/generated_cases.c.h" + #line 1456 "Python/generated_cases.c.h" Py_DECREF(seq); - #line 1060 "Python/bytecodes.c" + #line 1063 "Python/bytecodes.c" if (res == 0) goto pop_1_error; - #line 1457 "Python/generated_cases.c.h" + #line 1460 "Python/generated_cases.c.h" STACK_SHRINK(1); STACK_GROW(oparg); next_instr += 1; @@ -1463,14 +1466,14 @@ TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1064 "Python/bytecodes.c" + #line 1067 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - #line 1474 "Python/generated_cases.c.h" + #line 1477 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1481,7 +1484,7 @@ TARGET(UNPACK_SEQUENCE_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1074 "Python/bytecodes.c" + #line 1077 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1489,7 +1492,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - #line 1493 "Python/generated_cases.c.h" + #line 1496 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1500,7 +1503,7 @@ TARGET(UNPACK_SEQUENCE_LIST) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1085 "Python/bytecodes.c" + #line 1088 "Python/bytecodes.c" DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1508,7 +1511,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - #line 1512 "Python/generated_cases.c.h" + #line 1515 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1518,15 +1521,15 @@ TARGET(UNPACK_EX) { PyObject *seq = stack_pointer[-1]; - #line 1096 "Python/bytecodes.c" + #line 1099 "Python/bytecodes.c" int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - #line 1526 "Python/generated_cases.c.h" + #line 1529 "Python/generated_cases.c.h" Py_DECREF(seq); - #line 1100 "Python/bytecodes.c" + #line 1103 "Python/bytecodes.c" if (res == 0) goto pop_1_error; - #line 1530 "Python/generated_cases.c.h" + #line 1533 "Python/generated_cases.c.h" STACK_GROW((oparg & 0xFF) + (oparg >> 8)); DISPATCH(); } @@ -1537,7 +1540,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *v = stack_pointer[-2]; uint16_t counter = read_u16(&next_instr[0].cache); - #line 1111 "Python/bytecodes.c" + #line 1114 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(frame->f_code->co_names, oparg); @@ -1553,12 +1556,12 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, v); - #line 1557 "Python/generated_cases.c.h" + #line 1560 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(owner); - #line 1127 "Python/bytecodes.c" + #line 1130 "Python/bytecodes.c" if (err) goto pop_2_error; - #line 1562 "Python/generated_cases.c.h" + #line 1565 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -1566,34 +1569,34 @@ TARGET(DELETE_ATTR) { PyObject *owner = stack_pointer[-1]; - #line 1131 "Python/bytecodes.c" + #line 1134 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); - #line 1573 "Python/generated_cases.c.h" + #line 1576 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1134 "Python/bytecodes.c" + #line 1137 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1577 "Python/generated_cases.c.h" + #line 1580 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(STORE_GLOBAL) { PyObject *v = stack_pointer[-1]; - #line 1138 "Python/bytecodes.c" + #line 1141 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); - #line 1587 "Python/generated_cases.c.h" + #line 1590 "Python/generated_cases.c.h" Py_DECREF(v); - #line 1141 "Python/bytecodes.c" + #line 1144 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1591 "Python/generated_cases.c.h" + #line 1594 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_GLOBAL) { - #line 1145 "Python/bytecodes.c" + #line 1148 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err; err = PyDict_DelItem(GLOBALS(), name); @@ -1605,13 +1608,13 @@ } goto error; } - #line 1609 "Python/generated_cases.c.h" + #line 1612 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_NAME) { PyObject *v; - #line 1159 "Python/bytecodes.c" + #line 1162 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *locals = LOCALS(); if (locals == NULL) { @@ -1670,7 +1673,7 @@ } } } - #line 1674 "Python/generated_cases.c.h" + #line 1677 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = v; DISPATCH(); @@ -1681,7 +1684,7 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *null = NULL; PyObject *v; - #line 1226 "Python/bytecodes.c" + #line 1229 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1733,7 +1736,7 @@ } } null = NULL; - #line 1737 "Python/generated_cases.c.h" + #line 1740 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = v; @@ -1747,7 +1750,7 @@ PyObject *res; uint16_t index = read_u16(&next_instr[1].cache); uint16_t version = read_u16(&next_instr[2].cache); - #line 1280 "Python/bytecodes.c" + #line 1283 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); @@ -1758,7 +1761,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - #line 1762 "Python/generated_cases.c.h" + #line 1765 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1773,7 +1776,7 @@ uint16_t index = read_u16(&next_instr[1].cache); uint16_t mod_version = read_u16(&next_instr[2].cache); uint16_t bltn_version = read_u16(&next_instr[3].cache); - #line 1293 "Python/bytecodes.c" + #line 1296 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); PyDictObject *mdict = (PyDictObject *)GLOBALS(); @@ -1788,7 +1791,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - #line 1792 "Python/generated_cases.c.h" + #line 1795 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1798,16 +1801,16 @@ } TARGET(DELETE_FAST) { - #line 1310 "Python/bytecodes.c" + #line 1313 "Python/bytecodes.c" PyObject *v = GETLOCAL(oparg); if (v == NULL) goto unbound_local_error; SETLOCAL(oparg, NULL); - #line 1806 "Python/generated_cases.c.h" + #line 1809 "Python/generated_cases.c.h" DISPATCH(); } TARGET(MAKE_CELL) { - #line 1316 "Python/bytecodes.c" + #line 1319 "Python/bytecodes.c" // "initial" is probably NULL but not if it's an arg (or set // via PyFrame_LocalsToFast() before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); @@ -1816,12 +1819,12 @@ goto resume_with_error; } SETLOCAL(oparg, cell); - #line 1820 "Python/generated_cases.c.h" + #line 1823 "Python/generated_cases.c.h" DISPATCH(); } TARGET(DELETE_DEREF) { - #line 1327 "Python/bytecodes.c" + #line 1330 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); // Can't use ERROR_IF here. @@ -1832,13 +1835,13 @@ } PyCell_SET(cell, NULL); Py_DECREF(oldobj); - #line 1836 "Python/generated_cases.c.h" + #line 1839 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLASSDEREF) { PyObject *value; - #line 1340 "Python/bytecodes.c" + #line 1343 "Python/bytecodes.c" PyObject *name, *locals = LOCALS(); assert(locals); assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus); @@ -1870,7 +1873,7 @@ } Py_INCREF(value); } - #line 1874 "Python/generated_cases.c.h" + #line 1877 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1878,7 +1881,7 @@ TARGET(LOAD_DEREF) { PyObject *value; - #line 1374 "Python/bytecodes.c" + #line 1377 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { @@ -1886,7 +1889,7 @@ if (true) goto error; } Py_INCREF(value); - #line 1890 "Python/generated_cases.c.h" + #line 1893 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1894,18 +1897,18 @@ TARGET(STORE_DEREF) { PyObject *v = stack_pointer[-1]; - #line 1384 "Python/bytecodes.c" + #line 1387 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); - #line 1903 "Python/generated_cases.c.h" + #line 1906 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(COPY_FREE_VARS) { - #line 1391 "Python/bytecodes.c" + #line 1394 "Python/bytecodes.c" /* Copy closure variables to free variables */ PyCodeObject *co = frame->f_code; assert(PyFunction_Check(frame->f_funcobj)); @@ -1916,22 +1919,22 @@ PyObject *o = PyTuple_GET_ITEM(closure, i); frame->localsplus[offset + i] = Py_NewRef(o); } - #line 1920 "Python/generated_cases.c.h" + #line 1923 "Python/generated_cases.c.h" DISPATCH(); } TARGET(BUILD_STRING) { PyObject **pieces = (stack_pointer - oparg); PyObject *str; - #line 1404 "Python/bytecodes.c" + #line 1407 "Python/bytecodes.c" str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - #line 1929 "Python/generated_cases.c.h" + #line 1932 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } - #line 1406 "Python/bytecodes.c" + #line 1409 "Python/bytecodes.c" if (str == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1935 "Python/generated_cases.c.h" + #line 1938 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = str; @@ -1941,10 +1944,10 @@ TARGET(BUILD_TUPLE) { PyObject **values = (stack_pointer - oparg); PyObject *tup; - #line 1410 "Python/bytecodes.c" + #line 1413 "Python/bytecodes.c" tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1948 "Python/generated_cases.c.h" + #line 1951 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = tup; @@ -1954,10 +1957,10 @@ TARGET(BUILD_LIST) { PyObject **values = (stack_pointer - oparg); PyObject *list; - #line 1415 "Python/bytecodes.c" + #line 1418 "Python/bytecodes.c" list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1961 "Python/generated_cases.c.h" + #line 1964 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = list; @@ -1967,7 +1970,7 @@ TARGET(LIST_EXTEND) { PyObject *iterable = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 1420 "Python/bytecodes.c" + #line 1423 "Python/bytecodes.c" PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1978,13 +1981,13 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - #line 1982 "Python/generated_cases.c.h" + #line 1985 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1431 "Python/bytecodes.c" + #line 1434 "Python/bytecodes.c" if (true) goto pop_1_error; } Py_DECREF(none_val); - #line 1988 "Python/generated_cases.c.h" + #line 1991 "Python/generated_cases.c.h" Py_DECREF(iterable); STACK_SHRINK(1); DISPATCH(); @@ -1993,13 +1996,13 @@ TARGET(SET_UPDATE) { PyObject *iterable = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 1438 "Python/bytecodes.c" + #line 1441 "Python/bytecodes.c" int err = _PySet_Update(set, iterable); - #line 1999 "Python/generated_cases.c.h" + #line 2002 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1440 "Python/bytecodes.c" + #line 1443 "Python/bytecodes.c" if (err < 0) goto pop_1_error; - #line 2003 "Python/generated_cases.c.h" + #line 2006 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -2007,7 +2010,7 @@ TARGET(BUILD_SET) { PyObject **values = (stack_pointer - oparg); PyObject *set; - #line 1444 "Python/bytecodes.c" + #line 1447 "Python/bytecodes.c" set = PySet_New(NULL); if (set == NULL) goto error; @@ -2022,7 +2025,7 @@ Py_DECREF(set); if (true) { STACK_SHRINK(oparg); goto error; } } - #line 2026 "Python/generated_cases.c.h" + #line 2029 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = set; @@ -2032,7 +2035,7 @@ TARGET(BUILD_MAP) { PyObject **values = (stack_pointer - oparg*2); PyObject *map; - #line 1461 "Python/bytecodes.c" + #line 1464 "Python/bytecodes.c" map = _PyDict_FromItems( values, 2, values+1, 2, @@ -2040,13 +2043,13 @@ if (map == NULL) goto error; - #line 2044 "Python/generated_cases.c.h" + #line 2047 "Python/generated_cases.c.h" for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } - #line 1469 "Python/bytecodes.c" + #line 1472 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } - #line 2050 "Python/generated_cases.c.h" + #line 2053 "Python/generated_cases.c.h" STACK_SHRINK(oparg*2); STACK_GROW(1); stack_pointer[-1] = map; @@ -2054,7 +2057,7 @@ } TARGET(SETUP_ANNOTATIONS) { - #line 1473 "Python/bytecodes.c" + #line 1476 "Python/bytecodes.c" int err; PyObject *ann_dict; if (LOCALS() == NULL) { @@ -2094,7 +2097,7 @@ Py_DECREF(ann_dict); } } - #line 2098 "Python/generated_cases.c.h" + #line 2101 "Python/generated_cases.c.h" DISPATCH(); } @@ -2102,7 +2105,7 @@ PyObject *keys = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); PyObject *map; - #line 1515 "Python/bytecodes.c" + #line 1518 "Python/bytecodes.c" if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -2112,14 +2115,14 @@ map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); - #line 2116 "Python/generated_cases.c.h" + #line 2119 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); - #line 1525 "Python/bytecodes.c" + #line 1528 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } - #line 2123 "Python/generated_cases.c.h" + #line 2126 "Python/generated_cases.c.h" STACK_SHRINK(oparg); stack_pointer[-1] = map; DISPATCH(); @@ -2127,7 +2130,7 @@ TARGET(DICT_UPDATE) { PyObject *update = stack_pointer[-1]; - #line 1529 "Python/bytecodes.c" + #line 1532 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { @@ -2135,12 +2138,12 @@ "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } - #line 2139 "Python/generated_cases.c.h" + #line 2142 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1537 "Python/bytecodes.c" + #line 1540 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2144 "Python/generated_cases.c.h" + #line 2147 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); DISPATCH(); @@ -2148,17 +2151,17 @@ TARGET(DICT_MERGE) { PyObject *update = stack_pointer[-1]; - #line 1543 "Python/bytecodes.c" + #line 1546 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { format_kwargs_error(tstate, PEEK(3 + oparg), update); - #line 2157 "Python/generated_cases.c.h" + #line 2160 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1548 "Python/bytecodes.c" + #line 1551 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2162 "Python/generated_cases.c.h" + #line 2165 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); PREDICT(CALL_FUNCTION_EX); @@ -2168,13 +2171,13 @@ TARGET(MAP_ADD) { PyObject *value = stack_pointer[-1]; PyObject *key = stack_pointer[-2]; - #line 1555 "Python/bytecodes.c" + #line 1558 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - #line 2178 "Python/generated_cases.c.h" + #line 2181 "Python/generated_cases.c.h" STACK_SHRINK(2); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -2188,7 +2191,7 @@ PyObject *global_super = stack_pointer[-3]; PyObject *res2 = NULL; PyObject *res; - #line 1570 "Python/bytecodes.c" + #line 1573 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 2); int load_method = oparg & 1; #if ENABLE_SPECIALIZATION @@ -2206,16 +2209,16 @@ // handle any case whose performance we care about PyObject *stack[] = {class, self}; PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - #line 2210 "Python/generated_cases.c.h" + #line 2213 "Python/generated_cases.c.h" Py_DECREF(global_super); Py_DECREF(class); Py_DECREF(self); - #line 1588 "Python/bytecodes.c" + #line 1591 "Python/bytecodes.c" if (super == NULL) goto pop_3_error; res = PyObject_GetAttr(super, name); Py_DECREF(super); if (res == NULL) goto pop_3_error; - #line 2219 "Python/generated_cases.c.h" + #line 2222 "Python/generated_cases.c.h" STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2230,20 +2233,20 @@ PyObject *global_super = stack_pointer[-3]; PyObject *res2 = NULL; PyObject *res; - #line 1595 "Python/bytecodes.c" + #line 1598 "Python/bytecodes.c" assert(!(oparg & 1)); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 2); res = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - #line 2241 "Python/generated_cases.c.h" + #line 2244 "Python/generated_cases.c.h" Py_DECREF(global_super); Py_DECREF(class); Py_DECREF(self); - #line 1602 "Python/bytecodes.c" + #line 1605 "Python/bytecodes.c" if (res == NULL) goto pop_3_error; - #line 2247 "Python/generated_cases.c.h" + #line 2250 "Python/generated_cases.c.h" STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2258,7 +2261,7 @@ PyObject *global_super = stack_pointer[-3]; PyObject *res2; PyObject *res; - #line 1606 "Python/bytecodes.c" + #line 1609 "Python/bytecodes.c" assert(oparg & 1); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); @@ -2279,7 +2282,7 @@ res = res2; res2 = NULL; } - #line 2283 "Python/generated_cases.c.h" + #line 2286 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; stack_pointer[-2] = res2; @@ -2293,7 +2296,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; - #line 1643 "Python/bytecodes.c" + #line 1646 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2327,9 +2330,9 @@ NULL | meth | arg1 | ... | argN */ - #line 2331 "Python/generated_cases.c.h" + #line 2334 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1677 "Python/bytecodes.c" + #line 1680 "Python/bytecodes.c" if (meth == NULL) goto pop_1_error; res2 = NULL; res = meth; @@ -2338,12 +2341,12 @@ else { /* Classic, pushes one value. */ res = PyObject_GetAttr(owner, name); - #line 2342 "Python/generated_cases.c.h" + #line 2345 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1686 "Python/bytecodes.c" + #line 1689 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; } - #line 2347 "Python/generated_cases.c.h" + #line 2350 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -2357,7 +2360,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1691 "Python/bytecodes.c" + #line 1694 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2370,7 +2373,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2374 "Python/generated_cases.c.h" + #line 2377 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2385,7 +2388,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1707 "Python/bytecodes.c" + #line 1710 "Python/bytecodes.c" DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -2398,7 +2401,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2402 "Python/generated_cases.c.h" + #line 2405 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2413,7 +2416,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1723 "Python/bytecodes.c" + #line 1726 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2440,7 +2443,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2444 "Python/generated_cases.c.h" + #line 2447 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2455,7 +2458,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1753 "Python/bytecodes.c" + #line 1756 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2465,7 +2468,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2469 "Python/generated_cases.c.h" + #line 2472 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2480,7 +2483,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 1766 "Python/bytecodes.c" + #line 1769 "Python/bytecodes.c" DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, @@ -2492,7 +2495,7 @@ res = descr; assert(res != NULL); Py_INCREF(res); - #line 2496 "Python/generated_cases.c.h" + #line 2499 "Python/generated_cases.c.h" Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2506,7 +2509,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *fget = read_obj(&next_instr[5].cache); - #line 1781 "Python/bytecodes.c" + #line 1784 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -2530,7 +2533,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 2534 "Python/generated_cases.c.h" + #line 2537 "Python/generated_cases.c.h" } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { @@ -2538,7 +2541,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *getattribute = read_obj(&next_instr[5].cache); - #line 1807 "Python/bytecodes.c" + #line 1810 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -2564,7 +2567,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 2568 "Python/generated_cases.c.h" + #line 2571 "Python/generated_cases.c.h" } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -2572,7 +2575,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1835 "Python/bytecodes.c" + #line 1838 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2590,7 +2593,7 @@ Py_DECREF(old_value); } Py_DECREF(owner); - #line 2594 "Python/generated_cases.c.h" + #line 2597 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2601,7 +2604,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t hint = read_u16(&next_instr[3].cache); - #line 1855 "Python/bytecodes.c" + #line 1858 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2640,7 +2643,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - #line 2644 "Python/generated_cases.c.h" + #line 2647 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2651,7 +2654,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1896 "Python/bytecodes.c" + #line 1899 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2661,7 +2664,7 @@ *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - #line 2665 "Python/generated_cases.c.h" + #line 2668 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2673,7 +2676,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1915 "Python/bytecodes.c" + #line 1918 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2686,12 +2689,12 @@ #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); - #line 2690 "Python/generated_cases.c.h" + #line 2693 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1928 "Python/bytecodes.c" + #line 1931 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2695 "Python/generated_cases.c.h" + #line 2698 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2702,7 +2705,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1932 "Python/bytecodes.c" + #line 1935 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2714,7 +2717,7 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2718 "Python/generated_cases.c.h" + #line 2721 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2725,7 +2728,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1947 "Python/bytecodes.c" + #line 1950 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -2741,7 +2744,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2745 "Python/generated_cases.c.h" + #line 2748 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2752,7 +2755,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1966 "Python/bytecodes.c" + #line 1969 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2765,7 +2768,7 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2769 "Python/generated_cases.c.h" + #line 2772 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2776,14 +2779,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1981 "Python/bytecodes.c" + #line 1984 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; - #line 2782 "Python/generated_cases.c.h" + #line 2785 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1983 "Python/bytecodes.c" + #line 1986 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2787 "Python/generated_cases.c.h" + #line 2790 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2793,15 +2796,15 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1987 "Python/bytecodes.c" + #line 1990 "Python/bytecodes.c" int res = PySequence_Contains(right, left); - #line 2799 "Python/generated_cases.c.h" + #line 2802 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1989 "Python/bytecodes.c" + #line 1992 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = Py_NewRef((res^oparg) ? Py_True : Py_False); - #line 2805 "Python/generated_cases.c.h" + #line 2808 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2812,12 +2815,12 @@ PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; - #line 1994 "Python/bytecodes.c" + #line 1997 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { - #line 2818 "Python/generated_cases.c.h" + #line 2821 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1996 "Python/bytecodes.c" + #line 1999 "Python/bytecodes.c" if (true) goto pop_2_error; } @@ -2825,10 +2828,10 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); - #line 2829 "Python/generated_cases.c.h" + #line 2832 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 2004 "Python/bytecodes.c" + #line 2007 "Python/bytecodes.c" if (res < 0) goto pop_2_error; assert((match == NULL) == (rest == NULL)); @@ -2837,7 +2840,7 @@ if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } - #line 2841 "Python/generated_cases.c.h" + #line 2844 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -2847,21 +2850,21 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 2015 "Python/bytecodes.c" + #line 2018 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - #line 2854 "Python/generated_cases.c.h" + #line 2857 "Python/generated_cases.c.h" Py_DECREF(right); - #line 2018 "Python/bytecodes.c" + #line 2021 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - #line 2861 "Python/generated_cases.c.h" + #line 2864 "Python/generated_cases.c.h" Py_DECREF(right); - #line 2023 "Python/bytecodes.c" + #line 2026 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2865 "Python/generated_cases.c.h" + #line 2868 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -2870,15 +2873,15 @@ PyObject *fromlist = stack_pointer[-1]; PyObject *level = stack_pointer[-2]; PyObject *res; - #line 2027 "Python/bytecodes.c" + #line 2030 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); - #line 2877 "Python/generated_cases.c.h" + #line 2880 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); - #line 2030 "Python/bytecodes.c" + #line 2033 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2882 "Python/generated_cases.c.h" + #line 2885 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -2887,29 +2890,29 @@ TARGET(IMPORT_FROM) { PyObject *from = stack_pointer[-1]; PyObject *res; - #line 2034 "Python/bytecodes.c" + #line 2037 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - #line 2895 "Python/generated_cases.c.h" + #line 2898 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); } TARGET(JUMP_FORWARD) { - #line 2040 "Python/bytecodes.c" + #line 2043 "Python/bytecodes.c" JUMPBY(oparg); - #line 2904 "Python/generated_cases.c.h" + #line 2907 "Python/generated_cases.c.h" DISPATCH(); } TARGET(JUMP_BACKWARD) { PREDICTED(JUMP_BACKWARD); - #line 2044 "Python/bytecodes.c" + #line 2047 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); - #line 2913 "Python/generated_cases.c.h" + #line 2916 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2917,7 +2920,7 @@ TARGET(POP_JUMP_IF_FALSE) { PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = stack_pointer[-1]; - #line 2050 "Python/bytecodes.c" + #line 2053 "Python/bytecodes.c" if (Py_IsTrue(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2927,9 +2930,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2931 "Python/generated_cases.c.h" + #line 2934 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 2060 "Python/bytecodes.c" + #line 2063 "Python/bytecodes.c" if (err == 0) { JUMPBY(oparg); } @@ -2937,14 +2940,14 @@ if (err < 0) goto pop_1_error; } } - #line 2941 "Python/generated_cases.c.h" + #line 2944 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = stack_pointer[-1]; - #line 2070 "Python/bytecodes.c" + #line 2073 "Python/bytecodes.c" if (Py_IsFalse(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2954,9 +2957,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2958 "Python/generated_cases.c.h" + #line 2961 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 2080 "Python/bytecodes.c" + #line 2083 "Python/bytecodes.c" if (err > 0) { JUMPBY(oparg); } @@ -2964,67 +2967,67 @@ if (err < 0) goto pop_1_error; } } - #line 2968 "Python/generated_cases.c.h" + #line 2971 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = stack_pointer[-1]; - #line 2090 "Python/bytecodes.c" + #line 2093 "Python/bytecodes.c" if (!Py_IsNone(value)) { - #line 2977 "Python/generated_cases.c.h" + #line 2980 "Python/generated_cases.c.h" Py_DECREF(value); - #line 2092 "Python/bytecodes.c" + #line 2095 "Python/bytecodes.c" JUMPBY(oparg); } else { _Py_DECREF_NO_DEALLOC(value); } - #line 2985 "Python/generated_cases.c.h" + #line 2988 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = stack_pointer[-1]; - #line 2100 "Python/bytecodes.c" + #line 2103 "Python/bytecodes.c" if (Py_IsNone(value)) { _Py_DECREF_NO_DEALLOC(value); JUMPBY(oparg); } else { - #line 2998 "Python/generated_cases.c.h" + #line 3001 "Python/generated_cases.c.h" Py_DECREF(value); - #line 2106 "Python/bytecodes.c" + #line 2109 "Python/bytecodes.c" } - #line 3002 "Python/generated_cases.c.h" + #line 3005 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - #line 2110 "Python/bytecodes.c" + #line 2113 "Python/bytecodes.c" /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ JUMPBY(-oparg); - #line 3015 "Python/generated_cases.c.h" + #line 3018 "Python/generated_cases.c.h" DISPATCH(); } TARGET(GET_LEN) { PyObject *obj = stack_pointer[-1]; PyObject *len_o; - #line 2119 "Python/bytecodes.c" + #line 2122 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 3028 "Python/generated_cases.c.h" + #line 3031 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -3035,16 +3038,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; - #line 2127 "Python/bytecodes.c" + #line 2130 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 3044 "Python/generated_cases.c.h" + #line 3047 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); - #line 2132 "Python/bytecodes.c" + #line 2135 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -3052,7 +3055,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_NewRef(Py_None); // Failure! } - #line 3056 "Python/generated_cases.c.h" + #line 3059 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -3061,10 +3064,10 @@ TARGET(MATCH_MAPPING) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2142 "Python/bytecodes.c" + #line 2145 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = Py_NewRef(match ? Py_True : Py_False); - #line 3068 "Python/generated_cases.c.h" + #line 3071 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -3074,10 +3077,10 @@ TARGET(MATCH_SEQUENCE) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2148 "Python/bytecodes.c" + #line 2151 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = Py_NewRef(match ? Py_True : Py_False); - #line 3081 "Python/generated_cases.c.h" + #line 3084 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -3088,11 +3091,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; - #line 2154 "Python/bytecodes.c" + #line 2157 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 3096 "Python/generated_cases.c.h" + #line 3099 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -3101,14 +3104,14 @@ TARGET(GET_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2160 "Python/bytecodes.c" + #line 2163 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 3108 "Python/generated_cases.c.h" + #line 3111 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2163 "Python/bytecodes.c" + #line 2166 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 3112 "Python/generated_cases.c.h" + #line 3115 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -3116,7 +3119,7 @@ TARGET(GET_YIELD_FROM_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2167 "Python/bytecodes.c" + #line 2170 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -3139,11 +3142,11 @@ if (iter == NULL) { goto error; } - #line 3143 "Python/generated_cases.c.h" + #line 3146 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2190 "Python/bytecodes.c" + #line 2193 "Python/bytecodes.c" } - #line 3147 "Python/generated_cases.c.h" + #line 3150 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -3154,7 +3157,7 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2209 "Python/bytecodes.c" + #line 2212 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -3185,7 +3188,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator - #line 3189 "Python/generated_cases.c.h" + #line 3192 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3193,7 +3196,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - #line 2242 "Python/bytecodes.c" + #line 2245 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -3219,14 +3222,14 @@ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - #line 3223 "Python/generated_cases.c.h" + #line 3226 "Python/generated_cases.c.h" DISPATCH(); } TARGET(FOR_ITER_LIST) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2270 "Python/bytecodes.c" + #line 2273 "Python/bytecodes.c" DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; STAT_INC(FOR_ITER, hit); @@ -3246,7 +3249,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator - #line 3250 "Python/generated_cases.c.h" + #line 3253 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3256,7 +3259,7 @@ TARGET(FOR_ITER_TUPLE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2292 "Python/bytecodes.c" + #line 2295 "Python/bytecodes.c" _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3276,7 +3279,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator - #line 3280 "Python/generated_cases.c.h" + #line 3283 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3286,7 +3289,7 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2314 "Python/bytecodes.c" + #line 2317 "Python/bytecodes.c" _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3304,7 +3307,7 @@ if (next == NULL) { goto error; } - #line 3308 "Python/generated_cases.c.h" + #line 3311 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3313,7 +3316,8 @@ TARGET(FOR_ITER_GEN) { PyObject *iter = stack_pointer[-1]; - #line 2334 "Python/bytecodes.c" + #line 2337 "Python/bytecodes.c" + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -3328,14 +3332,14 @@ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); - #line 3332 "Python/generated_cases.c.h" + #line 3336 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2351 "Python/bytecodes.c" + #line 2355 "Python/bytecodes.c" PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -3358,16 +3362,16 @@ Py_DECREF(enter); goto error; } - #line 3362 "Python/generated_cases.c.h" + #line 3366 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2374 "Python/bytecodes.c" + #line 2378 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3371 "Python/generated_cases.c.h" + #line 3375 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3379,7 +3383,7 @@ PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2384 "Python/bytecodes.c" + #line 2388 "Python/bytecodes.c" /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -3405,16 +3409,16 @@ Py_DECREF(enter); goto error; } - #line 3409 "Python/generated_cases.c.h" + #line 3413 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2410 "Python/bytecodes.c" + #line 2414 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3418 "Python/generated_cases.c.h" + #line 3422 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3426,7 +3430,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; - #line 2419 "Python/bytecodes.c" + #line 2423 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3447,7 +3451,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 3451 "Python/generated_cases.c.h" + #line 3455 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3456,7 +3460,7 @@ TARGET(PUSH_EXC_INFO) { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; - #line 2442 "Python/bytecodes.c" + #line 2446 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3466,7 +3470,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 3470 "Python/generated_cases.c.h" + #line 3474 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3480,7 +3484,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2454 "Python/bytecodes.c" + #line 2458 "Python/bytecodes.c" /* Cached method object */ PyTypeObject *self_cls = Py_TYPE(self); assert(type_version != 0); @@ -3497,7 +3501,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); - #line 3501 "Python/generated_cases.c.h" + #line 3505 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3511,7 +3515,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2473 "Python/bytecodes.c" + #line 2477 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); @@ -3521,7 +3525,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3525 "Python/generated_cases.c.h" + #line 3529 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3535,7 +3539,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2485 "Python/bytecodes.c" + #line 2489 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; @@ -3549,7 +3553,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3553 "Python/generated_cases.c.h" + #line 3557 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3558,16 +3562,16 @@ } TARGET(KW_NAMES) { - #line 2501 "Python/bytecodes.c" + #line 2505 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); - #line 3566 "Python/generated_cases.c.h" + #line 3570 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_CALL) { - #line 2507 "Python/bytecodes.c" + #line 2511 "Python/bytecodes.c" int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); @@ -3580,7 +3584,7 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(CALL); - #line 3584 "Python/generated_cases.c.h" + #line 3588 "Python/generated_cases.c.h" } TARGET(CALL) { @@ -3590,7 +3594,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2552 "Python/bytecodes.c" + #line 2556 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3672,7 +3676,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3676 "Python/generated_cases.c.h" + #line 3680 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3684,7 +3688,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2640 "Python/bytecodes.c" + #line 2644 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3694,7 +3698,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3698 "Python/generated_cases.c.h" + #line 3702 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3703,7 +3707,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2652 "Python/bytecodes.c" + #line 2656 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3729,7 +3733,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 3733 "Python/generated_cases.c.h" + #line 3737 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3737,7 +3741,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2680 "Python/bytecodes.c" + #line 2684 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3773,7 +3777,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 3777 "Python/generated_cases.c.h" + #line 3781 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3781,7 +3785,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2718 "Python/bytecodes.c" + #line 2722 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3791,7 +3795,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 3795 "Python/generated_cases.c.h" + #line 3799 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3804,7 +3808,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2730 "Python/bytecodes.c" + #line 2734 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3815,7 +3819,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3819 "Python/generated_cases.c.h" + #line 3823 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3829,7 +3833,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2744 "Python/bytecodes.c" + #line 2748 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3840,7 +3844,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3844 "Python/generated_cases.c.h" + #line 3848 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3854,7 +3858,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2758 "Python/bytecodes.c" + #line 2762 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3876,7 +3880,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3880 "Python/generated_cases.c.h" + #line 3884 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3890,7 +3894,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2783 "Python/bytecodes.c" + #line 2787 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3918,7 +3922,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3922 "Python/generated_cases.c.h" + #line 3926 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3932,7 +3936,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2814 "Python/bytecodes.c" + #line 2818 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3964,7 +3968,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 3968 "Python/generated_cases.c.h" + #line 3972 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3978,7 +3982,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2849 "Python/bytecodes.c" + #line 2853 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -4010,7 +4014,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4014 "Python/generated_cases.c.h" + #line 4018 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4024,7 +4028,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2884 "Python/bytecodes.c" + #line 2888 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -4049,7 +4053,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4053 "Python/generated_cases.c.h" + #line 4057 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4062,7 +4066,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2911 "Python/bytecodes.c" + #line 2915 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -4089,7 +4093,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4093 "Python/generated_cases.c.h" + #line 4097 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4101,7 +4105,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2941 "Python/bytecodes.c" + #line 2945 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); assert(method != NULL); @@ -4119,14 +4123,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 4123 "Python/generated_cases.c.h" + #line 4127 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2961 "Python/bytecodes.c" + #line 2965 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4157,7 +4161,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4161 "Python/generated_cases.c.h" + #line 4165 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4170,7 +4174,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2995 "Python/bytecodes.c" + #line 2999 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4199,7 +4203,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4203 "Python/generated_cases.c.h" + #line 4207 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4212,7 +4216,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3027 "Python/bytecodes.c" + #line 3031 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4241,7 +4245,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4245 "Python/generated_cases.c.h" + #line 4249 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4254,7 +4258,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3059 "Python/bytecodes.c" + #line 3063 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4282,7 +4286,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4286 "Python/generated_cases.c.h" + #line 4290 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4292,9 +4296,9 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #line 3090 "Python/bytecodes.c" + #line 3094 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4298 "Python/generated_cases.c.h" + #line 4302 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4303,7 +4307,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 3094 "Python/bytecodes.c" + #line 3098 "Python/bytecodes.c" // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4365,14 +4369,14 @@ } result = PyObject_Call(func, callargs, kwargs); } - #line 4369 "Python/generated_cases.c.h" + #line 4373 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3156 "Python/bytecodes.c" + #line 3160 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4376 "Python/generated_cases.c.h" + #line 4380 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4387,7 +4391,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 3166 "Python/bytecodes.c" + #line 3170 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4416,14 +4420,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4420 "Python/generated_cases.c.h" + #line 4424 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3197 "Python/bytecodes.c" + #line 3201 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4444,7 +4448,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4448 "Python/generated_cases.c.h" + #line 4452 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4452,15 +4456,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3220 "Python/bytecodes.c" + #line 3224 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4458 "Python/generated_cases.c.h" + #line 4462 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3222 "Python/bytecodes.c" + #line 3226 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4464 "Python/generated_cases.c.h" + #line 4468 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4471,7 +4475,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 3226 "Python/bytecodes.c" + #line 3230 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4506,7 +4510,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4510 "Python/generated_cases.c.h" + #line 4514 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4515,10 +4519,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3263 "Python/bytecodes.c" + #line 3267 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4522 "Python/generated_cases.c.h" + #line 4526 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4530,7 +4534,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3268 "Python/bytecodes.c" + #line 3272 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4545,12 +4549,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4549 "Python/generated_cases.c.h" + #line 4553 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3283 "Python/bytecodes.c" + #line 3287 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4554 "Python/generated_cases.c.h" + #line 4558 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4560,16 +4564,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3288 "Python/bytecodes.c" + #line 3292 "Python/bytecodes.c" assert(oparg >= 2); - #line 4566 "Python/generated_cases.c.h" + #line 4570 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3292 "Python/bytecodes.c" + #line 3296 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4581,26 +4585,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4585 "Python/generated_cases.c.h" + #line 4589 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3306 "Python/bytecodes.c" + #line 3310 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4591 "Python/generated_cases.c.h" + #line 4595 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3310 "Python/bytecodes.c" + #line 3314 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4598 "Python/generated_cases.c.h" + #line 4602 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3315 "Python/bytecodes.c" + #line 3319 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4609,12 +4613,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4613 "Python/generated_cases.c.h" + #line 4617 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3326 "Python/bytecodes.c" + #line 3330 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4623,12 +4627,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4627 "Python/generated_cases.c.h" + #line 4631 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3337 "Python/bytecodes.c" + #line 3341 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4641,12 +4645,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4645 "Python/generated_cases.c.h" + #line 4649 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3352 "Python/bytecodes.c" + #line 3356 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4659,30 +4663,30 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4663 "Python/generated_cases.c.h" + #line 4667 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3367 "Python/bytecodes.c" + #line 3371 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4674 "Python/generated_cases.c.h" + #line 4678 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3375 "Python/bytecodes.c" + #line 3379 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); - #line 4681 "Python/generated_cases.c.h" + #line 4685 "Python/generated_cases.c.h" } TARGET(RESERVED) { - #line 3380 "Python/bytecodes.c" + #line 3384 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); - #line 4688 "Python/generated_cases.c.h" + #line 4692 "Python/generated_cases.c.h" } diff --git a/Python/specialize.c b/Python/specialize.c index 5071d8ef9a495d..f1684913b1bc30 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -783,6 +783,10 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) if (version == 0) { goto fail; } + if (_PyInterpreterState_GET()->eval_frame) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); + goto fail; + } write_u32(lm_cache->keys_version, version); assert(type->tp_version_tag != 0); write_u32(lm_cache->type_version, type->tp_version_tag); @@ -845,6 +849,10 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) if (version == 0) { goto fail; } + if (_PyInterpreterState_GET()->eval_frame) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); + goto fail; + } write_u32(lm_cache->keys_version, version); /* borrowed */ write_obj(lm_cache->descr, descr); @@ -1371,6 +1379,10 @@ _Py_Specialize_BinarySubscr( SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; } + if (_PyInterpreterState_GET()->eval_frame) { + SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OTHER); + goto fail; + } PyHeapTypeObject *ht = (PyHeapTypeObject *)container_type; // This pointer is invalidated by PyType_Modified (see the comment on // struct _specialization_cache): @@ -2192,11 +2204,16 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) assert(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == END_FOR || instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == INSTRUMENTED_END_FOR ); + if (_PyInterpreterState_GET()->eval_frame) { + SPECIALIZATION_FAIL(FOR_ITER, SPEC_FAIL_OTHER); + goto failure; + } instr->op.code = FOR_ITER_GEN; goto success; } SPECIALIZATION_FAIL(FOR_ITER, _PySpecialization_ClassifyIterator(iter)); +failure: STAT_INC(FOR_ITER, failure); instr->op.code = FOR_ITER; cache->counter = adaptive_counter_backoff(cache->counter); @@ -2214,11 +2231,16 @@ _Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr) _PySendCache *cache = (_PySendCache *)(instr + 1); PyTypeObject *tp = Py_TYPE(receiver); if (tp == &PyGen_Type || tp == &PyCoro_Type) { + if (_PyInterpreterState_GET()->eval_frame) { + SPECIALIZATION_FAIL(SEND, SPEC_FAIL_OTHER); + goto failure; + } instr->op.code = SEND_GEN; goto success; } SPECIALIZATION_FAIL(SEND, _PySpecialization_ClassifyIterator(receiver)); +failure: STAT_INC(SEND, failure); instr->op.code = SEND; cache->counter = adaptive_counter_backoff(cache->counter); From 563c7dcba0ea1070698b77129628e9e1c86d34e2 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Fri, 12 May 2023 18:42:04 -0600 Subject: [PATCH 183/204] gh-104404: fix crasher with nested comprehensions plus lambdas (#104442) --- Lib/test/test_listcomps.py | 8 ++++++++ Python/symtable.c | 21 ++++++++++++++------- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index b2a3b7ea3e49b6..23e1b8c1ce3193 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -338,6 +338,14 @@ def test_nested_3(self): outputs = {"y": [1, 3, 5]} self._check_in_scopes(code, outputs) + def test_nested_4(self): + code = """ + items = [([lambda: x for x in range(2)], lambda: x) for x in range(3)] + out = [([fn() for fn in fns], fn()) for fns, fn in items] + """ + outputs = {"out": [([1, 1], 2), ([1, 1], 2), ([1, 1], 2)]} + self._check_in_scopes(code, outputs) + def test_nameerror(self): code = """ [x for x in [1]] diff --git a/Python/symtable.c b/Python/symtable.c index 9361674bf16594..2c29f608413501 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -575,7 +575,8 @@ is_free_in_any_child(PySTEntryObject *entry, PyObject *key) static int inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, - PyObject *scopes, PyObject *comp_free) + PyObject *scopes, PyObject *comp_free, + PyObject *promote_to_cell) { PyObject *k, *v; Py_ssize_t pos = 0; @@ -611,7 +612,9 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, // cell vars in comprehension that are locals in outer scope // must be promoted to cell so u_cellvars isn't wrong if (scope == CELL && ste->ste_type == FunctionBlock) { - SET_SCOPE(scopes, k, scope); + if (PySet_Add(promote_to_cell, k) < 0) { + return 0; + } } // free vars in comprehension that are locals in outer scope can @@ -638,7 +641,7 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, */ static int -analyze_cells(PyObject *scopes, PyObject *free) +analyze_cells(PyObject *scopes, PyObject *free, PyObject *promote_to_cell) { PyObject *name, *v, *v_cell; int success = 0; @@ -653,7 +656,7 @@ analyze_cells(PyObject *scopes, PyObject *free) scope = PyLong_AS_LONG(v); if (scope != LOCAL) continue; - if (!PySet_Contains(free, name)) + if (!PySet_Contains(free, name) && !PySet_Contains(promote_to_cell, name)) continue; /* Replace LOCAL with CELL for this name, and remove from free. It is safe to replace the value of name @@ -803,7 +806,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, PyObject *global) { PyObject *name, *v, *local = NULL, *scopes = NULL, *newbound = NULL; - PyObject *newglobal = NULL, *newfree = NULL; + PyObject *newglobal = NULL, *newfree = NULL, *promote_to_cell = NULL; PyObject *temp; int success = 0; Py_ssize_t i, pos = 0; @@ -835,6 +838,9 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, newbound = PySet_New(NULL); if (!newbound) goto error; + promote_to_cell = PySet_New(NULL); + if (!promote_to_cell) + goto error; /* Class namespace has no effect on names visible in nested functions, so populate the global and bound @@ -915,7 +921,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, goto error; } if (inline_comp) { - if (!inline_comprehension(ste, entry, scopes, child_free)) { + if (!inline_comprehension(ste, entry, scopes, child_free, promote_to_cell)) { Py_DECREF(child_free); goto error; } @@ -946,7 +952,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, } /* Check if any local variables must be converted to cell variables */ - if (ste->ste_type == FunctionBlock && !analyze_cells(scopes, newfree)) + if (ste->ste_type == FunctionBlock && !analyze_cells(scopes, newfree, promote_to_cell)) goto error; else if (ste->ste_type == ClassBlock && !drop_class_free(ste, newfree)) goto error; @@ -966,6 +972,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, Py_XDECREF(newbound); Py_XDECREF(newglobal); Py_XDECREF(newfree); + Py_XDECREF(promote_to_cell); if (!success) assert(PyErr_Occurred()); return success; From 57139a6b5f0cfa04156d5c650026012a7c5a7aad Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 13 May 2023 00:35:45 -0400 Subject: [PATCH 184/204] gh-99836: IDLE - update news.txt for 3.12 (#104444) * gh-99836: IDLE 3.12 news.txt * gh-99836: IDLE - update news.txt for 3.12 --- Lib/idlelib/NEWS.txt | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index e64e96f75e6cfc..e97e3ca01ef30c 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -1,9 +1,20 @@ +What's New in IDLE 3.12.0 +(since 3.11.0) +Released on 2023-10-02 +========================= + + +gh-88496 Fix IDLE test hang on macOS. + +gh-103314 Support sys.last_exc after exceptions in Shell. +Patch by Irit Katriel. + + What's New in IDLE 3.11.0 (since 3.10.0) -Released on 2022-10-03 +Released on 2022-10-24 ========================= - gh-97527: Fix a bug in the previous bugfix that caused IDLE to not start when run with 3.10.8, 3.12.0a1, and at least Microsoft Python 3.10.2288.0 installed without the Lib/test package. 3.11.0 was never From 1be80ed107deb15b926f2794b8e6a7a527b8b84c Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Sat, 13 May 2023 00:12:03 -0700 Subject: [PATCH 185/204] Add a mention of PYTHONBREAKPOINT to breakpoint() docs (#104430) --- Doc/library/functions.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 48a832db60e919..3d2bb8efc95d8e 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -168,6 +168,13 @@ are always available. They are listed here in alphabetical order. If :func:`sys.breakpointhook` is not accessible, this function will raise :exc:`RuntimeError`. + By default, the behavior of :func:`breakpoint` can be changed with + the :envvar:`PYTHONBREAKPOINT` environment variable. + See :func:`sys.breakpointhook` for usage details. + + Note that this is not guaranteed if :func:`sys.breakpointhook` + has been replaced. + .. audit-event:: builtins.breakpoint breakpointhook breakpoint .. versionadded:: 3.7 From c527eb1c2a473df01c19195b378f780e9180fd1c Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sat, 13 May 2023 09:55:35 +0100 Subject: [PATCH 186/204] gh-91896: Revert some very noisy DeprecationWarnings for `ByteString` (#104424) --- Doc/library/collections.abc.rst | 5 +---- Doc/whatsnew/3.12.rst | 4 ++-- Lib/collections/abc.py | 9 --------- Lib/test/libregrtest/refleak.py | 15 ++++++--------- Lib/test/test_collections.py | 10 +--------- Lib/test/test_typing.py | 28 ++++++---------------------- Lib/typing.py | 27 +++------------------------ 7 files changed, 19 insertions(+), 79 deletions(-) diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 158f4851634652..43a3286ba832cf 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -14,10 +14,7 @@ .. testsetup:: * - import warnings - # Ignore warning when ByteString is imported - with warnings.catch_warnings(action='ignore', category=DeprecationWarning): - from collections.abc import * + from collections.abc import * import itertools __name__ = '' diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 546c7147bb3b27..dc1178811e75cc 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -831,8 +831,8 @@ Pending Removal in Python 3.14 For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. (Contributed by Shantanu Jain in :gh:`91896`.) -* :class:`typing.ByteString`, deprecated since Python 3.9, now causes an - :exc:`DeprecationWarning` to be emitted when it is used or accessed. +* :class:`typing.ByteString`, deprecated since Python 3.9, now causes a + :exc:`DeprecationWarning` to be emitted when it is used. * Creating immutable types (:data:`Py_TPFLAGS_IMMUTABLETYPE`) with mutable bases using the C API. diff --git a/Lib/collections/abc.py b/Lib/collections/abc.py index 60b1eb60fa6ae1..86ca8b8a8414b3 100644 --- a/Lib/collections/abc.py +++ b/Lib/collections/abc.py @@ -1,12 +1,3 @@ from _collections_abc import * from _collections_abc import __all__ from _collections_abc import _CallableGenericAlias - -_deprecated_ByteString = globals().pop("ByteString") - -def __getattr__(attr): - if attr == "ByteString": - import warnings - warnings._deprecated("collections.abc.ByteString", remove=(3, 14)) - return _deprecated_ByteString - raise AttributeError(f"module 'collections.abc' has no attribute {attr!r}") diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 776a9e9b587d32..cd11d385591f80 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -48,13 +48,11 @@ def dash_R(ns, test_name, test_func): else: zdc = zipimport._zip_directory_cache.copy() abcs = {} - # catch and ignore collections.abc.ByteString deprecation - with warnings.catch_warnings(action='ignore', category=DeprecationWarning): - for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: - if not isabstract(abc): - continue - for obj in abc.__subclasses__() + [abc]: - abcs[obj] = _get_dump(obj)[0] + for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: + if not isabstract(abc): + continue + for obj in abc.__subclasses__() + [abc]: + abcs[obj] = _get_dump(obj)[0] # bpo-31217: Integer pool to get a single integer object for the same # value. The pool is used to prevent false alarm when checking for memory @@ -176,8 +174,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): # Clear ABC registries, restoring previously saved ABC registries. # ignore deprecation warning for collections.abc.ByteString - with warnings.catch_warnings(action='ignore', category=DeprecationWarning): - abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] + abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] abs_classes = filter(isabstract, abs_classes) for abc in abs_classes: for obj in abc.__subclasses__() + [abc]: diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index f0736b8081feac..bb8b352518ef3e 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -11,7 +11,6 @@ import string import sys from test import support -from test.support.import_helper import import_fresh_module import types import unittest @@ -26,7 +25,7 @@ from collections.abc import Set, MutableSet from collections.abc import Mapping, MutableMapping, KeysView, ItemsView, ValuesView from collections.abc import Sequence, MutableSequence -from collections.abc import Buffer +from collections.abc import ByteString, Buffer class TestUserObjects(unittest.TestCase): @@ -1940,8 +1939,6 @@ def assert_index_same(seq1, seq2, index_args): nativeseq, seqseq, (letter, start, stop)) def test_ByteString(self): - with self.assertWarns(DeprecationWarning): - from collections.abc import ByteString for sample in [bytes, bytearray]: with self.assertWarns(DeprecationWarning): self.assertIsInstance(sample(), ByteString) @@ -1963,11 +1960,6 @@ class X(ByteString): pass # No metaclass conflict class Z(ByteString, Awaitable): pass - def test_ByteString_attribute_access(self): - collections_abc = import_fresh_module("collections.abc") - with self.assertWarns(DeprecationWarning): - collections_abc.ByteString - def test_Buffer(self): for sample in [bytes, bytearray, memoryview]: self.assertIsInstance(sample(b"x"), Buffer) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 3422dc1ed3f5f8..e1c6a8a7f376eb 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -8,7 +8,6 @@ import re import sys import warnings -from test.support.import_helper import import_fresh_module from unittest import TestCase, main, skipUnless, skip from unittest.mock import patch from copy import copy, deepcopy @@ -3909,14 +3908,7 @@ class MyChain(typing.ChainMap[str, T]): ... self.assertEqual(MyChain[int]().__orig_class__, MyChain[int]) def test_all_repr_eq_any(self): - typing = import_fresh_module("typing") - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - objs = [getattr(typing, el) for el in typing.__all__] - self.assertEqual( - [str(w.message) for w in wlog], - ["'typing.ByteString' is deprecated and slated for removal in Python 3.14"] - ) + objs = (getattr(typing, el) for el in typing.__all__) for obj in objs: self.assertNotEqual(repr(obj), '') self.assertEqual(obj, obj) @@ -6005,15 +5997,13 @@ def test_mutablesequence(self): def test_bytestring(self): with self.assertWarns(DeprecationWarning): - from typing import ByteString + self.assertIsInstance(b'', typing.ByteString) with self.assertWarns(DeprecationWarning): - self.assertIsInstance(b'', ByteString) + self.assertIsInstance(bytearray(b''), typing.ByteString) with self.assertWarns(DeprecationWarning): - self.assertIsInstance(bytearray(b''), ByteString) + class Foo(typing.ByteString): ... with self.assertWarns(DeprecationWarning): - class Foo(ByteString): ... - with self.assertWarns(DeprecationWarning): - class Bar(ByteString, typing.Awaitable): ... + class Bar(typing.ByteString, typing.Awaitable): ... def test_list(self): self.assertIsSubclass(list, typing.List) @@ -8309,10 +8299,6 @@ def test_no_isinstance(self): class SpecialAttrsTests(BaseTestCase): def test_special_attrs(self): - with warnings.catch_warnings( - action='ignore', category=DeprecationWarning - ): - typing_ByteString = typing.ByteString cls_to_check = { # ABC classes typing.AbstractSet: 'AbstractSet', @@ -8321,7 +8307,7 @@ def test_special_attrs(self): typing.AsyncIterable: 'AsyncIterable', typing.AsyncIterator: 'AsyncIterator', typing.Awaitable: 'Awaitable', - typing_ByteString: 'ByteString', + typing.ByteString: 'ByteString', typing.Callable: 'Callable', typing.ChainMap: 'ChainMap', typing.Collection: 'Collection', @@ -8646,8 +8632,6 @@ def test_all_exported_names(self): getattr(v, '__module__', None) == typing.__name__ ) } - # Deprecated; added dynamically via module __getattr__ - computed_all.add("ByteString") self.assertSetEqual(computed_all, actual_all) diff --git a/Lib/typing.py b/Lib/typing.py index 513d4d96dd6e1d..61aed0980ac2eb 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2772,6 +2772,9 @@ class Other(Leaf): # Error reported by type checker MutableMapping = _alias(collections.abc.MutableMapping, 2) Sequence = _alias(collections.abc.Sequence, 1) MutableSequence = _alias(collections.abc.MutableSequence, 1) +ByteString = _DeprecatedGenericAlias( + collections.abc.ByteString, 0, removal_version=(3, 14) # Not generic. +) # Tuple accepts variable number of parameters. Tuple = _TupleType(tuple, -1, inst=False, name='Tuple') Tuple.__doc__ = \ @@ -3571,27 +3574,3 @@ def method(self) -> None: # read-only property, TypeError if it's a builtin class. pass return method - - -def __getattr__(attr): - if attr == "ByteString": - import warnings - warnings._deprecated("typing.ByteString", remove=(3, 14)) - with warnings.catch_warnings( - action="ignore", category=DeprecationWarning - ): - # Not generic - ByteString = globals()["ByteString"] = _DeprecatedGenericAlias( - collections.abc.ByteString, 0, removal_version=(3, 14) - ) - return ByteString - raise AttributeError(f"module 'typing' has no attribute {attr!r}") - - -def _remove_cached_ByteString_from_globals(): - try: - del globals()["ByteString"] - except KeyError: - pass - -_cleanups.append(_remove_cached_ByteString_from_globals) From 46f1c78eebe08e96ed29d364b1804dd37364831d Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 13 May 2023 11:41:39 -0400 Subject: [PATCH 187/204] gh-75710: IDLE - add docstrings and comments to editor module (#104446) Commit extracted from PR #3669. Will edit more later. Co-authored-by: Cheryl Sabella --- Lib/idlelib/editor.py | 127 +++++++++++++++++++++++++++++++++--------- 1 file changed, 102 insertions(+), 25 deletions(-) diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 505815502600b1..21402ad7139173 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -446,6 +446,26 @@ def set_line_and_column(self, event=None): self.status_bar.set_label('column', 'Col: %s' % column) self.status_bar.set_label('line', 'Ln: %s' % line) + + """ Menu definitions and functions. + * self.menubar - the always visible horizontal menu bar. + * mainmenu.menudefs - a list of tuples, one for each menubar item. + Each tuple pairs a lower-case name and list of dropdown items. + Each item is a name, virtual event pair or None for separator. + * mainmenu.default_keydefs - maps events to keys. + * text.keydefs - same. + * cls.menu_specs - menubar name, titlecase display form pairs + with Alt-hotkey indicator. A subset of menudefs items. + * self.menudict - map menu name to dropdown menu. + * self.recent_files_menu - 2nd level cascade in the file cascade. + * self.wmenu_end - set in __init__ (purpose unclear). + + createmenubar, postwindowsmenu, update_menu_label, update_menu_state, + ApplyKeybings (2nd part), reset_help_menu_entries, + _extra_help_callback, update_recent_files_list, + apply_bindings, fill_menus, (other functions?) + """ + menu_specs = [ ("file", "_File"), ("edit", "_Edit"), @@ -456,8 +476,22 @@ def set_line_and_column(self, event=None): ("help", "_Help"), ] - def createmenubar(self): + """Populate the menu bar widget for the editor window. + + Each option on the menubar is itself a cascade-type Menu widget + with the menubar as the parent. The names, labels, and menu + shortcuts for the menubar items are stored in menu_specs. Each + submenu is subsequently populated in fill_menus(), except for + 'Recent Files' which is added to the File menu here. + + Instance variables: + menubar: Menu widget containing first level menu items. + menudict: Dictionary of {menuname: Menu instance} items. The keys + represent the valid menu items for this window and may be a + subset of all the menudefs available. + recent_files_menu: Menu widget contained within the 'file' menudict. + """ mbar = self.menubar self.menudict = menudict = {} for name, label in self.menu_specs: @@ -480,7 +514,10 @@ def createmenubar(self): self.reset_help_menu_entries() def postwindowsmenu(self): - # Only called when Window menu exists + """Callback to register window. + + Only called when Window menu exists. + """ menu = self.menudict['window'] end = menu.index("end") if end is None: @@ -859,8 +896,11 @@ def ResetFont(self): self.set_width() def RemoveKeybindings(self): - "Remove the keybindings before they are changed." - # Called from configdialog.py + """Remove the virtual, configurable keybindings. + + Leaves the default Tk Text keybindings. + """ + # Called from configdialog.deactivate_current_config. self.mainmenu.default_keydefs = keydefs = idleConf.GetCurrentKeySet() for event, keylist in keydefs.items(): self.text.event_delete(event, *keylist) @@ -871,15 +911,19 @@ def RemoveKeybindings(self): self.text.event_delete(event, *keylist) def ApplyKeybindings(self): - "Update the keybindings after they are changed" - # Called from configdialog.py + """Apply the virtual, configurable keybindings. + + Alse update hotkeys to current keyset. + """ + # Called from configdialog.activate_config_changes. self.mainmenu.default_keydefs = keydefs = idleConf.GetCurrentKeySet() self.apply_bindings() for extensionName in self.get_standard_extension_names(): xkeydefs = idleConf.GetExtensionBindings(extensionName) if xkeydefs: self.apply_bindings(xkeydefs) - #update menu accelerators + + # Update menu accelerators. menuEventDict = {} for menu in self.mainmenu.menudefs: menuEventDict[menu[0]] = {} @@ -914,25 +958,25 @@ def set_notabs_indentwidth(self): type='int') def reset_help_menu_entries(self): - "Update the additional help entries on the Help menu" + """Update the additional help entries on the Help menu.""" help_list = idleConf.GetAllExtraHelpSourcesList() helpmenu = self.menudict['help'] - # first delete the extra help entries, if any + # First delete the extra help entries, if any. helpmenu_length = helpmenu.index(END) if helpmenu_length > self.base_helpmenu_length: helpmenu.delete((self.base_helpmenu_length + 1), helpmenu_length) - # then rebuild them + # Then rebuild them. if help_list: helpmenu.add_separator() for entry in help_list: - cmd = self.__extra_help_callback(entry[1]) + cmd = self._extra_help_callback(entry[1]) helpmenu.add_command(label=entry[0], command=cmd) - # and update the menu dictionary + # And update the menu dictionary. self.menudict['help'] = helpmenu - def __extra_help_callback(self, helpfile): - "Create a callback with the helpfile value frozen at definition time" - def display_extra_help(helpfile=helpfile): + def _extra_help_callback(self, resource): + """Return a callback that loads resource (file or web page).""" + def display_extra_help(helpfile=resource): if not helpfile.startswith(('www', 'http')): helpfile = os.path.normpath(helpfile) if sys.platform[:3] == 'win': @@ -1158,6 +1202,7 @@ def load_extension(self, name): self.text.bind(vevent, getattr(ins, methodname)) def apply_bindings(self, keydefs=None): + """Add events with keys to self.text.""" if keydefs is None: keydefs = self.mainmenu.default_keydefs text = self.text @@ -1167,9 +1212,10 @@ def apply_bindings(self, keydefs=None): text.event_add(event, *keylist) def fill_menus(self, menudefs=None, keydefs=None): - """Add appropriate entries to the menus and submenus + """Fill in dropdown menus used by this window. - Menus that are absent or None in self.menudict are ignored. + Items whose name begins with '!' become checkbuttons. + Other names indicate commands. None becomes a separator. """ if menudefs is None: menudefs = self.mainmenu.menudefs @@ -1182,7 +1228,7 @@ def fill_menus(self, menudefs=None, keydefs=None): if not menu: continue for entry in entrylist: - if not entry: + if entry is None: menu.add_separator() else: label, eventname = entry @@ -1218,11 +1264,13 @@ def setvar(self, name, value, vartype=None): else: raise NameError(name) - def get_var_obj(self, name, vartype=None): - var = self.tkinter_vars.get(name) + def get_var_obj(self, eventname, vartype=None): + """Return a tkinter variable instance for the event. + """ + var = self.tkinter_vars.get(eventname) if not var and vartype: - # create a Tkinter variable object with self.text as master: - self.tkinter_vars[name] = var = vartype(self.text) + # Create a Tkinter variable object. + self.tkinter_vars[eventname] = var = vartype(self.text) return var # Tk implementations of "virtual text methods" -- each platform @@ -1613,8 +1661,16 @@ def run(self): ### end autoindent code ### def prepstr(s): - # Helper to extract the underscore from a string, e.g. - # prepstr("Co_py") returns (2, "Copy"). + """Extract the underscore from a string. + + For example, prepstr("Co_py") returns (2, "Copy"). + + Args: + s: String with underscore. + + Returns: + Tuple of (position of underscore, string without underscore). + """ i = s.find('_') if i >= 0: s = s[:i] + s[i+1:] @@ -1628,6 +1684,18 @@ def prepstr(s): } def get_accelerator(keydefs, eventname): + """Return a formatted string for the keybinding of an event. + + Convert the first keybinding for a given event to a form that + can be displayed as an accelerator on the menu. + + Args: + keydefs: Dictionary of valid events to keybindings. + eventname: Event to retrieve keybinding for. + + Returns: + Formatted string of the keybinding. + """ keylist = keydefs.get(eventname) # issue10940: temporary workaround to prevent hang with OS X Cocoa Tk 8.5 # if not keylist: @@ -1637,14 +1705,23 @@ def get_accelerator(keydefs, eventname): "<>"}): return "" s = keylist[0] + # Convert strings of the form -singlelowercase to -singleuppercase. s = re.sub(r"-[a-z]\b", lambda m: m.group().upper(), s) + # Convert certain keynames to their symbol. s = re.sub(r"\b\w+\b", lambda m: keynames.get(m.group(), m.group()), s) + # Remove Key- from string. s = re.sub("Key-", "", s) - s = re.sub("Cancel","Ctrl-Break",s) # dscherer@cmu.edu + # Convert Cancel to Ctrl-Break. + s = re.sub("Cancel", "Ctrl-Break", s) # dscherer@cmu.edu + # Convert Control to Ctrl-. s = re.sub("Control-", "Ctrl-", s) + # Change - to +. s = re.sub("-", "+", s) + # Change >< to space. s = re.sub("><", " ", s) + # Remove <. s = re.sub("<", "", s) + # Remove >. s = re.sub(">", "", s) return s From 7d2deafb73237a2175971a26cfb544974661de4b Mon Sep 17 00:00:00 2001 From: Charles Machalow Date: Sat, 13 May 2023 13:45:36 -0700 Subject: [PATCH 188/204] gh-104454: Fix refleak in AttributeError_reduce (#104455) * Fix the reference leak introduced by https://github.com/python/cpython/issues/103333 Co-authored-by: Kirill Podoprigora --- Objects/exceptions.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 59c63f4aa44958..a8d4e3a696ce8e 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -2324,7 +2324,9 @@ AttributeError_reduce(PyAttributeErrorObject *self, PyObject *Py_UNUSED(ignored) return NULL; } - return PyTuple_Pack(3, Py_TYPE(self), self->args, state); + PyObject *return_value = PyTuple_Pack(3, Py_TYPE(self), self->args, state); + Py_DECREF(state); + return return_value; } static PyMemberDef AttributeError_members[] = { From 080a5961527473af182b025bb29e0c52d43fd49e Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 13 May 2023 23:55:20 -0400 Subject: [PATCH 189/204] GH-71383: IDLE - Document testing subsets of modules (#104463) --- Lib/idlelib/idle_test/README.txt | 19 +++++++++++-------- Lib/idlelib/idle_test/__init__.py | 24 +++++++++++++++++------- Lib/test/test_idle.py | 6 +----- 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/Lib/idlelib/idle_test/README.txt b/Lib/idlelib/idle_test/README.txt index 566bfd179fdf1b..cacd06db873d03 100644 --- a/Lib/idlelib/idle_test/README.txt +++ b/Lib/idlelib/idle_test/README.txt @@ -146,14 +146,17 @@ python -m unittest -v idlelib.idle_test python -m test -v -ugui test_idle python -m test.test_idle -The idle tests are 'discovered' by -idlelib.idle_test.__init__.load_tests, which is also imported into -test.test_idle. Normally, neither file should be changed when working on -individual test modules. The third command runs unittest indirectly -through regrtest. The same happens when the entire test suite is run -with 'python -m test'. So that command must work for buildbots to stay -green. Idle tests must not disturb the environment in a way that makes -other tests fail (issue 18081). +IDLE tests are 'discovered' by idlelib.idle_test.__init__.load_tests +when this is imported into test.test_idle. Normally, neither file +should be changed when working on individual test modules. The third +command runs unittest indirectly through regrtest. The same happens when +the entire test suite is run with 'python -m test'. So that command must +work for buildbots to stay green. IDLE tests must not disturb the +environment in a way that makes other tests fail (GH-62281). + +To test subsets of modules, see idlelib.idle_test.__init__. This +can be used to find refleaks or possible sources of "Theme changed" +tcl messages (GH-71383). To run an individual Testcase or test method, extend the dotted name given to unittest on the command line or use the test -m option. The diff --git a/Lib/idlelib/idle_test/__init__.py b/Lib/idlelib/idle_test/__init__.py index ad067b405cab67..79b5d102dd7da5 100644 --- a/Lib/idlelib/idle_test/__init__.py +++ b/Lib/idlelib/idle_test/__init__.py @@ -1,17 +1,27 @@ -'''idlelib.idle_test is a private implementation of test.test_idle, -which tests the IDLE application as part of the stdlib test suite. -Run IDLE tests alone with "python -m test.test_idle". -Starting with Python 3.6, IDLE requires tcl/tk 8.5 or later. +"""idlelib.idle_test implements test.test_idle, which tests the IDLE +application as part of the stdlib test suite. +Run IDLE tests alone with "python -m test.test_idle (-v)". This package and its contained modules are subject to change and any direct use is at your own risk. -''' +""" from os.path import dirname +# test_idle imports load_tests for test discovery (default all). +# To run subsets of idlelib module tests, insert '[]' after '_'. +# Example: insert '[ac]' for modules beginning with 'a' or 'c'. +# Additional .discover/.addTest pairs with separate inserts work. +# Example: pairs with 'c' and 'g' test c* files and grep. + def load_tests(loader, standard_tests, pattern): this_dir = dirname(__file__) top_dir = dirname(dirname(this_dir)) - package_tests = loader.discover(start_dir=this_dir, pattern='test*.py', + module_tests = loader.discover(start_dir=this_dir, + pattern='test_*.py', # Insert here. top_level_dir=top_dir) - standard_tests.addTests(package_tests) + standard_tests.addTests(module_tests) +## module_tests = loader.discover(start_dir=this_dir, +## pattern='test_*.py', # Insert here. +## top_level_dir=top_dir) +## standard_tests.addTests(module_tests) return standard_tests diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py index 90cff9002b75b2..ebb1e8eb823b51 100644 --- a/Lib/test/test_idle.py +++ b/Lib/test/test_idle.py @@ -5,12 +5,8 @@ if check_sanitizer(address=True, memory=True): raise unittest.SkipTest("Tests involving libX11 can SEGFAULT on ASAN/MSAN builds") -# Skip test_idle if _tkinter wasn't built, if tkinter is missing, -# if tcl/tk is not the 8.5+ needed for ttk widgets, -# or if idlelib is missing (not installed). +# Skip test_idle if _tkinter, tkinter, or idlelib are missing. tk = import_module('tkinter') # Also imports _tkinter. -if tk.TkVersion < 8.5: - raise unittest.SkipTest("IDLE requires tk 8.5 or later.") idlelib = import_module('idlelib') # Before importing and executing more of idlelib, From 178153c9a6f32da8d132aae591c0cfebea7c6366 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Sun, 14 May 2023 20:03:30 +0900 Subject: [PATCH 190/204] gh-87092: avoid gcc warning on uninitialized struct field in assemble.c (#104460) --- Python/assemble.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/assemble.c b/Python/assemble.c index 6889831ae3fe0c..8789d8ef978c22 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -128,7 +128,7 @@ assemble_emit_exception_table_entry(struct assembler *a, int start, int end, assert(end > start); int target = handler->h_offset; int depth = handler->h_startdepth - 1; - if (handler->h_preserve_lasti) { + if (handler->h_preserve_lasti > 0) { depth -= 1; } assert(depth >= 0); @@ -146,6 +146,7 @@ assemble_exception_table(struct assembler *a, instr_sequence *instrs) int ioffset = 0; _PyCompile_ExceptHandlerInfo handler; handler.h_offset = -1; + handler.h_preserve_lasti = -1; int start = -1; for (int i = 0; i < instrs->s_used; i++) { instruction *instr = &instrs->s_instrs[i]; From 2f7b5e458e9189fa1ffd44339848aa1e52add3fa Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 14 May 2023 04:53:15 -0700 Subject: [PATCH 191/204] Minor improvements to typing docs (#104465) --- Doc/library/typing.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index fb1c916b141a25..c300c4257f0e81 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -19,7 +19,7 @@ This module provides runtime support for type hints. The most fundamental support consists of the types :data:`Any`, :data:`Union`, :data:`Callable`, -:class:`TypeVar`, and :class:`Generic`. For a full specification, please see +:class:`TypeVar`, and :class:`Generic`. For a specification, please see :pep:`484`. For a simplified introduction to type hints, see :pep:`483`. @@ -592,7 +592,7 @@ The module defines the following classes, functions and decorators. when the checked program targets Python 3.9 or newer. The deprecated types will be removed from the :mod:`typing` module - in the first Python version released 5 years after the release of Python 3.9.0. + no sooner than the first Python version released 5 years after the release of Python 3.9.0. See details in :pep:`585`—*Type Hinting Generics In Standard Collections*. @@ -1291,6 +1291,8 @@ These are not used in annotations. They are building blocks for creating generic U = TypeVar('U', bound=str|bytes) # Can be any subtype of the union str|bytes V = TypeVar('V', bound=SupportsAbs) # Can be anything with an __abs__ method +.. _typing-constrained-typevar: + Using a *constrained* type variable, however, means that the ``TypeVar`` can only ever be solved as being exactly one of the constraints given:: @@ -1550,7 +1552,7 @@ These are not used in annotations. They are building blocks for creating generic .. data:: AnyStr - ``AnyStr`` is a :class:`constrained type variable ` defined as + ``AnyStr`` is a :ref:`constrained type variable ` defined as ``AnyStr = TypeVar('AnyStr', str, bytes)``. It is meant to be used for functions that may accept any kind of string @@ -2112,7 +2114,7 @@ Other concrete types Python 2 is no longer supported, and most type checkers also no longer support type checking Python 2 code. Removal of the alias is not currently planned, but users are encouraged to use - :class:`str` instead of ``Text`` wherever possible. + :class:`str` instead of ``Text``. Abstract Base Classes --------------------- From 88c5c586708dcff369c49edae947d487a80f0346 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 14 May 2023 14:45:54 -0400 Subject: [PATCH 192/204] gh-104337: Clarify random.gammavariate doc entry (#104410) * gh-104337: Clarify random.gammavariate doc entry * Fix parameter markup. --- Doc/library/random.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Doc/library/random.rst b/Doc/library/random.rst index c192919ac62e54..76ae97a8be7e63 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -334,8 +334,10 @@ be found in any statistics text. .. function:: gammavariate(alpha, beta) - Gamma distribution. (*Not* the gamma function!) Conditions on the - parameters are ``alpha > 0`` and ``beta > 0``. + Gamma distribution. (*Not* the gamma function!) The shape and + scale parameters, *alpha* and *beta*, must have positive values. + (Calling conventions vary and some sources define 'beta' + as the inverse of the scale). The probability distribution function is:: @@ -346,7 +348,8 @@ be found in any statistics text. .. function:: gauss(mu=0.0, sigma=1.0) - Normal distribution, also called the Gaussian distribution. *mu* is the mean, + Normal distribution, also called the Gaussian distribution. + *mu* is the mean, and *sigma* is the standard deviation. This is slightly faster than the :func:`normalvariate` function defined below. From fb8739f0b6291fb048a94d6312f59ba4d10a20ca Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sun, 14 May 2023 19:58:13 +0100 Subject: [PATCH 193/204] gh-98539: Make _SSLTransportProtocol.abort() safe to call when closed (#104474) --- Lib/asyncio/sslproto.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index bbf9cad6bc7f86..488e17d8bccd5b 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -244,7 +244,8 @@ def abort(self): called with None as its argument. """ self._closed = True - self._ssl_protocol._abort() + if self._ssl_protocol is not None: + self._ssl_protocol._abort() def _force_close(self, exc): self._closed = True From 2cd1c87d2a23ffd00730b5d1648304593530326c Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Mon, 15 May 2023 00:05:35 +0300 Subject: [PATCH 194/204] gh-104456: Fix ref leak in _ctypes.COMError (#104457) --- Modules/_ctypes/_ctypes.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index f6cda45eaeac27..534ef8c1d6cf8f 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -5476,11 +5476,17 @@ comerror_init(PyObject *self, PyObject *args, PyObject *kwds) return 0; } +static int +comerror_clear(PyObject *self) +{ + return ((PyTypeObject *)PyExc_BaseException)->tp_clear(self); +} + static int comerror_traverse(PyObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); - return 0; + return ((PyTypeObject *)PyExc_BaseException)->tp_traverse(self, visit, arg); } static void @@ -5488,6 +5494,7 @@ comerror_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); + (void)comerror_clear(self); tp->tp_free(self); Py_DECREF(tp); } @@ -5497,6 +5504,7 @@ static PyType_Slot comerror_slots[] = { {Py_tp_init, comerror_init}, {Py_tp_traverse, comerror_traverse}, {Py_tp_dealloc, comerror_dealloc}, + {Py_tp_clear, comerror_clear}, {0, NULL}, }; From 48b3617de491f00a3bf978b355074cc8e228d61b Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Mon, 15 May 2023 12:44:00 +0900 Subject: [PATCH 195/204] gh-104469 Convert _testcapi/float.c to use AC (gh-104470) --- Modules/_testcapi/clinic/float.c.h | 88 ++++++++++++++++++++++++++++++ Modules/_testcapi/float.c | 54 +++++++++++------- 2 files changed, 123 insertions(+), 19 deletions(-) create mode 100644 Modules/_testcapi/clinic/float.c.h diff --git a/Modules/_testcapi/clinic/float.c.h b/Modules/_testcapi/clinic/float.c.h new file mode 100644 index 00000000000000..c1dff2a9ac5789 --- /dev/null +++ b/Modules/_testcapi/clinic/float.c.h @@ -0,0 +1,88 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif + + +PyDoc_STRVAR(_testcapi_float_pack__doc__, +"float_pack($module, size, d, le, /)\n" +"--\n" +"\n" +"Test PyFloat_Pack2(), PyFloat_Pack4() and PyFloat_Pack8()"); + +#define _TESTCAPI_FLOAT_PACK_METHODDEF \ + {"float_pack", _PyCFunction_CAST(_testcapi_float_pack), METH_FASTCALL, _testcapi_float_pack__doc__}, + +static PyObject * +_testcapi_float_pack_impl(PyObject *module, int size, double d, int le); + +static PyObject * +_testcapi_float_pack(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int size; + double d; + int le; + + if (!_PyArg_CheckPositional("float_pack", nargs, 3, 3)) { + goto exit; + } + size = _PyLong_AsInt(args[0]); + if (size == -1 && PyErr_Occurred()) { + goto exit; + } + if (PyFloat_CheckExact(args[1])) { + d = PyFloat_AS_DOUBLE(args[1]); + } + else + { + d = PyFloat_AsDouble(args[1]); + if (d == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + le = _PyLong_AsInt(args[2]); + if (le == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = _testcapi_float_pack_impl(module, size, d, le); + +exit: + return return_value; +} + +PyDoc_STRVAR(_testcapi_float_unpack__doc__, +"float_unpack($module, data, le, /)\n" +"--\n" +"\n" +"Test PyFloat_Unpack2(), PyFloat_Unpack4() and PyFloat_Unpack8()"); + +#define _TESTCAPI_FLOAT_UNPACK_METHODDEF \ + {"float_unpack", _PyCFunction_CAST(_testcapi_float_unpack), METH_FASTCALL, _testcapi_float_unpack__doc__}, + +static PyObject * +_testcapi_float_unpack_impl(PyObject *module, const char *data, + Py_ssize_t data_length, int le); + +static PyObject * +_testcapi_float_unpack(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + const char *data; + Py_ssize_t data_length; + int le; + + if (!_PyArg_ParseStack(args, nargs, "y#i:float_unpack", + &data, &data_length, &le)) { + goto exit; + } + return_value = _testcapi_float_unpack_impl(module, data, data_length, le); + +exit: + return return_value; +} +/*[clinic end generated code: output=083e5df26cd5fbeb input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/float.c b/Modules/_testcapi/float.c index 26d99d990e0042..33cbda83a81af7 100644 --- a/Modules/_testcapi/float.c +++ b/Modules/_testcapi/float.c @@ -1,18 +1,29 @@ #define PY_SSIZE_T_CLEAN #include "parts.h" +#include "clinic/float.c.h" -// Test PyFloat_Pack2(), PyFloat_Pack4() and PyFloat_Pack8() +/*[clinic input] +module _testcapi +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/ + +/*[clinic input] +_testcapi.float_pack + + size: int + d: double + le: int + / + +Test PyFloat_Pack2(), PyFloat_Pack4() and PyFloat_Pack8() +[clinic start generated code]*/ + static PyObject * -test_float_pack(PyObject *self, PyObject *args) +_testcapi_float_pack_impl(PyObject *module, int size, double d, int le) +/*[clinic end generated code: output=7899bd98f8b6cb04 input=52c9115121999c98]*/ { - int size; - double d; - int le; - if (!PyArg_ParseTuple(args, "idi", &size, &d, &le)) { - return NULL; - } switch (size) { case 2: @@ -47,19 +58,24 @@ test_float_pack(PyObject *self, PyObject *args) } -// Test PyFloat_Unpack2(), PyFloat_Unpack4() and PyFloat_Unpack8() +/*[clinic input] +_testcapi.float_unpack + + data: str(accept={robuffer}, zeroes=True) + le: int + / + +Test PyFloat_Unpack2(), PyFloat_Unpack4() and PyFloat_Unpack8() +[clinic start generated code]*/ + static PyObject * -test_float_unpack(PyObject *self, PyObject *args) +_testcapi_float_unpack_impl(PyObject *module, const char *data, + Py_ssize_t data_length, int le) +/*[clinic end generated code: output=617059f889ddbfe4 input=c095e4bb75a696cd]*/ { assert(!PyErr_Occurred()); - const char *data; - Py_ssize_t size; - int le; - if (!PyArg_ParseTuple(args, "y#i", &data, &size, &le)) { - return NULL; - } double d; - switch (size) + switch (data_length) { case 2: d = PyFloat_Unpack2(data, le); @@ -82,8 +98,8 @@ test_float_unpack(PyObject *self, PyObject *args) } static PyMethodDef test_methods[] = { - {"float_pack", test_float_pack, METH_VARARGS, NULL}, - {"float_unpack", test_float_unpack, METH_VARARGS, NULL}, + _TESTCAPI_FLOAT_PACK_METHODDEF + _TESTCAPI_FLOAT_UNPACK_METHODDEF {NULL}, }; From 27d8ecd7f3325a40a967d2d6b6b36b21d5328753 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Sun, 14 May 2023 21:29:44 -0700 Subject: [PATCH 196/204] gh-101282: move BOLT config after PGO (gh-104493) --- configure | 360 +++++++++++++++++++++++++-------------------------- configure.ac | 124 +++++++++--------- 2 files changed, 242 insertions(+), 242 deletions(-) diff --git a/configure b/configure index c9ea72cf6efacf..4d3520aee43def 100755 --- a/configure +++ b/configure @@ -883,6 +883,11 @@ CFLAGS_NODIST BASECFLAGS CFLAGS_ALIASING OPT +MERGE_FDATA +LLVM_BOLT +ac_ct_READELF +READELF +PREBOLT_RULE LLVM_PROF_FOUND LLVM_PROFDATA LLVM_PROF_ERR @@ -890,11 +895,6 @@ LLVM_PROF_FILE LLVM_PROF_MERGER PGO_PROF_USE_FLAG PGO_PROF_GEN_FLAG -MERGE_FDATA -LLVM_BOLT -ac_ct_READELF -READELF -PREBOLT_RULE LLVM_AR_FOUND LLVM_AR PROFILE_TASK @@ -7903,7 +7903,181 @@ fi LDFLAGS_NODIST="$LDFLAGS_NODIST $LTOFLAGS" fi -# Enable bolt flags +# Enable PGO flags. + + + + + + +if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}llvm-profdata", so it can be a program name with args. +set dummy ${ac_tool_prefix}llvm-profdata; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_LLVM_PROFDATA+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $LLVM_PROFDATA in + [\\/]* | ?:[\\/]*) + ac_cv_path_LLVM_PROFDATA="$LLVM_PROFDATA" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in ${llvm_path} +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_LLVM_PROFDATA="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +LLVM_PROFDATA=$ac_cv_path_LLVM_PROFDATA +if test -n "$LLVM_PROFDATA"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LLVM_PROFDATA" >&5 +$as_echo "$LLVM_PROFDATA" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_path_LLVM_PROFDATA"; then + ac_pt_LLVM_PROFDATA=$LLVM_PROFDATA + # Extract the first word of "llvm-profdata", so it can be a program name with args. +set dummy llvm-profdata; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_ac_pt_LLVM_PROFDATA+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $ac_pt_LLVM_PROFDATA in + [\\/]* | ?:[\\/]*) + ac_cv_path_ac_pt_LLVM_PROFDATA="$ac_pt_LLVM_PROFDATA" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in ${llvm_path} +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_ac_pt_LLVM_PROFDATA="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +ac_pt_LLVM_PROFDATA=$ac_cv_path_ac_pt_LLVM_PROFDATA +if test -n "$ac_pt_LLVM_PROFDATA"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_pt_LLVM_PROFDATA" >&5 +$as_echo "$ac_pt_LLVM_PROFDATA" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_pt_LLVM_PROFDATA" = x; then + LLVM_PROFDATA="''" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + LLVM_PROFDATA=$ac_pt_LLVM_PROFDATA + fi +else + LLVM_PROFDATA="$ac_cv_path_LLVM_PROFDATA" +fi + + +if test -n "${LLVM_PROFDATA}" -a -x "${LLVM_PROFDATA}" +then + LLVM_PROF_FOUND="found" +else + LLVM_PROF_FOUND="not-found" +fi +if test "$ac_sys_system" = "Darwin" -a "${LLVM_PROF_FOUND}" = "not-found" +then + found_llvm_profdata=`/usr/bin/xcrun -find llvm-profdata 2>/dev/null` + if test -n "${found_llvm_profdata}" + then + # llvm-profdata isn't directly in $PATH in some cases. + # https://apple.stackexchange.com/questions/197053/ + LLVM_PROFDATA='/usr/bin/xcrun llvm-profdata' + LLVM_PROF_FOUND=found + { $as_echo "$as_me:${as_lineno-$LINENO}: llvm-profdata found via xcrun: ${LLVM_PROFDATA}" >&5 +$as_echo "$as_me: llvm-profdata found via xcrun: ${LLVM_PROFDATA}" >&6;} + fi +fi +LLVM_PROF_ERR=no +case $CC in + *clang*) + # Any changes made here should be reflected in the GCC+Darwin case below + PGO_PROF_GEN_FLAG="-fprofile-instr-generate" + PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" + LLVM_PROF_MERGER="${LLVM_PROFDATA} merge -output=code.profclangd *.profclangr" + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + if test $LLVM_PROF_FOUND = not-found + then + LLVM_PROF_ERR=yes + if test "${REQUIRE_PGO}" = "yes" + then + as_fn_error $? "llvm-profdata is required for a --enable-optimizations build but could not be found." "$LINENO" 5 + fi + fi + ;; + *gcc*) + case $ac_sys_system in + Darwin*) + PGO_PROF_GEN_FLAG="-fprofile-instr-generate" + PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" + LLVM_PROF_MERGER="${LLVM_PROFDATA} merge -output=code.profclangd *.profclangr" + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + if test "${LLVM_PROF_FOUND}" = "not-found" + then + LLVM_PROF_ERR=yes + if test "${REQUIRE_PGO}" = "yes" + then + as_fn_error $? "llvm-profdata is required for a --enable-optimizations build but could not be found." "$LINENO" 5 + fi + fi + ;; + *) + PGO_PROF_GEN_FLAG="-fprofile-generate" + PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction" + LLVM_PROF_MERGER="true" + LLVM_PROF_FILE="" + ;; + esac + ;; + *icc*) + PGO_PROF_GEN_FLAG="-prof-gen" + PGO_PROF_USE_FLAG="-prof-use" + LLVM_PROF_MERGER="true" + LLVM_PROF_FILE="" + ;; +esac + +# BOLT optimization. Always configured after PGO since it always runs after PGO. Py_BOLT='false' { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --enable-bolt" >&5 $as_echo_n "checking for --enable-bolt... " >&6; } @@ -8300,180 +8474,6 @@ $as_echo "\"Found merge-fdata\"" >&6; } fi fi -# Enable PGO flags. - - - - - - -if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}llvm-profdata", so it can be a program name with args. -set dummy ${ac_tool_prefix}llvm-profdata; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_path_LLVM_PROFDATA+:} false; then : - $as_echo_n "(cached) " >&6 -else - case $LLVM_PROFDATA in - [\\/]* | ?:[\\/]*) - ac_cv_path_LLVM_PROFDATA="$LLVM_PROFDATA" # Let the user override the test with a path. - ;; - *) - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in ${llvm_path} -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_path_LLVM_PROFDATA="$as_dir/$ac_word$ac_exec_ext" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - - ;; -esac -fi -LLVM_PROFDATA=$ac_cv_path_LLVM_PROFDATA -if test -n "$LLVM_PROFDATA"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LLVM_PROFDATA" >&5 -$as_echo "$LLVM_PROFDATA" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - -fi -if test -z "$ac_cv_path_LLVM_PROFDATA"; then - ac_pt_LLVM_PROFDATA=$LLVM_PROFDATA - # Extract the first word of "llvm-profdata", so it can be a program name with args. -set dummy llvm-profdata; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_path_ac_pt_LLVM_PROFDATA+:} false; then : - $as_echo_n "(cached) " >&6 -else - case $ac_pt_LLVM_PROFDATA in - [\\/]* | ?:[\\/]*) - ac_cv_path_ac_pt_LLVM_PROFDATA="$ac_pt_LLVM_PROFDATA" # Let the user override the test with a path. - ;; - *) - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in ${llvm_path} -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_path_ac_pt_LLVM_PROFDATA="$as_dir/$ac_word$ac_exec_ext" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - - ;; -esac -fi -ac_pt_LLVM_PROFDATA=$ac_cv_path_ac_pt_LLVM_PROFDATA -if test -n "$ac_pt_LLVM_PROFDATA"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_pt_LLVM_PROFDATA" >&5 -$as_echo "$ac_pt_LLVM_PROFDATA" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - if test "x$ac_pt_LLVM_PROFDATA" = x; then - LLVM_PROFDATA="''" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; -esac - LLVM_PROFDATA=$ac_pt_LLVM_PROFDATA - fi -else - LLVM_PROFDATA="$ac_cv_path_LLVM_PROFDATA" -fi - - -if test -n "${LLVM_PROFDATA}" -a -x "${LLVM_PROFDATA}" -then - LLVM_PROF_FOUND="found" -else - LLVM_PROF_FOUND="not-found" -fi -if test "$ac_sys_system" = "Darwin" -a "${LLVM_PROF_FOUND}" = "not-found" -then - found_llvm_profdata=`/usr/bin/xcrun -find llvm-profdata 2>/dev/null` - if test -n "${found_llvm_profdata}" - then - # llvm-profdata isn't directly in $PATH in some cases. - # https://apple.stackexchange.com/questions/197053/ - LLVM_PROFDATA='/usr/bin/xcrun llvm-profdata' - LLVM_PROF_FOUND=found - { $as_echo "$as_me:${as_lineno-$LINENO}: llvm-profdata found via xcrun: ${LLVM_PROFDATA}" >&5 -$as_echo "$as_me: llvm-profdata found via xcrun: ${LLVM_PROFDATA}" >&6;} - fi -fi -LLVM_PROF_ERR=no -case $CC in - *clang*) - # Any changes made here should be reflected in the GCC+Darwin case below - PGO_PROF_GEN_FLAG="-fprofile-instr-generate" - PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" - LLVM_PROF_MERGER="${LLVM_PROFDATA} merge -output=code.profclangd *.profclangr" - LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" - if test $LLVM_PROF_FOUND = not-found - then - LLVM_PROF_ERR=yes - if test "${REQUIRE_PGO}" = "yes" - then - as_fn_error $? "llvm-profdata is required for a --enable-optimizations build but could not be found." "$LINENO" 5 - fi - fi - ;; - *gcc*) - case $ac_sys_system in - Darwin*) - PGO_PROF_GEN_FLAG="-fprofile-instr-generate" - PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" - LLVM_PROF_MERGER="${LLVM_PROFDATA} merge -output=code.profclangd *.profclangr" - LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" - if test "${LLVM_PROF_FOUND}" = "not-found" - then - LLVM_PROF_ERR=yes - if test "${REQUIRE_PGO}" = "yes" - then - as_fn_error $? "llvm-profdata is required for a --enable-optimizations build but could not be found." "$LINENO" 5 - fi - fi - ;; - *) - PGO_PROF_GEN_FLAG="-fprofile-generate" - PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction" - LLVM_PROF_MERGER="true" - LLVM_PROF_FILE="" - ;; - esac - ;; - *icc*) - PGO_PROF_GEN_FLAG="-prof-gen" - PGO_PROF_USE_FLAG="-prof-use" - LLVM_PROF_MERGER="true" - LLVM_PROF_FILE="" - ;; -esac - # XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be # merged with this chunk of code? diff --git a/configure.ac b/configure.ac index 10672bd3761da8..0c63cde0363e2e 100644 --- a/configure.ac +++ b/configure.ac @@ -1929,68 +1929,6 @@ if test "$Py_LTO" = 'true' ; then LDFLAGS_NODIST="$LDFLAGS_NODIST $LTOFLAGS" fi -# Enable bolt flags -Py_BOLT='false' -AC_MSG_CHECKING(for --enable-bolt) -AC_ARG_ENABLE(bolt, AS_HELP_STRING( - [--enable-bolt], - [enable usage of the llvm-bolt post-link optimizer (default is no)]), -[ -if test "$enableval" != no -then - Py_BOLT='true' - AC_MSG_RESULT(yes); -else - Py_BOLT='false' - AC_MSG_RESULT(no); -fi], -[AC_MSG_RESULT(no)]) - -AC_SUBST(PREBOLT_RULE) -if test "$Py_BOLT" = 'true' ; then - PREBOLT_RULE="${DEF_MAKE_ALL_RULE}" - DEF_MAKE_ALL_RULE="bolt-opt" - DEF_MAKE_RULE="build_all" - - AC_SUBST(READELF) - AC_CHECK_TOOLS(READELF, [readelf], "notfound") - if test "$READELF" == "notfound" - then - AC_MSG_ERROR([readelf is required for a --enable-bolt build but could not be found.]) - fi - - # -fno-reorder-blocks-and-partition is required for bolt to work. - # Possibly GCC only. - AX_CHECK_COMPILE_FLAG([-fno-reorder-blocks-and-partition],[ - CFLAGS_NODIST="$CFLAGS_NODIST -fno-reorder-blocks-and-partition" - ]) - - # These flags are required for bolt to work: - LDFLAGS_NODIST="$LDFLAGS_NODIST -Wl,--emit-relocs" - - # These flags are required to get good performance from bolt: - CFLAGS_NODIST="$CFLAGS_NODIST -fno-pie" - # We want to add these no-pie flags to linking executables but not shared libraries: - LINKCC="$LINKCC -fno-pie -no-pie" - AC_SUBST(LLVM_BOLT) - AC_PATH_TOOL(LLVM_BOLT, llvm-bolt, '', ${llvm_path}) - if test -n "${LLVM_BOLT}" -a -x "${LLVM_BOLT}" - then - AC_MSG_RESULT("Found llvm-bolt") - else - AC_MSG_ERROR([llvm-bolt is required for a --enable-bolt build but could not be found.]) - fi - - AC_SUBST(MERGE_FDATA) - AC_PATH_TOOL(MERGE_FDATA, merge-fdata, '', ${llvm_path}) - if test -n "${MERGE_FDATA}" -a -x "${MERGE_FDATA}" - then - AC_MSG_RESULT("Found merge-fdata") - else - AC_MSG_ERROR([merge-fdata is required for a --enable-bolt build but could not be found.]) - fi -fi - # Enable PGO flags. AC_SUBST(PGO_PROF_GEN_FLAG) AC_SUBST(PGO_PROF_USE_FLAG) @@ -2067,6 +2005,68 @@ case $CC in ;; esac +# BOLT optimization. Always configured after PGO since it always runs after PGO. +Py_BOLT='false' +AC_MSG_CHECKING(for --enable-bolt) +AC_ARG_ENABLE(bolt, AS_HELP_STRING( + [--enable-bolt], + [enable usage of the llvm-bolt post-link optimizer (default is no)]), +[ +if test "$enableval" != no +then + Py_BOLT='true' + AC_MSG_RESULT(yes); +else + Py_BOLT='false' + AC_MSG_RESULT(no); +fi], +[AC_MSG_RESULT(no)]) + +AC_SUBST(PREBOLT_RULE) +if test "$Py_BOLT" = 'true' ; then + PREBOLT_RULE="${DEF_MAKE_ALL_RULE}" + DEF_MAKE_ALL_RULE="bolt-opt" + DEF_MAKE_RULE="build_all" + + AC_SUBST(READELF) + AC_CHECK_TOOLS(READELF, [readelf], "notfound") + if test "$READELF" == "notfound" + then + AC_MSG_ERROR([readelf is required for a --enable-bolt build but could not be found.]) + fi + + # -fno-reorder-blocks-and-partition is required for bolt to work. + # Possibly GCC only. + AX_CHECK_COMPILE_FLAG([-fno-reorder-blocks-and-partition],[ + CFLAGS_NODIST="$CFLAGS_NODIST -fno-reorder-blocks-and-partition" + ]) + + # These flags are required for bolt to work: + LDFLAGS_NODIST="$LDFLAGS_NODIST -Wl,--emit-relocs" + + # These flags are required to get good performance from bolt: + CFLAGS_NODIST="$CFLAGS_NODIST -fno-pie" + # We want to add these no-pie flags to linking executables but not shared libraries: + LINKCC="$LINKCC -fno-pie -no-pie" + AC_SUBST(LLVM_BOLT) + AC_PATH_TOOL(LLVM_BOLT, llvm-bolt, '', ${llvm_path}) + if test -n "${LLVM_BOLT}" -a -x "${LLVM_BOLT}" + then + AC_MSG_RESULT("Found llvm-bolt") + else + AC_MSG_ERROR([llvm-bolt is required for a --enable-bolt build but could not be found.]) + fi + + AC_SUBST(MERGE_FDATA) + AC_PATH_TOOL(MERGE_FDATA, merge-fdata, '', ${llvm_path}) + if test -n "${MERGE_FDATA}" -a -x "${MERGE_FDATA}" + then + AC_MSG_RESULT("Found merge-fdata") + else + AC_MSG_ERROR([merge-fdata is required for a --enable-bolt build but could not be found.]) + fi +fi + # XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be # merged with this chunk of code? From 146106a0f1cc61815fa33f0d3f808a3e3e3275be Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 15 May 2023 10:02:44 +0200 Subject: [PATCH 197/204] gh-104487: PYTHON_FOR_REGEN must be minimum Python 3.10 (#104488) Also include Python 3.12 in the list of accepted versions. --- Doc/whatsnew/3.12.rst | 2 ++ .../next/Build/2023-05-15-09-34-08.gh-issue-99017.nToOQu.rst | 1 + configure | 2 +- configure.ac | 2 +- 4 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2023-05-15-09-34-08.gh-issue-99017.nToOQu.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index dc1178811e75cc..3e55b3fa0f4734 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1204,6 +1204,8 @@ Build Changes (Contributed by Zhang Na in :gh:`90656`.) +* ``PYTHON_FOR_REGEN`` now require Python 3.10 or newer. + C API Changes ============= diff --git a/Misc/NEWS.d/next/Build/2023-05-15-09-34-08.gh-issue-99017.nToOQu.rst b/Misc/NEWS.d/next/Build/2023-05-15-09-34-08.gh-issue-99017.nToOQu.rst new file mode 100644 index 00000000000000..a3517ac0204b1d --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-05-15-09-34-08.gh-issue-99017.nToOQu.rst @@ -0,0 +1 @@ +``PYTHON_FOR_REGEN`` now require Python 3.10 or newer. diff --git a/configure b/configure index 4d3520aee43def..7aad4fe89e3cbf 100755 --- a/configure +++ b/configure @@ -3359,7 +3359,7 @@ fi -for ac_prog in python$PACKAGE_VERSION python3.11 python3.10 python3.9 python3.8 python3.7 python3.6 python3 python +for ac_prog in python$PACKAGE_VERSION python3.12 python3.11 python3.10 python3 python do # Extract the first word of "$ac_prog", so it can be a program name with args. set dummy $ac_prog; ac_word=$2 diff --git a/configure.ac b/configure.ac index 0c63cde0363e2e..115998e0753b26 100644 --- a/configure.ac +++ b/configure.ac @@ -202,7 +202,7 @@ AC_SUBST([FREEZE_MODULE_DEPS]) AC_SUBST([PYTHON_FOR_BUILD_DEPS]) AC_CHECK_PROGS([PYTHON_FOR_REGEN], - [python$PACKAGE_VERSION python3.11 python3.10 python3.9 python3.8 python3.7 python3.6 python3 python], + [python$PACKAGE_VERSION python3.12 python3.11 python3.10 python3 python], [python3]) AC_SUBST(PYTHON_FOR_REGEN) From b15a1a6ac6ea0d7792036e639e90f0e51400c2ee Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 15 May 2023 09:12:52 +0100 Subject: [PATCH 198/204] gh-67056: document that registering/unregistering an atexit func from within an atexit func is undefined (#104473) --- Doc/library/atexit.rst | 3 +++ .../2023-05-14-12-11-28.gh-issue-67056.nVC2Rf.rst | 2 ++ 2 files changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Documentation/2023-05-14-12-11-28.gh-issue-67056.nVC2Rf.rst diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst index f7f038107d11fe..a2bd85b31c9a8d 100644 --- a/Doc/library/atexit.rst +++ b/Doc/library/atexit.rst @@ -20,6 +20,9 @@ at interpreter termination time they will be run in the order ``C``, ``B``, program is killed by a signal not handled by Python, when a Python fatal internal error is detected, or when :func:`os._exit` is called. +**Note:** The effect of registering or unregistering functions from within +a cleanup function is undefined. + .. versionchanged:: 3.7 When used with C-API subinterpreters, registered functions are local to the interpreter they were registered in. diff --git a/Misc/NEWS.d/next/Documentation/2023-05-14-12-11-28.gh-issue-67056.nVC2Rf.rst b/Misc/NEWS.d/next/Documentation/2023-05-14-12-11-28.gh-issue-67056.nVC2Rf.rst new file mode 100644 index 00000000000000..2c6ef17810723d --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2023-05-14-12-11-28.gh-issue-67056.nVC2Rf.rst @@ -0,0 +1,2 @@ +Document that the effect of registering or unregistering an :mod:`atexit` +cleanup function from within a registered cleanup function is undefined. From a6bcc8fb92ffb75bb1907cc568ba9fff516979c3 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Mon, 15 May 2023 01:48:34 -0700 Subject: [PATCH 199/204] gh-104490: Consistently define phony make targets (#104491) By convention make targets that don't refer to a file have a dependency on the fake .PHONY target/file. This ensures that these targets are always evaluated because there is no rule to create a .PHONY file and that will force make to think the rule is out of date and needs to be rebuilt. This commit consistently associates virtual targets with .PHONY by declaring the .PHONY dependency immediately above the make rule. This should avoid race conditions and avoidable rebuilds across multiple make invocations. --- Makefile.pre.in | 97 +++++++++++++++---- ...-05-14-19-00-19.gh-issue-104490.1tA4AF.rst | 1 + 2 files changed, 78 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2023-05-14-19-00-19.gh-issue-104490.1tA4AF.rst diff --git a/Makefile.pre.in b/Makefile.pre.in index 52cafabd1ab999..7c44b7be5dbe67 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -603,12 +603,21 @@ LIBHACL_SHA2_HEADERS= \ # Default target all: @DEF_MAKE_ALL_RULE@ + +# First target in Makefile is implicit default. So .PHONY needs to come after +# all. +.PHONY: all + +.PHONY: build_all build_all: check-clean-src $(BUILDPYTHON) platform sharedmods \ gdbhooks Programs/_testembed scripts checksharedmods rundsymutil + +.PHONY: build_wasm build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \ python-config checksharedmods # Check that the source is clean when building out of source. +.PHONY: check-clean-src check-clean-src: @if test -n "$(VPATH)" -a \( \ -f "$(srcdir)/Programs/python.o" \ @@ -652,23 +661,28 @@ profile-run-stamp: # to record its completion and avoid re-running it. touch $@ +.PHONY: build_all_generate_profile build_all_generate_profile: $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS_NODIST) $(PGO_PROF_GEN_FLAG)" LDFLAGS_NODIST="$(LDFLAGS_NODIST) $(PGO_PROF_GEN_FLAG)" LIBS="$(LIBS)" +.PHONY: run_profile_task run_profile_task: @ # FIXME: can't run for a cross build $(LLVM_PROF_FILE) $(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK) || true +.PHONY: build_all_merge_profile build_all_merge_profile: $(LLVM_PROF_MERGER) # Compile Python binary with profile guided optimization. # To force re-running of the profile task, remove the profile-run-stamp file. +.PHONY: profile-opt profile-opt: profile-run-stamp @echo "Rebuilding with profile guided optimizations:" -rm -f profile-clean-stamp $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS_NODIST) $(PGO_PROF_USE_FLAG)" LDFLAGS_NODIST="$(LDFLAGS_NODIST)" +.PHONY: bolt-opt bolt-opt: @PREBOLT_RULE@ rm -f *.fdata @if $(READELF) -p .note.bolt_info $(BUILDPYTHON) | grep BOLT > /dev/null; then\ @@ -685,12 +699,13 @@ bolt-opt: @PREBOLT_RULE@ # Compile and run with gcov -.PHONY=coverage coverage-lcov coverage-report +.PHONY: coverage coverage: @echo "Building with support for coverage checking:" $(MAKE) clean $(MAKE) @DEF_MAKE_RULE@ CFLAGS="$(CFLAGS) -O0 -pg --coverage" LDFLAGS="$(LDFLAGS) --coverage" +.PHONY: coverage-lcov coverage-lcov: @echo "Creating Coverage HTML report with LCOV:" @rm -f $(COVERAGE_INFO) @@ -722,6 +737,7 @@ coverage-lcov: @echo # Force regeneration of parser and frozen modules +.PHONY: coverage-report coverage-report: regen-token regen-frozen @ # build with coverage info $(MAKE) coverage @@ -731,7 +747,7 @@ coverage-report: regen-token regen-frozen $(MAKE) coverage-lcov # Run "Argument Clinic" over all source files -.PHONY=clinic +.PHONY: clinic clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --make --srcdir $(srcdir) $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_global_objects.py @@ -796,6 +812,7 @@ Modules/python.exp: $(LIBRARY) # # Distributors are likely to want to install this somewhere else e.g. relative # to the stripped DWARF data for the shared library. +.PHONY: gdbhooks gdbhooks: $(BUILDPYTHON)-gdb.py SRC_GDB_HOOKS=$(srcdir)/Tools/gdb/libpython.py @@ -936,6 +953,7 @@ $(LIBHACL_SHA2_A): $(LIBHACL_SHA2_OBJS) # create relative links from build/lib.platform/egg.so to Modules/egg.so # pybuilddir.txt is created too late. We cannot use it in Makefile # targets. ln --relative is not portable. +.PHONY: sharedmods sharedmods: $(SHAREDMODS) pybuilddir.txt @target=`cat pybuilddir.txt`; \ $(MKDIR_P) $$target; \ @@ -946,9 +964,11 @@ sharedmods: $(SHAREDMODS) pybuilddir.txt done # dependency on BUILDPYTHON ensures that the target is run last +.PHONY: checksharedmods checksharedmods: sharedmods $(PYTHON_FOR_BUILD_DEPS) $(BUILDPYTHON) @$(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/check_extension_modules.py +.PHONY: rundsymutil rundsymutil: sharedmods $(PYTHON_FOR_BUILD_DEPS) $(BUILDPYTHON) @if [ ! -z $(DSYMUTIL) ] ; then \ echo $(DSYMUTIL_PATH) $(BUILDPYTHON); \ @@ -1252,20 +1272,24 @@ regen-global-objects: $(srcdir)/Tools/build/generate_global_objects.py ############################################################################ # ABI +.PHONY: regen-abidump regen-abidump: all @$(MKDIR_P) $(srcdir)/Doc/data/ abidw "libpython$(LDVERSION).so" --no-architecture --out-file $(srcdir)/Doc/data/python$(LDVERSION).abi.new @$(UPDATE_FILE) --create $(srcdir)/Doc/data/python$(LDVERSION).abi $(srcdir)/Doc/data/python$(LDVERSION).abi.new +.PHONY: check-abidump check-abidump: all abidiff $(srcdir)/Doc/data/python$(LDVERSION).abi "libpython$(LDVERSION).so" --drop-private-types --no-architecture --no-added-syms +.PHONY: regen-limited-abi regen-limited-abi: all $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/stable_abi.py --generate-all $(srcdir)/Misc/stable_abi.toml ############################################################################ # Regenerate all generated files +.PHONY: regen-all regen-all: regen-cases regen-opcode regen-opcode-targets regen-typeslots \ regen-token regen-ast regen-keyword regen-sre regen-frozen clinic \ regen-pegen-metaparser regen-pegen regen-test-frozenmain \ @@ -1351,7 +1375,7 @@ regen-pegen: -o $(srcdir)/Parser/parser.new.c $(UPDATE_FILE) $(srcdir)/Parser/parser.c $(srcdir)/Parser/parser.new.c -.PHONY=regen-ast +.PHONY: regen-ast regen-ast: # Regenerate 3 files using using Parser/asdl_c.py: # - Include/internal/pycore_ast.h @@ -1424,6 +1448,7 @@ regen-stdlib-module-names: all Programs/_testembed > $(srcdir)/Python/stdlib_module_names.h.new $(UPDATE_FILE) $(srcdir)/Python/stdlib_module_names.h $(srcdir)/Python/stdlib_module_names.h.new +.PHONY: regen-sre regen-sre: # Regenerate Modules/_sre/sre_constants.h and Modules/_sre/sre_targets.h # from Lib/re/_constants.py using Tools/build/generate_sre_constants.py @@ -1762,15 +1787,15 @@ TESTPYTHON= $(RUNSHARED) $(PYTHON_FOR_BUILD) $(TESTPYTHONOPTS) TESTRUNNER= $(TESTPYTHON) $(srcdir)/Tools/scripts/run_tests.py TESTTIMEOUT= 1200 -.PHONY: test testall testuniversal buildbottest pythoninfo - # Remove "test_python_*" directories of previous failed test jobs. # Pass TESTOPTS options because it can contain --tempdir option. +.PHONY: cleantest cleantest: all $(TESTRUNNER) $(TESTOPTS) --cleanup # Run a basic set of regression tests. # This excludes some tests that are particularly resource-intensive. +.PHONY: test test: all $(TESTRUNNER) $(TESTOPTS) @@ -1781,6 +1806,7 @@ test: all # the bytecode read from a .pyc file had the bug, sometimes the directly # generated bytecode. This is sometimes a very shy bug needing a lot of # sample data. +.PHONY: testall testall: all -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py @@ -1790,6 +1816,7 @@ testall: all # Run the test suite for both architectures in a Universal build on OSX. # Must be run on an Intel box. +.PHONY: testuniversal testuniversal: all @if [ `arch` != 'i386' ]; then \ echo "This can only be used on OSX/i386" ;\ @@ -1801,6 +1828,7 @@ testuniversal: all # Like testall, but with only one pass and without multiple processes. # Run an optional script to include information about the build environment. +.PHONY: buildbottest buildbottest: all -@if which pybuildbot.identify >/dev/null 2>&1; then \ pybuildbot.identify "CC='$(CC)'" "CXX='$(CXX)'"; \ @@ -1808,9 +1836,11 @@ buildbottest: all $(TESTRUNNER) -j 1 -u all -W --slowest --fail-env-changed --timeout=$(TESTTIMEOUT) $(TESTOPTS) # Like testall, but run Python tests with HOSTRUNNER directly. +.PHONY: hostrunnertest hostrunnertest: all $(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test -u all $(TESTOPTS) +.PHONY: pythoninfo pythoninfo: all $(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test.pythoninfo @@ -1820,14 +1850,17 @@ QUICKTESTOPTS= $(TESTOPTS) -x test_subprocess test_io test_lib2to3 \ test_multiprocessing_forkserver \ test_mailbox test_nntplib test_socket test_poll \ test_select test_zipfile test_concurrent_futures + +.PHONY: quicktest quicktest: all $(TESTRUNNER) $(QUICKTESTOPTS) # SSL tests -.PHONY: multisslcompile multissltest +.PHONY: multisslcompile multisslcompile: all $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/ssl/multissltests.py --steps=modules +.PHONY: multissltest multissltest: all $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/ssl/multissltests.py @@ -1835,6 +1868,7 @@ multissltest: all # prevent race conditions with PGO builds. PGO builds use recursive make, # which can lead to two parallel `./python setup.py build` processes that # step on each others toes. +.PHONY: install install: @FRAMEWORKINSTALLFIRST@ commoninstall bininstall maninstall @FRAMEWORKINSTALLLAST@ if test "x$(ENSUREPIP)" != "xno" ; then \ case $(ENSUREPIP) in \ @@ -1845,6 +1879,7 @@ install: @FRAMEWORKINSTALLFIRST@ commoninstall bininstall maninstall @FRAMEWORKI $$ensurepip --root=$(DESTDIR)/ ; \ fi +.PHONY: altinstall altinstall: commoninstall if test "x$(ENSUREPIP)" != "xno" ; then \ case $(ENSUREPIP) in \ @@ -1855,6 +1890,7 @@ altinstall: commoninstall $$ensurepip --root=$(DESTDIR)/ ; \ fi +.PHONY: commoninstall commoninstall: check-clean-src @FRAMEWORKALTINSTALLFIRST@ \ altbininstall libinstall inclinstall libainstall \ sharedinstall altmaninstall \ @@ -1863,6 +1899,7 @@ commoninstall: check-clean-src @FRAMEWORKALTINSTALLFIRST@ \ # Install shared libraries enabled by Setup DESTDIRS= $(exec_prefix) $(LIBDIR) $(BINLIBDEST) $(DESTSHARED) +.PHONY: sharedinstall sharedinstall: all @for i in $(DESTDIRS); \ do \ @@ -1885,6 +1922,7 @@ sharedinstall: all # Install the interpreter with $(VERSION) affixed # This goes into $(exec_prefix) +.PHONY: altbininstall altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@ @for i in $(BINDIR) $(LIBDIR); \ do \ @@ -1950,7 +1988,7 @@ altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@ fi \ fi - +.PHONY: bininstall bininstall: altbininstall if test ! -d $(DESTDIR)$(LIBPC); then \ echo "Creating directory $(LIBPC)"; \ @@ -1991,6 +2029,7 @@ bininstall: altbininstall fi # Install the versioned manual page +.PHONY: altmaninstall altmaninstall: @for i in $(MANDIR) $(MANDIR)/man1; \ do \ @@ -2004,6 +2043,7 @@ altmaninstall: $(DESTDIR)$(MANDIR)/man1/python$(VERSION).1 # Install the unversioned manual page +.PHONY: maninstall maninstall: altmaninstall -rm -f $(DESTDIR)$(MANDIR)/man1/python3.1 (cd $(DESTDIR)$(MANDIR)/man1; $(LN) -s python$(VERSION).1 python3.1) @@ -2162,6 +2202,8 @@ TESTSUBDIRS= idlelib/idle_test \ COMPILEALL_OPTS=-j0 TEST_MODULES=@TEST_MODULES@ + +.PHONY: libinstall libinstall: all $(srcdir)/Modules/xxmodule.c @for i in $(SCRIPTDIR) $(LIBDEST); \ do \ @@ -2282,10 +2324,13 @@ $(SCRIPT_PYDOC): $(srcdir)/Tools/scripts/pydoc3 sed -e "s,/usr/bin/env python3,$(EXENAME)," < $(srcdir)/Tools/scripts/pydoc3 > $@ @chmod +x $@ +.PHONY: scripts scripts: $(SCRIPT_2TO3) $(SCRIPT_IDLE) $(SCRIPT_PYDOC) python-config # Install the include files INCLDIRSTOMAKE=$(INCLUDEDIR) $(CONFINCLUDEDIR) $(INCLUDEPY) $(CONFINCLUDEPY) + +.PHONY: inclinstall inclinstall: @for i in $(INCLDIRSTOMAKE); \ do \ @@ -2329,6 +2374,7 @@ LIBPL= @LIBPL@ # pkgconfig directory LIBPC= $(LIBDIR)/pkgconfig +.PHONY: libainstall libainstall: all scripts @for i in $(LIBDIR) $(LIBPL) $(LIBPC) $(BINDIR); \ do \ @@ -2392,6 +2438,7 @@ libainstall: all scripts # # This target is here for backward compatibility, previous versions of Python # hadn't integrated framework installation in the normal install process. +.PHONY: frameworkinstall frameworkinstall: install # On install, we re-make the framework @@ -2400,8 +2447,10 @@ frameworkinstall: install # automatically set prefix to the location deep down in the framework, so we # only have to cater for the structural bits of the framework. +.PHONY: frameworkinstallframework frameworkinstallframework: frameworkinstallstructure install frameworkinstallmaclib +.PHONY: frameworkinstallstructure frameworkinstallstructure: $(LDLIBRARY) @if test "$(PYTHONFRAMEWORKDIR)" = no-framework; then \ echo Not configured with --enable-framework; \ @@ -2426,6 +2475,7 @@ frameworkinstallstructure: $(LDLIBRARY) # This installs Mac/Lib into the framework # Install a number of symlinks to keep software that expects a normal unix # install (which includes python-config) happy. +.PHONY: frameworkinstallmaclib frameworkinstallmaclib: $(LN) -fs "../../../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(LIBPL)/libpython$(LDVERSION).a" $(LN) -fs "../../../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(LIBPL)/libpython$(LDVERSION).dylib" @@ -2435,25 +2485,30 @@ frameworkinstallmaclib: $(LN) -fs "../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/libpython$(VERSION).dylib" # This installs the IDE, the Launcher and other apps into /Applications +.PHONY: frameworkinstallapps frameworkinstallapps: cd Mac && $(MAKE) installapps DESTDIR="$(DESTDIR)" # Build the bootstrap executable that will spawn the interpreter inside # an app bundle within the framework. This allows the interpreter to # run OS X GUI APIs. +.PHONY: frameworkpythonw frameworkpythonw: cd Mac && $(MAKE) pythonw # This installs the python* and other bin symlinks in $prefix/bin or in # a bin directory relative to the framework root +.PHONY: frameworkinstallunixtools frameworkinstallunixtools: cd Mac && $(MAKE) installunixtools DESTDIR="$(DESTDIR)" +.PHONY: frameworkaltinstallunixtools frameworkaltinstallunixtools: cd Mac && $(MAKE) altinstallunixtools DESTDIR="$(DESTDIR)" # This installs the Tools into the applications directory. # It is not part of a normal frameworkinstall +.PHONY: frameworkinstallextras frameworkinstallextras: cd Mac && $(MAKE) installextras DESTDIR="$(DESTDIR)" @@ -2483,11 +2538,13 @@ Python/dtoa.o: Python/dtoa.c $(CC) -c $(PY_CORE_CFLAGS) $(CFLAGS_ALIASING) -o $@ $< # Run reindent on the library +.PHONY: reindent reindent: ./$(BUILDPYTHON) $(srcdir)/Tools/patchcheck/reindent.py -r $(srcdir)/Lib # Rerun configure with the same options as it was run last time, # provided the config.status script exists +.PHONY: recheck recheck: ./config.status --recheck ./config.status @@ -2524,10 +2581,12 @@ TAGS:: # Sanitation targets -- clean leaves libraries, executables and tags # files, which clobber removes as well +.PHONY: pycremoval pycremoval: -find $(srcdir) -depth -name '__pycache__' -exec rm -rf {} ';' -find $(srcdir) -name '*.py[co]' -exec rm -f {} ';' +.PHONY: rmtestturds rmtestturds: -rm -f *BAD *GOOD *SKIPPED -rm -rf OUT @@ -2535,11 +2594,13 @@ rmtestturds: -rm -f *.txt -rm -f gb-18030-2000.xml +.PHONY: docclean docclean: $(MAKE) -C $(srcdir)/Doc clean # like the 'clean' target but retain the profile guided optimization (PGO) # data. The PGO data is only valid if source code remains unchanged. +.PHONY: clean-retain-profile clean-retain-profile: pycremoval find . -name '*.[oa]' -exec rm -f {} ';' find . -name '*.s[ol]' -exec rm -f {} ';' @@ -2563,6 +2624,7 @@ clean-retain-profile: pycremoval -rm -f Include/pydtrace_probes.h -rm -f profile-gen-stamp +.PHONY: profile-removal profile-removal: find . -name '*.gc??' -exec rm -f {} ';' find . -name '*.profclang?' -exec rm -f {} ';' @@ -2571,12 +2633,14 @@ profile-removal: rm -rf $(COVERAGE_REPORT) rm -f profile-run-stamp +.PHONY: clean clean: clean-retain-profile @if test @DEF_MAKE_ALL_RULE@ = profile-opt; then \ rm -f profile-gen-stamp profile-clean-stamp; \ $(MAKE) profile-removal; \ fi +.PHONY: clobber clobber: clean -rm -f $(BUILDPYTHON) $(LIBRARY) $(LDLIBRARY) $(DLLLIBRARY) \ tags TAGS \ @@ -2588,6 +2652,7 @@ clobber: clean # Make things extra clean, before making a distribution: # remove all generated files, even Makefile[.pre] # Keep configure and Python-ast.[ch], it's possible they can't be generated +.PHONY: distclean distclean: clobber docclean for file in $(srcdir)/Lib/test/data/* ; do \ if test "$$file" != "$(srcdir)/Lib/test/data/README"; then rm "$$file"; fi; \ @@ -2608,16 +2673,19 @@ distclean: clobber docclean -exec rm -f {} ';' # Check that all symbols exported by libpython start with "Py" or "_Py" +.PHONY: smelly smelly: all $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/smelly.py # Check if any unsupported C global variables have been added. +.PHONY: check-c-globals check-c-globals: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/c-analyzer/check-c-globals.py \ --format summary \ --traceback # Find files with funny names +.PHONY: funny funny: find $(SUBDIRS) $(SUBDIRSTOO) \ -type d \ @@ -2649,9 +2717,11 @@ funny: -o -print # Perform some verification checks on any modified files. +.PHONY: patchcheck patchcheck: all $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/patchcheck/patchcheck.py +.PHONY: check-limited-abi check-limited-abi: all $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/stable_abi.py --all $(srcdir)/Misc/stable_abi.toml @@ -2665,19 +2735,6 @@ update-config: Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h -# Declare targets that aren't real files -.PHONY: all build_all build_wasm check-clean-src -.PHONY: sharedmods checksharedmods test quicktest rundsymutil -.PHONY: install altinstall sharedinstall bininstall altbininstall -.PHONY: maninstall libinstall inclinstall libainstall -.PHONY: frameworkinstall frameworkinstallframework frameworkinstallstructure -.PHONY: frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools -.PHONY: frameworkaltinstallunixtools recheck clean clobber distclean -.PHONY: smelly funny patchcheck touch altmaninstall commoninstall -.PHONY: clean-retain-profile profile-removal run_profile_task -.PHONY: build_all_generate_profile build_all_merge_profile -.PHONY: gdbhooks scripts - ########################################################################## # Module dependencies and platform-specific files diff --git a/Misc/NEWS.d/next/Build/2023-05-14-19-00-19.gh-issue-104490.1tA4AF.rst b/Misc/NEWS.d/next/Build/2023-05-14-19-00-19.gh-issue-104490.1tA4AF.rst new file mode 100644 index 00000000000000..1315b5cb3c1969 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-05-14-19-00-19.gh-issue-104490.1tA4AF.rst @@ -0,0 +1 @@ +Define ``.PHONY`` / virtual make targets consistently and properly. From 9d41f83c58e6dc2fc6eb4b91f803551850b0adeb Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 15 May 2023 09:49:28 +0100 Subject: [PATCH 200/204] gh-104050: Run mypy on `clinic.py` in CI (#104421) * Add basic mypy workflow to CI * Make the type check pass --------- Co-authored-by: Erlend E. Aasland Co-authored-by: Nikita Sobolev Co-authored-by: Hugo van Kemenade --- .github/dependabot.yml | 7 ++++++ .github/workflows/mypy.yml | 39 ++++++++++++++++++++++++++++++ Tools/clinic/clinic.py | 40 ++++++++++++++++++++----------- Tools/clinic/cpp.py | 26 ++++++++++++-------- Tools/clinic/mypy.ini | 11 +++++++++ Tools/clinic/requirements-dev.txt | 2 ++ 6 files changed, 101 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/mypy.yml create mode 100644 Tools/clinic/mypy.ini create mode 100644 Tools/clinic/requirements-dev.txt diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 555e246e402bf9..f026b0f5f9454a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,3 +12,10 @@ updates: update-types: - "version-update:semver-minor" - "version-update:semver-patch" + - package-ecosystem: "pip" + directory: "/Tools/clinic/" + schedule: + interval: "monthly" + labels: + - "skip issue" + - "skip news" diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml new file mode 100644 index 00000000000000..1315bb5a966f01 --- /dev/null +++ b/.github/workflows/mypy.yml @@ -0,0 +1,39 @@ +# Workflow to run mypy on select parts of the CPython repo +name: mypy + +on: + push: + branches: + - main + pull_request: + paths: + - "Tools/clinic/**" + - ".github/workflows/mypy.yml" + workflow_dispatch: + +permissions: + contents: read + +env: + PIP_DISABLE_PIP_VERSION_CHECK: 1 + FORCE_COLOR: 1 + TERM: xterm-256color # needed for FORCE_COLOR to work on mypy on Ubuntu, see https://github.com/python/mypy/issues/13817 + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + mypy: + name: Run mypy on Tools/clinic/ + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.x" + cache: pip + cache-dependency-path: Tools/clinic/requirements-dev.txt + - run: pip install -r Tools/clinic/requirements-dev.txt + - run: mypy --config-file Tools/clinic/mypy.ini diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 19c4cd299f0bbb..4270fb3cc56613 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -7,6 +7,7 @@ import abc import ast +import builtins as bltns import collections import contextlib import copy @@ -26,7 +27,9 @@ import traceback import types +from collections.abc import Callable from types import * +from typing import Any, NamedTuple # TODO: # @@ -78,8 +81,13 @@ def __repr__(self): sig_end_marker = '--' +Appender = Callable[[str], None] +Outputter = Callable[[None], str] -_text_accumulator_nt = collections.namedtuple("_text_accumulator", "text append output") +class _TextAccumulator(NamedTuple): + text: list[str] + append: Appender + output: Outputter def _text_accumulator(): text = [] @@ -87,10 +95,12 @@ def output(): s = ''.join(text) text.clear() return s - return _text_accumulator_nt(text, text.append, output) + return _TextAccumulator(text, text.append, output) -text_accumulator_nt = collections.namedtuple("text_accumulator", "text append") +class TextAccumulator(NamedTuple): + text: list[str] + append: Appender def text_accumulator(): """ @@ -104,7 +114,7 @@ def text_accumulator(): empties the accumulator. """ text, append, output = _text_accumulator() - return text_accumulator_nt(append, output) + return TextAccumulator(append, output) def warn_or_fail(fail=False, *args, filename=None, line_number=None): @@ -1925,8 +1935,10 @@ def dump(self): # maps strings to Language objects. # "languages" maps the name of the language ("C", "Python"). # "extensions" maps the file extension ("c", "py"). +LangDict = dict[str, Callable[[str], Language]] + languages = { 'C': CLanguage, 'Python': PythonLanguage } -extensions = { name: CLanguage for name in "c cc cpp cxx h hh hpp hxx".split() } +extensions: LangDict = { name: CLanguage for name in "c cc cpp cxx h hh hpp hxx".split() } extensions['py'] = PythonLanguage @@ -2558,15 +2570,15 @@ class CConverter(metaclass=CConverterAutoRegister): """ # The C name to use for this variable. - name = None + name: str | None = None # The Python name to use for this variable. - py_name = None + py_name: str | None = None # The C type to use for this variable. # 'type' should be a Python string specifying the type, e.g. "int". # If this is a pointer type, the type string should end with ' *'. - type = None + type: str | None = None # The Python default value for this parameter, as a Python value. # Or the magic value "unspecified" if there is no default. @@ -2577,15 +2589,15 @@ class CConverter(metaclass=CConverterAutoRegister): # If not None, default must be isinstance() of this type. # (You can also specify a tuple of types.) - default_type = None + default_type: bltns.type[Any] | tuple[bltns.type[Any], ...] | None = None # "default" converted into a C value, as a string. # Or None if there is no default. - c_default = None + c_default: str | None = None # "default" converted into a Python value, as a string. # Or None if there is no default. - py_default = None + py_default: str | None = None # The default value used to initialize the C variable when # there is no default, but not specifying a default may @@ -2597,14 +2609,14 @@ class CConverter(metaclass=CConverterAutoRegister): # # This value is specified as a string. # Every non-abstract subclass should supply a valid value. - c_ignored_default = 'NULL' + c_ignored_default: str = 'NULL' # If true, wrap with Py_UNUSED. unused = False # The C converter *function* to be used, if any. # (If this is not None, format_unit must be 'O&'.) - converter = None + converter: str | None = None # Should Argument Clinic add a '&' before the name of # the variable when passing it into the _impl function? @@ -3432,7 +3444,7 @@ class robuffer: pass def str_converter_key(types, encoding, zeroes): return (frozenset(types), bool(encoding), bool(zeroes)) -str_converter_argument_map = {} +str_converter_argument_map: dict[str, str] = {} class str_converter(CConverter): type = 'const char *' diff --git a/Tools/clinic/cpp.py b/Tools/clinic/cpp.py index 77f5f9696a6d84..bc2cc713aac394 100644 --- a/Tools/clinic/cpp.py +++ b/Tools/clinic/cpp.py @@ -1,7 +1,12 @@ import re import sys +from collections.abc import Callable -def negate(condition): + +TokenAndCondition = tuple[str, str] +TokenStack = list[TokenAndCondition] + +def negate(condition: str) -> str: """ Returns a CPP conditional that is the opposite of the conditional passed in. """ @@ -22,17 +27,18 @@ class Monitor: Anyway this implementation seems to work well enough for the CPython sources. """ + is_a_simple_defined: Callable[[str], re.Match[str] | None] is_a_simple_defined = re.compile(r'^defined\s*\(\s*[A-Za-z0-9_]+\s*\)$').match - def __init__(self, filename=None, *, verbose=False): - self.stack = [] + def __init__(self, filename=None, *, verbose: bool = False): + self.stack: TokenStack = [] self.in_comment = False - self.continuation = None + self.continuation: str | None = None self.line_number = 0 self.filename = filename self.verbose = verbose - def __repr__(self): + def __repr__(self) -> str: return ''.join(( '")) - def status(self): + def status(self) -> str: return str(self.line_number).rjust(4) + ": " + self.condition() - def condition(self): + def condition(self) -> str: """ Returns the current preprocessor state, as a single #if condition. """ @@ -62,15 +68,15 @@ def close(self): if self.stack: self.fail("Ended file while still in a preprocessor conditional block!") - def write(self, s): + def write(self, s: str) -> None: for line in s.split("\n"): self.writeline(line) - def writeline(self, line): + def writeline(self, line: str) -> None: self.line_number += 1 line = line.strip() - def pop_stack(): + def pop_stack() -> TokenAndCondition: if not self.stack: self.fail("#" + token + " without matching #if / #ifdef / #ifndef!") return self.stack.pop() diff --git a/Tools/clinic/mypy.ini b/Tools/clinic/mypy.ini new file mode 100644 index 00000000000000..3c5643e789be37 --- /dev/null +++ b/Tools/clinic/mypy.ini @@ -0,0 +1,11 @@ +[mypy] +# make sure clinic can still be run on Python 3.10 +python_version = 3.10 +pretty = True +enable_error_code = ignore-without-code +disallow_any_generics = True +strict_concatenate = True +warn_redundant_casts = True +warn_unused_ignores = True +warn_unused_configs = True +files = Tools/clinic/ diff --git a/Tools/clinic/requirements-dev.txt b/Tools/clinic/requirements-dev.txt new file mode 100644 index 00000000000000..7e0aa37a46d7bc --- /dev/null +++ b/Tools/clinic/requirements-dev.txt @@ -0,0 +1,2 @@ +# Requirements file for external linters and checks we run on Tools/clinic/ in CI +mypy==1.2.0 From 3cba61f111db9b5e8ef35632915309f81fff8c6c Mon Sep 17 00:00:00 2001 From: Christopher Chavez Date: Mon, 15 May 2023 03:54:41 -0500 Subject: [PATCH 201/204] gh-104494: Update certain Tkinter pack/place tests for Tk 8.7 errors (#104495) Co-authored-by: Terry Jan Reedy Co-authored-by: Erlend E. Aasland --- Lib/test/test_tkinter/test_geometry_managers.py | 10 ++++++---- .../2023-05-15-02-22-44.gh-issue-104494.Bkrbfn.rst | 2 ++ 2 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-05-15-02-22-44.gh-issue-104494.Bkrbfn.rst diff --git a/Lib/test/test_tkinter/test_geometry_managers.py b/Lib/test/test_tkinter/test_geometry_managers.py index 3663048a145ab1..59fe592b492adc 100644 --- a/Lib/test/test_tkinter/test_geometry_managers.py +++ b/Lib/test/test_tkinter/test_geometry_managers.py @@ -108,8 +108,8 @@ def test_pack_configure_in(self): a.pack_configure(in_=c) self.assertEqual(pack.pack_slaves(), [b, c, d]) self.assertEqual(c.pack_slaves(), [a]) - with self.assertRaisesRegex(TclError, - 'can\'t pack %s inside itself' % (a,)): + with self.assertRaisesRegex( + TclError, """can't pack "?%s"? inside itself""" % (a,)): a.pack_configure(in_=a) with self.assertRaisesRegex(TclError, 'bad window path name ".foo"'): a.pack_configure(in_='.foo') @@ -292,8 +292,10 @@ def create2(self): def test_place_configure_in(self): t, f, f2 = self.create2() self.assertEqual(f2.winfo_manager(), '') - with self.assertRaisesRegex(TclError, "can't place %s relative to " - "itself" % re.escape(str(f2))): + with self.assertRaisesRegex( + TclError, + """can't place "?%s"? relative to itself""" + % re.escape(str(f2))): f2.place_configure(in_=f2) self.assertEqual(f2.winfo_manager(), '') with self.assertRaisesRegex(TclError, 'bad window path name'): diff --git a/Misc/NEWS.d/next/Tests/2023-05-15-02-22-44.gh-issue-104494.Bkrbfn.rst b/Misc/NEWS.d/next/Tests/2023-05-15-02-22-44.gh-issue-104494.Bkrbfn.rst new file mode 100644 index 00000000000000..a320c48428b58b --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-05-15-02-22-44.gh-issue-104494.Bkrbfn.rst @@ -0,0 +1,2 @@ +Update ``test_pack_configure_in`` and ``test_place_configure_in`` +for changes to error message formatting in Tk 8.7. From 35bf0916d913b83cf1883b76e67ad8b1c9a25bbe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 May 2023 09:17:48 +0000 Subject: [PATCH 202/204] Bump mypy from 1.2.0 to 1.3.0 in /Tools/clinic (#104501) build(deps-dev): bump mypy from 1.2.0 to 1.3.0 in /Tools/clinic Bumps [mypy](https://github.com/python/mypy) from 1.2.0 to 1.3.0. - [Commits](https://github.com/python/mypy/compare/v1.2.0...v1.3.0) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Tools/clinic/requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/clinic/requirements-dev.txt b/Tools/clinic/requirements-dev.txt index 7e0aa37a46d7bc..154934003c31c4 100644 --- a/Tools/clinic/requirements-dev.txt +++ b/Tools/clinic/requirements-dev.txt @@ -1,2 +1,2 @@ # Requirements file for external linters and checks we run on Tools/clinic/ in CI -mypy==1.2.0 +mypy==1.3.0 From 186bf39f5c6003912e4f445430c504db51b9a743 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 15 May 2023 11:26:27 +0200 Subject: [PATCH 203/204] gh-101819: Isolate `_io` (#101948) Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Co-authored-by: Victor Stinner --- Lib/test/test_io.py | 7 +- ...-03-14-11-20-19.gh-issue-101819.0-h0it.rst | 2 + Modules/_io/_iomodule.c | 138 ++++---------- Modules/_io/_iomodule.h | 15 +- Modules/_io/bufferedio.c | 116 +++++------- Modules/_io/bytesio.c | 2 + Modules/_io/fileio.c | 4 +- Modules/_io/iobase.c | 176 +++++++----------- Modules/_io/stringio.c | 11 +- Modules/_io/textio.c | 80 +++----- Modules/_io/winconsoleio.c | 2 +- Python/pylifecycle.c | 10 - Tools/c-analyzer/cpython/globals-to-fix.tsv | 4 - 13 files changed, 205 insertions(+), 362 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-14-11-20-19.gh-issue-101819.0-h0it.rst diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 9bcc92a40c61df..cc16804fe21829 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -4242,6 +4242,7 @@ def test_warn_on_dealloc_fd(self): def test_pickling(self): # Pickling file objects is forbidden + msg = "cannot pickle" for kwargs in [ {"mode": "w"}, {"mode": "wb"}, @@ -4256,8 +4257,10 @@ def test_pickling(self): if "b" not in kwargs["mode"]: kwargs["encoding"] = "utf-8" for protocol in range(pickle.HIGHEST_PROTOCOL + 1): - with self.open(os_helper.TESTFN, **kwargs) as f: - self.assertRaises(TypeError, pickle.dumps, f, protocol) + with self.subTest(protocol=protocol, kwargs=kwargs): + with self.open(os_helper.TESTFN, **kwargs) as f: + with self.assertRaisesRegex(TypeError, msg): + pickle.dumps(f, protocol) @unittest.skipIf( support.is_emscripten, "fstat() of a pipe fd is not supported" diff --git a/Misc/NEWS.d/next/Library/2023-03-14-11-20-19.gh-issue-101819.0-h0it.rst b/Misc/NEWS.d/next/Library/2023-03-14-11-20-19.gh-issue-101819.0-h0it.rst new file mode 100644 index 00000000000000..4a73bbf32b370e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-14-11-20-19.gh-issue-101819.0-h0it.rst @@ -0,0 +1,2 @@ +Isolate the :mod:`io` extension module by applying :pep:`687`. Patch by +Kumar Aditya, Victor Stinner, and Erlend E. Aasland. diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 2457cb124036fe..7b06c1bee5a832 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -561,25 +561,9 @@ PyNumber_AsOff_t(PyObject *item, PyObject *err) return result; } -_PyIO_State * -_PyIO_get_module_state(void) -{ - PyObject *mod = PyState_FindModule(&_PyIO_Module); - _PyIO_State *state; - if (mod == NULL || (state = get_io_state(mod)) == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "could not find io module state " - "(interpreter shutdown?)"); - return NULL; - } - return state; -} - static int iomodule_traverse(PyObject *mod, visitproc visit, void *arg) { _PyIO_State *state = get_io_state(mod); - if (!state->initialized) - return 0; Py_VISIT(state->unsupported_operation); Py_VISIT(state->PyIOBase_Type); @@ -606,8 +590,6 @@ iomodule_traverse(PyObject *mod, visitproc visit, void *arg) { static int iomodule_clear(PyObject *mod) { _PyIO_State *state = get_io_state(mod); - if (!state->initialized) - return 0; Py_CLEAR(state->unsupported_operation); Py_CLEAR(state->PyIOBase_Type); @@ -652,115 +634,57 @@ static PyMethodDef module_methods[] = { {NULL, NULL} }; -struct PyModuleDef _PyIO_Module = { - PyModuleDef_HEAD_INIT, - "io", - module_doc, - sizeof(_PyIO_State), - module_methods, - NULL, - iomodule_traverse, - iomodule_clear, - (freefunc)iomodule_free, -}; - - -static PyTypeObject* static_types[] = { - // Base classes - &PyIOBase_Type, - - // PyIOBase_Type subclasses - &PyBufferedIOBase_Type, - &PyRawIOBase_Type, - &PyTextIOBase_Type, -}; - - -PyStatus -_PyIO_InitTypes(PyInterpreterState *interp) -{ - for (size_t i=0; i < Py_ARRAY_LENGTH(static_types); i++) { - PyTypeObject *type = static_types[i]; - if (_PyStaticType_InitBuiltin(interp, type) < 0) { - return _PyStatus_ERR("Can't initialize builtin type"); - } - } - - return _PyStatus_OK(); -} - -void -_PyIO_FiniTypes(PyInterpreterState *interp) -{ - // Deallocate types in the reverse order to deallocate subclasses before - // their base classes. - for (Py_ssize_t i=Py_ARRAY_LENGTH(static_types) - 1; i >= 0; i--) { - PyTypeObject *type = static_types[i]; - _PyStaticType_Dealloc(interp, type); - } -} - #define ADD_TYPE(module, type, spec, base) \ do { \ type = (PyTypeObject *)PyType_FromModuleAndSpec(module, spec, \ (PyObject *)base); \ if (type == NULL) { \ - goto fail; \ + return -1; \ } \ if (PyModule_AddType(module, type) < 0) { \ - goto fail; \ + return -1; \ } \ } while (0) -PyMODINIT_FUNC -PyInit__io(void) +static int +iomodule_exec(PyObject *m) { - PyObject *m = PyModule_Create(&_PyIO_Module); - _PyIO_State *state = NULL; - if (m == NULL) - return NULL; - state = get_io_state(m); - state->initialized = 0; + _PyIO_State *state = get_io_state(m); /* DEFAULT_BUFFER_SIZE */ if (PyModule_AddIntMacro(m, DEFAULT_BUFFER_SIZE) < 0) - goto fail; + return -1; /* UnsupportedOperation inherits from ValueError and OSError */ state->unsupported_operation = PyObject_CallFunction( (PyObject *)&PyType_Type, "s(OO){}", "UnsupportedOperation", PyExc_OSError, PyExc_ValueError); if (state->unsupported_operation == NULL) - goto fail; + return -1; if (PyModule_AddObjectRef(m, "UnsupportedOperation", state->unsupported_operation) < 0) { - goto fail; + return -1; } /* BlockingIOError, for compatibility */ if (PyModule_AddObjectRef(m, "BlockingIOError", (PyObject *) PyExc_BlockingIOError) < 0) { - goto fail; - } - - // Add types - for (size_t i=0; i < Py_ARRAY_LENGTH(static_types); i++) { - PyTypeObject *type = static_types[i]; - if (PyModule_AddType(m, type) < 0) { - goto fail; - } + return -1; } // Base classes - state->PyIOBase_Type = (PyTypeObject *)Py_NewRef(&PyIOBase_Type); ADD_TYPE(m, state->PyIncrementalNewlineDecoder_Type, &nldecoder_spec, NULL); ADD_TYPE(m, state->PyBytesIOBuffer_Type, &bytesiobuf_spec, NULL); + ADD_TYPE(m, state->PyIOBase_Type, &iobase_spec, NULL); // PyIOBase_Type subclasses - state->PyRawIOBase_Type = (PyTypeObject *)Py_NewRef(&PyRawIOBase_Type); - state->PyBufferedIOBase_Type = (PyTypeObject *)Py_NewRef(&PyBufferedIOBase_Type); - state->PyTextIOBase_Type = (PyTypeObject *)Py_NewRef(&PyTextIOBase_Type); + ADD_TYPE(m, state->PyTextIOBase_Type, &textiobase_spec, + state->PyIOBase_Type); + ADD_TYPE(m, state->PyBufferedIOBase_Type, &bufferediobase_spec, + state->PyIOBase_Type); + ADD_TYPE(m, state->PyRawIOBase_Type, &rawiobase_spec, + state->PyIOBase_Type); // PyBufferedIOBase_Type(PyIOBase_Type) subclasses ADD_TYPE(m, state->PyBytesIO_Type, &bytesio_spec, state->PyBufferedIOBase_Type); @@ -775,6 +699,7 @@ PyInit__io(void) // PyRawIOBase_Type(PyIOBase_Type) subclasses ADD_TYPE(m, state->PyFileIO_Type, &fileio_spec, state->PyRawIOBase_Type); + #ifdef HAVE_WINDOWS_CONSOLE_IO ADD_TYPE(m, state->PyWindowsConsoleIO_Type, &winconsoleio_spec, state->PyRawIOBase_Type); @@ -785,11 +710,30 @@ PyInit__io(void) ADD_TYPE(m, state->PyTextIOWrapper_Type, &textiowrapper_spec, state->PyTextIOBase_Type); - state->initialized = 1; +#undef ADD_TYPE + return 0; +} - return m; +static struct PyModuleDef_Slot iomodule_slots[] = { + {Py_mod_exec, iomodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL}, +}; - fail: - Py_DECREF(m); - return NULL; +struct PyModuleDef _PyIO_Module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "io", + .m_doc = module_doc, + .m_size = sizeof(_PyIO_State), + .m_methods = module_methods, + .m_traverse = iomodule_traverse, + .m_clear = iomodule_clear, + .m_free = iomodule_free, + .m_slots = iomodule_slots, +}; + +PyMODINIT_FUNC +PyInit__io(void) +{ + return PyModuleDef_Init(&_PyIO_Module); } diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index ae06fecc48b450..afd638a120ba08 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -8,13 +8,8 @@ #include "pycore_typeobject.h" // _PyType_GetModuleState() #include "structmember.h" -/* ABCs */ -extern PyTypeObject PyIOBase_Type; -extern PyTypeObject PyRawIOBase_Type; -extern PyTypeObject PyBufferedIOBase_Type; -extern PyTypeObject PyTextIOBase_Type; - /* Type specs */ +extern PyType_Spec bufferediobase_spec; extern PyType_Spec bufferedrandom_spec; extern PyType_Spec bufferedreader_spec; extern PyType_Spec bufferedrwpair_spec; @@ -22,8 +17,11 @@ extern PyType_Spec bufferedwriter_spec; extern PyType_Spec bytesio_spec; extern PyType_Spec bytesiobuf_spec; extern PyType_Spec fileio_spec; +extern PyType_Spec iobase_spec; extern PyType_Spec nldecoder_spec; +extern PyType_Spec rawiobase_spec; extern PyType_Spec stringio_spec; +extern PyType_Spec textiobase_spec; extern PyType_Spec textiowrapper_spec; #ifdef HAVE_WINDOWS_CONSOLE_IO @@ -168,9 +166,6 @@ struct _io_state { #endif }; -#define IO_MOD_STATE(mod) ((_PyIO_State *)PyModule_GetState(mod)) -#define IO_STATE() _PyIO_get_module_state() - static inline _PyIO_State * get_io_state(PyObject *module) { @@ -195,7 +190,7 @@ find_io_state_by_def(PyTypeObject *type) return get_io_state(mod); } -extern _PyIO_State *_PyIO_get_module_state(void); +extern PyObject *_PyIOBase_cannot_pickle(PyObject *self, PyObject *args); #ifdef HAVE_WINDOWS_CONSOLE_IO extern char _PyIO_get_console_type(PyObject *); diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 00e228bca4375b..d5cc047bd8d563 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -16,14 +16,14 @@ /*[clinic input] module _io -class _io._BufferedIOBase "PyObject *" "&PyBufferedIOBase_Type" -class _io._Buffered "buffered *" "&PyBufferedIOBase_Type" +class _io._BufferedIOBase "PyObject *" "clinic_state()->PyBufferedIOBase_Type" +class _io._Buffered "buffered *" "clinic_state()->PyBufferedIOBase_Type" class _io.BufferedReader "buffered *" "clinic_state()->PyBufferedReader_Type" class _io.BufferedWriter "buffered *" "clinic_state()->PyBufferedWriter_Type" class _io.BufferedRWPair "rwpair *" "clinic_state()->PyBufferedRWPair_Type" class _io.BufferedRandom "buffered *" "clinic_state()->PyBufferedRandom_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=abd685b9d94b9888]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=3b3ef9cbbbad4590]*/ /* * BufferedIOBase class, inherits from IOBase. @@ -128,7 +128,7 @@ static PyObject * _io__BufferedIOBase_detach_impl(PyObject *self, PyTypeObject *cls) /*[clinic end generated code: output=b87b135d67cd4448 input=0b61a7b4357c1ea7]*/ { - _PyIO_State *state = IO_STATE(); + _PyIO_State *state = get_io_state_by_cls(cls); return bufferediobase_unsupported(state, "detach"); } @@ -162,7 +162,7 @@ _io__BufferedIOBase_read_impl(PyObject *self, PyTypeObject *cls, PyObject *args) /*[clinic end generated code: output=4521b30940fd7b67 input=390205758adc8510]*/ { - _PyIO_State *state = IO_STATE(); + _PyIO_State *state = get_io_state_by_cls(cls); return bufferediobase_unsupported(state, "read"); } @@ -184,7 +184,7 @@ _io__BufferedIOBase_read1_impl(PyObject *self, PyTypeObject *cls, PyObject *args) /*[clinic end generated code: output=636fd241c21e050a input=ef546a1238c5b41c]*/ { - _PyIO_State *state = IO_STATE(); + _PyIO_State *state = get_io_state_by_cls(cls); return bufferediobase_unsupported(state, "read1"); } @@ -209,7 +209,7 @@ _io__BufferedIOBase_write_impl(PyObject *self, PyTypeObject *cls, PyObject *args) /*[clinic end generated code: output=d51feea4bcac9892 input=f79b72c4dccb3dc2]*/ { - _PyIO_State *state = IO_STATE(); + _PyIO_State *state = get_io_state_by_cls(cls); return bufferediobase_unsupported(state, "write"); } @@ -394,6 +394,15 @@ _enter_buffered_busy(buffered *self) (self->buffer_size * (size / self->buffer_size))) +static int +buffered_clear(buffered *self) +{ + self->ok = 0; + Py_CLEAR(self->raw); + Py_CLEAR(self->dict); + return 0; +} + static void buffered_dealloc(buffered *self) { @@ -405,7 +414,6 @@ buffered_dealloc(buffered *self) self->ok = 0; if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *)self); - Py_CLEAR(self->raw); if (self->buffer) { PyMem_Free(self->buffer); self->buffer = NULL; @@ -414,7 +422,7 @@ buffered_dealloc(buffered *self) PyThread_free_lock(self->lock); self->lock = NULL; } - Py_CLEAR(self->dict); + (void)buffered_clear(self); tp->tp_free((PyObject *)self); Py_DECREF(tp); } @@ -443,15 +451,6 @@ buffered_traverse(buffered *self, visitproc visit, void *arg) return 0; } -static int -buffered_clear(buffered *self) -{ - self->ok = 0; - Py_CLEAR(self->raw); - Py_CLEAR(self->dict); - return 0; -} - /* Because this can call arbitrary code, it shouldn't be called when the refcount is 0 (that is, not directly from tp_dealloc unless the refcount has been temporarily re-incremented). */ @@ -2220,6 +2219,8 @@ bufferedrwpair_traverse(rwpair *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); Py_VISIT(self->dict); + Py_VISIT(self->reader); + Py_VISIT(self->writer); return 0; } @@ -2239,9 +2240,7 @@ bufferedrwpair_dealloc(rwpair *self) _PyObject_GC_UNTRACK(self); if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *)self); - Py_CLEAR(self->reader); - Py_CLEAR(self->writer); - Py_CLEAR(self->dict); + (void)bufferedrwpair_clear(self); tp->tp_free((PyObject *) self); Py_DECREF(tp); } @@ -2424,6 +2423,12 @@ _io_BufferedRandom___init___impl(buffered *self, PyObject *raw, #include "clinic/bufferedio.c.h" #undef clinic_state +static int +bufferediobase_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} static PyMethodDef bufferediobase_methods[] = { _IO__BUFFEREDIOBASE_DETACH_METHODDEF @@ -2435,57 +2440,19 @@ static PyMethodDef bufferediobase_methods[] = { {NULL, NULL} }; -PyTypeObject PyBufferedIOBase_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io._BufferedIOBase", /*tp_name*/ - 0, /*tp_basicsize*/ - 0, /*tp_itemsize*/ - 0, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ - bufferediobase_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - bufferediobase_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - &PyIOBase_Type, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0, /* tp_finalize */ +static PyType_Slot bufferediobase_slots[] = { + {Py_tp_doc, (void *)bufferediobase_doc}, + {Py_tp_methods, bufferediobase_methods}, + {Py_tp_traverse, bufferediobase_traverse}, + {0, NULL}, }; +PyType_Spec bufferediobase_spec = { + .name = "_io._BufferedIOBase", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = bufferediobase_slots, +}; static PyMethodDef bufferedreader_methods[] = { /* BufferedIOMixin methods */ @@ -2508,6 +2475,9 @@ static PyMethodDef bufferedreader_methods[] = { _IO__BUFFERED_TELL_METHODDEF _IO__BUFFERED_TRUNCATE_METHODDEF _IO__BUFFERED___SIZEOF___METHODDEF + + {"__reduce__", _PyIOBase_cannot_pickle, METH_VARARGS}, + {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_VARARGS}, {NULL, NULL} }; @@ -2565,6 +2535,9 @@ static PyMethodDef bufferedwriter_methods[] = { _IO__BUFFERED_SEEK_METHODDEF _IO__BUFFERED_TELL_METHODDEF _IO__BUFFERED___SIZEOF___METHODDEF + + {"__reduce__", _PyIOBase_cannot_pickle, METH_VARARGS}, + {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_VARARGS}, {NULL, NULL} }; @@ -2680,6 +2653,9 @@ static PyMethodDef bufferedrandom_methods[] = { _IO__BUFFERED_PEEK_METHODDEF _IO_BUFFEREDWRITER_WRITE_METHODDEF _IO__BUFFERED___SIZEOF___METHODDEF + + {"__reduce__", _PyIOBase_cannot_pickle, METH_VARARGS}, + {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_VARARGS}, {NULL, NULL} }; diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 3fddfc2ed0bc9c..80773058693259 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -979,6 +979,7 @@ bytesio_traverse(bytesio *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); Py_VISIT(self->dict); + Py_VISIT(self->buf); return 0; } @@ -986,6 +987,7 @@ static int bytesio_clear(bytesio *self) { Py_CLEAR(self->dict); + Py_CLEAR(self->buf); return 0; } diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 473f0a8a6befcb..30944fc56bf70e 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -536,7 +536,7 @@ fileio_dealloc(fileio *self) _PyObject_GC_UNTRACK(self); if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) self); - Py_CLEAR(self->dict); + (void)fileio_clear(self); tp->tp_free((PyObject *)self); Py_DECREF(tp); } @@ -1166,6 +1166,8 @@ static PyMethodDef fileio_methods[] = { _IO_FILEIO_FILENO_METHODDEF _IO_FILEIO_ISATTY_METHODDEF {"_dealloc_warn", (PyCFunction)fileio_dealloc_warn, METH_O, NULL}, + {"__reduce__", _PyIOBase_cannot_pickle, METH_VARARGS}, + {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index 26f2a3155bda62..764c5fb3320072 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -17,10 +17,10 @@ /*[clinic input] module _io -class _io._IOBase "PyObject *" "&PyIOBase_Type" -class _io._RawIOBase "PyObject *" "&PyRawIOBase_Type" +class _io._IOBase "PyObject *" "clinic_state()->PyIOBase_Type" +class _io._RawIOBase "PyObject *" "clinic_state()->PyRawIOBase_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=d29a4d076c2b211c]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=9006b7802ab8ea85]*/ /* * IOBase class, an abstract class @@ -101,7 +101,7 @@ static PyObject * _io__IOBase_seek_impl(PyObject *self, PyTypeObject *cls, PyObject *args) /*[clinic end generated code: output=1dd694ac9de260fa input=ebb5476eb22fc5d4]*/ { - _PyIO_State *state = IO_STATE(); + _PyIO_State *state = get_io_state_by_cls(cls); return iobase_unsupported(state, "seek"); } @@ -134,7 +134,7 @@ static PyObject * _io__IOBase_truncate_impl(PyObject *self, PyTypeObject *cls, PyObject *args) /*[clinic end generated code: output=b7eed4649cbe22c1 input=ad90582a1d8b5cc9]*/ { - _PyIO_State *state = IO_STATE(); + _PyIO_State *state = get_io_state_by_cls(cls); return iobase_unsupported(state, "truncate"); } @@ -220,24 +220,32 @@ _PyIOBase_check_closed(PyObject *self, PyObject *args) static PyObject * iobase_check_seekable(PyObject *self, PyObject *args) { - _PyIO_State *state = IO_STATE(); + _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); return _PyIOBase_check_seekable(state, self, args); } static PyObject * iobase_check_readable(PyObject *self, PyObject *args) { - _PyIO_State *state = IO_STATE(); + _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); return _PyIOBase_check_readable(state, self, args); } static PyObject * iobase_check_writable(PyObject *self, PyObject *args) { - _PyIO_State *state = IO_STATE(); + _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); return _PyIOBase_check_writable(state, self, args); } +PyObject * +_PyIOBase_cannot_pickle(PyObject *self, PyObject *args) +{ + PyErr_Format(PyExc_TypeError, + "cannot pickle '%.100s' instances", _PyType_Name(Py_TYPE(self))); + return NULL; +} + /* XXX: IOBase thinks it has to maintain its own internal state in `__IOBase_closed` and call flush() by itself, but it is redundant with whatever behaviour a non-trivial derived class will implement. */ @@ -351,6 +359,7 @@ _PyIOBase_finalize(PyObject *self) static int iobase_traverse(iobase *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); Py_VISIT(self->dict); return 0; } @@ -380,11 +389,13 @@ iobase_dealloc(iobase *self) } return; } + PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) self); Py_CLEAR(self->dict); - Py_TYPE(self)->tp_free((PyObject *) self); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); } /* Inquiry methods */ @@ -523,7 +534,7 @@ static PyObject * _io__IOBase_fileno_impl(PyObject *self, PyTypeObject *cls) /*[clinic end generated code: output=7caaa32a6f4ada3d input=1927c8bea5c85099]*/ { - _PyIO_State *state = IO_STATE(); + _PyIO_State *state = get_io_state_by_cls(cls); return iobase_unsupported(state, "fileno"); } @@ -821,7 +832,9 @@ _io__IOBase_writelines(PyObject *self, PyObject *lines) Py_RETURN_NONE; } +#define clinic_state() (find_io_state_by_def(Py_TYPE(self))) #include "clinic/iobase.c.h" +#undef clinic_state static PyMethodDef iobase_methods[] = { _IO__IOBASE_SEEK_METHODDEF @@ -858,59 +871,34 @@ static PyGetSetDef iobase_getset[] = { {NULL} }; +static struct PyMemberDef iobase_members[] = { + {"__weaklistoffset__", T_PYSSIZET, offsetof(iobase, weakreflist), READONLY}, + {"__dictoffset__", T_PYSSIZET, offsetof(iobase, dict), READONLY}, + {NULL}, +}; + -PyTypeObject PyIOBase_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io._IOBase", /*tp_name*/ - sizeof(iobase), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)iobase_dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - iobase_doc, /* tp_doc */ - (traverseproc)iobase_traverse, /* tp_traverse */ - (inquiry)iobase_clear, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(iobase, weakreflist), /* tp_weaklistoffset */ - iobase_iter, /* tp_iter */ - iobase_iternext, /* tp_iternext */ - iobase_methods, /* tp_methods */ - 0, /* tp_members */ - iobase_getset, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - offsetof(iobase, dict), /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - iobase_finalize, /* tp_finalize */ +static PyType_Slot iobase_slots[] = { + {Py_tp_dealloc, iobase_dealloc}, + {Py_tp_doc, (void *)iobase_doc}, + {Py_tp_traverse, iobase_traverse}, + {Py_tp_clear, iobase_clear}, + {Py_tp_iter, iobase_iter}, + {Py_tp_iternext, iobase_iternext}, + {Py_tp_methods, iobase_methods}, + {Py_tp_members, iobase_members}, + {Py_tp_getset, iobase_getset}, + {Py_tp_finalize, iobase_finalize}, + {0, NULL}, }; +PyType_Spec iobase_spec = { + .name = "_io._IOBase", + .basicsize = sizeof(iobase), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = iobase_slots, +}; /* * RawIOBase class, Inherits from IOBase. @@ -1045,6 +1033,13 @@ rawiobase_write(PyObject *self, PyObject *args) return NULL; } +static int +rawiobase_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + static PyMethodDef rawiobase_methods[] = { _IO__RAWIOBASE_READ_METHODDEF _IO__RAWIOBASE_READALL_METHODDEF @@ -1053,53 +1048,16 @@ static PyMethodDef rawiobase_methods[] = { {NULL, NULL} }; -PyTypeObject PyRawIOBase_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io._RawIOBase", /*tp_name*/ - 0, /*tp_basicsize*/ - 0, /*tp_itemsize*/ - 0, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ - rawiobase_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - rawiobase_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - &PyIOBase_Type, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0, /* tp_finalize */ +static PyType_Slot rawiobase_slots[] = { + {Py_tp_doc, (void *)rawiobase_doc}, + {Py_tp_methods, rawiobase_methods}, + {Py_tp_traverse, rawiobase_traverse}, + {0, NULL}, +}; + +PyType_Spec rawiobase_spec = { + .name = "_io._RawIOBase", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = rawiobase_slots, }; diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 13d3b870b39a81..3eb25704b4aa55 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -583,6 +583,9 @@ static int stringio_traverse(stringio *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); + Py_VISIT(self->readnl); + Py_VISIT(self->writenl); + Py_VISIT(self->decoder); Py_VISIT(self->dict); return 0; } @@ -590,6 +593,9 @@ stringio_traverse(stringio *self, visitproc visit, void *arg) static int stringio_clear(stringio *self) { + Py_CLEAR(self->readnl); + Py_CLEAR(self->writenl); + Py_CLEAR(self->decoder); Py_CLEAR(self->dict); return 0; } @@ -605,10 +611,7 @@ stringio_dealloc(stringio *self) self->buf = NULL; } _PyUnicodeWriter_Dealloc(&self->writer); - Py_CLEAR(self->readnl); - Py_CLEAR(self->writenl); - Py_CLEAR(self->decoder); - Py_CLEAR(self->dict); + (void)stringio_clear(self); if (self->weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject *) self); } diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 3cc292cc35102e..81dd3bed005a61 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -62,7 +62,7 @@ static PyObject * _io__TextIOBase_detach_impl(PyObject *self, PyTypeObject *cls) /*[clinic end generated code: output=50915f40c609eaa4 input=987ca3640d0a3776]*/ { - _PyIO_State *state = IO_STATE(); + _PyIO_State *state = get_io_state_by_cls(cls); return _unsupported(state, "detach"); } @@ -82,7 +82,7 @@ static PyObject * _io__TextIOBase_read_impl(PyObject *self, PyTypeObject *cls, PyObject *args) /*[clinic end generated code: output=3adf28998831f461 input=cee1e84664a20de0]*/ { - _PyIO_State *state = IO_STATE(); + _PyIO_State *state = get_io_state_by_cls(cls); return _unsupported(state, "read"); } @@ -102,7 +102,7 @@ _io__TextIOBase_readline_impl(PyObject *self, PyTypeObject *cls, PyObject *args) /*[clinic end generated code: output=3073a948d02319f3 input=58f801259f7ff3ef]*/ { - _PyIO_State *state = IO_STATE(); + _PyIO_State *state = get_io_state_by_cls(cls); return _unsupported(state, "readline"); } @@ -122,7 +122,7 @@ static PyObject * _io__TextIOBase_write_impl(PyObject *self, PyTypeObject *cls, PyObject *args) /*[clinic end generated code: output=5d985eb529472bc4 input=21b6961b5cba9496]*/ { - _PyIO_State *state = IO_STATE(); + _PyIO_State *state = get_io_state_by_cls(cls); return _unsupported(state, "write"); } @@ -164,6 +164,12 @@ textiobase_errors_get(PyObject *self, void *context) Py_RETURN_NONE; } +static int +textiobase_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} static PyMethodDef textiobase_methods[] = { _IO__TEXTIOBASE_DETACH_METHODDEF @@ -180,57 +186,20 @@ static PyGetSetDef textiobase_getset[] = { {NULL} }; -PyTypeObject PyTextIOBase_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io._TextIOBase", /*tp_name*/ - 0, /*tp_basicsize*/ - 0, /*tp_itemsize*/ - 0, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ - textiobase_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - textiobase_methods, /* tp_methods */ - 0, /* tp_members */ - textiobase_getset, /* tp_getset */ - &PyIOBase_Type, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0, /* tp_finalize */ +static PyType_Slot textiobase_slots[] = { + {Py_tp_doc, (void *)textiobase_doc}, + {Py_tp_methods, textiobase_methods}, + {Py_tp_getset, textiobase_getset}, + {Py_tp_traverse, textiobase_traverse}, + {0, NULL}, }; +PyType_Spec textiobase_spec = { + .name = "_io._TextIOBase", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = textiobase_slots, +}; /* IncrementalNewlineDecoder */ @@ -1456,7 +1425,7 @@ textiowrapper_dealloc(textio *self) _PyObject_GC_UNTRACK(self); if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *)self); - textiowrapper_clear(self); + (void)textiowrapper_clear(self); tp->tp_free((PyObject *)self); Py_DECREF(tp); } @@ -3267,6 +3236,9 @@ static PyMethodDef textiowrapper_methods[] = { _IO_TEXTIOWRAPPER_SEEK_METHODDEF _IO_TEXTIOWRAPPER_TELL_METHODDEF _IO_TEXTIOWRAPPER_TRUNCATE_METHODDEF + + {"__reduce__", _PyIOBase_cannot_pickle, METH_VARARGS}, + {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_VARARGS}, {NULL, NULL} }; diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index d65e247737a071..15f3053957da61 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -1096,7 +1096,7 @@ _io__WindowsConsoleIO_isatty_impl(winconsoleio *self) Py_RETURN_TRUE; } -#define clinic_state() (IO_STATE()) +#define clinic_state() (find_io_state_by_def(Py_TYPE(self))) #include "clinic/winconsoleio.c.h" #undef clinic_state diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 61f87c5eba60ed..c5dc0f44a38068 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -29,9 +29,6 @@ #include "pycore_unicodeobject.h" // _PyUnicode_InitTypes() #include "opcode.h" -extern PyStatus _PyIO_InitTypes(PyInterpreterState *interp); -extern void _PyIO_FiniTypes(PyInterpreterState *interp); - #include // setlocale() #include // getenv() @@ -706,11 +703,6 @@ pycore_init_types(PyInterpreterState *interp) return _PyStatus_ERR("failed to initialize an exception type"); } - status = _PyIO_InitTypes(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - status = _PyExc_InitGlobalObjects(interp); if (_PyStatus_EXCEPTION(status)) { return status; @@ -1667,8 +1659,6 @@ flush_std_files(void) static void finalize_interp_types(PyInterpreterState *interp) { - _PyIO_FiniTypes(interp); - _PyUnicode_FiniTypes(interp); _PySys_FiniTypes(interp); _PyExc_Fini(interp); diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 8afa92ef25d376..9863acdade308b 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -317,10 +317,6 @@ Python/instrumentation.c - _PyInstrumentation_MISSING - ##----------------------- ## static types -Modules/_io/bufferedio.c - PyBufferedIOBase_Type - -Modules/_io/iobase.c - PyIOBase_Type - -Modules/_io/iobase.c - PyRawIOBase_Type - -Modules/_io/textio.c - PyTextIOBase_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorBase_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorDerived_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorNopGet_Type - From b378d991f8cd41c33416e590cb83472cce1d6b98 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 15 May 2023 13:21:38 +0200 Subject: [PATCH 204/204] gh-101819: Fix _io clinic input for unused base class method stubs (#104418) When preparing the _io extension module for isolation, many methods were adapted to Argument Clinic. Some of these used the '*args: object' signature, which is incorrect. These are now corrected to an exact signature, and marked unused, since they are stub methods. --- Modules/_io/bufferedio.c | 26 +++++----- Modules/_io/clinic/bufferedio.c.h | 69 +++++++++++++++----------- Modules/_io/clinic/iobase.c.h | 58 ++++++++++++++-------- Modules/_io/clinic/textio.c.h | 80 ++++++++++++++++++++----------- Modules/_io/iobase.c | 21 ++++---- Modules/_io/textio.c | 23 +++++---- 6 files changed, 168 insertions(+), 109 deletions(-) diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index d5cc047bd8d563..6f291c34496064 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -136,15 +136,15 @@ _io__BufferedIOBase_detach_impl(PyObject *self, PyTypeObject *cls) _io._BufferedIOBase.read cls: defining_class + size: int(unused=True) = -1 / - *args: object Read and return up to n bytes. -If the argument is omitted, None, or negative, read and +If the size argument is omitted, None, or negative, read and return all data until EOF. -If the argument is positive, and the underlying raw stream is +If the size argument is positive, and the underlying raw stream is not 'interactive', multiple raw reads may be issued to satisfy the byte count (unless EOF is reached first). However, for interactive raw streams (as well as sockets and pipes), @@ -159,8 +159,8 @@ mode and no data is available at the moment. static PyObject * _io__BufferedIOBase_read_impl(PyObject *self, PyTypeObject *cls, - PyObject *args) -/*[clinic end generated code: output=4521b30940fd7b67 input=390205758adc8510]*/ + int Py_UNUSED(size)) +/*[clinic end generated code: output=aceb2765587b0a29 input=824f6f910465e61a]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return bufferediobase_unsupported(state, "read"); @@ -170,10 +170,10 @@ _io__BufferedIOBase_read_impl(PyObject *self, PyTypeObject *cls, _io._BufferedIOBase.read1 cls: defining_class + size: int(unused=True) = -1 / - *args: object -Read and return up to n bytes, with at most one read() call to the underlying raw stream. +Read and return up to size bytes, with at most one read() call to the underlying raw stream. Return an empty bytes object on EOF. A short result does not imply that EOF is imminent. @@ -181,8 +181,8 @@ A short result does not imply that EOF is imminent. static PyObject * _io__BufferedIOBase_read1_impl(PyObject *self, PyTypeObject *cls, - PyObject *args) -/*[clinic end generated code: output=636fd241c21e050a input=ef546a1238c5b41c]*/ + int Py_UNUSED(size)) +/*[clinic end generated code: output=2e7fc62972487eaa input=af76380e020fd9e6]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return bufferediobase_unsupported(state, "read1"); @@ -192,10 +192,10 @@ _io__BufferedIOBase_read1_impl(PyObject *self, PyTypeObject *cls, _io._BufferedIOBase.write cls: defining_class + b: object(unused=True) / - *args: object -Write the given buffer to the IO stream. +Write buffer b to the IO stream. Return the number of bytes written, which is always the length of b in bytes. @@ -206,8 +206,8 @@ underlying raw stream cannot accept more data at the moment. static PyObject * _io__BufferedIOBase_write_impl(PyObject *self, PyTypeObject *cls, - PyObject *args) -/*[clinic end generated code: output=d51feea4bcac9892 input=f79b72c4dccb3dc2]*/ + PyObject *Py_UNUSED(b)) +/*[clinic end generated code: output=712c635246bf2306 input=9793f5c8f71029ad]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return bufferediobase_unsupported(state, "write"); diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index e2ac90c54a10f0..4f40fdadf8ec85 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -108,15 +108,15 @@ _io__BufferedIOBase_detach(PyObject *self, PyTypeObject *cls, PyObject *const *a } PyDoc_STRVAR(_io__BufferedIOBase_read__doc__, -"read($self, /, *args)\n" +"read($self, size=-1, /)\n" "--\n" "\n" "Read and return up to n bytes.\n" "\n" -"If the argument is omitted, None, or negative, read and\n" +"If the size argument is omitted, None, or negative, read and\n" "return all data until EOF.\n" "\n" -"If the argument is positive, and the underlying raw stream is\n" +"If the size argument is positive, and the underlying raw stream is\n" "not \'interactive\', multiple raw reads may be issued to satisfy\n" "the byte count (unless EOF is reached first).\n" "However, for interactive raw streams (as well as sockets and pipes),\n" @@ -133,7 +133,7 @@ PyDoc_STRVAR(_io__BufferedIOBase_read__doc__, static PyObject * _io__BufferedIOBase_read_impl(PyObject *self, PyTypeObject *cls, - PyObject *args); + int Py_UNUSED(size)); static PyObject * _io__BufferedIOBase_read(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -145,7 +145,7 @@ _io__BufferedIOBase_read(PyObject *self, PyTypeObject *cls, PyObject *const *arg # define KWTUPLE NULL #endif - static const char * const _keywords[] = { NULL}; + static const char * const _keywords[] = {"", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "read", @@ -153,25 +153,31 @@ _io__BufferedIOBase_read(PyObject *self, PyTypeObject *cls, PyObject *const *arg }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *__clinic_args = NULL; + int size = -1; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); if (!args) { goto exit; } - __clinic_args = args[0]; - return_value = _io__BufferedIOBase_read_impl(self, cls, __clinic_args); + if (nargs < 1) { + goto skip_optional_posonly; + } + size = _PyLong_AsInt(args[0]); + if (size == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_posonly: + return_value = _io__BufferedIOBase_read_impl(self, cls, size); exit: - Py_XDECREF(__clinic_args); return return_value; } PyDoc_STRVAR(_io__BufferedIOBase_read1__doc__, -"read1($self, /, *args)\n" +"read1($self, size=-1, /)\n" "--\n" "\n" -"Read and return up to n bytes, with at most one read() call to the underlying raw stream.\n" +"Read and return up to size bytes, with at most one read() call to the underlying raw stream.\n" "\n" "Return an empty bytes object on EOF.\n" "A short result does not imply that EOF is imminent."); @@ -181,7 +187,7 @@ PyDoc_STRVAR(_io__BufferedIOBase_read1__doc__, static PyObject * _io__BufferedIOBase_read1_impl(PyObject *self, PyTypeObject *cls, - PyObject *args); + int Py_UNUSED(size)); static PyObject * _io__BufferedIOBase_read1(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -193,7 +199,7 @@ _io__BufferedIOBase_read1(PyObject *self, PyTypeObject *cls, PyObject *const *ar # define KWTUPLE NULL #endif - static const char * const _keywords[] = { NULL}; + static const char * const _keywords[] = {"", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "read1", @@ -201,25 +207,31 @@ _io__BufferedIOBase_read1(PyObject *self, PyTypeObject *cls, PyObject *const *ar }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *__clinic_args = NULL; + int size = -1; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); if (!args) { goto exit; } - __clinic_args = args[0]; - return_value = _io__BufferedIOBase_read1_impl(self, cls, __clinic_args); + if (nargs < 1) { + goto skip_optional_posonly; + } + size = _PyLong_AsInt(args[0]); + if (size == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_posonly: + return_value = _io__BufferedIOBase_read1_impl(self, cls, size); exit: - Py_XDECREF(__clinic_args); return return_value; } PyDoc_STRVAR(_io__BufferedIOBase_write__doc__, -"write($self, /, *args)\n" +"write($self, b, /)\n" "--\n" "\n" -"Write the given buffer to the IO stream.\n" +"Write buffer b to the IO stream.\n" "\n" "Return the number of bytes written, which is always\n" "the length of b in bytes.\n" @@ -232,7 +244,7 @@ PyDoc_STRVAR(_io__BufferedIOBase_write__doc__, static PyObject * _io__BufferedIOBase_write_impl(PyObject *self, PyTypeObject *cls, - PyObject *args); + PyObject *Py_UNUSED(b)); static PyObject * _io__BufferedIOBase_write(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -244,7 +256,7 @@ _io__BufferedIOBase_write(PyObject *self, PyTypeObject *cls, PyObject *const *ar # define KWTUPLE NULL #endif - static const char * const _keywords[] = { NULL}; + static const char * const _keywords[] = {"", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "write", @@ -252,17 +264,16 @@ _io__BufferedIOBase_write(PyObject *self, PyTypeObject *cls, PyObject *const *ar }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *__clinic_args = NULL; + PyObject *b; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - __clinic_args = args[0]; - return_value = _io__BufferedIOBase_write_impl(self, cls, __clinic_args); + b = args[0]; + return_value = _io__BufferedIOBase_write_impl(self, cls, b); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -1087,4 +1098,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=d770e392e8702e12 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b7ddf84a5bc2bf34 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/iobase.c.h b/Modules/_io/clinic/iobase.c.h index 7e6b3b5b78e8f6..773e0010477053 100644 --- a/Modules/_io/clinic/iobase.c.h +++ b/Modules/_io/clinic/iobase.c.h @@ -9,7 +9,7 @@ preserve PyDoc_STRVAR(_io__IOBase_seek__doc__, -"seek($self, /, *args)\n" +"seek($self, offset, whence=os.SEEK_SET, /)\n" "--\n" "\n" "Change the stream position to the given byte offset.\n" @@ -17,9 +17,9 @@ PyDoc_STRVAR(_io__IOBase_seek__doc__, "The offset is interpreted relative to the position indicated by whence.\n" "Values for whence are:\n" "\n" -"* 0 -- start of stream (the default); offset should be zero or positive\n" -"* 1 -- current stream position; offset may be negative\n" -"* 2 -- end of stream; offset is usually negative\n" +"* os.SEEK_SET or 0 -- start of stream (the default); offset should be zero or positive\n" +"* os.SEEK_CUR or 1 -- current stream position; offset may be negative\n" +"* os.SEEK_END or 2 -- end of stream; offset is usually negative\n" "\n" "Return the new absolute position."); @@ -27,7 +27,8 @@ PyDoc_STRVAR(_io__IOBase_seek__doc__, {"seek", _PyCFunction_CAST(_io__IOBase_seek), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__IOBase_seek__doc__}, static PyObject * -_io__IOBase_seek_impl(PyObject *self, PyTypeObject *cls, PyObject *args); +_io__IOBase_seek_impl(PyObject *self, PyTypeObject *cls, + int Py_UNUSED(offset), int Py_UNUSED(whence)); static PyObject * _io__IOBase_seek(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -39,25 +40,36 @@ _io__IOBase_seek(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ss # define KWTUPLE NULL #endif - static const char * const _keywords[] = { NULL}; + static const char * const _keywords[] = {"", "", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "seek", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject *__clinic_args = NULL; + PyObject *argsbuf[2]; + int offset; + int whence = 0; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); if (!args) { goto exit; } - __clinic_args = args[0]; - return_value = _io__IOBase_seek_impl(self, cls, __clinic_args); + offset = _PyLong_AsInt(args[0]); + if (offset == -1 && PyErr_Occurred()) { + goto exit; + } + if (nargs < 2) { + goto skip_optional_posonly; + } + whence = _PyLong_AsInt(args[1]); + if (whence == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_posonly: + return_value = _io__IOBase_seek_impl(self, cls, offset, whence); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -80,7 +92,7 @@ _io__IOBase_tell(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(_io__IOBase_truncate__doc__, -"truncate($self, /, *args)\n" +"truncate($self, size=None, /)\n" "--\n" "\n" "Truncate file to size bytes.\n" @@ -92,7 +104,8 @@ PyDoc_STRVAR(_io__IOBase_truncate__doc__, {"truncate", _PyCFunction_CAST(_io__IOBase_truncate), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__IOBase_truncate__doc__}, static PyObject * -_io__IOBase_truncate_impl(PyObject *self, PyTypeObject *cls, PyObject *args); +_io__IOBase_truncate_impl(PyObject *self, PyTypeObject *cls, + PyObject *Py_UNUSED(size)); static PyObject * _io__IOBase_truncate(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -104,7 +117,7 @@ _io__IOBase_truncate(PyObject *self, PyTypeObject *cls, PyObject *const *args, P # define KWTUPLE NULL #endif - static const char * const _keywords[] = { NULL}; + static const char * const _keywords[] = {"", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "truncate", @@ -112,17 +125,20 @@ _io__IOBase_truncate(PyObject *self, PyTypeObject *cls, PyObject *const *args, P }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *__clinic_args = NULL; + PyObject *size = Py_None; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); if (!args) { goto exit; } - __clinic_args = args[0]; - return_value = _io__IOBase_truncate_impl(self, cls, __clinic_args); + if (nargs < 1) { + goto skip_optional_posonly; + } + size = args[0]; +skip_optional_posonly: + return_value = _io__IOBase_truncate_impl(self, cls, size); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -420,4 +436,4 @@ _io__RawIOBase_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _io__RawIOBase_readall_impl(self); } -/*[clinic end generated code: output=63bc25a5bfcecaf0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=301b22f8f75ce3dc input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 01965013ec6abc..33fc23bd4c0c69 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -33,7 +33,7 @@ _io__TextIOBase_detach(PyObject *self, PyTypeObject *cls, PyObject *const *args, } PyDoc_STRVAR(_io__TextIOBase_read__doc__, -"read($self, /, *args)\n" +"read($self, size=-1, /)\n" "--\n" "\n" "Read at most size characters from stream.\n" @@ -45,7 +45,8 @@ PyDoc_STRVAR(_io__TextIOBase_read__doc__, {"read", _PyCFunction_CAST(_io__TextIOBase_read), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__TextIOBase_read__doc__}, static PyObject * -_io__TextIOBase_read_impl(PyObject *self, PyTypeObject *cls, PyObject *args); +_io__TextIOBase_read_impl(PyObject *self, PyTypeObject *cls, + int Py_UNUSED(size)); static PyObject * _io__TextIOBase_read(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -57,7 +58,7 @@ _io__TextIOBase_read(PyObject *self, PyTypeObject *cls, PyObject *const *args, P # define KWTUPLE NULL #endif - static const char * const _keywords[] = { NULL}; + static const char * const _keywords[] = {"", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "read", @@ -65,34 +66,41 @@ _io__TextIOBase_read(PyObject *self, PyTypeObject *cls, PyObject *const *args, P }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *__clinic_args = NULL; + int size = -1; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); if (!args) { goto exit; } - __clinic_args = args[0]; - return_value = _io__TextIOBase_read_impl(self, cls, __clinic_args); + if (nargs < 1) { + goto skip_optional_posonly; + } + size = _PyLong_AsInt(args[0]); + if (size == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_posonly: + return_value = _io__TextIOBase_read_impl(self, cls, size); exit: - Py_XDECREF(__clinic_args); return return_value; } PyDoc_STRVAR(_io__TextIOBase_readline__doc__, -"readline($self, /, *args)\n" +"readline($self, size=-1, /)\n" "--\n" "\n" "Read until newline or EOF.\n" "\n" -"Return an empty string if EOF is hit immediately."); +"Return an empty string if EOF is hit immediately.\n" +"If size is specified, at most size characters will be read."); #define _IO__TEXTIOBASE_READLINE_METHODDEF \ {"readline", _PyCFunction_CAST(_io__TextIOBase_readline), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__TextIOBase_readline__doc__}, static PyObject * _io__TextIOBase_readline_impl(PyObject *self, PyTypeObject *cls, - PyObject *args); + int Py_UNUSED(size)); static PyObject * _io__TextIOBase_readline(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -104,7 +112,7 @@ _io__TextIOBase_readline(PyObject *self, PyTypeObject *cls, PyObject *const *arg # define KWTUPLE NULL #endif - static const char * const _keywords[] = { NULL}; + static const char * const _keywords[] = {"", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "readline", @@ -112,25 +120,31 @@ _io__TextIOBase_readline(PyObject *self, PyTypeObject *cls, PyObject *const *arg }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *__clinic_args = NULL; + int size = -1; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); if (!args) { goto exit; } - __clinic_args = args[0]; - return_value = _io__TextIOBase_readline_impl(self, cls, __clinic_args); + if (nargs < 1) { + goto skip_optional_posonly; + } + size = _PyLong_AsInt(args[0]); + if (size == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_posonly: + return_value = _io__TextIOBase_readline_impl(self, cls, size); exit: - Py_XDECREF(__clinic_args); return return_value; } PyDoc_STRVAR(_io__TextIOBase_write__doc__, -"write($self, /, *args)\n" +"write($self, s, /)\n" "--\n" "\n" -"Write string to stream.\n" +"Write string s to stream.\n" "\n" "Return the number of characters written\n" "(which is always equal to the length of the string)."); @@ -139,7 +153,8 @@ PyDoc_STRVAR(_io__TextIOBase_write__doc__, {"write", _PyCFunction_CAST(_io__TextIOBase_write), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__TextIOBase_write__doc__}, static PyObject * -_io__TextIOBase_write_impl(PyObject *self, PyTypeObject *cls, PyObject *args); +_io__TextIOBase_write_impl(PyObject *self, PyTypeObject *cls, + const char *Py_UNUSED(s)); static PyObject * _io__TextIOBase_write(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -151,7 +166,7 @@ _io__TextIOBase_write(PyObject *self, PyTypeObject *cls, PyObject *const *args, # define KWTUPLE NULL #endif - static const char * const _keywords[] = { NULL}; + static const char * const _keywords[] = {"", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "write", @@ -159,17 +174,28 @@ _io__TextIOBase_write(PyObject *self, PyTypeObject *cls, PyObject *const *args, }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *__clinic_args = NULL; + const char *s; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - __clinic_args = args[0]; - return_value = _io__TextIOBase_write_impl(self, cls, __clinic_args); + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("write", "argument 1", "str", args[0]); + goto exit; + } + Py_ssize_t s_length; + s = PyUnicode_AsUTF8AndSize(args[0], &s_length); + if (s == NULL) { + goto exit; + } + if (strlen(s) != (size_t)s_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + return_value = _io__TextIOBase_write_impl(self, cls, s); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -934,4 +960,4 @@ _io_TextIOWrapper_close(textio *self, PyObject *Py_UNUSED(ignored)) { return _io_TextIOWrapper_close_impl(self); } -/*[clinic end generated code: output=d800e5a8a50d6720 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=42f592331302973f input=a9049054013a1b77]*/ diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index 764c5fb3320072..14d48813aefe83 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -82,24 +82,26 @@ iobase_unsupported(_PyIO_State *state, const char *message) /*[clinic input] _io._IOBase.seek cls: defining_class + offset: int(unused=True) + whence: int(unused=True, c_default='0') = os.SEEK_SET / - *args: object Change the stream position to the given byte offset. The offset is interpreted relative to the position indicated by whence. Values for whence are: -* 0 -- start of stream (the default); offset should be zero or positive -* 1 -- current stream position; offset may be negative -* 2 -- end of stream; offset is usually negative +* os.SEEK_SET or 0 -- start of stream (the default); offset should be zero or positive +* os.SEEK_CUR or 1 -- current stream position; offset may be negative +* os.SEEK_END or 2 -- end of stream; offset is usually negative Return the new absolute position. [clinic start generated code]*/ static PyObject * -_io__IOBase_seek_impl(PyObject *self, PyTypeObject *cls, PyObject *args) -/*[clinic end generated code: output=1dd694ac9de260fa input=ebb5476eb22fc5d4]*/ +_io__IOBase_seek_impl(PyObject *self, PyTypeObject *cls, + int Py_UNUSED(offset), int Py_UNUSED(whence)) +/*[clinic end generated code: output=8bd74ea6538ded53 input=8d4e6adcd08292f2]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return iobase_unsupported(state, "seek"); @@ -121,8 +123,8 @@ _io__IOBase_tell_impl(PyObject *self) /*[clinic input] _io._IOBase.truncate cls: defining_class + size: object(unused=True) = None / - *args: object Truncate file to size bytes. @@ -131,8 +133,9 @@ as reported by tell(). Return the new size. [clinic start generated code]*/ static PyObject * -_io__IOBase_truncate_impl(PyObject *self, PyTypeObject *cls, PyObject *args) -/*[clinic end generated code: output=b7eed4649cbe22c1 input=ad90582a1d8b5cc9]*/ +_io__IOBase_truncate_impl(PyObject *self, PyTypeObject *cls, + PyObject *Py_UNUSED(size)) +/*[clinic end generated code: output=2013179bff1fe8ef input=660ac20936612c27]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return iobase_unsupported(state, "truncate"); diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 81dd3bed005a61..e858a1fb498f82 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -69,8 +69,8 @@ _io__TextIOBase_detach_impl(PyObject *self, PyTypeObject *cls) /*[clinic input] _io._TextIOBase.read cls: defining_class + size: int(unused=True) = -1 / - *args: object Read at most size characters from stream. @@ -79,8 +79,9 @@ If size is negative or omitted, read until EOF. [clinic start generated code]*/ static PyObject * -_io__TextIOBase_read_impl(PyObject *self, PyTypeObject *cls, PyObject *args) -/*[clinic end generated code: output=3adf28998831f461 input=cee1e84664a20de0]*/ +_io__TextIOBase_read_impl(PyObject *self, PyTypeObject *cls, + int Py_UNUSED(size)) +/*[clinic end generated code: output=51a5178a309ce647 input=f5e37720f9fc563f]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return _unsupported(state, "read"); @@ -89,18 +90,19 @@ _io__TextIOBase_read_impl(PyObject *self, PyTypeObject *cls, PyObject *args) /*[clinic input] _io._TextIOBase.readline cls: defining_class + size: int(unused=True) = -1 / - *args: object Read until newline or EOF. Return an empty string if EOF is hit immediately. +If size is specified, at most size characters will be read. [clinic start generated code]*/ static PyObject * _io__TextIOBase_readline_impl(PyObject *self, PyTypeObject *cls, - PyObject *args) -/*[clinic end generated code: output=3073a948d02319f3 input=58f801259f7ff3ef]*/ + int Py_UNUSED(size)) +/*[clinic end generated code: output=3f47d7966d6d074e input=42eafec94107fa27]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return _unsupported(state, "readline"); @@ -109,18 +111,19 @@ _io__TextIOBase_readline_impl(PyObject *self, PyTypeObject *cls, /*[clinic input] _io._TextIOBase.write cls: defining_class + s: str(unused=True) / - *args: object -Write string to stream. +Write string s to stream. Return the number of characters written (which is always equal to the length of the string). [clinic start generated code]*/ static PyObject * -_io__TextIOBase_write_impl(PyObject *self, PyTypeObject *cls, PyObject *args) -/*[clinic end generated code: output=5d985eb529472bc4 input=21b6961b5cba9496]*/ +_io__TextIOBase_write_impl(PyObject *self, PyTypeObject *cls, + const char *Py_UNUSED(s)) +/*[clinic end generated code: output=18b28231460275de input=e9cabaa5f6732b07]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return _unsupported(state, "write");