From 04492cbc9aa45ac2c12d22083c406a0364c39f5b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 22 Jun 2023 09:48:19 +0100 Subject: [PATCH] GH-91095: Specialize calls to normal Python classes. (GH-99331) --- Include/cpython/object.h | 1 + Include/internal/pycore_frame.h | 24 ++ Include/internal/pycore_object.h | 2 + Include/internal/pycore_opcode.h | 46 ++-- Include/opcode.h | 108 ++++---- Lib/_opcode_metadata.py | 1 + Lib/opcode.py | 1 + Lib/test/test_sys.py | 2 +- Lib/test/test_sys_settrace.py | 24 +- ...2-11-10-13-04-35.gh-issue-91095.4E3Pwn.rst | 11 + Objects/dictobject.c | 7 +- Objects/typeobject.c | 20 ++ Python/bytecodes.c | 63 +++++ Python/ceval.c | 4 +- Python/compile.c | 3 + Python/generated_cases.c.h | 230 ++++++++++++------ Python/instrumentation.c | 2 +- Python/opcode_metadata.h | 10 + Python/opcode_targets.h | 42 ++-- Python/specialize.c | 99 +++++++- 20 files changed, 511 insertions(+), 189 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-11-10-13-04-35.gh-issue-91095.4E3Pwn.rst diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 7d69231aaa3119..71d268fae1a6dc 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -246,6 +246,7 @@ struct _specialization_cache { // *args nor **kwargs (as required by BINARY_SUBSCR_GETITEM): PyObject *getitem; uint32_t getitem_version; + PyObject *init; }; /* The *real* layout of a type object when allocated on the heap */ diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index caff86c64acfe2..952a96009dda6c 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -272,6 +272,30 @@ _PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func, int null_l return new_frame; } +/* Pushes a trampoline frame without checking for space. + * Must be guarded by _PyThreadState_HasStackSpace() */ +static inline _PyInterpreterFrame * +_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth, int prev_instr) +{ + CALL_STAT_INC(frames_pushed); + _PyInterpreterFrame *frame = (_PyInterpreterFrame *)tstate->datastack_top; + tstate->datastack_top += code->co_framesize; + assert(tstate->datastack_top < tstate->datastack_limit); + frame->f_funcobj = Py_None; + frame->f_executable = Py_NewRef(code); +#ifdef Py_DEBUG + frame->f_builtins = NULL; + frame->f_globals = NULL; +#endif + frame->f_locals = NULL; + frame->stacktop = code->co_nlocalsplus + stackdepth; + frame->frame_obj = NULL; + frame->prev_instr = _PyCode_CODE(code) + prev_instr; + frame->owner = FRAME_OWNED_BY_THREAD; + frame->return_offset = 0; + return frame; +} + static inline PyGenObject *_PyFrame_GetGenerator(_PyInterpreterFrame *frame) { diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 3ac7e89bbf5345..2358f48738a905 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -355,8 +355,10 @@ static inline int _PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) { } extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems); +PyObject *_PyType_NewManagedObject(PyTypeObject *type); extern int _PyObject_InitializeDict(PyObject *obj); +int _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, PyObject *name, PyObject *value); PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 6b269e3c02d6c1..cab5b587eda87d 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -68,6 +68,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_INTRINSIC_1] = CALL_INTRINSIC_1, [CALL_INTRINSIC_2] = CALL_INTRINSIC_2, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL, + [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = CALL, [CALL_NO_KW_BUILTIN_FAST] = CALL, [CALL_NO_KW_BUILTIN_O] = CALL, [CALL_NO_KW_ISINSTANCE] = CALL, @@ -104,6 +105,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [END_FOR] = END_FOR, [END_SEND] = END_SEND, [ENTER_EXECUTOR] = ENTER_EXECUTOR, + [EXIT_INIT_CHECK] = EXIT_INIT_CHECK, [EXTENDED_ARG] = EXTENDED_ARG, [FORMAT_SIMPLE] = FORMAT_SIMPLE, [FORMAT_WITH_SPEC] = FORMAT_WITH_SPEC, @@ -251,39 +253,39 @@ static const char *const _PyOpcode_OpName[267] = { [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", [UNARY_INVERT] = "UNARY_INVERT", - [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", + [EXIT_INIT_CHECK] = "EXIT_INIT_CHECK", [RESERVED] = "RESERVED", + [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", - [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [MAKE_FUNCTION] = "MAKE_FUNCTION", [BINARY_SUBSCR] = "BINARY_SUBSCR", [BINARY_SLICE] = "BINARY_SLICE", [STORE_SLICE] = "STORE_SLICE", + [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", - [SEND_GEN] = "SEND_GEN", [GET_LEN] = "GET_LEN", [MATCH_MAPPING] = "MATCH_MAPPING", [MATCH_SEQUENCE] = "MATCH_SEQUENCE", [MATCH_KEYS] = "MATCH_KEYS", - [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", + [SEND_GEN] = "SEND_GEN", [PUSH_EXC_INFO] = "PUSH_EXC_INFO", [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", [CHECK_EG_MATCH] = "CHECK_EG_MATCH", + [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", - [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [FORMAT_SIMPLE] = "FORMAT_SIMPLE", [FORMAT_WITH_SPEC] = "FORMAT_WITH_SPEC", + [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [LOAD_SUPER_ATTR_ATTR] = "LOAD_SUPER_ATTR_ATTR", - [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", [WITH_EXCEPT_START] = "WITH_EXCEPT_START", [GET_AITER] = "GET_AITER", [GET_ANEXT] = "GET_ANEXT", @@ -291,39 +293,39 @@ static const char *const _PyOpcode_OpName[267] = { [BEFORE_WITH] = "BEFORE_WITH", [END_ASYNC_FOR] = "END_ASYNC_FOR", [CLEANUP_THROW] = "CLEANUP_THROW", + [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", - [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", [STORE_SUBSCR] = "STORE_SUBSCR", [DELETE_SUBSCR] = "DELETE_SUBSCR", + [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", - [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [GET_ITER] = "GET_ITER", [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", - [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", + [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", + [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", [COMPARE_OP_INT] = "COMPARE_OP_INT", - [COMPARE_OP_STR] = "COMPARE_OP_STR", [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [RETURN_GENERATOR] = "RETURN_GENERATOR", + [COMPARE_OP_STR] = "COMPARE_OP_STR", [FOR_ITER_LIST] = "FOR_ITER_LIST", [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [FOR_ITER_GEN] = "FOR_ITER_GEN", [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", - [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", [RETURN_VALUE] = "RETURN_VALUE", - [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", + [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", + [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", [LOAD_LOCALS] = "LOAD_LOCALS", - [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", + [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", [POP_EXCEPT] = "POP_EXCEPT", [STORE_NAME] = "STORE_NAME", [DELETE_NAME] = "DELETE_NAME", @@ -346,9 +348,9 @@ static const char *const _PyOpcode_OpName[267] = { [IMPORT_NAME] = "IMPORT_NAME", [IMPORT_FROM] = "IMPORT_FROM", [JUMP_FORWARD] = "JUMP_FORWARD", + [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", - [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -367,7 +369,7 @@ static const char *const _PyOpcode_OpName[267] = { [POP_JUMP_IF_NONE] = "POP_JUMP_IF_NONE", [RAISE_VARARGS] = "RAISE_VARARGS", [GET_AWAITABLE] = "GET_AWAITABLE", - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", + [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", [BUILD_SLICE] = "BUILD_SLICE", [JUMP_BACKWARD_NO_INTERRUPT] = "JUMP_BACKWARD_NO_INTERRUPT", [MAKE_CELL] = "MAKE_CELL", @@ -383,26 +385,26 @@ static const char *const _PyOpcode_OpName[267] = { [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", [MAP_ADD] = "MAP_ADD", - [CALL_NO_KW_LEN] = "CALL_NO_KW_LEN", + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", [COPY_FREE_VARS] = "COPY_FREE_VARS", [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", + [CALL_NO_KW_LEN] = "CALL_NO_KW_LEN", [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", [CALL_NO_KW_LIST_APPEND] = "CALL_NO_KW_LIST_APPEND", - [CALL_NO_KW_METHOD_DESCRIPTOR_O] = "CALL_NO_KW_METHOD_DESCRIPTOR_O", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", [CONVERT_VALUE] = "CONVERT_VALUE", + [CALL_NO_KW_METHOD_DESCRIPTOR_O] = "CALL_NO_KW_METHOD_DESCRIPTOR_O", [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", - [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", - [166] = "<166>", - [167] = "<167>", + [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", + [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = "CALL_NO_KW_ALLOC_AND_ENTER_INIT", [LOAD_FAST_LOAD_FAST] = "LOAD_FAST_LOAD_FAST", [STORE_FAST_LOAD_FAST] = "STORE_FAST_LOAD_FAST", [STORE_FAST_STORE_FAST] = "STORE_FAST_STORE_FAST", @@ -506,8 +508,6 @@ static const char *const _PyOpcode_OpName[267] = { #endif #define EXTRA_CASES \ - case 166: \ - case 167: \ case 178: \ case 179: \ case 180: \ diff --git a/Include/opcode.h b/Include/opcode.h index 39bb70a8f2842f..3fe4fc0bb3404f 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -18,6 +18,7 @@ extern "C" { #define UNARY_NEGATIVE 11 #define UNARY_NOT 12 #define UNARY_INVERT 15 +#define EXIT_INIT_CHECK 16 #define RESERVED 17 #define MAKE_FUNCTION 24 #define BINARY_SUBSCR 25 @@ -164,59 +165,60 @@ extern "C" { #define BINARY_OP_MULTIPLY_FLOAT 10 #define BINARY_OP_ADD_FLOAT 13 #define BINARY_OP_SUBTRACT_FLOAT 14 -#define BINARY_OP_ADD_UNICODE 16 -#define BINARY_OP_INPLACE_ADD_UNICODE 18 -#define BINARY_SUBSCR_DICT 19 -#define BINARY_SUBSCR_GETITEM 20 -#define BINARY_SUBSCR_LIST_INT 21 -#define BINARY_SUBSCR_TUPLE_INT 22 -#define STORE_SUBSCR_DICT 23 -#define STORE_SUBSCR_LIST_INT 28 -#define SEND_GEN 29 -#define UNPACK_SEQUENCE_TWO_TUPLE 34 -#define UNPACK_SEQUENCE_TUPLE 38 -#define UNPACK_SEQUENCE_LIST 39 -#define STORE_ATTR_INSTANCE_VALUE 42 -#define STORE_ATTR_SLOT 43 -#define STORE_ATTR_WITH_HINT 44 -#define LOAD_GLOBAL_MODULE 45 -#define LOAD_GLOBAL_BUILTIN 46 -#define LOAD_SUPER_ATTR_ATTR 47 -#define LOAD_SUPER_ATTR_METHOD 48 -#define LOAD_ATTR_INSTANCE_VALUE 56 -#define LOAD_ATTR_MODULE 57 -#define LOAD_ATTR_WITH_HINT 58 -#define LOAD_ATTR_SLOT 59 -#define LOAD_ATTR_CLASS 62 -#define LOAD_ATTR_PROPERTY 63 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 64 -#define LOAD_ATTR_METHOD_WITH_VALUES 65 -#define LOAD_ATTR_METHOD_NO_DICT 66 -#define LOAD_ATTR_METHOD_LAZY_DICT 67 -#define COMPARE_OP_FLOAT 70 -#define COMPARE_OP_INT 72 -#define COMPARE_OP_STR 73 -#define FOR_ITER_LIST 76 -#define FOR_ITER_TUPLE 77 -#define FOR_ITER_RANGE 78 -#define FOR_ITER_GEN 79 -#define CALL_BOUND_METHOD_EXACT_ARGS 80 -#define CALL_PY_EXACT_ARGS 81 -#define CALL_PY_WITH_DEFAULTS 82 -#define CALL_NO_KW_TYPE_1 84 -#define CALL_NO_KW_STR_1 86 -#define CALL_NO_KW_TUPLE_1 88 -#define CALL_BUILTIN_CLASS 111 -#define CALL_NO_KW_BUILTIN_O 112 -#define CALL_NO_KW_BUILTIN_FAST 113 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 132 -#define CALL_NO_KW_LEN 148 -#define CALL_NO_KW_ISINSTANCE 153 -#define CALL_NO_KW_LIST_APPEND 154 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 155 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 159 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 160 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 161 +#define BINARY_OP_ADD_UNICODE 18 +#define BINARY_OP_INPLACE_ADD_UNICODE 19 +#define BINARY_SUBSCR_DICT 20 +#define BINARY_SUBSCR_GETITEM 21 +#define BINARY_SUBSCR_LIST_INT 22 +#define BINARY_SUBSCR_TUPLE_INT 23 +#define STORE_SUBSCR_DICT 28 +#define STORE_SUBSCR_LIST_INT 29 +#define SEND_GEN 34 +#define UNPACK_SEQUENCE_TWO_TUPLE 38 +#define UNPACK_SEQUENCE_TUPLE 39 +#define UNPACK_SEQUENCE_LIST 42 +#define STORE_ATTR_INSTANCE_VALUE 43 +#define STORE_ATTR_SLOT 44 +#define STORE_ATTR_WITH_HINT 45 +#define LOAD_GLOBAL_MODULE 46 +#define LOAD_GLOBAL_BUILTIN 47 +#define LOAD_SUPER_ATTR_ATTR 48 +#define LOAD_SUPER_ATTR_METHOD 56 +#define LOAD_ATTR_INSTANCE_VALUE 57 +#define LOAD_ATTR_MODULE 58 +#define LOAD_ATTR_WITH_HINT 59 +#define LOAD_ATTR_SLOT 62 +#define LOAD_ATTR_CLASS 63 +#define LOAD_ATTR_PROPERTY 64 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 65 +#define LOAD_ATTR_METHOD_WITH_VALUES 66 +#define LOAD_ATTR_METHOD_NO_DICT 67 +#define LOAD_ATTR_METHOD_LAZY_DICT 70 +#define COMPARE_OP_FLOAT 72 +#define COMPARE_OP_INT 73 +#define COMPARE_OP_STR 76 +#define FOR_ITER_LIST 77 +#define FOR_ITER_TUPLE 78 +#define FOR_ITER_RANGE 79 +#define FOR_ITER_GEN 80 +#define CALL_BOUND_METHOD_EXACT_ARGS 81 +#define CALL_PY_EXACT_ARGS 82 +#define CALL_PY_WITH_DEFAULTS 84 +#define CALL_NO_KW_TYPE_1 86 +#define CALL_NO_KW_STR_1 88 +#define CALL_NO_KW_TUPLE_1 111 +#define CALL_BUILTIN_CLASS 112 +#define CALL_NO_KW_BUILTIN_O 113 +#define CALL_NO_KW_BUILTIN_FAST 132 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 148 +#define CALL_NO_KW_LEN 153 +#define CALL_NO_KW_ISINSTANCE 154 +#define CALL_NO_KW_LIST_APPEND 155 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 159 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 160 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 161 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 166 +#define CALL_NO_KW_ALLOC_AND_ENTER_INIT 167 #define NB_ADD 0 #define NB_AND 1 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index 92ed4037d8ddb7..c0a68256802c37 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -85,6 +85,7 @@ "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", + "CALL_NO_KW_ALLOC_AND_ENTER_INIT", ], } diff --git a/Lib/opcode.py b/Lib/opcode.py index ed01d2cdad9ebf..392464ddab9728 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -97,6 +97,7 @@ def pseudo_op(name, op, real_ops): def_op('UNARY_NOT', 12) def_op('UNARY_INVERT', 15) +def_op('EXIT_INIT_CHECK', 16) # We reserve 17 as it is the initial value for the specializing counter # This helps us catch cases where we attempt to execute a cache. diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index d81501f6f1df6a..37f75ad54387a0 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1557,7 +1557,7 @@ def delx(self): del self.__x '10P' # PySequenceMethods '2P' # PyBufferProcs '6P' - '1PI' # Specializer cache + '1PIP' # Specializer cache ) class newstyleclass(object): pass # Separate block for PyDictKeysObject with 8 keys and 5 entries diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 5603c3cdbf3c5e..4462b5c712d662 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -1614,8 +1614,30 @@ def func(): self.run_and_compare(func, EXPECTED_EVENTS) - def test_settrace_error(self): + def test_correct_tracing_quickened_call_class_init(self): + + class C: + def __init__(self): + self + + def func(): + C() + EXPECTED_EVENTS = [ + (0, 'call'), + (1, 'line'), + (-3, 'call'), + (-2, 'line'), + (-2, 'return'), + (1, 'return')] + + self.run_and_compare(func, EXPECTED_EVENTS) + # Quicken + for _ in range(100): + func() + self.run_and_compare(func, EXPECTED_EVENTS) + + def test_settrace_error(self): raised = False def error_once(frame, event, arg): nonlocal raised diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-10-13-04-35.gh-issue-91095.4E3Pwn.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-10-13-04-35.gh-issue-91095.4E3Pwn.rst new file mode 100644 index 00000000000000..5633097f4a3fdd --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-10-13-04-35.gh-issue-91095.4E3Pwn.rst @@ -0,0 +1,11 @@ +Specializes calls to most Python classes. Specifically, any class that +inherits from ``object``, or another Python class, and does not override +``__new__``. + +The specialized instruction does the following: + +1. Creates the object (by calling ``object.__new__``) +2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) +3. Pushes the frame for ``__init__`` to the frame stack + +Speeds up the instantiation of most Python classes. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 254cd9ad2f9bda..194081db2f290a 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5334,11 +5334,10 @@ _PyDict_NewKeysForClass(void) #define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys) -static int -init_inline_values(PyObject *obj, PyTypeObject *tp) +int +_PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp) { assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); - // assert(type->tp_dictoffset > 0); -- TO DO Update this assert. assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictKeysObject *keys = CACHED_KEYS(tp); assert(keys != NULL); @@ -5370,7 +5369,7 @@ _PyObject_InitializeDict(PyObject *obj) } if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { OBJECT_STAT_INC(new_values); - return init_inline_values(obj, tp); + return _PyObject_InitInlineValues(obj, tp); } PyObject *dict; if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { diff --git a/Objects/typeobject.c b/Objects/typeobject.c index cbba6f0fa80d12..d427ecd50a76fc 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1681,6 +1681,26 @@ type_call(PyTypeObject *type, PyObject *args, PyObject *kwds) return obj; } +PyObject * +_PyType_NewManagedObject(PyTypeObject *type) +{ + assert(type->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(_PyType_IS_GC(type)); + assert(type->tp_new == PyBaseObject_Type.tp_new); + assert(type->tp_alloc == PyType_GenericAlloc); + assert(type->tp_itemsize == 0); + PyObject *obj = PyType_GenericAlloc(type, 0); + if (obj == NULL) { + return PyErr_NoMemory(); + } + _PyObject_DictOrValuesPointer(obj)->dict = NULL; + if (_PyObject_InitInlineValues(obj, type)) { + Py_DECREF(obj); + return NULL; + } + return obj; +} + PyObject * _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems) { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a7acff65e13e3a..3ef8ed0ccaafdc 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2685,6 +2685,7 @@ dummy_func( CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, CALL_NO_KW_METHOD_DESCRIPTOR_FAST, + CALL_NO_KW_ALLOC_AND_ENTER_INIT, }; // On entry, the stack is either @@ -2900,6 +2901,68 @@ dummy_func( CHECK_EVAL_BREAKER(); } + inst(CALL_NO_KW_ALLOC_AND_ENTER_INIT, (unused/1, unused/2, null, callable, args[oparg] -- unused)) { + /* This instruction does the following: + * 1. Creates the object (by calling ``object.__new__``) + * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) + * 3. Pushes the frame for ``__init__`` to the frame stack + * */ + assert(kwnames == NULL); + _PyCallCache *cache = (_PyCallCache *)next_instr; + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(!PyType_Check(callable), CALL); + PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; + PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init->func_code; + DEOPT_IF(code->co_argcount != oparg+1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + STAT_INC(CALL, hit); + PyObject *self = _PyType_NewManagedObject(tp); + if (self == NULL) { + goto error; + } + Py_DECREF(tp); + if (_Py_EnterRecursivePy(tstate)) { + goto exit_unwind; + } + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, 0); + assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[1].op.code == EXIT_INIT_CHECK); + /* Push self onto stack of shim */ + Py_INCREF(self); + shim->localsplus[0] = self; + Py_INCREF(init); + _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); + /* Copy self followed by args to __init__ frame */ + init_frame->localsplus[0] = self; + for (int i = 0; i < oparg; i++) { + init_frame->localsplus[i+1] = args[i]; + } + SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); + frame->prev_instr = next_instr - 1; + frame->return_offset = 0; + STACK_SHRINK(oparg+2); + _PyFrame_SetStackPointer(frame, stack_pointer); + /* Link frames */ + init_frame->previous = shim; + shim->previous = frame; + frame = cframe.current_frame = init_frame; + CALL_STAT_INC(inlined_py_calls); + goto start_frame; + } + + inst(EXIT_INIT_CHECK, (should_be_none -- )) { + assert(STACK_LEVEL() == 2); + if (should_be_none != Py_None) { + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(should_be_none)->tp_name); + goto error; + } + } + inst(CALL_BUILTIN_CLASS, (unused/1, unused/2, method, callable, args[oparg] -- res)) { int is_meth = method != NULL; int total_args = oparg; diff --git a/Python/ceval.c b/Python/ceval.c index b1ac4de3f625da..53107018978c0b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -641,6 +641,8 @@ static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { { .op.code = RESUME, .op.arg = 0 } }; +extern const struct _PyCode_DEF(8) _Py_InitCleanup; + /* Disable unused label warnings. They are handy for debugging, even if computed gotos aren't used. */ @@ -746,7 +748,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #ifdef LLTRACE { - if (frame != &entry_frame) { + if (frame != &entry_frame && GLOBALS()) { int r = PyDict_Contains(GLOBALS(), &_Py_ID(__lltrace__)); if (r < 0) { goto exit_unwind; diff --git a/Python/compile.c b/Python/compile.c index 69468f755a1d26..5a0560521b99d5 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -825,6 +825,9 @@ stack_effect(int opcode, int oparg, int jump) case JUMP_NO_INTERRUPT: return 0; + case EXIT_INIT_CHECK: + return -1; + /* Exception handling pseudo-instructions */ case SETUP_FINALLY: /* 0 in the normal flow. diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 717b29e811d309..11ca535adfb19b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3765,7 +3765,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2698 "Python/bytecodes.c" + #line 2699 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3859,7 +3859,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2786 "Python/bytecodes.c" + #line 2787 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3878,7 +3878,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2798 "Python/bytecodes.c" + #line 2799 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3912,7 +3912,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2826 "Python/bytecodes.c" + #line 2827 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3956,7 +3956,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2864 "Python/bytecodes.c" + #line 2865 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3979,7 +3979,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2876 "Python/bytecodes.c" + #line 2877 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -4004,7 +4004,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2890 "Python/bytecodes.c" + #line 2891 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -4024,12 +4024,84 @@ DISPATCH(); } + TARGET(CALL_NO_KW_ALLOC_AND_ENTER_INIT) { + PyObject **args = (stack_pointer - oparg); + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *null = stack_pointer[-(2 + oparg)]; + #line 2905 "Python/bytecodes.c" + /* This instruction does the following: + * 1. Creates the object (by calling ``object.__new__``) + * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) + * 3. Pushes the frame for ``__init__`` to the frame stack + * */ + assert(kwnames == NULL); + _PyCallCache *cache = (_PyCallCache *)next_instr; + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(!PyType_Check(callable), CALL); + PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; + PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init->func_code; + DEOPT_IF(code->co_argcount != oparg+1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + STAT_INC(CALL, hit); + PyObject *self = _PyType_NewManagedObject(tp); + if (self == NULL) { + goto error; + } + Py_DECREF(tp); + if (_Py_EnterRecursivePy(tstate)) { + goto exit_unwind; + } + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, 0); + assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[1].op.code == EXIT_INIT_CHECK); + /* Push self onto stack of shim */ + Py_INCREF(self); + shim->localsplus[0] = self; + Py_INCREF(init); + _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); + /* Copy self followed by args to __init__ frame */ + init_frame->localsplus[0] = self; + for (int i = 0; i < oparg; i++) { + init_frame->localsplus[i+1] = args[i]; + } + SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); + frame->prev_instr = next_instr - 1; + frame->return_offset = 0; + STACK_SHRINK(oparg+2); + _PyFrame_SetStackPointer(frame, stack_pointer); + /* Link frames */ + init_frame->previous = shim; + shim->previous = frame; + frame = cframe.current_frame = init_frame; + CALL_STAT_INC(inlined_py_calls); + goto start_frame; + #line 4082 "Python/generated_cases.c.h" + } + + TARGET(EXIT_INIT_CHECK) { + PyObject *should_be_none = stack_pointer[-1]; + #line 2957 "Python/bytecodes.c" + assert(STACK_LEVEL() == 2); + if (should_be_none != Py_None) { + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(should_be_none)->tp_name); + goto error; + } + #line 4095 "Python/generated_cases.c.h" + STACK_SHRINK(1); + DISPATCH(); + } + TARGET(CALL_BUILTIN_CLASS) { PyObject **args = (stack_pointer - oparg); PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2904 "Python/bytecodes.c" + #line 2967 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4051,7 +4123,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4055 "Python/generated_cases.c.h" + #line 4127 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4065,7 +4137,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2929 "Python/bytecodes.c" + #line 2992 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -4093,7 +4165,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4097 "Python/generated_cases.c.h" + #line 4169 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4107,7 +4179,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2960 "Python/bytecodes.c" + #line 3023 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -4139,7 +4211,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 4143 "Python/generated_cases.c.h" + #line 4215 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4153,7 +4225,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2995 "Python/bytecodes.c" + #line 3058 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -4185,7 +4257,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4189 "Python/generated_cases.c.h" + #line 4261 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4199,7 +4271,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3030 "Python/bytecodes.c" + #line 3093 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -4224,7 +4296,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4228 "Python/generated_cases.c.h" + #line 4300 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4237,7 +4309,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3057 "Python/bytecodes.c" + #line 3120 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -4264,7 +4336,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4268 "Python/generated_cases.c.h" + #line 4340 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4276,7 +4348,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 3087 "Python/bytecodes.c" + #line 3150 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); assert(method != NULL); @@ -4294,14 +4366,14 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 4298 "Python/generated_cases.c.h" + #line 4370 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3107 "Python/bytecodes.c" + #line 3170 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4332,7 +4404,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4336 "Python/generated_cases.c.h" + #line 4408 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4345,7 +4417,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3141 "Python/bytecodes.c" + #line 3204 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4374,7 +4446,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4378 "Python/generated_cases.c.h" + #line 4450 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4387,7 +4459,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3173 "Python/bytecodes.c" + #line 3236 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4416,7 +4488,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4420 "Python/generated_cases.c.h" + #line 4492 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4429,7 +4501,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3205 "Python/bytecodes.c" + #line 3268 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4457,7 +4529,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4461 "Python/generated_cases.c.h" + #line 4533 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4467,9 +4539,9 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #line 3236 "Python/bytecodes.c" + #line 3299 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4473 "Python/generated_cases.c.h" + #line 4545 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4478,7 +4550,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 3240 "Python/bytecodes.c" + #line 3303 "Python/bytecodes.c" // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4540,14 +4612,14 @@ } result = PyObject_Call(func, callargs, kwargs); } - #line 4544 "Python/generated_cases.c.h" + #line 4616 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3302 "Python/bytecodes.c" + #line 3365 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4551 "Python/generated_cases.c.h" + #line 4623 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4558,7 +4630,7 @@ TARGET(MAKE_FUNCTION) { PyObject *codeobj = stack_pointer[-1]; PyObject *func; - #line 3308 "Python/bytecodes.c" + #line 3371 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4570,7 +4642,7 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4574 "Python/generated_cases.c.h" + #line 4646 "Python/generated_cases.c.h" stack_pointer[-1] = func; DISPATCH(); } @@ -4578,7 +4650,7 @@ TARGET(SET_FUNCTION_ATTRIBUTE) { PyObject *func = stack_pointer[-1]; PyObject *attr = stack_pointer[-2]; - #line 3322 "Python/bytecodes.c" + #line 3385 "Python/bytecodes.c" assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { @@ -4603,14 +4675,14 @@ default: Py_UNREACHABLE(); } - #line 4607 "Python/generated_cases.c.h" + #line 4679 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3349 "Python/bytecodes.c" + #line 3412 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4631,7 +4703,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4635 "Python/generated_cases.c.h" + #line 4707 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4639,15 +4711,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3372 "Python/bytecodes.c" + #line 3435 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4645 "Python/generated_cases.c.h" + #line 4717 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3374 "Python/bytecodes.c" + #line 3437 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4651 "Python/generated_cases.c.h" + #line 4723 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4657,14 +4729,14 @@ TARGET(CONVERT_VALUE) { PyObject *value = stack_pointer[-1]; PyObject *result; - #line 3378 "Python/bytecodes.c" + #line 3441 "Python/bytecodes.c" convertion_func_ptr conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = CONVERSION_FUNCTIONS[oparg]; result = conv_fn(value); Py_DECREF(value); if (result == NULL) goto pop_1_error; - #line 4668 "Python/generated_cases.c.h" + #line 4740 "Python/generated_cases.c.h" stack_pointer[-1] = result; DISPATCH(); } @@ -4672,7 +4744,7 @@ TARGET(FORMAT_SIMPLE) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 3387 "Python/bytecodes.c" + #line 3450 "Python/bytecodes.c" /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value)) { @@ -4683,7 +4755,7 @@ else { res = value; } - #line 4687 "Python/generated_cases.c.h" + #line 4759 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -4692,12 +4764,12 @@ PyObject *fmt_spec = stack_pointer[-1]; PyObject *value = stack_pointer[-2]; PyObject *res; - #line 3400 "Python/bytecodes.c" + #line 3463 "Python/bytecodes.c" res = PyObject_Format(value, fmt_spec); Py_DECREF(value); Py_DECREF(fmt_spec); if (res == NULL) goto pop_2_error; - #line 4701 "Python/generated_cases.c.h" + #line 4773 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -4706,10 +4778,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3407 "Python/bytecodes.c" + #line 3470 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4713 "Python/generated_cases.c.h" + #line 4785 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4721,7 +4793,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3412 "Python/bytecodes.c" + #line 3475 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4736,12 +4808,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4740 "Python/generated_cases.c.h" + #line 4812 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3427 "Python/bytecodes.c" + #line 3490 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4745 "Python/generated_cases.c.h" + #line 4817 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4751,16 +4823,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3432 "Python/bytecodes.c" + #line 3495 "Python/bytecodes.c" assert(oparg >= 2); - #line 4757 "Python/generated_cases.c.h" + #line 4829 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3436 "Python/bytecodes.c" + #line 3499 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4772,26 +4844,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4776 "Python/generated_cases.c.h" + #line 4848 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3450 "Python/bytecodes.c" + #line 3513 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4782 "Python/generated_cases.c.h" + #line 4854 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3454 "Python/bytecodes.c" + #line 3517 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+1-oparg, PY_MONITORING_EVENT_JUMP); - #line 4789 "Python/generated_cases.c.h" + #line 4861 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3459 "Python/bytecodes.c" + #line 3522 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4800,12 +4872,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4804 "Python/generated_cases.c.h" + #line 4876 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3470 "Python/bytecodes.c" + #line 3533 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4814,12 +4886,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4818 "Python/generated_cases.c.h" + #line 4890 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3481 "Python/bytecodes.c" + #line 3544 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4831,12 +4903,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4835 "Python/generated_cases.c.h" + #line 4907 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3495 "Python/bytecodes.c" + #line 3558 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4848,30 +4920,30 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4852 "Python/generated_cases.c.h" + #line 4924 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3509 "Python/bytecodes.c" + #line 3572 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4863 "Python/generated_cases.c.h" + #line 4935 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3517 "Python/bytecodes.c" + #line 3580 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); - #line 4870 "Python/generated_cases.c.h" + #line 4942 "Python/generated_cases.c.h" } TARGET(RESERVED) { - #line 3522 "Python/bytecodes.c" + #line 3585 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); - #line 4877 "Python/generated_cases.c.h" + #line 4949 "Python/generated_cases.c.h" } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index fcaccc022c8b2d..524a82c076e99d 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1526,7 +1526,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) return 0; } /* Insert instrumentation */ - for (int i = 0; i < code_len; i+= _PyInstruction_GetLength(code, i)) { + for (int i = code->_co_firsttraceable; i < code_len; i+= _PyInstruction_GetLength(code, i)) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; CHECK(instr->op.code != 0); int base_opcode = _Py_GetBaseOpcode(code, i); diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index aa1db721d1ae71..421d9795e7da76 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -368,6 +368,10 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return oparg + 2; case CALL_NO_KW_TUPLE_1: return oparg + 2; + case CALL_NO_KW_ALLOC_AND_ENTER_INIT: + return oparg + 2; + case EXIT_INIT_CHECK: + return 1; case CALL_BUILTIN_CLASS: return oparg + 2; case CALL_NO_KW_BUILTIN_O: @@ -790,6 +794,10 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case CALL_NO_KW_TUPLE_1: return 1; + case CALL_NO_KW_ALLOC_AND_ENTER_INIT: + return 1; + case EXIT_INIT_CHECK: + return 0; case CALL_BUILTIN_CLASS: return 1; case CALL_NO_KW_BUILTIN_O: @@ -1057,6 +1065,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[512] = { [CALL_NO_KW_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, [CALL_NO_KW_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, [CALL_NO_KW_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, + [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, + [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, 0 }, [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, [CALL_NO_KW_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, [CALL_NO_KW_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 4de4bf0608f35e..781d72fa7f5333 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -15,39 +15,39 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_SUBTRACT_FLOAT, &&TARGET_UNARY_INVERT, - &&TARGET_BINARY_OP_ADD_UNICODE, + &&TARGET_EXIT_INIT_CHECK, &&TARGET_RESERVED, + &&TARGET_BINARY_OP_ADD_UNICODE, &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, &&TARGET_BINARY_SUBSCR_DICT, &&TARGET_BINARY_SUBSCR_GETITEM, &&TARGET_BINARY_SUBSCR_LIST_INT, &&TARGET_BINARY_SUBSCR_TUPLE_INT, - &&TARGET_STORE_SUBSCR_DICT, &&TARGET_MAKE_FUNCTION, &&TARGET_BINARY_SUBSCR, &&TARGET_BINARY_SLICE, &&TARGET_STORE_SLICE, + &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, - &&TARGET_SEND_GEN, &&TARGET_GET_LEN, &&TARGET_MATCH_MAPPING, &&TARGET_MATCH_SEQUENCE, &&TARGET_MATCH_KEYS, - &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, + &&TARGET_SEND_GEN, &&TARGET_PUSH_EXC_INFO, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CHECK_EG_MATCH, + &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&TARGET_UNPACK_SEQUENCE_TUPLE, - &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_FORMAT_SIMPLE, &&TARGET_FORMAT_WITH_SPEC, + &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_LOAD_SUPER_ATTR_ATTR, - &&TARGET_LOAD_SUPER_ATTR_METHOD, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, @@ -55,39 +55,39 @@ static void *opcode_targets[256] = { &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, &&TARGET_CLEANUP_THROW, + &&TARGET_LOAD_SUPER_ATTR_METHOD, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_LOAD_ATTR_MODULE, &&TARGET_LOAD_ATTR_WITH_HINT, - &&TARGET_LOAD_ATTR_SLOT, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, + &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_LOAD_ATTR_METHOD_NO_DICT, - &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, - &&TARGET_COMPARE_OP_FLOAT, + &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_LOAD_BUILD_CLASS, + &&TARGET_COMPARE_OP_FLOAT, &&TARGET_COMPARE_OP_INT, - &&TARGET_COMPARE_OP_STR, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_COMPARE_OP_STR, &&TARGET_FOR_ITER_LIST, &&TARGET_FOR_ITER_TUPLE, &&TARGET_FOR_ITER_RANGE, &&TARGET_FOR_ITER_GEN, &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, &&TARGET_CALL_PY_EXACT_ARGS, - &&TARGET_CALL_PY_WITH_DEFAULTS, &&TARGET_RETURN_VALUE, - &&TARGET_CALL_NO_KW_TYPE_1, + &&TARGET_CALL_PY_WITH_DEFAULTS, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_CALL_NO_KW_STR_1, + &&TARGET_CALL_NO_KW_TYPE_1, &&TARGET_LOAD_LOCALS, - &&TARGET_CALL_NO_KW_TUPLE_1, + &&TARGET_CALL_NO_KW_STR_1, &&TARGET_POP_EXCEPT, &&TARGET_STORE_NAME, &&TARGET_DELETE_NAME, @@ -110,9 +110,9 @@ static void *opcode_targets[256] = { &&TARGET_IMPORT_NAME, &&TARGET_IMPORT_FROM, &&TARGET_JUMP_FORWARD, + &&TARGET_CALL_NO_KW_TUPLE_1, &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_CALL_NO_KW_BUILTIN_O, - &&TARGET_CALL_NO_KW_BUILTIN_FAST, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -131,7 +131,7 @@ static void *opcode_targets[256] = { &&TARGET_POP_JUMP_IF_NONE, &&TARGET_RAISE_VARARGS, &&TARGET_GET_AWAITABLE, - &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, + &&TARGET_CALL_NO_KW_BUILTIN_FAST, &&TARGET_BUILD_SLICE, &&TARGET_JUMP_BACKWARD_NO_INTERRUPT, &&TARGET_MAKE_CELL, @@ -147,26 +147,26 @@ static void *opcode_targets[256] = { &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, &&TARGET_MAP_ADD, - &&TARGET_CALL_NO_KW_LEN, + &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, &&TARGET_COPY_FREE_VARS, &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, + &&TARGET_CALL_NO_KW_LEN, &&TARGET_CALL_NO_KW_ISINSTANCE, &&TARGET_CALL_NO_KW_LIST_APPEND, - &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, &&TARGET_CONVERT_VALUE, + &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O, &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, - &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_FAST, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_FAST, + &&TARGET_CALL_NO_KW_ALLOC_AND_ENTER_INIT, &&TARGET_LOAD_FAST_LOAD_FAST, &&TARGET_STORE_FAST_LOAD_FAST, &&TARGET_STORE_FAST_STORE_FAST, diff --git a/Python/specialize.c b/Python/specialize.c index 44b14c5952315f..0006aa733bd6cb 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -391,7 +391,7 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_CALL_METH_DESCR_VARARGS_KEYWORDS 18 #define SPEC_FAIL_CALL_METH_DESCR_METHOD_FASTCALL_KEYWORDS 19 #define SPEC_FAIL_CALL_BAD_CALL_FLAGS 20 -#define SPEC_FAIL_CALL_PYTHON_CLASS 21 +#define SPEC_FAIL_CALL_INIT_NOT_PYTHON 21 #define SPEC_FAIL_CALL_PEP_523 22 #define SPEC_FAIL_CALL_BOUND_METHOD 23 #define SPEC_FAIL_CALL_STR 24 @@ -400,6 +400,7 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_CALL_KWNAMES 27 #define SPEC_FAIL_CALL_METHOD_WRAPPER 28 #define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29 +#define SPEC_FAIL_CALL_INIT_NOT_SIMPLE 30 /* COMPARE_OP */ #define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12 @@ -1491,15 +1492,46 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins cache->counter = adaptive_counter_cooldown(); } +/* Returns a borrowed reference. + * The reference is only valid if guarded by a type version check. + */ +static PyFunctionObject * +get_init_for_simple_managed_python_class(PyTypeObject *tp) +{ + assert(tp->tp_new == PyBaseObject_Type.tp_new); + if (tp->tp_alloc != PyType_GenericAlloc) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OVERRIDDEN); + return NULL; + } + if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_NO_DICT); + return NULL; + } + if (!(tp->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + /* Is this possible? */ + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_EXPECTED_ERROR); + return NULL; + } + PyObject *init = _PyType_Lookup(tp, &_Py_ID(__init__)); + if (init == NULL || !PyFunction_Check(init)) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_PYTHON); + return NULL; + } + int kind = function_kind((PyCodeObject *)PyFunction_GET_CODE(init)); + if (kind != SIMPLE_FUNCTION) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_SIMPLE); + return NULL; + } + ((PyHeapTypeObject *)tp)->_spec_cache.init = init; + return (PyFunctionObject *)init; +} + static int specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames) { + assert(PyType_Check(callable)); PyTypeObject *tp = _PyType_CAST(callable); - if (tp->tp_new == PyBaseObject_Type.tp_new) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PYTHON_CLASS); - return -1; - } if (tp->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) { int oparg = instr->op.arg; if (nargs == 1 && kwnames == NULL && oparg == 1) { @@ -1524,6 +1556,24 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, SPEC_FAIL_CALL_STR : SPEC_FAIL_CALL_CLASS_NO_VECTORCALL); return -1; } + if (tp->tp_new == PyBaseObject_Type.tp_new) { + PyFunctionObject *init = get_init_for_simple_managed_python_class(tp); + if (init != NULL) { + if (((PyCodeObject *)init->func_code)->co_argcount != nargs+1) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); + return -1; + } + if (kwnames) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES); + return -1; + } + _PyCallCache *cache = (_PyCallCache *)(instr + 1); + write_u32(cache->func_version, tp->tp_version_tag); + _Py_SET_OPCODE(*instr, CALL_NO_KW_ALLOC_AND_ENTER_INIT); + return 0; + } + return -1; + } SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_CLASS_MUTABLE); return -1; } @@ -2229,3 +2279,42 @@ _Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr) STAT_INC(SEND, success); cache->counter = adaptive_counter_cooldown(); } + +/* Code init cleanup. + * CALL_NO_KW_ALLOC_AND_ENTER_INIT will set up + * the frame to execute the EXIT_INIT_CHECK + * instruction. + * Ends with a RESUME so that it is not traced. + * This is used as a plain code object, not a function, + * so must not access globals or builtins. + */ + +#define NO_LOC_4 (128 | (PY_CODE_LOCATION_INFO_NONE << 3) | 3) + +static const PyBytesObject no_location = { + PyVarObject_HEAD_INIT(&PyBytes_Type, 1) + .ob_sval = { NO_LOC_4 } +}; + +const struct _PyCode_DEF(8) _Py_InitCleanup = { + _PyVarObject_HEAD_INIT(&PyCode_Type, 4) + .co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty), + .co_names = (PyObject *)&_Py_SINGLETON(tuple_empty), + .co_exceptiontable = (PyObject *)&_Py_SINGLETON(bytes_empty), + .co_flags = CO_OPTIMIZED, + .co_localsplusnames = (PyObject *)&_Py_SINGLETON(tuple_empty), + .co_localspluskinds = (PyObject *)&_Py_SINGLETON(bytes_empty), + .co_filename = &_Py_ID(__init__), + .co_name = &_Py_ID(__init__), + .co_qualname = &_Py_ID(__init__), + .co_linetable = (PyObject *)&no_location, + ._co_firsttraceable = 4, + .co_stacksize = 2, + .co_framesize = 2 + FRAME_SPECIALS_SIZE, + .co_code_adaptive = { + NOP, 0, + EXIT_INIT_CHECK, 0, + RETURN_VALUE, 0, + RESUME, 0, + } +};