diff --git a/Doc/Makefile b/Doc/Makefile index 9ddf97fd775dec..8020884447983c 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -188,54 +188,69 @@ dist: mkdir -p dist # archive the HTML - make html + @echo "Building HTML..." + $(MAKE) html cp -pPR build/html dist/python-$(DISTVERSION)-docs-html tar -C dist -cf dist/python-$(DISTVERSION)-docs-html.tar python-$(DISTVERSION)-docs-html bzip2 -9 -k dist/python-$(DISTVERSION)-docs-html.tar (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-html.zip python-$(DISTVERSION)-docs-html) rm -r dist/python-$(DISTVERSION)-docs-html rm dist/python-$(DISTVERSION)-docs-html.tar + @echo "Build finished and archived!" # archive the text build - make text + @echo "Building text..." + $(MAKE) text cp -pPR build/text dist/python-$(DISTVERSION)-docs-text tar -C dist -cf dist/python-$(DISTVERSION)-docs-text.tar python-$(DISTVERSION)-docs-text bzip2 -9 -k dist/python-$(DISTVERSION)-docs-text.tar (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-text.zip python-$(DISTVERSION)-docs-text) rm -r dist/python-$(DISTVERSION)-docs-text rm dist/python-$(DISTVERSION)-docs-text.tar + @echo "Build finished and archived!" # archive the A4 latex + @echo "Building LaTeX (A4 paper)..." rm -rf build/latex - make latex PAPER=a4 - -sed -i 's/makeindex/makeindex -q/' build/latex/Makefile - (cd build/latex; make clean && make all-pdf && make FMT=pdf zip bz2) + $(MAKE) latex PAPER=a4 + # remove zip & bz2 dependency on all-pdf, + # as otherwise the full latexmk process is run twice. + # ($$ is needed to escape the $; https://www.gnu.org/software/make/manual/make.html#Basics-of-Variable-References) + -sed -i 's/: all-$$(FMT)/:/' build/latex/Makefile + (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$((`nproc`+1)) --output-sync LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-a4.zip cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-a4.tar.bz2 + @echo "Build finished and archived!" # archive the letter latex + @echo "Building LaTeX (US paper)..." rm -rf build/latex - make latex PAPER=letter - -sed -i 's/makeindex/makeindex -q/' build/latex/Makefile - (cd build/latex; make clean && make all-pdf && make FMT=pdf zip bz2) + $(MAKE) latex PAPER=letter + -sed -i 's/: all-$$(FMT)/:/' build/latex/Makefile + (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$((`nproc`+1)) --output-sync LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-letter.zip cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-letter.tar.bz2 + @echo "Build finished and archived!" # copy the epub build + @echo "Building EPUB..." rm -rf build/epub - make epub + $(MAKE) epub cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub + @echo "Build finished and archived!" # archive the texinfo build + @echo "Building Texinfo..." rm -rf build/texinfo - make texinfo - make info --directory=build/texinfo + $(MAKE) texinfo + $(MAKE) info --directory=build/texinfo cp -pPR build/texinfo dist/python-$(DISTVERSION)-docs-texinfo tar -C dist -cf dist/python-$(DISTVERSION)-docs-texinfo.tar python-$(DISTVERSION)-docs-texinfo bzip2 -9 -k dist/python-$(DISTVERSION)-docs-texinfo.tar (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-texinfo.zip python-$(DISTVERSION)-docs-texinfo) rm -r dist/python-$(DISTVERSION)-docs-texinfo rm dist/python-$(DISTVERSION)-docs-texinfo.tar + @echo "Build finished and archived!" .PHONY: _ensure-package _ensure-package: venv @@ -247,11 +262,11 @@ _ensure-package: venv .PHONY: _ensure-pre-commit _ensure-pre-commit: - make _ensure-package PACKAGE=pre-commit + $(MAKE) _ensure-package PACKAGE=pre-commit .PHONY: _ensure-sphinx-autobuild _ensure-sphinx-autobuild: - make _ensure-package PACKAGE=sphinx-autobuild + $(MAKE) _ensure-package PACKAGE=sphinx-autobuild .PHONY: check check: _ensure-pre-commit @@ -271,12 +286,12 @@ serve: # for development releases: always build .PHONY: autobuild-dev autobuild-dev: - make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' + $(MAKE) dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' # for quick rebuilds (HTML only) .PHONY: autobuild-dev-html autobuild-dev-html: - make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' + $(MAKE) html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' # for stable releases: only build if not in pre-release stage (alpha, beta) # release candidate downloads are okay, since the stable tree can be in that stage @@ -286,7 +301,7 @@ autobuild-stable: echo "Not building; $(DISTVERSION) is not a release version."; \ exit 1;; \ esac - @make autobuild-dev + @$(MAKE) autobuild-dev .PHONY: autobuild-stable-html autobuild-stable-html: @@ -294,4 +309,4 @@ autobuild-stable-html: echo "Not building; $(DISTVERSION) is not a release version."; \ exit 1;; \ esac - @make autobuild-dev-html + @$(MAKE) autobuild-dev-html diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst index 10cb5e424a623b..de134f8e2ee9d3 100644 --- a/Doc/deprecations/pending-removal-in-3.16.rst +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -8,3 +8,8 @@ Pending Removal in Python 3.16 * :mod:`symtable`: Deprecate :meth:`symtable.Class.get_methods` due to the lack of interest. (Contributed by Bénédikt Tran in :gh:`119698`.) + +* :mod:`shutil`: Deprecate :class:`!shutil.ExecError`, which hasn't + been raised by any :mod:`!shutil` function since Python 3.4. It's + now an alias for :exc:`RuntimeError`. + diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index 7aaad932c0104a..587fe50fd51164 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -1314,13 +1314,19 @@ RawConfigParser Objects .. method:: add_section(section) - Add a section named *section* to the instance. If a section by the given - name already exists, :exc:`DuplicateSectionError` is raised. If the - *default section* name is passed, :exc:`ValueError` is raised. + Add a section named *section* or :const:`UNNAMED_SECTION` to the instance. + + If the given section already exists, :exc:`DuplicateSectionError` is + raised. If the *default section* name is passed, :exc:`ValueError` is + raised. If :const:`UNNAMED_SECTION` is passed and support is disabled, + :exc:`UnnamedSectionDisabledError` is raised. Type of *section* is not checked which lets users create non-string named sections. This behaviour is unsupported and may cause internal errors. + .. versionchanged:: 3.14 + Added support for :const:`UNNAMED_SECTION`. + .. method:: set(section, option, value) @@ -1405,7 +1411,6 @@ Exceptions Exception raised when attempting to parse a file which has no section headers. - .. exception:: ParsingError Exception raised when errors occur attempting to parse a file. @@ -1421,6 +1426,13 @@ Exceptions .. versionadded:: 3.13 +.. exception:: UnnamedSectionDisabledError + + Exception raised when attempting to use the + :const:`UNNAMED_SECTION` without enabling it. + + .. versionadded:: 3.14 + .. rubric:: Footnotes .. [1] Config parsers allow for heavy customization. If you are interested in diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index a770ad87ab02d3..cc8f6363d78be3 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -56,6 +56,10 @@ interpreter. for jump targets and exception handlers. The ``-O`` command line option and the ``show_offsets`` argument were added. + .. versionchanged:: 3.14 + The :option:`-P ` command-line option + and the ``show_positions`` argument were added. + Example: Given the function :func:`!myfunc`:: def myfunc(alist): @@ -85,7 +89,7 @@ The :mod:`dis` module can be invoked as a script from the command line: .. code-block:: sh - python -m dis [-h] [-C] [-O] [infile] + python -m dis [-h] [-C] [-O] [-P] [infile] The following options are accepted: @@ -103,6 +107,10 @@ The following options are accepted: Show offsets of instructions. +.. cmdoption:: -P, --show-positions + + Show positions of instructions in the source code. + If :file:`infile` is specified, its disassembled code will be written to stdout. Otherwise, disassembly is performed on compiled source code received from stdin. @@ -116,7 +124,8 @@ The bytecode analysis API allows pieces of Python code to be wrapped in a code. .. class:: Bytecode(x, *, first_line=None, current_offset=None,\ - show_caches=False, adaptive=False, show_offsets=False) + show_caches=False, adaptive=False, show_offsets=False,\ + show_positions=False) Analyse the bytecode corresponding to a function, generator, asynchronous generator, coroutine, method, string of source code, or a code object (as @@ -144,6 +153,9 @@ code. If *show_offsets* is ``True``, :meth:`.dis` will include instruction offsets in the output. + If *show_positions* is ``True``, :meth:`.dis` will include instruction + source code positions in the output. + .. classmethod:: from_traceback(tb, *, show_caches=False) Construct a :class:`Bytecode` instance from the given traceback, setting @@ -173,6 +185,12 @@ code. .. versionchanged:: 3.11 Added the *show_caches* and *adaptive* parameters. + .. versionchanged:: 3.13 + Added the *show_offsets* parameter + + .. versionchanged:: 3.14 + Added the *show_positions* parameter. + Example: .. doctest:: @@ -226,7 +244,8 @@ operation is being performed, so the intermediate analysis object isn't useful: Added *file* parameter. -.. function:: dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False) +.. function:: dis(x=None, *, file=None, depth=None, show_caches=False,\ + adaptive=False, show_offsets=False, show_positions=False) Disassemble the *x* object. *x* can denote either a module, a class, a method, a function, a generator, an asynchronous generator, a coroutine, @@ -265,9 +284,14 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.11 Added the *show_caches* and *adaptive* parameters. + .. versionchanged:: 3.13 + Added the *show_offsets* parameter. + + .. versionchanged:: 3.14 + Added the *show_positions* parameter. -.. function:: distb(tb=None, *, file=None, show_caches=False, adaptive=False, - show_offset=False) +.. function:: distb(tb=None, *, file=None, show_caches=False, adaptive=False,\ + show_offset=False, show_positions=False) Disassemble the top-of-stack function of a traceback, using the last traceback if none was passed. The instruction causing the exception is @@ -285,14 +309,19 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.13 Added the *show_offsets* parameter. + .. versionchanged:: 3.14 + Added the *show_positions* parameter. + .. function:: disassemble(code, lasti=-1, *, file=None, show_caches=False, adaptive=False) - disco(code, lasti=-1, *, file=None, show_caches=False, adaptive=False, - show_offsets=False) + disco(code, lasti=-1, *, file=None, show_caches=False, adaptive=False,\ + show_offsets=False, show_positions=False) Disassemble a code object, indicating the last instruction if *lasti* was provided. The output is divided in the following columns: - #. the line number, for the first instruction of each line + #. the source code location of the instruction. Complete location information + is shown if *show_positions* is true. Otherwise (the default) only the + line number is displayed. #. the current instruction, indicated as ``-->``, #. a labelled instruction, indicated with ``>>``, #. the address of the instruction, @@ -315,6 +344,9 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.13 Added the *show_offsets* parameter. + .. versionchanged:: 3.14 + Added the *show_positions* parameter. + .. function:: get_instructions(x, *, first_line=None, show_caches=False, adaptive=False) Return an iterator over the instructions in the supplied function, method, diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index a2c293443e23d3..4769affdf1d666 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -279,9 +279,9 @@ Notes: (1) .. index:: single: ? (question mark); in struct format strings - The ``'?'`` conversion code corresponds to the :c:expr:`_Bool` type defined by - C99. If this type is not available, it is simulated using a :c:expr:`char`. In - standard mode, it is always represented by one byte. + The ``'?'`` conversion code corresponds to the :c:expr:`_Bool` type + defined by C standards since C99. In standard mode, it is + represented by one byte. (2) When attempting to pack a non-integer using any of the integer conversion diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 8d2c4e77b6568f..c5e11458733508 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -242,6 +242,7 @@ Windows support contributed by Dino Viehland and Anthony Shaw.) .. _`PyPy project`: https://pypy.org/ + .. _whatsnew313-improved-error-messages: Improved error messages @@ -492,138 +493,123 @@ Incremental garbage collection The cycle garbage collector is now incremental. This means that maximum pause times are reduced by an order of magnitude or more for larger heaps. +(Contributed by Mark Shannon in :gh:`108362`.) Other Language Changes ====================== -* Classes have a new :attr:`~class.__static_attributes__` attribute, populated by the compiler, - with a tuple of names of attributes of this class which are assigned - through ``self.X`` from any function in its body. (Contributed by Irit Katriel - in :gh:`115775`.) +* The compiler now strips common leading whitespace + from every line in a docstring. + This reduces the size of the :term:`bytecode cache ` + (such as ``.pyc`` files), with reductions in file size of around 5%, + for example in :mod:`!sqlalchemy.orm.session` from SQLAlchemy 2.0. + This change affects tools that use docstrings, such as :mod:`doctest`. + + .. doctest:: + + >>> def spam(): + ... """ + ... This is a docstring with + ... leading whitespace. + ... + ... It even has multiple paragraphs! + ... """ + ... + >>> spam.__doc__ + '\nThis is a docstring with\n leading whitespace.\n\nIt even has multiple paragraphs!\n' -* The :func:`exec` and :func:`eval` built-ins now accept their ``globals`` - and ``locals`` namespace arguments as keywords. - (Contributed by Raphael Gaschignard in :gh:`105879`) + (Contributed by Inada Naoki in :gh:`81283`.) -* Allow the *count* argument of :meth:`str.replace` to be a keyword. - (Contributed by Hugo van Kemenade in :gh:`106487`.) +* :ref:`Annotation scopes ` within class scopes + can now contain lambdas and comprehensions. + Comprehensions that are located within class scopes + are not inlined into their parent scope. -* Compiler now strip indents from docstrings. - This will reduce the size of :term:`bytecode cache ` (e.g. ``.pyc`` file). - For example, cache file size for ``sqlalchemy.orm.session`` in SQLAlchemy 2.0 - is reduced by about 5%. - This change will affect tools using docstrings, like :mod:`doctest`. - (Contributed by Inada Naoki in :gh:`81283`.) + .. code-block:: python -* The :func:`compile` built-in can now accept a new flag, - ``ast.PyCF_OPTIMIZED_AST``, which is similar to ``ast.PyCF_ONLY_AST`` - except that the returned ``AST`` is optimized according to the value - of the ``optimize`` argument. - (Contributed by Irit Katriel in :gh:`108113`). + class C[T]: + type Alias = lambda: T -* :mod:`multiprocessing`, :mod:`concurrent.futures`, :mod:`compileall`: - Replace :func:`os.cpu_count` with :func:`os.process_cpu_count` to select the - default number of worker threads and processes. Get the CPU affinity - if supported. - (Contributed by Victor Stinner in :gh:`109649`.) + (Contributed by Jelle Zijlstra in :gh:`109118` and :gh:`118160`.) -* :func:`os.path.realpath` now resolves MS-DOS style file names even if - the file is not accessible. - (Contributed by Moonsik Park in :gh:`82367`.) +* :ref:`Future statements ` are no longer triggered by + relative imports of the :mod:`__future__` module, + meaning that statements of the form ``from .__future__ import ...`` + are now simply standard relative imports, with no special features activated. + (Contributed by Jeremiah Gabriel Pascual in :gh:`118216`.) -* Fixed a bug where a :keyword:`global` declaration in an :keyword:`except` block - is rejected when the global is used in the :keyword:`else` block. +* :keyword:`global` declarations are now permitted in :keyword:`except` blocks + when that global is used in the :keyword:`else` block. + Previously this raised an erroneous :exc:`SyntaxError`. (Contributed by Irit Katriel in :gh:`111123`.) -* Many functions now emit a warning if a boolean value is passed as - a file descriptor argument. - This can help catch some errors earlier. - (Contributed by Serhiy Storchaka in :gh:`82626`.) - -* Added a new environment variable :envvar:`PYTHON_FROZEN_MODULES`. It - determines whether or not frozen modules are ignored by the import machinery, - equivalent of the :option:`-X frozen_modules <-X>` command-line option. +* Add :envvar:`PYTHON_FROZEN_MODULES`, a new environment variable that + determines whether frozen modules are ignored by the import machinery, + equivalent to the :option:`-X frozen_modules <-X>` command-line option. (Contributed by Yilei Yang in :gh:`111374`.) -* Add :ref:`support for the perf profiler ` working without - frame pointers through the new environment variable - :envvar:`PYTHON_PERF_JIT_SUPPORT` and command-line option :option:`-X perf_jit - <-X>` (Contributed by Pablo Galindo in :gh:`118518`.) - -* The new :envvar:`PYTHON_HISTORY` environment variable can be used to change - the location of a ``.python_history`` file. - (Contributed by Levi Sabah, Zackery Spytz and Hugo van Kemenade in - :gh:`73965`.) - -* Add :exc:`PythonFinalizationError` exception. This exception derived from - :exc:`RuntimeError` is raised when an operation is blocked during - the :term:`Python finalization `. - - The following functions now raise PythonFinalizationError, instead of - :exc:`RuntimeError`: - - * :func:`_thread.start_new_thread`. - * :class:`subprocess.Popen`. - * :func:`os.fork`. - * :func:`os.forkpty`. - - (Contributed by Victor Stinner in :gh:`114570`.) - -* Added :attr:`!name` and :attr:`!mode` attributes for compressed - and archived file-like objects in modules :mod:`bz2`, :mod:`lzma`, - :mod:`tarfile` and :mod:`zipfile`. - (Contributed by Serhiy Storchaka in :gh:`115961`.) - -* Allow controlling Expat >=2.6.0 reparse deferral (:cve:`2023-52425`) - by adding five new methods: +* Add :ref:`support for the perf profiler ` working + without `frame pointers `_ through + the new environment variable :envvar:`PYTHON_PERF_JIT_SUPPORT` + and command-line option :option:`-X perf_jit <-X>`. + (Contributed by Pablo Galindo in :gh:`118518`.) - * :meth:`xml.etree.ElementTree.XMLParser.flush` - * :meth:`xml.etree.ElementTree.XMLPullParser.flush` - * :meth:`xml.parsers.expat.xmlparser.GetReparseDeferralEnabled` - * :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` - * :meth:`!xml.sax.expatreader.ExpatParser.flush` +* The location of a :file:`.python_history` file can be changed via the + new :envvar:`PYTHON_HISTORY` environment variable. + (Contributed by Levi Sabah, Zackery Spytz and Hugo van Kemenade + in :gh:`73965`.) - (Contributed by Sebastian Pipping in :gh:`115623`.) +* Classes have a new :attr:`~class.__static_attributes__` attribute. + This is populated by the compiler with a tuple of the class's attribute names + which are assigned through ``self.`` from any function in its body. + (Contributed by Irit Katriel in :gh:`115775`.) -* The :func:`ssl.create_default_context` API now includes - :data:`ssl.VERIFY_X509_PARTIAL_CHAIN` and :data:`ssl.VERIFY_X509_STRICT` - in its default flags. +* The compiler now creates a :attr:`!__firstlineno__` attribute on classes + with the line number of the first line of the class definition. + (Contributed by Serhiy Storchaka in :gh:`118465`.) - .. note:: +* The :func:`exec` and :func:`eval` builtins now accept + the *globals* and *locals* arguments as keywords. + (Contributed by Raphael Gaschignard in :gh:`105879`) - :data:`ssl.VERIFY_X509_STRICT` may reject pre-:rfc:`5280` or malformed - certificates that the underlying OpenSSL implementation otherwise would - accept. While disabling this is not recommended, you can do so using:: +* The :func:`compile` builtin now accepts a new flag, + ``ast.PyCF_OPTIMIZED_AST``, which is similar to ``ast.PyCF_ONLY_AST`` + except that the returned AST is optimized according to + the value of the *optimize* argument. + (Contributed by Irit Katriel in :gh:`108113`). - ctx = ssl.create_default_context() - ctx.verify_flags &= ~ssl.VERIFY_X509_STRICT +* Add :exc:`PythonFinalizationError`, a new exception derived from + :exc:`RuntimeError` and used to signal when operations are blocked + during :term:`finalization `. + The following callables now raise :exc:`!PythonFinalizationError`, + instead of :exc:`RuntimeError`: - (Contributed by William Woodruff in :gh:`112389`.) + * :func:`_thread.start_new_thread` + * :func:`os.fork` + * :func:`os.forkpty` + * :class:`subprocess.Popen` -* The :class:`configparser.ConfigParser` now accepts unnamed sections before named - ones if configured to do so. - (Contributed by Pedro Sousa Lacerda in :gh:`66449`.) + (Contributed by Victor Stinner in :gh:`114570`.) -* :ref:`annotation scope ` within class scopes can now - contain lambdas and comprehensions. Comprehensions that are located within - class scopes are not inlined into their parent scope. (Contributed by - Jelle Zijlstra in :gh:`109118` and :gh:`118160`.) +* Allow the *count* argument of :meth:`str.replace` to be a keyword. + (Contributed by Hugo van Kemenade in :gh:`106487`.) -* Classes have a new :attr:`!__firstlineno__` attribute, - populated by the compiler, with the line number of the first line - of the class definition. - (Contributed by Serhiy Storchaka in :gh:`118465`.) +* Many functions now emit a warning if a boolean value is passed as + a file descriptor argument. + This can help catch some errors earlier. + (Contributed by Serhiy Storchaka in :gh:`82626`.) -* ``from __future__ import ...`` statements are now just normal - relative imports if dots are present before the module name. - (Contributed by Jeremiah Gabriel Pascual in :gh:`118216`.) +* Added :attr:`!name` and :attr:`!mode` attributes + for compressed and archived file-like objects in + the :mod:`bz2`, :mod:`lzma`, :mod:`tarfile`, and :mod:`zipfile` modules. + (Contributed by Serhiy Storchaka in :gh:`115961`.) New Modules =========== -* :mod:`dbm.sqlite3`: SQLite backend for :mod:`dbm`. +* :mod:`dbm.sqlite3`: An SQLite backend for :mod:`dbm`. (Contributed by Raymond Hettinger and Erlend E. Aasland in :gh:`100414`.) @@ -646,7 +632,7 @@ array It can be used instead of ``'u'`` type code, which is deprecated. (Contributed by Inada Naoki in :gh:`80480`.) -* Add ``clear()`` method in order to implement ``MutableSequence``. +* Add ``clear()`` method in order to implement :class:`~collections.abc.MutableSequence`. (Contributed by Mike Zimin in :gh:`114894`.) ast @@ -748,6 +734,23 @@ base64 See the `Z85 specification `_ for more information. (Contributed by Matan Perelman in :gh:`75299`.) + +compileall +---------- + +* Select the default number of worker threads and processes using + :func:`os.process_cpu_count` instead of :func:`os.cpu_count`. + (Contributed by Victor Stinner in :gh:`109649`.) + + +concurrent.futures +------------------ + +* Select the default number of worker threads and processes using + :func:`os.process_cpu_count` instead of :func:`os.cpu_count`. + (Contributed by Victor Stinner in :gh:`109649`.) + + copy ---- @@ -901,7 +904,7 @@ importlib io -- -* The :class:`io.IOBase` finalizer now logs the ``close()`` method errors with +* The :class:`io.IOBase` finalizer now logs errors raised by the ``close()`` method with :data:`sys.unraisablehook`. Previously, errors were ignored silently by default, and only logged in :ref:`Python Development Mode ` or on :ref:`Python built on debug mode `. @@ -954,7 +957,7 @@ mimetypes mmap ---- -* The :class:`mmap.mmap` class now has an :meth:`~mmap.mmap.seekable` method +* The :class:`mmap.mmap` class now has a :meth:`~mmap.mmap.seekable` method that can be used when a seekable file-like object is required. The :meth:`~mmap.mmap.seek` method now returns the new absolute position. (Contributed by Donghee Na and Sylvie Liberman in :gh:`111835`.) @@ -965,6 +968,15 @@ mmap is inaccessible due to file system errors or access violations. (Contributed by Jannis Weigend in :gh:`118209`.) + +multiprocessing +--------------- + +* Select the default number of worker threads and processes using + :func:`os.process_cpu_count` instead of :func:`os.cpu_count`. + (Contributed by Victor Stinner in :gh:`109649`.) + + opcode ------ @@ -1029,10 +1041,15 @@ os.path * Add :func:`os.path.isreserved` to check if a path is reserved on the current system. This function is only available on Windows. (Contributed by Barney Gale in :gh:`88569`.) + * On Windows, :func:`os.path.isabs` no longer considers paths starting with exactly one (back)slash to be absolute. (Contributed by Barney Gale and Jon Foster in :gh:`44626`.) +* :func:`os.path.realpath` now resolves MS-DOS style file names even if + the file is not accessible. + (Contributed by Moonsik Park in :gh:`82367`.) + * Add support of *dir_fd* and *follow_symlinks* keyword arguments in :func:`shutil.chown`. (Contributed by Berker Peksag and Tahia K in :gh:`62308`) @@ -1074,7 +1091,7 @@ pathlib pdb --- -* Add ability to move between chained exceptions during post mortem debugging in :func:`~pdb.pm` using +* Add ability to move between chained exceptions during post-mortem debugging in :func:`~pdb.pm` using the new ``exceptions [exc_number]`` command for Pdb. (Contributed by Matthias Bussonnier in :gh:`106676`.) @@ -1132,6 +1149,26 @@ sqlite3 for filtering database objects to dump. (Contributed by Mariusz Felisiak in :gh:`91602`.) + +ssl +--- + +* The :func:`ssl.create_default_context` API now includes + :data:`ssl.VERIFY_X509_PARTIAL_CHAIN` and :data:`ssl.VERIFY_X509_STRICT` + in its default flags. + + .. note:: + + :data:`ssl.VERIFY_X509_STRICT` may reject pre-:rfc:`5280` or malformed + certificates that the underlying OpenSSL implementation otherwise would + accept. While disabling this is not recommended, you can do so using:: + + ctx = ssl.create_default_context() + ctx.verify_flags &= ~ssl.VERIFY_X509_STRICT + + (Contributed by William Woodruff in :gh:`112389`.) + + statistics ---------- @@ -1150,7 +1187,7 @@ subprocess more situations. Notably in the default case of ``close_fds=True`` on more recent versions of platforms including Linux, FreeBSD, and Solaris where the C library provides :c:func:`!posix_spawn_file_actions_addclosefrom_np`. - On Linux this should perform similar to our existing Linux :c:func:`!vfork` + On Linux this should perform similarly to the existing Linux :c:func:`!vfork` based code. A private control knob :attr:`!subprocess._USE_POSIX_SPAWN` can be set to ``False`` if you need to force :mod:`subprocess` not to ever use :func:`os.posix_spawn`. Please report your reason and platform details in @@ -1161,7 +1198,7 @@ subprocess sys --- -* Add the :func:`sys._is_interned` function to test if the string was interned. +* Add the :func:`sys._is_interned` function to test if a string was interned. This function is not guaranteed to exist in all implementations of Python. (Contributed by Serhiy Storchaka in :gh:`78573`.) @@ -1177,7 +1214,7 @@ time ---- * On Windows, :func:`time.monotonic()` now uses the - ``QueryPerformanceCounter()`` clock to have a resolution better than 1 us, + ``QueryPerformanceCounter()`` clock to have a resolution better than 1 μs, instead of the ``GetTickCount64()`` clock which has a resolution of 15.6 ms. (Contributed by Victor Stinner in :gh:`88494`.) @@ -1208,7 +1245,7 @@ tkinter * Add new optional keyword-only parameter *return_ints* in the :meth:`!Text.count` method. - Passing ``return_ints=True`` makes it always returning the single count + Passing ``return_ints=True`` makes it always return the single count as an integer instead of a 1-tuple or ``None``. (Contributed by Serhiy Storchaka in :gh:`97928`.) @@ -1251,8 +1288,8 @@ traceback types ----- -* :class:`~types.SimpleNamespace` constructor now allows specifying initial - values of attributes as a positional argument which must be a mapping or +* The :class:`~types.SimpleNamespace` constructor now allows specifying initial + values of attributes as a positional argument, which must be a mapping or an iterable of key-value pairs. (Contributed by Serhiy Storchaka in :gh:`108191`.) @@ -1297,13 +1334,26 @@ warnings warning may also be emitted when a decorated function or class is used at runtime. See :pep:`702`. (Contributed by Jelle Zijlstra in :gh:`104003`.) -xml.etree.ElementTree ---------------------- + +xml +--- + +* Allow controlling Expat >=2.6.0 reparse deferral (:cve:`2023-52425`) + by adding five new methods: + + * :meth:`xml.etree.ElementTree.XMLParser.flush` + * :meth:`xml.etree.ElementTree.XMLPullParser.flush` + * :meth:`xml.parsers.expat.xmlparser.GetReparseDeferralEnabled` + * :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` + * :meth:`!xml.sax.expatreader.ExpatParser.flush` + + (Contributed by Sebastian Pipping in :gh:`115623`.) * Add the :meth:`!close` method for the iterator returned by :func:`~xml.etree.ElementTree.iterparse` for explicit cleaning up. (Contributed by Serhiy Storchaka in :gh:`69893`.) + zipimport --------- @@ -1459,6 +1509,10 @@ PEP 594: dead batteries (and other module removals) configparser ------------ +* The :class:`configparser.ConfigParser` now accepts unnamed sections + before named ones if configured to do so. + (Contributed by Pedro Sousa Lacerda in :gh:`66449`.) + * Remove the undocumented :class:`!configparser.LegacyInterpolation` class, deprecated in the docstring since Python 3.2, and with a deprecation warning since Python 3.11. @@ -1661,7 +1715,7 @@ New Deprecations (Contributed by Erlend E. Aasland in :gh:`107948` and :gh:`108278`.) -* :mod:`sys`: :func:`sys._enablelegacywindowsfsencoding` function. +* :mod:`sys`: The :func:`sys._enablelegacywindowsfsencoding` function is deprecated. Replace it with the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable. (Contributed by Inada Naoki in :gh:`73427`.) @@ -1685,8 +1739,8 @@ New Deprecations deprecated. Passing ``None`` to the 'fields' parameter (``NT = NamedTuple("NT", None)`` or ``TD = TypedDict("TD", None)``) is also deprecated. Both will be disallowed in Python 3.15. To create a NamedTuple - class with 0 fields, use ``class NT(NamedTuple): pass`` or - ``NT = NamedTuple("NT", [])``. To create a TypedDict class with 0 fields, use + class with zero fields, use ``class NT(NamedTuple): pass`` or + ``NT = NamedTuple("NT", [])``. To create a TypedDict class with zero fields, use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. (Contributed by Alex Waygood in :gh:`105566` and :gh:`105570`.) @@ -1703,7 +1757,7 @@ New Deprecations * :ref:`user-defined-funcs`: Assignment to a function's :attr:`~function.__code__` attribute where the new code - object's type does not match the function's type, is deprecated. The + object's type does not match the function's type is deprecated. The different types are: plain function, generator, async generator and coroutine. (Contributed by Irit Katriel in :gh:`81137`.) @@ -1944,7 +1998,7 @@ New Features (Contributed by Victor Stinner in :gh:`116936`.) * Add two new functions to the C-API, :c:func:`PyRefTracer_SetTracer` and - :c:func:`PyRefTracer_GetTracer`, that allows to track object creation and + :c:func:`PyRefTracer_GetTracer`, that allow to track object creation and destruction the same way the :mod:`tracemalloc` module does. (Contributed by Pablo Galindo in :gh:`93502`.) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 088f70d9e9fad4..a34dc639ad2a94 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -110,6 +110,22 @@ ast (Contributed by Bénédikt Tran in :gh:`121141`.) +dis +--- + +* Added support for rendering full source location information of + :class:`instructions `, rather than only the line number. + This feature is added to the following interfaces via the ``show_positions`` + keyword argument: + + - :class:`dis.Bytecode`, + - :func:`dis.dis`, :func:`dis.distb`, and + - :func:`dis.disassemble`. + + This feature is also exposed via :option:`dis --show-positions`. + + (Contributed by Bénédikt Tran in :gh:`123165`.) + fractions --------- diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 2e76e9f64d4aec..a722a2e2527862 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -593,6 +593,10 @@ extern _Py_CODEUNIT _Py_GetBaseCodeUnit(PyCodeObject *code, int offset); extern int _PyInstruction_GetLength(PyCodeObject *code, int offset); +struct _PyCode8 _PyCode_DEF(8); + +PyAPI_DATA(const struct _PyCode8) _Py_InitCleanup; + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index d3d745574667aa..a6f7c1735b349f 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -144,8 +144,9 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * static inline void _PyFrame_Initialize( _PyInterpreterFrame *frame, PyFunctionObject *func, - PyObject *locals, PyCodeObject *code, int null_locals_from) + PyObject *locals, PyCodeObject *code, int null_locals_from, _PyInterpreterFrame *previous) { + frame->previous = previous; frame->f_funcobj = (PyObject *)func; frame->f_executable = Py_NewRef(code); frame->f_builtins = func->func_builtins; @@ -298,26 +299,27 @@ PyAPI_FUNC(void) _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFr * Must be guarded by _PyThreadState_HasStackSpace() * Consumes reference to func. */ static inline _PyInterpreterFrame * -_PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func, int null_locals_from) +_PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func, int null_locals_from, _PyInterpreterFrame * previous) { CALL_STAT_INC(frames_pushed); PyCodeObject *code = (PyCodeObject *)func->func_code; _PyInterpreterFrame *new_frame = (_PyInterpreterFrame *)tstate->datastack_top; tstate->datastack_top += code->co_framesize; assert(tstate->datastack_top < tstate->datastack_limit); - _PyFrame_Initialize(new_frame, func, NULL, code, null_locals_from); + _PyFrame_Initialize(new_frame, func, NULL, code, null_locals_from, previous); return new_frame; } /* Pushes a trampoline frame without checking for space. * Must be guarded by _PyThreadState_HasStackSpace() */ static inline _PyInterpreterFrame * -_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth) +_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth, _PyInterpreterFrame * previous) { CALL_STAT_INC(frames_pushed); _PyInterpreterFrame *frame = (_PyInterpreterFrame *)tstate->datastack_top; tstate->datastack_top += code->co_framesize; assert(tstate->datastack_top < tstate->datastack_limit); + frame->previous = previous; frame->f_funcobj = Py_None; frame->f_executable = Py_NewRef(code); #ifdef Py_DEBUG @@ -344,7 +346,8 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int PyAPI_FUNC(_PyInterpreterFrame *) _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, _PyStackRef const* args, - size_t argcount, PyObject *kwnames); + size_t argcount, PyObject *kwnames, + _PyInterpreterFrame *previous); #ifdef __cplusplus } diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index a5640b7bcb7d60..ee33da77f01f2d 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -301,6 +301,14 @@ _Py_INCREF_TYPE(PyTypeObject *type) return; } + // gh-122974: GCC 11 warns about the access to PyHeapTypeObject fields when + // _Py_INCREF_TYPE() is called on a statically allocated type. The + // _PyType_HasFeature check above ensures that the type is a heap type. +#if defined(__GNUC__) && __GNUC__ >= 11 +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Warray-bounds" +#endif + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); PyHeapTypeObject *ht = (PyHeapTypeObject *)type; @@ -319,6 +327,10 @@ _Py_INCREF_TYPE(PyTypeObject *type) // It handles the unique_id=-1 case to keep the inlinable function smaller. _PyType_IncrefSlow(ht); } + +#if defined(__GNUC__) && __GNUC__ >= 11 +# pragma GCC diagnostic pop +#endif } static inline void @@ -749,7 +761,7 @@ static inline int _PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) { } extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems); -extern PyObject *_PyType_NewManagedObject(PyTypeObject *type); +PyAPI_FUNC(PyObject *) _PyType_NewManagedObject(PyTypeObject *type); extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 8e38855175a646..236e6b414611f3 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -537,7 +537,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case CALL: return 1; case CALL_ALLOC_AND_ENTER_INIT: - return 1; + return 0; case CALL_BOUND_METHOD_EXACT_ARGS: return 0; case CALL_BOUND_METHOD_GENERAL: @@ -1261,6 +1261,7 @@ _PyOpcode_macro_expansion[256] = { [BUILD_SLICE] = { .nuops = 1, .uops = { { _BUILD_SLICE, 0, 0 } } }, [BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, 0, 0 } } }, [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, 0, 0 } } }, + [CALL_ALLOC_AND_ENTER_INIT] = { .nuops = 4, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_AND_ALLOCATE_OBJECT, 2, 1 }, { _CREATE_INIT_FRAME, 0, 0 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 9, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, 0, 0 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_BOUND_METHOD_GENERAL] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, 0, 0 }, { _PY_FRAME_GENERAL, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_BUILTIN_CLASS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_CLASS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 2a255143c3e0c9..226f4a63460b37 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -34,7 +34,6 @@ extern "C" { #define _BUILD_SLICE BUILD_SLICE #define _BUILD_STRING BUILD_STRING #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL_ALLOC_AND_ENTER_INIT CALL_ALLOC_AND_ENTER_INIT #define _CALL_BUILTIN_CLASS 314 #define _CALL_BUILTIN_FAST 315 #define _CALL_BUILTIN_FAST_WITH_KEYWORDS 316 @@ -53,97 +52,99 @@ extern "C" { #define _CALL_STR_1 324 #define _CALL_TUPLE_1 325 #define _CALL_TYPE_1 CALL_TYPE_1 -#define _CHECK_ATTR_CLASS 326 -#define _CHECK_ATTR_METHOD_LAZY_DICT 327 -#define _CHECK_ATTR_MODULE 328 -#define _CHECK_ATTR_WITH_HINT 329 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 330 +#define _CHECK_AND_ALLOCATE_OBJECT 326 +#define _CHECK_ATTR_CLASS 327 +#define _CHECK_ATTR_METHOD_LAZY_DICT 328 +#define _CHECK_ATTR_MODULE 329 +#define _CHECK_ATTR_WITH_HINT 330 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 331 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 331 -#define _CHECK_FUNCTION_EXACT_ARGS 332 -#define _CHECK_FUNCTION_VERSION 333 -#define _CHECK_FUNCTION_VERSION_KW 334 -#define _CHECK_IS_NOT_PY_CALLABLE 335 -#define _CHECK_IS_NOT_PY_CALLABLE_KW 336 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 337 -#define _CHECK_METHOD_VERSION 338 -#define _CHECK_METHOD_VERSION_KW 339 -#define _CHECK_PEP_523 340 -#define _CHECK_PERIODIC 341 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 342 -#define _CHECK_STACK_SPACE 343 -#define _CHECK_STACK_SPACE_OPERAND 344 -#define _CHECK_VALIDITY 345 -#define _CHECK_VALIDITY_AND_SET_IP 346 -#define _COMPARE_OP 347 -#define _COMPARE_OP_FLOAT 348 -#define _COMPARE_OP_INT 349 -#define _COMPARE_OP_STR 350 -#define _CONTAINS_OP 351 +#define _CHECK_FUNCTION 332 +#define _CHECK_FUNCTION_EXACT_ARGS 333 +#define _CHECK_FUNCTION_VERSION 334 +#define _CHECK_FUNCTION_VERSION_KW 335 +#define _CHECK_IS_NOT_PY_CALLABLE 336 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 337 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 338 +#define _CHECK_METHOD_VERSION 339 +#define _CHECK_METHOD_VERSION_KW 340 +#define _CHECK_PEP_523 341 +#define _CHECK_PERIODIC 342 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 343 +#define _CHECK_STACK_SPACE 344 +#define _CHECK_STACK_SPACE_OPERAND 345 +#define _CHECK_VALIDITY 346 +#define _CHECK_VALIDITY_AND_SET_IP 347 +#define _COMPARE_OP 348 +#define _COMPARE_OP_FLOAT 349 +#define _COMPARE_OP_INT 350 +#define _COMPARE_OP_STR 351 +#define _CONTAINS_OP 352 #define _CONTAINS_OP_DICT CONTAINS_OP_DICT #define _CONTAINS_OP_SET CONTAINS_OP_SET #define _CONVERT_VALUE CONVERT_VALUE #define _COPY COPY #define _COPY_FREE_VARS COPY_FREE_VARS +#define _CREATE_INIT_FRAME 353 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 352 +#define _DEOPT 354 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 353 -#define _DO_CALL_KW 354 -#define _DYNAMIC_EXIT 355 +#define _DO_CALL 355 +#define _DO_CALL_KW 356 +#define _DYNAMIC_EXIT 357 #define _END_SEND END_SEND -#define _ERROR_POP_N 356 +#define _ERROR_POP_N 358 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 357 -#define _EXPAND_METHOD_KW 358 -#define _FATAL_ERROR 359 +#define _EXPAND_METHOD 359 +#define _EXPAND_METHOD_KW 360 +#define _FATAL_ERROR 361 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 360 -#define _FOR_ITER_GEN_FRAME 361 -#define _FOR_ITER_TIER_TWO 362 +#define _FOR_ITER 362 +#define _FOR_ITER_GEN_FRAME 363 +#define _FOR_ITER_TIER_TWO 364 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 363 -#define _GUARD_BOTH_INT 364 -#define _GUARD_BOTH_UNICODE 365 -#define _GUARD_BUILTINS_VERSION 366 -#define _GUARD_DORV_NO_DICT 367 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 368 -#define _GUARD_GLOBALS_VERSION 369 -#define _GUARD_IS_FALSE_POP 370 -#define _GUARD_IS_NONE_POP 371 -#define _GUARD_IS_NOT_NONE_POP 372 -#define _GUARD_IS_TRUE_POP 373 -#define _GUARD_KEYS_VERSION 374 -#define _GUARD_NOS_FLOAT 375 -#define _GUARD_NOS_INT 376 -#define _GUARD_NOT_EXHAUSTED_LIST 377 -#define _GUARD_NOT_EXHAUSTED_RANGE 378 -#define _GUARD_NOT_EXHAUSTED_TUPLE 379 -#define _GUARD_TOS_FLOAT 380 -#define _GUARD_TOS_INT 381 -#define _GUARD_TYPE_VERSION 382 +#define _GUARD_BOTH_FLOAT 365 +#define _GUARD_BOTH_INT 366 +#define _GUARD_BOTH_UNICODE 367 +#define _GUARD_BUILTINS_VERSION 368 +#define _GUARD_DORV_NO_DICT 369 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 370 +#define _GUARD_GLOBALS_VERSION 371 +#define _GUARD_IS_FALSE_POP 372 +#define _GUARD_IS_NONE_POP 373 +#define _GUARD_IS_NOT_NONE_POP 374 +#define _GUARD_IS_TRUE_POP 375 +#define _GUARD_KEYS_VERSION 376 +#define _GUARD_NOS_FLOAT 377 +#define _GUARD_NOS_INT 378 +#define _GUARD_NOT_EXHAUSTED_LIST 379 +#define _GUARD_NOT_EXHAUSTED_RANGE 380 +#define _GUARD_NOT_EXHAUSTED_TUPLE 381 +#define _GUARD_TOS_FLOAT 382 +#define _GUARD_TOS_INT 383 +#define _GUARD_TYPE_VERSION 384 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 383 -#define _INIT_CALL_PY_EXACT_ARGS 384 -#define _INIT_CALL_PY_EXACT_ARGS_0 385 -#define _INIT_CALL_PY_EXACT_ARGS_1 386 -#define _INIT_CALL_PY_EXACT_ARGS_2 387 -#define _INIT_CALL_PY_EXACT_ARGS_3 388 -#define _INIT_CALL_PY_EXACT_ARGS_4 389 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 385 +#define _INIT_CALL_PY_EXACT_ARGS 386 +#define _INIT_CALL_PY_EXACT_ARGS_0 387 +#define _INIT_CALL_PY_EXACT_ARGS_1 388 +#define _INIT_CALL_PY_EXACT_ARGS_2 389 +#define _INIT_CALL_PY_EXACT_ARGS_3 390 +#define _INIT_CALL_PY_EXACT_ARGS_4 391 #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER @@ -155,65 +156,65 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 390 -#define _IS_NONE 391 +#define _INTERNAL_INCREMENT_OPT_COUNTER 392 +#define _IS_NONE 393 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 392 -#define _ITER_CHECK_RANGE 393 -#define _ITER_CHECK_TUPLE 394 -#define _ITER_JUMP_LIST 395 -#define _ITER_JUMP_RANGE 396 -#define _ITER_JUMP_TUPLE 397 -#define _ITER_NEXT_LIST 398 -#define _ITER_NEXT_RANGE 399 -#define _ITER_NEXT_TUPLE 400 -#define _JUMP_TO_TOP 401 +#define _ITER_CHECK_LIST 394 +#define _ITER_CHECK_RANGE 395 +#define _ITER_CHECK_TUPLE 396 +#define _ITER_JUMP_LIST 397 +#define _ITER_JUMP_RANGE 398 +#define _ITER_JUMP_TUPLE 399 +#define _ITER_NEXT_LIST 400 +#define _ITER_NEXT_RANGE 401 +#define _ITER_NEXT_TUPLE 402 +#define _JUMP_TO_TOP 403 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 402 -#define _LOAD_ATTR_CLASS 403 -#define _LOAD_ATTR_CLASS_0 404 -#define _LOAD_ATTR_CLASS_1 405 +#define _LOAD_ATTR 404 +#define _LOAD_ATTR_CLASS 405 +#define _LOAD_ATTR_CLASS_0 406 +#define _LOAD_ATTR_CLASS_1 407 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 406 -#define _LOAD_ATTR_INSTANCE_VALUE_0 407 -#define _LOAD_ATTR_INSTANCE_VALUE_1 408 -#define _LOAD_ATTR_METHOD_LAZY_DICT 409 -#define _LOAD_ATTR_METHOD_NO_DICT 410 -#define _LOAD_ATTR_METHOD_WITH_VALUES 411 -#define _LOAD_ATTR_MODULE 412 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 413 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 414 -#define _LOAD_ATTR_PROPERTY_FRAME 415 -#define _LOAD_ATTR_SLOT 416 -#define _LOAD_ATTR_SLOT_0 417 -#define _LOAD_ATTR_SLOT_1 418 -#define _LOAD_ATTR_WITH_HINT 419 +#define _LOAD_ATTR_INSTANCE_VALUE 408 +#define _LOAD_ATTR_INSTANCE_VALUE_0 409 +#define _LOAD_ATTR_INSTANCE_VALUE_1 410 +#define _LOAD_ATTR_METHOD_LAZY_DICT 411 +#define _LOAD_ATTR_METHOD_NO_DICT 412 +#define _LOAD_ATTR_METHOD_WITH_VALUES 413 +#define _LOAD_ATTR_MODULE 414 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 415 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 416 +#define _LOAD_ATTR_PROPERTY_FRAME 417 +#define _LOAD_ATTR_SLOT 418 +#define _LOAD_ATTR_SLOT_0 419 +#define _LOAD_ATTR_SLOT_1 420 +#define _LOAD_ATTR_WITH_HINT 421 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 420 -#define _LOAD_CONST_INLINE_BORROW 421 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 422 -#define _LOAD_CONST_INLINE_WITH_NULL 423 +#define _LOAD_CONST_INLINE 422 +#define _LOAD_CONST_INLINE_BORROW 423 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 424 +#define _LOAD_CONST_INLINE_WITH_NULL 425 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 424 -#define _LOAD_FAST_0 425 -#define _LOAD_FAST_1 426 -#define _LOAD_FAST_2 427 -#define _LOAD_FAST_3 428 -#define _LOAD_FAST_4 429 -#define _LOAD_FAST_5 430 -#define _LOAD_FAST_6 431 -#define _LOAD_FAST_7 432 +#define _LOAD_FAST 426 +#define _LOAD_FAST_0 427 +#define _LOAD_FAST_1 428 +#define _LOAD_FAST_2 429 +#define _LOAD_FAST_3 430 +#define _LOAD_FAST_4 431 +#define _LOAD_FAST_5 432 +#define _LOAD_FAST_6 433 +#define _LOAD_FAST_7 434 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 433 -#define _LOAD_GLOBAL_BUILTINS 434 -#define _LOAD_GLOBAL_MODULE 435 +#define _LOAD_GLOBAL 435 +#define _LOAD_GLOBAL_BUILTINS 436 +#define _LOAD_GLOBAL_MODULE 437 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SPECIAL LOAD_SPECIAL @@ -226,59 +227,59 @@ extern "C" { #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 436 -#define _MONITOR_CALL 437 -#define _MONITOR_JUMP_BACKWARD 438 -#define _MONITOR_RESUME 439 +#define _MAYBE_EXPAND_METHOD 438 +#define _MONITOR_CALL 439 +#define _MONITOR_JUMP_BACKWARD 440 +#define _MONITOR_RESUME 441 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 440 -#define _POP_JUMP_IF_TRUE 441 +#define _POP_JUMP_IF_FALSE 442 +#define _POP_JUMP_IF_TRUE 443 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 442 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 444 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 443 +#define _PUSH_FRAME 445 #define _PUSH_NULL PUSH_NULL -#define _PY_FRAME_GENERAL 444 -#define _PY_FRAME_KW 445 -#define _QUICKEN_RESUME 446 -#define _REPLACE_WITH_TRUE 447 +#define _PY_FRAME_GENERAL 446 +#define _PY_FRAME_KW 447 +#define _QUICKEN_RESUME 448 +#define _REPLACE_WITH_TRUE 449 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 448 -#define _SEND 449 -#define _SEND_GEN_FRAME 450 +#define _SAVE_RETURN_OFFSET 450 +#define _SEND 451 +#define _SEND_GEN_FRAME 452 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 451 -#define _STORE_ATTR 452 -#define _STORE_ATTR_INSTANCE_VALUE 453 -#define _STORE_ATTR_SLOT 454 -#define _STORE_ATTR_WITH_HINT 455 +#define _START_EXECUTOR 453 +#define _STORE_ATTR 454 +#define _STORE_ATTR_INSTANCE_VALUE 455 +#define _STORE_ATTR_SLOT 456 +#define _STORE_ATTR_WITH_HINT 457 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 456 -#define _STORE_FAST_0 457 -#define _STORE_FAST_1 458 -#define _STORE_FAST_2 459 -#define _STORE_FAST_3 460 -#define _STORE_FAST_4 461 -#define _STORE_FAST_5 462 -#define _STORE_FAST_6 463 -#define _STORE_FAST_7 464 +#define _STORE_FAST 458 +#define _STORE_FAST_0 459 +#define _STORE_FAST_1 460 +#define _STORE_FAST_2 461 +#define _STORE_FAST_3 462 +#define _STORE_FAST_4 463 +#define _STORE_FAST_5 464 +#define _STORE_FAST_6 465 +#define _STORE_FAST_7 466 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 465 +#define _STORE_SUBSCR 467 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 466 -#define _TO_BOOL 467 +#define _TIER2_RESUME_CHECK 468 +#define _TO_BOOL 469 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -288,14 +289,14 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 468 +#define _UNPACK_SEQUENCE 470 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE #define __DO_CALL_FUNCTION_EX _DO_CALL_FUNCTION_EX -#define MAX_UOP_ID 468 +#define MAX_UOP_ID 470 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 98757b6c8d8321..e2cba4dc0dfc81 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -225,6 +225,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_CALL_STR_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_AND_ALLOCATE_OBJECT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CREATE_INIT_FRAME] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -327,6 +329,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CALL_STR_1] = "_CALL_STR_1", [_CALL_TUPLE_1] = "_CALL_TUPLE_1", [_CALL_TYPE_1] = "_CALL_TYPE_1", + [_CHECK_AND_ALLOCATE_OBJECT] = "_CHECK_AND_ALLOCATE_OBJECT", [_CHECK_ATTR_CLASS] = "_CHECK_ATTR_CLASS", [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", [_CHECK_ATTR_MODULE] = "_CHECK_ATTR_MODULE", @@ -360,6 +363,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CONVERT_VALUE] = "_CONVERT_VALUE", [_COPY] = "_COPY", [_COPY_FREE_VARS] = "_COPY_FREE_VARS", + [_CREATE_INIT_FRAME] = "_CREATE_INIT_FRAME", [_DELETE_ATTR] = "_DELETE_ATTR", [_DELETE_DEREF] = "_DELETE_DEREF", [_DELETE_FAST] = "_DELETE_FAST", @@ -960,6 +964,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 3; case _CALL_TUPLE_1: return 3; + case _CHECK_AND_ALLOCATE_OBJECT: + return 2 + oparg; + case _CREATE_INIT_FRAME: + return 2 + oparg; case _EXIT_INIT_CHECK: return 1; case _CALL_BUILTIN_CLASS: diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index a8d3f520340dcf..330ebbd6185679 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -161,17 +161,22 @@ def __init__( super().__init__(locals=locals, filename=filename, local_exit=local_exit) # type: ignore[call-arg] self.can_colorize = _colorize.can_colorize() - def showsyntaxerror(self, filename=None): - super().showsyntaxerror(colorize=self.can_colorize) + def showsyntaxerror(self, filename=None, **kwargs): + super().showsyntaxerror(**kwargs) - def showtraceback(self): - super().showtraceback(colorize=self.can_colorize) + def _excepthook(self, typ, value, tb): + import traceback + lines = traceback.format_exception( + typ, value, tb, + colorize=self.can_colorize, + limit=traceback.BUILTIN_EXCEPTION_LIMIT) + self.write(''.join(lines)) def runsource(self, source, filename="", symbol="single"): try: tree = ast.parse(source) except (SyntaxError, OverflowError, ValueError): - self.showsyntaxerror(filename) + self.showsyntaxerror(filename, source=source) return False if tree.body: *_, last_stmt = tree.body @@ -188,10 +193,10 @@ def runsource(self, source, filename="", symbol="single"): f"Try the asyncio REPL ({python} -m asyncio) to use" f" top-level 'await' and run background asyncio tasks." ) - self.showsyntaxerror(filename) + self.showsyntaxerror(filename, source=source) return False except (OverflowError, ValueError): - self.showsyntaxerror(filename) + self.showsyntaxerror(filename, source=source) return False if code is None: diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 3d94f91753587e..143770a885a2c2 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -342,10 +342,10 @@ def do(self) -> None: class _ReadlineWrapper: f_in: int = -1 f_out: int = -1 - reader: ReadlineAlikeReader | None = None + reader: ReadlineAlikeReader | None = field(default=None, repr=False) saved_history_length: int = -1 startup_hook: Callback | None = None - config: ReadlineConfig = field(default_factory=ReadlineConfig) + config: ReadlineConfig = field(default_factory=ReadlineConfig, repr=False) def __post_init__(self) -> None: if self.f_in == -1: diff --git a/Lib/code.py b/Lib/code.py index a1fd389b5a1015..b1079824a75414 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -64,7 +64,7 @@ def runsource(self, source, filename="", symbol="single"): code = self.compile(source, filename, symbol) except (OverflowError, SyntaxError, ValueError): # Case 1 - self.showsyntaxerror(filename) + self.showsyntaxerror(filename, source=source) return False if code is None: @@ -106,7 +106,6 @@ def showsyntaxerror(self, filename=None, **kwargs): The output is written by self.write(), below. """ - colorize = kwargs.pop('colorize', False) try: typ, value, tb = sys.exc_info() if filename and typ is SyntaxError: @@ -119,11 +118,12 @@ def showsyntaxerror(self, filename=None, **kwargs): else: # Stuff in the right filename value = SyntaxError(msg, (filename, lineno, offset, line)) - self._showtraceback(typ, value, None, colorize) + source = kwargs.pop('source', "") + self._showtraceback(typ, value, None, source) finally: typ = value = tb = None - def showtraceback(self, **kwargs): + def showtraceback(self): """Display the exception that just occurred. We remove the first stack item because it is our own code. @@ -131,21 +131,24 @@ def showtraceback(self, **kwargs): The output is written by self.write(), below. """ - colorize = kwargs.pop('colorize', False) try: typ, value, tb = sys.exc_info() - self._showtraceback(typ, value, tb.tb_next, colorize) + self._showtraceback(typ, value, tb.tb_next, "") finally: typ = value = tb = None - def _showtraceback(self, typ, value, tb, colorize): + def _showtraceback(self, typ, value, tb, source): sys.last_type = typ sys.last_traceback = tb - sys.last_exc = sys.last_value = value = value.with_traceback(tb) + value = value.with_traceback(tb) + # Set the line of text that the exception refers to + lines = source.splitlines() + if (source and typ is SyntaxError + and not value.text and len(lines) >= value.lineno): + value.text = lines[value.lineno - 1] + sys.last_exc = sys.last_value = value if sys.excepthook is sys.__excepthook__: - lines = traceback.format_exception(typ, value, tb, - colorize=colorize) - self.write(''.join(lines)) + self._excepthook(typ, value, tb) else: # If someone has set sys.excepthook, we let that take precedence # over self.write @@ -162,6 +165,12 @@ def _showtraceback(self, typ, value, tb, colorize): print('Original exception was:', file=sys.stderr) sys.__excepthook__(typ, value, tb) + def _excepthook(self, typ, value, tb): + # This method is being overwritten in + # _pyrepl.console.InteractiveColoredConsole + lines = traceback.format_exception(typ, value, tb) + self.write(''.join(lines)) + def write(self, data): """Write a string. diff --git a/Lib/configparser.py b/Lib/configparser.py index 4344a9e8baca44..420dce77c234e1 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -160,7 +160,7 @@ "NoOptionError", "InterpolationError", "InterpolationDepthError", "InterpolationMissingOptionError", "InterpolationSyntaxError", "ParsingError", "MissingSectionHeaderError", - "MultilineContinuationError", + "MultilineContinuationError", "UnnamedSectionDisabledError", "ConfigParser", "RawConfigParser", "Interpolation", "BasicInterpolation", "ExtendedInterpolation", "SectionProxy", "ConverterMapping", @@ -362,6 +362,14 @@ def __init__(self, filename, lineno, line): self.line = line self.args = (filename, lineno, line) + +class UnnamedSectionDisabledError(Error): + """Raised when an attempt to use UNNAMED_SECTION is made with the + feature disabled.""" + def __init__(self): + Error.__init__(self, "Support for UNNAMED_SECTION is disabled.") + + class _UnnamedSection: def __repr__(self): @@ -692,6 +700,10 @@ def add_section(self, section): if section == self.default_section: raise ValueError('Invalid section name: %r' % section) + if section is UNNAMED_SECTION: + if not self._allow_unnamed_section: + raise UnnamedSectionDisabledError + if section in self._sections: raise DuplicateSectionError(section) self._sections[section] = self._dict() @@ -1203,20 +1215,20 @@ def _convert_to_boolean(self, value): return self.BOOLEAN_STATES[value.lower()] def _validate_value_types(self, *, section="", option="", value=""): - """Raises a TypeError for non-string values. + """Raises a TypeError for illegal non-string values. - The only legal non-string value if we allow valueless - options is None, so we need to check if the value is a - string if: - - we do not allow valueless options, or - - we allow valueless options but the value is not None + Legal non-string values are UNNAMED_SECTION and falsey values if + they are allowed. For compatibility reasons this method is not used in classic set() for RawConfigParsers. It is invoked in every case for mapping protocol access and in ConfigParser.set(). """ - if not isinstance(section, str): - raise TypeError("section names must be strings") + if section is UNNAMED_SECTION: + if not self._allow_unnamed_section: + raise UnnamedSectionDisabledError + elif not isinstance(section, str): + raise TypeError("section names must be strings or UNNAMED_SECTION") if not isinstance(option, str): raise TypeError("option keys must be strings") if not self._allow_no_value or value: diff --git a/Lib/dis.py b/Lib/dis.py index bb922b786f5307..077c4035ca6511 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -80,7 +80,7 @@ def _try_compile(source, name): return compile(source, name, 'exec') def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, - show_offsets=False): + show_offsets=False, show_positions=False): """Disassemble classes, methods, functions, and other compiled objects. With no argument, disassemble the last traceback. @@ -91,7 +91,7 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, """ if x is None: distb(file=file, show_caches=show_caches, adaptive=adaptive, - show_offsets=show_offsets) + show_offsets=show_offsets, show_positions=show_positions) return # Extract functions from methods. if hasattr(x, '__func__'): @@ -112,12 +112,12 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, if isinstance(x1, _have_code): print("Disassembly of %s:" % name, file=file) try: - dis(x1, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) + dis(x1, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) except TypeError as msg: print("Sorry:", msg, file=file) print(file=file) elif hasattr(x, 'co_code'): # Code object - _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) + _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) elif isinstance(x, (bytes, bytearray)): # Raw bytecode labels_map = _make_labels_map(x) label_width = 4 + len(str(len(labels_map))) @@ -128,12 +128,12 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, arg_resolver = ArgResolver(labels_map=labels_map) _disassemble_bytes(x, arg_resolver=arg_resolver, formatter=formatter) elif isinstance(x, str): # Source code - _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) + _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) else: raise TypeError("don't know how to disassemble %s objects" % type(x).__name__) -def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets=False): +def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False): """Disassemble a traceback (default: last traceback).""" if tb is None: try: @@ -144,7 +144,7 @@ def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets except AttributeError: raise RuntimeError("no last traceback to disassemble") from None while tb.tb_next: tb = tb.tb_next - disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) + disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) # The inspect module interrogates this dictionary to build its # list of CO_* constants. It is also used by pretty_flags to @@ -427,21 +427,25 @@ def __str__(self): class Formatter: def __init__(self, file=None, lineno_width=0, offset_width=0, label_width=0, - line_offset=0, show_caches=False): + line_offset=0, show_caches=False, *, show_positions=False): """Create a Formatter *file* where to write the output - *lineno_width* sets the width of the line number field (0 omits it) + *lineno_width* sets the width of the source location field (0 omits it). + Should be large enough for a line number or full positions (depending + on the value of *show_positions*). *offset_width* sets the width of the instruction offset field *label_width* sets the width of the label field *show_caches* is a boolean indicating whether to display cache lines - + *show_positions* is a boolean indicating whether full positions should + be reported instead of only the line numbers. """ self.file = file self.lineno_width = lineno_width self.offset_width = offset_width self.label_width = label_width self.show_caches = show_caches + self.show_positions = show_positions def print_instruction(self, instr, mark_as_current=False): self.print_instruction_line(instr, mark_as_current) @@ -474,15 +478,27 @@ def print_instruction_line(self, instr, mark_as_current): print(file=self.file) fields = [] - # Column: Source code line number + # Column: Source code locations information if lineno_width: - if instr.starts_line: - lineno_fmt = "%%%dd" if instr.line_number is not None else "%%%ds" - lineno_fmt = lineno_fmt % lineno_width - lineno = _NO_LINENO if instr.line_number is None else instr.line_number - fields.append(lineno_fmt % lineno) + if self.show_positions: + # reporting positions instead of just line numbers + if instr_positions := instr.positions: + if all(p is None for p in instr_positions): + positions_str = _NO_LINENO + else: + ps = tuple('?' if p is None else p for p in instr_positions) + positions_str = f"{ps[0]}:{ps[2]}-{ps[1]}:{ps[3]}" + fields.append(f'{positions_str:{lineno_width}}') + else: + fields.append(' ' * lineno_width) else: - fields.append(' ' * lineno_width) + if instr.starts_line: + lineno_fmt = "%%%dd" if instr.line_number is not None else "%%%ds" + lineno_fmt = lineno_fmt % lineno_width + lineno = _NO_LINENO if instr.line_number is None else instr.line_number + fields.append(lineno_fmt % lineno) + else: + fields.append(' ' * lineno_width) # Column: Label if instr.label is not None: lbl = f"L{instr.label}:" @@ -769,17 +785,22 @@ def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=N def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False, - show_offsets=False): + show_offsets=False, show_positions=False): """Disassemble a code object.""" linestarts = dict(findlinestarts(co)) exception_entries = _parse_exception_table(co) + if show_positions: + lineno_width = _get_positions_width(co) + else: + lineno_width = _get_lineno_width(linestarts) labels_map = _make_labels_map(co.co_code, exception_entries=exception_entries) label_width = 4 + len(str(len(labels_map))) formatter = Formatter(file=file, - lineno_width=_get_lineno_width(linestarts), + lineno_width=lineno_width, offset_width=len(str(max(len(co.co_code) - 2, 9999))) if show_offsets else 0, label_width=label_width, - show_caches=show_caches) + show_caches=show_caches, + show_positions=show_positions) arg_resolver = ArgResolver(co_consts=co.co_consts, names=co.co_names, varname_from_oparg=co._varname_from_oparg, @@ -788,8 +809,8 @@ def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False, exception_entries=exception_entries, co_positions=co.co_positions(), original_code=co.co_code, arg_resolver=arg_resolver, formatter=formatter) -def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adaptive=False, show_offsets=False): - disassemble(co, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) +def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False): + disassemble(co, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) if depth is None or depth > 0: if depth is not None: depth = depth - 1 @@ -799,7 +820,7 @@ def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adap print("Disassembly of %r:" % (x,), file=file) _disassemble_recursive( x, file=file, depth=depth, show_caches=show_caches, - adaptive=adaptive, show_offsets=show_offsets + adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions ) @@ -832,6 +853,22 @@ def _get_lineno_width(linestarts): lineno_width = len(_NO_LINENO) return lineno_width +def _get_positions_width(code): + # Positions are formatted as 'LINE:COL-ENDLINE:ENDCOL ' (note trailing space). + # A missing component appears as '?', and when all components are None, we + # render '_NO_LINENO'. thus the minimum width is 1 + len(_NO_LINENO). + # + # If all values are missing, positions are not printed (i.e. positions_width = 0). + has_value = False + values_width = 0 + for positions in code.co_positions(): + has_value |= any(isinstance(p, int) for p in positions) + width = sum(1 if p is None else len(str(p)) for p in positions) + values_width = max(width, values_width) + if has_value: + # 3 = number of separators in a normal format + return 1 + max(len(_NO_LINENO), 3 + values_width) + return 0 def _disassemble_bytes(code, lasti=-1, linestarts=None, *, line_offset=0, exception_entries=(), @@ -978,7 +1015,7 @@ class Bytecode: Iterating over this yields the bytecode operations as Instruction instances. """ - def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False, adaptive=False, show_offsets=False): + def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False): self.codeobj = co = _get_code_object(x) if first_line is None: self.first_line = co.co_firstlineno @@ -993,6 +1030,7 @@ def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False self.show_caches = show_caches self.adaptive = adaptive self.show_offsets = show_offsets + self.show_positions = show_positions def __iter__(self): co = self.codeobj @@ -1036,16 +1074,19 @@ def dis(self): with io.StringIO() as output: code = _get_code_array(co, self.adaptive) offset_width = len(str(max(len(code) - 2, 9999))) if self.show_offsets else 0 - - + if self.show_positions: + lineno_width = _get_positions_width(co) + else: + lineno_width = _get_lineno_width(self._linestarts) labels_map = _make_labels_map(co.co_code, self.exception_entries) label_width = 4 + len(str(len(labels_map))) formatter = Formatter(file=output, - lineno_width=_get_lineno_width(self._linestarts), + lineno_width=lineno_width, offset_width=offset_width, label_width=label_width, line_offset=self._line_offset, - show_caches=self.show_caches) + show_caches=self.show_caches, + show_positions=self.show_positions) arg_resolver = ArgResolver(co_consts=co.co_consts, names=co.co_names, @@ -1071,6 +1112,8 @@ def main(): help='show inline caches') parser.add_argument('-O', '--show-offsets', action='store_true', help='show instruction offsets') + parser.add_argument('-P', '--show-positions', action='store_true', + help='show instruction positions') parser.add_argument('infile', nargs='?', default='-') args = parser.parse_args() if args.infile == '-': @@ -1081,7 +1124,7 @@ def main(): with open(args.infile, 'rb') as infile: source = infile.read() code = compile(source, name, "exec") - dis(code, show_caches=args.show_caches, show_offsets=args.show_offsets) + dis(code, show_caches=args.show_caches, show_offsets=args.show_offsets, show_positions=args.show_positions) if __name__ == "__main__": main() diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index 351faf428a20cd..6b9ed24ad8ec78 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -184,8 +184,13 @@ def _quote(str): return '"' + str.translate(_Translator) + '"' -_OctalPatt = re.compile(r"\\[0-3][0-7][0-7]") -_QuotePatt = re.compile(r"[\\].") +_unquote_sub = re.compile(r'\\(?:([0-3][0-7][0-7])|(.))').sub + +def _unquote_replace(m): + if m[1]: + return chr(int(m[1], 8)) + else: + return m[2] def _unquote(str): # If there aren't any doublequotes, @@ -205,30 +210,7 @@ def _unquote(str): # \012 --> \n # \" --> " # - i = 0 - n = len(str) - res = [] - while 0 <= i < n: - o_match = _OctalPatt.search(str, i) - q_match = _QuotePatt.search(str, i) - if not o_match and not q_match: # Neither matched - res.append(str[i:]) - break - # else: - j = k = -1 - if o_match: - j = o_match.start(0) - if q_match: - k = q_match.start(0) - if q_match and (not o_match or k < j): # QuotePatt matched - res.append(str[i:k]) - res.append(str[k+1]) - i = k + 2 - else: # OctalPatt matched - res.append(str[i:j]) - res.append(chr(int(str[j+1:j+4], 8))) - i = j + 4 - return _nulljoin(res) + return _unquote_sub(_unquote_replace, str) # The _getdate() routine is used to set the expiration time in the cookie's HTTP # header. By default, _getdate() returns the current time in the appropriate diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index d8b2652d5d7979..e882c6cb3b8d19 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -706,7 +706,7 @@ def prepend_syspath(self, filename): del _filename, _sys, _dirname, _dir \n""".format(filename)) - def showsyntaxerror(self, filename=None): + def showsyntaxerror(self, filename=None, **kwargs): """Override Interactive Interpreter method: Use Colorizing Color the offending position instead of printing it and pointing at it diff --git a/Lib/inspect.py b/Lib/inspect.py index 0a50189ade41c1..90c44cf74007a8 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -970,10 +970,10 @@ def findsource(object): if isclass(object): try: - firstlineno = object.__firstlineno__ - except AttributeError: + firstlineno = vars(object)['__firstlineno__'] + except (TypeError, KeyError): raise OSError('source code not available') - return lines, object.__firstlineno__ - 1 + return lines, firstlineno - 1 if ismethod(object): object = object.__func__ diff --git a/Lib/shutil.py b/Lib/shutil.py index 72b2d834dc387e..6037092a5e09f2 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -56,7 +56,7 @@ __all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2", "copytree", "move", "rmtree", "Error", "SpecialFileError", - "ExecError", "make_archive", "get_archive_formats", + "make_archive", "get_archive_formats", "register_archive_format", "unregister_archive_format", "get_unpack_formats", "register_unpack_format", "unregister_unpack_format", "unpack_archive", @@ -74,8 +74,6 @@ class SpecialFileError(OSError): """Raised when trying to do a kind of operation (e.g. copying) which is not supported on a special file (e.g. a named pipe)""" -class ExecError(OSError): - """Raised when a command could not be executed""" class ReadError(OSError): """Raised when an archive cannot be read""" @@ -1582,3 +1580,15 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): if _access_check(name, mode): return name return None + +def __getattr__(name): + if name == "ExecError": + import warnings + warnings._deprecated( + "shutil.ExecError", + f"{warnings._DEPRECATED_MSG}; it " + "isn't raised by any shutil function.", + remove=(3, 16) + ) + return RuntimeError + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/test/test_asyncio/test_timeouts.py b/Lib/test/test_asyncio/test_timeouts.py index 1f7f9ee696a525..f5543e191d07ff 100644 --- a/Lib/test/test_asyncio/test_timeouts.py +++ b/Lib/test/test_asyncio/test_timeouts.py @@ -220,7 +220,7 @@ async def test_nested_timeouts_loop_busy(self): # Pretend the loop is busy for a while. time.sleep(0.1) await asyncio.sleep(1) - # TimeoutError was cought by (2) + # TimeoutError was caught by (2) await asyncio.sleep(10) # This sleep should be interrupted by (1) t1 = loop.time() self.assertTrue(t0 <= t1 <= t0 + 1) diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py index c475b6d78d0c56..b22ddd8ad858d4 100644 --- a/Lib/test/test_capi/test_exceptions.py +++ b/Lib/test/test_capi/test_exceptions.py @@ -104,7 +104,7 @@ def __del__(self): ''') proc = assert_python_ok("-c", code) warnings = proc.err.splitlines() - # Due to the finalization of the interpreter, the source will be ommited + # Due to the finalization of the interpreter, the source will be omitted # because the ``warnings`` module cannot be imported at this time self.assertEqual(warnings, [ b':7: RuntimeWarning: Testing PyErr_WarnEx', diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index d1f828b1ed824d..f6ec213b84e664 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -787,6 +787,19 @@ def __init__(self, obj): Type(i) self.assertEqual(calls, 100) + def test_specialization_class_call_doesnt_crash(self): + # gh-123185 + + class Foo: + def __init__(self, arg): + pass + + for _ in range(8): + try: + Foo() + except: + pass + from _testinternalcapi import has_inline_values diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index a934e493a76391..e3c5d08dd1e7d1 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -2161,6 +2161,19 @@ def test_no_section(self): self.assertEqual('1', cfg2[configparser.UNNAMED_SECTION]['a']) self.assertEqual('2', cfg2[configparser.UNNAMED_SECTION]['b']) + def test_add_section(self): + cfg = configparser.ConfigParser(allow_unnamed_section=True) + cfg.add_section(configparser.UNNAMED_SECTION) + cfg.set(configparser.UNNAMED_SECTION, 'a', '1') + self.assertEqual('1', cfg[configparser.UNNAMED_SECTION]['a']) + + def test_disabled_error(self): + with self.assertRaises(configparser.MissingSectionHeaderError): + configparser.ConfigParser().read_string("a = 1") + + with self.assertRaises(configparser.UnnamedSectionDisabledError): + configparser.ConfigParser().add_section(configparser.UNNAMED_SECTION) + class MiscTestCase(unittest.TestCase): def test__all__(self): diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 80f66c168bab60..5ec06d14af6500 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -127,6 +127,16 @@ def _f(a): _f.__code__.co_firstlineno + 1, _f.__code__.co_firstlineno + 2) +dis_f_with_positions_format = f"""\ +%-14s RESUME 0 + +%-14s LOAD_GLOBAL 1 (print + NULL) +%-14s LOAD_FAST 0 (a) +%-14s CALL 1 +%-14s POP_TOP + +%-14s RETURN_CONST 1 (1) +""" dis_f_co_code = """\ RESUME 0 @@ -950,6 +960,76 @@ def test_dis(self): def test_dis_with_offsets(self): self.do_disassembly_test(_f, dis_f_with_offsets, show_offsets=True) + @requires_debug_ranges() + def test_dis_with_all_positions(self): + def format_instr_positions(instr): + values = tuple('?' if p is None else p for p in instr.positions) + return '%s:%s-%s:%s' % (values[0], values[2], values[1], values[3]) + + instrs = list(dis.get_instructions(_f)) + for instr in instrs: + with self.subTest(instr=instr): + self.assertTrue(all(p is not None for p in instr.positions)) + positions = tuple(map(format_instr_positions, instrs)) + expected = dis_f_with_positions_format % positions + self.do_disassembly_test(_f, expected, show_positions=True) + + @requires_debug_ranges() + def test_dis_with_some_positions(self): + def f(): + pass + + PY_CODE_LOCATION_INFO_NO_COLUMNS = 13 + PY_CODE_LOCATION_INFO_WITH_COLUMNS = 14 + PY_CODE_LOCATION_INFO_NO_LOCATION = 15 + + f.__code__ = f.__code__.replace( + co_stacksize=1, + co_firstlineno=42, + co_code=bytes([ + dis.opmap["RESUME"], 0, + dis.opmap["NOP"], 0, + dis.opmap["RETURN_CONST"], 0, + ]), + co_linetable=bytes([ + (1 << 7) + | (PY_CODE_LOCATION_INFO_NO_COLUMNS << 3) + | (1 - 1), # 1 code unit (RESUME) + (1 << 1), # start line offset is 0 (encoded as an svarint) + (1 << 7) + | (PY_CODE_LOCATION_INFO_NO_LOCATION << 3) + | (1 - 1), # 1 code unit (NOP) + (1 << 7) + | (PY_CODE_LOCATION_INFO_WITH_COLUMNS << 3) + | (1 - 1), # 1 code unit (RETURN CONST) + (2 << 1), # start line offset is 0 (encoded as an svarint) + 3, # end line offset is 0 (varint encoded) + 1, # 1-based start column (reported as COL - 1) + 5, # 1-based end column (reported as ENDCOL - 1) + ] + )) + expect = '\n'.join([ + '43:?-43:? RESUME 0', + '', + ' -- NOP', + '', + '45:0-48:4 RETURN_CONST 0 (None)', + '', + ]) + self.do_disassembly_test(f, expect, show_positions=True) + + def test_dis_with_no_positions(self): + def f(): + pass + + f.__code__ = f.__code__.replace(co_linetable=b'') + expect = '\n'.join([ + ' RESUME 0', + ' RETURN_CONST 0 (None)', + '', + ]) + self.do_disassembly_test(f, expect, show_positions=True) + def test_bug_708901(self): self.do_disassembly_test(bug708901, dis_bug708901) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index aab43338ece02d..7860c67f082b1f 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1,6 +1,6 @@ # Run the tests in Programs/_testembed.c (tests for the CPython embedding APIs) from test import support -from test.support import import_helper, os_helper, MS_WINDOWS +from test.support import import_helper, os_helper, threading_helper, MS_WINDOWS import unittest from collections import namedtuple @@ -1802,6 +1802,13 @@ def test_init_main_interpreter_settings(self): self.assertEqual(out, expected) + @threading_helper.requires_working_threading() + def test_init_in_background_thread(self): + # gh-123022: Check that running Py_Initialize() in a background + # thread doesn't crash. + out, err = self.run_embedded_interpreter("test_init_in_background_thread") + self.assertEqual(err, "") + class SetConfigTests(unittest.TestCase): def test_set_config(self): diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index 925c8697f60de6..8879902a6e2f41 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -5,6 +5,7 @@ import doctest from http import cookies import pickle +from test import support class CookieTests(unittest.TestCase): @@ -58,6 +59,43 @@ def test_basic(self): for k, v in sorted(case['dict'].items()): self.assertEqual(C[k].value, v) + def test_unquote(self): + cases = [ + (r'a="b=\""', 'b="'), + (r'a="b=\\"', 'b=\\'), + (r'a="b=\="', 'b=='), + (r'a="b=\n"', 'b=n'), + (r'a="b=\042"', 'b="'), + (r'a="b=\134"', 'b=\\'), + (r'a="b=\377"', 'b=\xff'), + (r'a="b=\400"', 'b=400'), + (r'a="b=\42"', 'b=42'), + (r'a="b=\\042"', 'b=\\042'), + (r'a="b=\\134"', 'b=\\134'), + (r'a="b=\\\""', 'b=\\"'), + (r'a="b=\\\042"', 'b=\\"'), + (r'a="b=\134\""', 'b=\\"'), + (r'a="b=\134\042"', 'b=\\"'), + ] + for encoded, decoded in cases: + with self.subTest(encoded): + C = cookies.SimpleCookie() + C.load(encoded) + self.assertEqual(C['a'].value, decoded) + + @support.requires_resource('cpu') + def test_unquote_large(self): + n = 10**6 + for encoded in r'\\', r'\134': + with self.subTest(encoded): + data = 'a="b=' + encoded*n + ';"' + C = cookies.SimpleCookie() + C.load(data) + value = C['a'].value + self.assertEqual(value[:3], 'b=\\') + self.assertEqual(value[-2:], '\\;') + self.assertEqual(len(value), n + 3) + def test_load(self): C = cookies.SimpleCookie() C.load('Customer="WILE_E_COYOTE"; Version=1; Path=/acme') diff --git a/Lib/test/test_inspect/inspect_fodder2.py b/Lib/test/test_inspect/inspect_fodder2.py index bb9d3e88cfbee1..43e9f852022934 100644 --- a/Lib/test/test_inspect/inspect_fodder2.py +++ b/Lib/test/test_inspect/inspect_fodder2.py @@ -315,3 +315,45 @@ def g(): class ClassWithCodeObject: import sys code = sys._getframe(0).f_code + +import enum + +# line 321 +class enum322(enum.Enum): + A = 'a' + +# line 325 +class enum326(enum.IntEnum): + A = 1 + +# line 329 +class flag330(enum.Flag): + A = 1 + +# line 333 +class flag334(enum.IntFlag): + A = 1 + +# line 337 +simple_enum338 = enum.Enum('simple_enum338', 'A') +simple_enum339 = enum.IntEnum('simple_enum339', 'A') +simple_flag340 = enum.Flag('simple_flag340', 'A') +simple_flag341 = enum.IntFlag('simple_flag341', 'A') + +import typing + +# line 345 +class nt346(typing.NamedTuple): + x: int + y: int + +# line 350 +nt351 = typing.NamedTuple('nt351', (('x', int), ('y', int))) + +# line 353 +class td354(typing.TypedDict): + x: int + y: int + +# line 358 +td359 = typing.TypedDict('td359', (('x', int), ('y', int))) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 730c192d1aa260..81188ad4d1fbe1 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -821,7 +821,7 @@ def test_getsource_on_code_object(self): self.assertSourceEqual(mod.eggs.__code__, 12, 18) def test_getsource_on_generated_class(self): - A = type('A', (), {}) + A = type('A', (unittest.TestCase,), {}) self.assertEqual(inspect.getsourcefile(A), __file__) self.assertEqual(inspect.getfile(A), __file__) self.assertIs(inspect.getmodule(A), sys.modules[__name__]) @@ -929,6 +929,24 @@ def test_anonymous(self): # as argument to another function. self.assertSourceEqual(mod2.anonymous, 55, 55) + def test_enum(self): + self.assertSourceEqual(mod2.enum322, 322, 323) + self.assertSourceEqual(mod2.enum326, 326, 327) + self.assertSourceEqual(mod2.flag330, 330, 331) + self.assertSourceEqual(mod2.flag334, 334, 335) + self.assertRaises(OSError, inspect.getsource, mod2.simple_enum338) + self.assertRaises(OSError, inspect.getsource, mod2.simple_enum339) + self.assertRaises(OSError, inspect.getsource, mod2.simple_flag340) + self.assertRaises(OSError, inspect.getsource, mod2.simple_flag341) + + def test_namedtuple(self): + self.assertSourceEqual(mod2.nt346, 346, 348) + self.assertRaises(OSError, inspect.getsource, mod2.nt351) + + def test_typeddict(self): + self.assertSourceEqual(mod2.td354, 354, 356) + self.assertRaises(OSError, inspect.getsource, mod2.td359) + class TestBlockComments(GetSourceBase): fodderModule = mod @@ -1550,6 +1568,56 @@ def f(self): self.assertIn(('f', b.f), inspect.getmembers(b)) self.assertIn(('f', b.f), inspect.getmembers(b, inspect.ismethod)) + def test_getmembers_custom_dir(self): + class CorrectDir: + def __init__(self, attr): + self.attr = attr + def method(self): + return self.attr + 1 + def __dir__(self): + return ['attr', 'method'] + + cd = CorrectDir(5) + self.assertEqual(inspect.getmembers(cd), [ + ('attr', 5), + ('method', cd.method), + ]) + self.assertEqual(inspect.getmembers(cd, inspect.ismethod), [ + ('method', cd.method), + ]) + + def test_getmembers_custom_broken_dir(self): + # inspect.getmembers calls `dir()` on the passed object inside. + # if `__dir__` mentions some non-existent attribute, + # we still need to return others correctly. + class BrokenDir: + existing = 1 + def method(self): + return self.existing + 1 + def __dir__(self): + return ['method', 'missing', 'existing'] + + bd = BrokenDir() + self.assertEqual(inspect.getmembers(bd), [ + ('existing', 1), + ('method', bd.method), + ]) + self.assertEqual(inspect.getmembers(bd, inspect.ismethod), [ + ('method', bd.method), + ]) + + def test_getmembers_custom_duplicated_dir(self): + # Duplicates in `__dir__` must not fail and return just one result. + class DuplicatedDir: + attr = 1 + def __dir__(self): + return ['attr', 'attr'] + + dd = DuplicatedDir() + self.assertEqual(inspect.getmembers(dd), [ + ('attr', 1), + ]) + def test_getmembers_VirtualAttribute(self): class M(type): def __getattr__(cls, name): diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py index 1bdab125dc6ef0..8325b83a5932b9 100644 --- a/Lib/test/test_patma.py +++ b/Lib/test/test_patma.py @@ -1,6 +1,7 @@ import array import collections import dataclasses +import dis import enum import inspect import sys @@ -3377,6 +3378,24 @@ class Keys: self.assertIs(y, None) self.assertIs(z, None) +class TestSourceLocations(unittest.TestCase): + def test_jump_threading(self): + # See gh-123048 + def f(): + x = 0 + v = 1 + match v: + case 1: + if x < 0: + x = 1 + case 2: + if x < 0: + x = 1 + x += 1 + + for inst in dis.get_instructions(f): + if inst.opcode in dis.hasjump: + self.assertIsNotNone(inst.positions.lineno, "jump without location") class TestTracing(unittest.TestCase): diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 9395e9b133f50d..db7d1b1e9cd935 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -1263,7 +1263,7 @@ def test_post_mortem_context_of_the_cause(): def test_post_mortem_from_none(): """Test post mortem traceback debugging of chained exception - In particular that cause from None (which sets __supress_context__ to True) + In particular that cause from None (which sets __suppress_context__ to True) does not show context. diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py index 369dab316af132..b7adaffbac0e22 100644 --- a/Lib/test/test_pyrepl/test_interact.py +++ b/Lib/test/test_pyrepl/test_interact.py @@ -88,6 +88,20 @@ def test_runsource_returns_false_for_failed_compilation(self): self.assertFalse(result) self.assertIn('SyntaxError', f.getvalue()) + @force_not_colorized + def test_runsource_show_syntax_error_location(self): + console = InteractiveColoredConsole() + source = "def f(x, x): ..." + f = io.StringIO() + with contextlib.redirect_stderr(f): + result = console.runsource(source) + self.assertFalse(result) + r = """ + def f(x, x): ... + ^ +SyntaxError: duplicate argument 'x' in function definition""" + self.assertIn(r, f.getvalue()) + def test_runsource_shows_syntax_error_for_failed_compilation(self): console = InteractiveColoredConsole() source = "print('Hello, world!'" diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index d5eafc5eb58cac..e11ec74aa14058 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -1020,7 +1020,7 @@ def test_dumb_terminal_exits_cleanly(self): env.update({"TERM": "dumb"}) output, exit_code = self.run_repl("exit()\n", env=env) self.assertEqual(exit_code, 0) - self.assertIn("warning: can\'t use pyrepl", output) + self.assertIn("warning: can't use pyrepl", output) self.assertNotIn("Exception", output) self.assertNotIn("Traceback", output) @@ -1100,6 +1100,38 @@ def test_not_wiping_history_file(self): self.assertIn("spam", output) self.assertNotEqual(pathlib.Path(hfile.name).stat().st_size, 0) + @force_not_colorized + def test_proper_tracebacklimit(self): + env = os.environ.copy() + for set_tracebacklimit in [True, False]: + commands = ("import sys\n" + + ("sys.tracebacklimit = 1\n" if set_tracebacklimit else "") + + "def x1(): 1/0\n\n" + "def x2(): x1()\n\n" + "def x3(): x2()\n\n" + "x3()\n" + "exit()\n") + + for basic_repl in [True, False]: + if basic_repl: + env["PYTHON_BASIC_REPL"] = "1" + else: + env.pop("PYTHON_BASIC_REPL", None) + with self.subTest(set_tracebacklimit=set_tracebacklimit, + basic_repl=basic_repl): + output, exit_code = self.run_repl(commands, env=env) + if "can't use pyrepl" in output: + self.skipTest("pyrepl not available") + self.assertIn("in x1", output) + if set_tracebacklimit: + self.assertNotIn("in x2", output) + self.assertNotIn("in x3", output) + self.assertNotIn("in ", output) + else: + self.assertIn("in x2", output) + self.assertIn("in x3", output) + self.assertIn("in ", output) + def run_repl( self, repl_input: str | list[str], diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index c770be21b41c2b..80e1d73b6b2aab 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -3393,7 +3393,7 @@ def test_module_all_attribute(self): self.assertTrue(hasattr(shutil, '__all__')) target_api = ['copyfileobj', 'copyfile', 'copymode', 'copystat', 'copy', 'copy2', 'copytree', 'move', 'rmtree', 'Error', - 'SpecialFileError', 'ExecError', 'make_archive', + 'SpecialFileError', 'make_archive', 'get_archive_formats', 'register_archive_format', 'unregister_archive_format', 'get_unpack_formats', 'register_unpack_format', 'unregister_unpack_format', @@ -3402,6 +3402,8 @@ def test_module_all_attribute(self): if hasattr(os, 'statvfs') or os.name == 'nt': target_api.append('disk_usage') self.assertEqual(set(shutil.__all__), set(target_api)) + with self.assertWarns(DeprecationWarning): + from shutil import ExecError if __name__ == '__main__': diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 89201c647dfe51..c69baa4bf4d1b1 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2799,7 +2799,7 @@ def test_sqrtprod_helper_function_fundamentals(self): @requires_IEEE_754 @unittest.skipIf(HAVE_DOUBLE_ROUNDING, "accuracy not guaranteed on machines with double rounding") - @support.cpython_only # Allow for a weaker sumprod() implmentation + @support.cpython_only # Allow for a weaker sumprod() implementation def test_sqrtprod_helper_function_improved_accuracy(self): # Test a known example where accuracy is improved x, y, target = 0.8035720646477457, 0.7957468097636939, 0.7996498651651661 diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 1895c88d23b70d..b568221212d71f 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -699,6 +699,35 @@ def f_with_multiline(): result_lines = self.get_exception(f_with_multiline) self.assertEqual(result_lines, expected_f.splitlines()) + # Check custom error messages covering multiple lines + code = textwrap.dedent(""" + dummy_call( + "dummy value" + foo="bar", + ) + """) + + def f_with_multiline(): + # Need to defer the compilation until in self.get_exception(..) + return compile(code, "?", "exec") + + lineno_f = f_with_multiline.__code__.co_firstlineno + + expected_f = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_multiline\n' + ' return compile(code, "?", "exec")\n' + ' File "?", line 3\n' + ' "dummy value"\n' + ' ^^^^^^^^^^^^^' + ) + + result_lines = self.get_exception(f_with_multiline) + self.assertEqual(result_lines, expected_f.splitlines()) + def test_caret_multiline_expression_bin_op(self): # Make sure no carets are printed for expressions spanning multiple # lines. @@ -2312,19 +2341,22 @@ def test_message_none(self): def test_syntax_error_various_offsets(self): for offset in range(-5, 10): for add in [0, 2]: - text = " "*add + "text%d" % offset + text = " " * add + "text%d" % offset expected = [' File "file.py", line 1'] if offset < 1: expected.append(" %s" % text.lstrip()) elif offset <= 6: expected.append(" %s" % text.lstrip()) - expected.append(" %s^" % (" "*(offset-1))) + # Set the caret length to match the length of the text minus the offset. + caret_length = max(1, len(text.lstrip()) - offset + 1) + expected.append(" %s%s" % (" " * (offset - 1), "^" * caret_length)) else: + caret_length = max(1, len(text.lstrip()) - 4) expected.append(" %s" % text.lstrip()) - expected.append(" %s^" % (" "*5)) + expected.append(" %s%s" % (" " * 5, "^" * caret_length)) expected.append("SyntaxError: msg") expected.append("") - err = self.get_report(SyntaxError("msg", ("file.py", 1, offset+add, text))) + err = self.get_report(SyntaxError("msg", ("file.py", 1, offset + add, text))) exp = "\n".join(expected) self.assertEqual(exp, err) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index ef2ad30317bf34..6e036b600330c1 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -2500,7 +2500,7 @@ def test_concatenate(self): def test_nested_paramspec(self): # Since Callable has some special treatment, we want to be sure - # that substituion works correctly, see gh-103054 + # that substitution works correctly, see gh-103054 Callable = self.Callable P = ParamSpec('P') P2 = ParamSpec('P2') diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index d3bf4ea7c7d437..5f572c4cf9a5c5 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -114,7 +114,7 @@ def test_no_names_in_pua(self): def test_lookup_nonexistant(self): # just make sure that lookup can fail - for nonexistant in [ + for nonexistent in [ "LATIN SMLL LETR A", "OPEN HANDS SIGHS", "DREGS", @@ -122,7 +122,7 @@ def test_lookup_nonexistant(self): "MODIFIER LETTER CYRILLIC SMALL QUESTION MARK", "???", ]: - self.assertRaises(KeyError, self.db.lookup, nonexistant) + self.assertRaises(KeyError, self.db.lookup, nonexistent) def test_digit(self): self.assertEqual(self.db.digit('A', None), None) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index eed0599642edfb..19179fdc9508ca 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -8,6 +8,7 @@ import os import io +import ftplib import socket import array import sys @@ -754,7 +755,6 @@ def connect_ftp(self, user, passwd, host, port, dirs, self.ftpwrapper = MockFTPWrapper(self.data) return self.ftpwrapper - import ftplib data = "rheum rhaponicum" h = NullFTPHandler(data) h.parent = MockOpener() @@ -794,6 +794,27 @@ def connect_ftp(self, user, passwd, host, port, dirs, self.assertEqual(headers.get("Content-type"), mimetype) self.assertEqual(int(headers["Content-length"]), len(data)) + def test_ftp_error(self): + class ErrorFTPHandler(urllib.request.FTPHandler): + def __init__(self, exception): + self._exception = exception + + def connect_ftp(self, user, passwd, host, port, dirs, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + raise self._exception + + exception = ftplib.error_perm( + "500 OOPS: cannot change directory:/nonexistent") + h = ErrorFTPHandler(exception) + urlopen = urllib.request.build_opener(h).open + try: + urlopen("ftp://www.pythontest.net/") + except urllib.error.URLError as raised: + self.assertEqual(raised.reason, + f"ftp error: {exception.args[0]}") + else: + self.fail("Did not raise ftplib exception") + def test_file(self): import email.utils h = urllib.request.FileHandler() diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index d6c83a75c1c03a..3dbbd9cf1d3364 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -207,6 +207,9 @@ def test_roundtrips(self): ('scheme://///path/to/file', ('scheme', '', '///path/to/file', '', '', ''), ('scheme', '', '///path/to/file', '', '')), + ('file:tmp/junk.txt', + ('file', '', 'tmp/junk.txt', '', '', ''), + ('file', '', 'tmp/junk.txt', '', '')), ('file:///tmp/junk.txt', ('file', '', '/tmp/junk.txt', '', '', ''), ('file', '', '/tmp/junk.txt', '', '')), @@ -216,6 +219,18 @@ def test_roundtrips(self): ('file://///tmp/junk.txt', ('file', '', '///tmp/junk.txt', '', '', ''), ('file', '', '///tmp/junk.txt', '', '')), + ('http:tmp/junk.txt', + ('http', '', 'tmp/junk.txt', '', '', ''), + ('http', '', 'tmp/junk.txt', '', '')), + ('http://example.com/tmp/junk.txt', + ('http', 'example.com', '/tmp/junk.txt', '', '', ''), + ('http', 'example.com', '/tmp/junk.txt', '', '')), + ('http:///example.com/tmp/junk.txt', + ('http', '', '/example.com/tmp/junk.txt', '', '', ''), + ('http', '', '/example.com/tmp/junk.txt', '', '')), + ('http:////example.com/tmp/junk.txt', + ('http', '', '//example.com/tmp/junk.txt', '', '', ''), + ('http', '', '//example.com/tmp/junk.txt', '', '')), ('imap://mail.python.org/mbox1', ('imap', 'mail.python.org', '/mbox1', '', '', ''), ('imap', 'mail.python.org', '/mbox1', '', '')), @@ -260,7 +275,8 @@ def _encode(t): ('', '', 'schème:path/to/file', '', '')), ] for url, parsed, split in str_cases + bytes_cases: - self.checkRoundtrips(url, parsed, split) + with self.subTest(url): + self.checkRoundtrips(url, parsed, split) def test_roundtrips_normalization(self): str_cases = [ @@ -292,7 +308,8 @@ def _encode(t): tuple(x.encode('ascii') for x in t[3])) bytes_cases = [_encode(x) for x in str_cases] for url, url2, parsed, split in str_cases + bytes_cases: - self.checkRoundtrips(url, parsed, split, url2) + with self.subTest(url): + self.checkRoundtrips(url, parsed, split, url2) def test_http_roundtrips(self): # urllib.parse.urlsplit treats 'http:' as an optimized special case, @@ -333,11 +350,17 @@ def _encode(t): self.checkRoundtrips(url, parsed, split) def checkJoin(self, base, relurl, expected): - str_components = (base, relurl, expected) - self.assertEqual(urllib.parse.urljoin(base, relurl), expected) - bytes_components = baseb, relurlb, expectedb = [ - x.encode('ascii') for x in str_components] - self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) + with self.subTest(base=base, relurl=relurl): + self.assertEqual(urllib.parse.urljoin(base, relurl), expected) + baseb = base.encode('ascii') + relurlb = relurl.encode('ascii') + expectedb = expected.encode('ascii') + self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) + + relurl = urllib.parse.urlunsplit(urllib.parse.urlsplit(relurl)) + self.assertEqual(urllib.parse.urljoin(base, relurl), expected) + relurlb = urllib.parse.urlunsplit(urllib.parse.urlsplit(relurlb)) + self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) def test_unparse_parse(self): str_cases = ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',] diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 5352276e874bf5..2e5affb15e3f61 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -4490,7 +4490,7 @@ def data(self, format=None, *, from_coords=None, The FORMAT option specifies the name of the image file format handler to be used. If this option is not given, this method uses a format that consists of a tuple (one element per row) of strings - containings space separated (one element per pixel/column) colors + containing space-separated (one element per pixel/column) colors in “#RRGGBB” format (where RR is a pair of hexadecimal digits for the red channel, GG for green, and BB for blue). diff --git a/Lib/traceback.py b/Lib/traceback.py index 6ee1a50ca6804a..3e708c6f86a4c5 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -1292,11 +1292,15 @@ def _format_syntax_error(self, stype, **kwargs): yield ' {}\n'.format(ltext) else: offset = self.offset - end_offset = self.end_offset if self.end_offset not in {None, 0} else offset + if self.lineno == self.end_lineno: + end_offset = self.end_offset if self.end_offset not in {None, 0} else offset + else: + end_offset = len(rtext) + 1 + if self.text and offset > len(self.text): - offset = len(self.text) + 1 + offset = len(rtext) + 1 if self.text and end_offset > len(self.text): - end_offset = len(self.text) + 1 + end_offset = len(rtext) + 1 if offset >= end_offset or end_offset < 0: end_offset = offset + 1 diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 8f724f907d4217..33165309daf0c4 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -525,9 +525,13 @@ def urlunsplit(components): empty query; the RFC states that these are equivalent).""" scheme, netloc, url, query, fragment, _coerce_result = ( _coerce_args(*components)) - if netloc or (scheme and scheme in uses_netloc) or url[:2] == '//': + if netloc: if url and url[:1] != '/': url = '/' + url - url = '//' + (netloc or '') + url + url = '//' + netloc + url + elif url[:2] == '//': + url = '//' + url + elif scheme and scheme in uses_netloc and (not url or url[:1] == '/'): + url = '//' + url if scheme: url = scheme + ':' + url if query: diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 58b0cb574a764a..bc35d8a80e5d03 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1555,7 +1555,7 @@ def ftp_open(self, req): headers = email.message_from_string(headers) return addinfourl(fp, headers, req.full_url) except ftplib.all_errors as exp: - raise URLError(exp) from exp + raise URLError(f"ftp error: {exp}") from exp def connect_ftp(self, user, passwd, host, port, dirs, timeout): return ftpwrapper(user, passwd, host, port, dirs, timeout, diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-08-18-18-25-54.gh-issue-123123.0ZcaEB.rst b/Misc/NEWS.d/next/Core and Builtins/2024-08-18-18-25-54.gh-issue-123123.0ZcaEB.rst new file mode 100644 index 00000000000000..824d307bb270a2 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-08-18-18-25-54.gh-issue-123123.0ZcaEB.rst @@ -0,0 +1,2 @@ +Fix displaying :exc:`SyntaxError` exceptions covering multiple lines. Patch +by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-08-20-11-09-16.gh-issue-123048.2TISpv.rst b/Misc/NEWS.d/next/Core and Builtins/2024-08-20-11-09-16.gh-issue-123048.2TISpv.rst new file mode 100644 index 00000000000000..f0b756febbc1b8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-08-20-11-09-16.gh-issue-123048.2TISpv.rst @@ -0,0 +1,2 @@ +Fix a bug where pattern matching code could emit a :opcode:`JUMP_FORWARD` +with no source location. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-15-19-28-43.gh-issue-123022.m3EF9E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-15-19-28-43.gh-issue-123022.m3EF9E.rst new file mode 100644 index 00000000000000..47107dee44eec3 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-15-19-28-43.gh-issue-123022.m3EF9E.rst @@ -0,0 +1,2 @@ +Fix crash in free-threaded build when calling :c:func:`Py_Initialize` from +a non-main thread. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-19-15-13-13.gh-issue-118093.dLZ8qS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-19-15-13-13.gh-issue-118093.dLZ8qS.rst new file mode 100644 index 00000000000000..d8127d8b5054fe --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-19-15-13-13.gh-issue-118093.dLZ8qS.rst @@ -0,0 +1,3 @@ +Break up ``CALL_ALLOC_AND_ENTER_INIT`` into micro-ops and relax +requirement for exact args, in order to increase the amount of code +supported by tier 2. diff --git a/Misc/NEWS.d/next/Library/2024-07-16-20-49-07.gh-issue-121804.gYN-In.rst b/Misc/NEWS.d/next/Library/2024-07-16-20-49-07.gh-issue-121804.gYN-In.rst new file mode 100644 index 00000000000000..1cc1cde7c22704 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-16-20-49-07.gh-issue-121804.gYN-In.rst @@ -0,0 +1,2 @@ +Correctly show error locations, when :exc:`SyntaxError` raised in new repl. +Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Library/2024-07-21-10-45-24.gh-issue-122081.dNrYMq.rst b/Misc/NEWS.d/next/Library/2024-07-21-10-45-24.gh-issue-122081.dNrYMq.rst new file mode 100644 index 00000000000000..4c988b16168047 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-21-10-45-24.gh-issue-122081.dNrYMq.rst @@ -0,0 +1,2 @@ +Fix a crash in the :func:`!decimal.IEEEContext` optional function +available via the ``EXTRA_FUNCTIONALITY`` configuration flag. diff --git a/Misc/NEWS.d/next/Library/2024-07-31-14-55-41.gh-issue-82378.eZvYmR.rst b/Misc/NEWS.d/next/Library/2024-07-31-14-55-41.gh-issue-82378.eZvYmR.rst new file mode 100644 index 00000000000000..8af016e7c82fcb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-31-14-55-41.gh-issue-82378.eZvYmR.rst @@ -0,0 +1,2 @@ +Make sure that the new :term:`REPL` interprets :data:`sys.tracebacklimit` in +the same way that the classic REPL did. diff --git a/Misc/NEWS.d/next/Library/2024-08-14-10-41-11.gh-issue-122981.BHV0Z9.rst b/Misc/NEWS.d/next/Library/2024-08-14-10-41-11.gh-issue-122981.BHV0Z9.rst new file mode 100644 index 00000000000000..7713d805155f9a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-14-10-41-11.gh-issue-122981.BHV0Z9.rst @@ -0,0 +1,2 @@ +Fix :func:`inspect.getsource` for generated classes with Python base classes +(e.g. enums). diff --git a/Misc/NEWS.d/next/Library/2024-08-16-16-53-52.gh-issue-123049.izx_fH.rst b/Misc/NEWS.d/next/Library/2024-08-16-16-53-52.gh-issue-123049.izx_fH.rst new file mode 100644 index 00000000000000..2faf85092a0f8a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-16-16-53-52.gh-issue-123049.izx_fH.rst @@ -0,0 +1,2 @@ +Add support for :const:`~configparser.UNNAMED_SECTION` +in :meth:`configparser.ConfigParser.add_section`. diff --git a/Misc/NEWS.d/next/Library/2024-08-16-19-13-21.gh-issue-123067.Nx9O4R.rst b/Misc/NEWS.d/next/Library/2024-08-16-19-13-21.gh-issue-123067.Nx9O4R.rst new file mode 100644 index 00000000000000..6a234561fe31a3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-16-19-13-21.gh-issue-123067.Nx9O4R.rst @@ -0,0 +1 @@ +Fix quadratic complexity in parsing ``"``-quoted cookie values with backslashes by :mod:`http.cookies`. diff --git a/Misc/NEWS.d/next/Library/2024-08-18-08-25-32.gh-issue-123084.rf8izX.rst b/Misc/NEWS.d/next/Library/2024-08-18-08-25-32.gh-issue-123084.rf8izX.rst new file mode 100644 index 00000000000000..eb01d66d98aef6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-18-08-25-32.gh-issue-123084.rf8izX.rst @@ -0,0 +1,4 @@ +Deprecate :class:`!shutil.ExecError`, which hasn't been +raised by any :mod:`shutil` function since Python 3.4. It's +now an alias for :exc:`RuntimeError`. + diff --git a/Misc/NEWS.d/next/Library/2024-08-19-17-37-18.gh-issue-122909.kP12SK.rst b/Misc/NEWS.d/next/Library/2024-08-19-17-37-18.gh-issue-122909.kP12SK.rst new file mode 100644 index 00000000000000..50eb4afd10791b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-19-17-37-18.gh-issue-122909.kP12SK.rst @@ -0,0 +1,3 @@ +In urllib.request when URLError is raised opening an ftp URL, the exception +argument is now consistently a string. Earlier versions passed either a +string or an ftplib exception instance as the argument to URLError. diff --git a/Misc/NEWS.d/next/Library/2024-08-20-14-22-49.gh-issue-123165.vOZZOA.rst b/Misc/NEWS.d/next/Library/2024-08-20-14-22-49.gh-issue-123165.vOZZOA.rst new file mode 100644 index 00000000000000..05728adc0be388 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-20-14-22-49.gh-issue-123165.vOZZOA.rst @@ -0,0 +1 @@ +Add support for rendering :class:`~dis.Positions` in :mod:`dis`. diff --git a/Misc/NEWS.d/next/Library/2024-08-20-18-02-27.gh-issue-85110.8_iDQy.rst b/Misc/NEWS.d/next/Library/2024-08-20-18-02-27.gh-issue-85110.8_iDQy.rst new file mode 100644 index 00000000000000..f22fac16b79c0b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-20-18-02-27.gh-issue-85110.8_iDQy.rst @@ -0,0 +1,2 @@ +Preserve relative path in URL without netloc in +:func:`urllib.parse.urlunsplit` and :func:`urllib.parse.urlunparse`. diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index 0719a1b921f040..8e07584f074936 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -187,8 +187,8 @@ _testfunc_array_in_struct3B_set_defaults(void) /* * Test3C struct tests the MAX_STRUCT_SIZE 32. Structs containing arrays of up * to four floating-point types are passed in registers on Arm platforms. - * This struct is used for within bounds test on Arm platfroms and for an - * out-of-bounds tests for platfroms where MAX_STRUCT_SIZE is less than 32. + * This struct is used for within-bounds tests on Arm platforms and for an + * out-of-bounds test for platforms where MAX_STRUCT_SIZE is less than 32. * See gh-110190. */ typedef struct { diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 94a2cc2c8e5f8a..15855e2cf1bb32 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -1519,7 +1519,7 @@ init_extended_context(PyObject *v) #ifdef EXTRA_FUNCTIONALITY /* Factory function for creating IEEE interchange format contexts */ static PyObject * -ieee_context(PyObject *dummy UNUSED, PyObject *v) +ieee_context(PyObject *module, PyObject *v) { PyObject *context; mpd_ssize_t bits; @@ -1536,7 +1536,7 @@ ieee_context(PyObject *dummy UNUSED, PyObject *v) goto error; } - decimal_state *state = get_module_state_by_def(Py_TYPE(v)); + decimal_state *state = get_module_state(module); context = PyObject_CallObject((PyObject *)state->PyDecContext_Type, NULL); if (context == NULL) { return NULL; diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 621f8996ccbb8e..85c24550d0b409 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1796,8 +1796,7 @@ init_frame(_PyInterpreterFrame *frame, PyFunctionObject *func, PyObject *locals) { PyCodeObject *code = (PyCodeObject *)func->func_code; _PyFrame_Initialize(frame, (PyFunctionObject*)Py_NewRef(func), - Py_XNewRef(locals), code, 0); - frame->previous = NULL; + Py_XNewRef(locals), code, 0, NULL); } PyFrameObject* diff --git a/Objects/mimalloc/os.c b/Objects/mimalloc/os.c index f3bc7184c41c5b..c9103168c12507 100644 --- a/Objects/mimalloc/os.c +++ b/Objects/mimalloc/os.c @@ -115,8 +115,12 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize uintptr_t init = MI_HINT_BASE; #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode - uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap()); - init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB + mi_heap_t* heap = mi_prim_get_default_heap(); + // gh-123022: default heap may not be initialized in CPython in background threads + if (mi_heap_is_initialized(heap)) { + uintptr_t r = _mi_heap_random_next(heap); + init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB + } #endif uintptr_t expected = hint + size; mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init); @@ -553,8 +557,12 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { // Initialize the start address after the 32TiB area start = ((uintptr_t)32 << 40); // 32TiB virtual start address #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode - uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap()); - start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB + mi_heap_t* heap = mi_prim_get_default_heap(); + // gh-123022: default heap may not be initialized in CPython in background threads + if (mi_heap_is_initialized(heap)) { + uintptr_t r = _mi_heap_random_next(heap); + start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB + } #endif } end = start + size; diff --git a/Objects/object.c b/Objects/object.c index c6d46caa0bb62b..97403227625971 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1258,9 +1258,9 @@ PyObject_GetOptionalAttr(PyObject *v, PyObject *name, PyObject **result) return 0; } if (tp->tp_getattro == _Py_type_getattro) { - int supress_missing_attribute_exception = 0; - *result = _Py_type_getattro_impl((PyTypeObject*)v, name, &supress_missing_attribute_exception); - if (supress_missing_attribute_exception) { + int suppress_missing_attribute_exception = 0; + *result = _Py_type_getattro_impl((PyTypeObject*)v, name, &suppress_missing_attribute_exception); + if (suppress_missing_attribute_exception) { // return 0 without having to clear the exception return 0; } diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 2c726c915c9df5..e341f0c6bfc595 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -8,6 +8,7 @@ #include #include "pycore_initconfig.h" // _PyConfig_InitCompatConfig() #include "pycore_runtime.h" // _PyRuntime +#include "pycore_pythread.h" // PyThread_start_joinable_thread() #include "pycore_import.h" // _PyImport_FrozenBootstrap #include #include @@ -2022,6 +2023,22 @@ static int test_init_main_interpreter_settings(void) return 0; } +static void do_init(void *unused) +{ + _testembed_Py_Initialize(); + Py_Finalize(); +} + +static int test_init_in_background_thread(void) +{ + PyThread_handle_t handle; + PyThread_ident_t ident; + if (PyThread_start_joinable_thread(&do_init, NULL, &ident, &handle) < 0) { + return -1; + } + return PyThread_join_thread(handle); +} + #ifndef MS_WINDOWS #include "test_frozenmain.h" // M_test_frozenmain @@ -2211,6 +2228,7 @@ static struct TestCase TestCases[] = { {"test_get_argc_argv", test_get_argc_argv}, {"test_init_use_frozen_modules", test_init_use_frozen_modules}, {"test_init_main_interpreter_settings", test_init_main_interpreter_settings}, + {"test_init_in_background_thread", test_init_in_background_thread}, // Audit {"test_open_code_hook", test_open_code_hook}, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 38b5cf4d2b649a..ff05842403dce4 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -797,7 +797,7 @@ dummy_func( PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem = ht->_spec_cache.getitem; - new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2); + new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2, frame); SYNC_SP(); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; @@ -1071,6 +1071,8 @@ dummy_func( tstate->exc_info = &gen->gi_exc_state; assert(next_instr - this_instr + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + assert(gen_frame->previous == NULL); + gen_frame->previous = frame; DISPATCH_INLINED(gen_frame); } if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) { @@ -1113,6 +1115,7 @@ dummy_func( tstate->exc_info = &gen->gi_exc_state; assert(1 + INLINE_CACHE_ENTRIES_SEND + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); + gen_frame->previous = frame; } macro(SEND_GEN) = @@ -2143,7 +2146,7 @@ dummy_func( DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(LOAD_ATTR, hit); Py_INCREF(fget); - new_frame = _PyFrame_PushUnchecked(tstate, f, 1); + new_frame = _PyFrame_PushUnchecked(tstate, f, 1, frame); new_frame->localsplus[0] = owner; } @@ -2175,7 +2178,7 @@ dummy_func( PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); Py_INCREF(f); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2, frame); // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; @@ -2957,6 +2960,7 @@ dummy_func( gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; + gen_frame->previous = frame; // oparg is the return offset from the next instruction. frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg); } @@ -3226,7 +3230,7 @@ dummy_func( PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, total_args, NULL, frame ); // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); @@ -3316,7 +3320,7 @@ dummy_func( PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, total_args, NULL, frame ); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. @@ -3455,7 +3459,7 @@ dummy_func( int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -3469,7 +3473,7 @@ dummy_func( assert(tstate->interp->eval_frame == NULL); SYNC_SP(); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -3551,58 +3555,55 @@ dummy_func( _CALL_TUPLE_1 + _CHECK_PERIODIC; - inst(CALL_ALLOC_AND_ENTER_INIT, (unused/1, unused/2, callable, null, args[oparg] -- unused)) { + op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, null, args[oparg] -- self, init, args[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - /* This instruction does the following: - * 1. Creates the object (by calling ``object.__new__``) - * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) - * 3. Pushes the frame for ``__init__`` to the frame stack - * */ - _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; DEOPT_IF(!PyStackRef_IsNull(null)); DEOPT_IF(!PyType_Check(callable_o)); PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version)); + DEOPT_IF(tp->tp_version_tag != type_version); assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; - PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; - PyCodeObject *code = (PyCodeObject *)init->func_code; - DEOPT_IF(code->co_argcount != oparg+1); + PyFunctionObject *init_func = (PyFunctionObject *)cls->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init_func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)); STAT_INC(CALL, hit); - PyObject *self = _PyType_NewManagedObject(tp); - if (self == NULL) { + self = PyStackRef_FromPyObjectSteal(_PyType_NewManagedObject(tp)); + if (PyStackRef_IsNull(self)) { ERROR_NO_POP(); } PyStackRef_CLOSE(callable); + init = PyStackRef_FromPyObjectNew(init_func); + } + + op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame: _PyInterpreterFrame *)) { _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1); + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ - Py_INCREF(self); - shim->localsplus[0] = PyStackRef_FromPyObjectSteal(self); - Py_INCREF(init); - _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); - /* Copy self followed by args to __init__ frame */ - init_frame->localsplus[0] = PyStackRef_FromPyObjectSteal(self); - for (int i = 0; i < oparg; i++) { - init_frame->localsplus[i+1] = args[i]; + shim->localsplus[0] = PyStackRef_DUP(self); + PyFunctionObject *init_func = (PyFunctionObject *)PyStackRef_AsPyObjectSteal(init); + args[-1] = self; + init_frame = _PyEvalFramePushAndInit( + tstate, init_func, NULL, args-1, oparg+1, NULL, shim); + SYNC_SP(); + if (init_frame == NULL) { + _PyEval_FrameClearAndPop(tstate, shim); + ERROR_NO_POP(); } - frame->return_offset = (uint16_t)(next_instr - this_instr); - STACK_SHRINK(oparg+2); - _PyFrame_SetStackPointer(frame, stack_pointer); - /* Link frames */ - init_frame->previous = shim; - shim->previous = frame; - frame = tstate->current_frame = init_frame; - CALL_STAT_INC(inlined_py_calls); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; /* Account for pushing the extra frame. * We don't check recursion depth here, * as it will be checked after start_frame */ tstate->py_recursion_remaining--; - goto start_frame; } + macro(CALL_ALLOC_AND_ENTER_INIT) = + unused/1 + + _CHECK_PEP_523 + + _CHECK_AND_ALLOCATE_OBJECT + + _CREATE_INIT_FRAME + + _PUSH_FRAME; + inst(EXIT_INIT_CHECK, (should_be_none -- )) { assert(STACK_LEVEL() == 2); if (!PyStackRef_Is(should_be_none, PyStackRef_None)) { @@ -4061,7 +4062,7 @@ dummy_func( PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, positional_args, kwnames_o + args, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); // Manipulate stack directly since we leave using DISPATCH_INLINED(). @@ -4130,7 +4131,7 @@ dummy_func( PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, positional_args, kwnames_o + args, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, @@ -4194,7 +4195,7 @@ dummy_func( _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); DISPATCH_SAME_OPARG(); } - STAT_INC(CALL, deferred); + STAT_INC(CALL_KW, deferred); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } @@ -4316,7 +4317,7 @@ dummy_func( _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, - nargs, callargs, kwargs); + nargs, callargs, kwargs, frame); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { diff --git a/Python/ceval.c b/Python/ceval.c index c685a95b2ef088..b615deec3c4242 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -246,7 +246,7 @@ static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg static int get_exception_handler(PyCodeObject *, int, int*, int*, int*); static _PyInterpreterFrame * _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, - PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs); + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous); #ifdef HAVE_ERRNO_H #include @@ -671,8 +671,6 @@ static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { { .op.code = RESUME, .op.arg = RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START } }; -extern const struct _PyCode_DEF(8) _Py_InitCleanup; - #ifdef Py_DEBUG extern void _PyUOpPrint(const _PyUOpInstruction *uop); #endif @@ -1691,7 +1689,7 @@ _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) _PyInterpreterFrame * _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, _PyStackRef const* args, - size_t argcount, PyObject *kwnames) + size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous) { PyCodeObject * code = (PyCodeObject *)func->func_code; CALL_STAT_INC(frames_pushed); @@ -1699,7 +1697,7 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, if (frame == NULL) { goto fail; } - _PyFrame_Initialize(frame, func, locals, code, 0); + _PyFrame_Initialize(frame, func, locals, code, 0, previous); if (initialize_locals(tstate, func, frame->localsplus, args, argcount, kwnames)) { assert(frame->owner == FRAME_OWNED_BY_THREAD); clear_thread_frame(tstate, frame); @@ -1726,7 +1724,7 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, static _PyInterpreterFrame * _PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, PyObject *const* args, - size_t argcount, PyObject *kwnames) + size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous) { #if defined(Py_GIL_DISABLED) size_t kw_count = kwnames == NULL ? 0 : PyTuple_GET_SIZE(kwnames); @@ -1742,11 +1740,11 @@ _PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func, for (size_t i = 0; i < kw_count; i++) { tagged_args_buffer[argcount + i] = PyStackRef_FromPyObjectSteal(args[argcount + i]); } - _PyInterpreterFrame *res = _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)tagged_args_buffer, argcount, kwnames); + _PyInterpreterFrame *res = _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)tagged_args_buffer, argcount, kwnames, previous); PyMem_Free(tagged_args_buffer); return res; #else - return _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)args, argcount, kwnames); + return _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)args, argcount, kwnames, previous); #endif } @@ -1755,7 +1753,7 @@ _PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func, */ static _PyInterpreterFrame * _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, - PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs) + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous) { bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); PyObject *kwnames = NULL; @@ -1776,7 +1774,7 @@ _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, } _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_UnTagged( tstate, (PyFunctionObject *)func, locals, - newargs, nargs, kwnames + newargs, nargs, kwnames, previous ); if (has_dict) { _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); @@ -1813,7 +1811,7 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, } } _PyInterpreterFrame *frame = _PyEvalFramePushAndInit_UnTagged( - tstate, func, locals, args, argcount, kwnames); + tstate, func, locals, args, argcount, kwnames, NULL); if (frame == NULL) { return NULL; } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index ed146a10b2af4b..9e1540674d4219 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -124,7 +124,7 @@ do { \ do { \ assert(tstate->interp->eval_frame == NULL); \ _PyFrame_SetStackPointer(frame, stack_pointer); \ - (NEW_FRAME)->previous = frame; \ + assert((NEW_FRAME)->previous == frame); \ frame = tstate->current_frame = (NEW_FRAME); \ CALL_STAT_INC(inlined_py_calls); \ goto start_frame; \ diff --git a/Python/compile.c b/Python/compile.c index 744aaeb8f7982b..c369202d53b384 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -307,26 +307,26 @@ static PyCodeObject *compiler_mod(struct compiler *, mod_ty); static int compiler_visit_stmt(struct compiler *, stmt_ty); static int compiler_visit_keyword(struct compiler *, keyword_ty); static int compiler_visit_expr(struct compiler *, expr_ty); -static int compiler_augassign(struct compiler *, stmt_ty); +static int codegen_augassign(struct compiler *, stmt_ty); static int compiler_annassign(struct compiler *, stmt_ty); -static int compiler_subscript(struct compiler *, expr_ty); -static int compiler_slice(struct compiler *, expr_ty); +static int codegen_subscript(struct compiler *, expr_ty); +static int codegen_slice(struct compiler *, expr_ty); static bool are_all_items_const(asdl_expr_seq *, Py_ssize_t, Py_ssize_t); -static int compiler_with(struct compiler *, stmt_ty, int); -static int compiler_async_with(struct compiler *, stmt_ty, int); -static int compiler_async_for(struct compiler *, stmt_ty); -static int compiler_call_simple_kw_helper(struct compiler *c, - location loc, - asdl_keyword_seq *keywords, - Py_ssize_t nkwelts); -static int compiler_call_helper(struct compiler *c, location loc, - int n, asdl_expr_seq *args, - asdl_keyword_seq *keywords); -static int compiler_try_except(struct compiler *, stmt_ty); -static int compiler_try_star_except(struct compiler *, stmt_ty); +static int codegen_with(struct compiler *, stmt_ty, int); +static int codegen_async_with(struct compiler *, stmt_ty, int); +static int codegen_async_for(struct compiler *, stmt_ty); +static int codegen_call_simple_kw_helper(struct compiler *c, + location loc, + asdl_keyword_seq *keywords, + Py_ssize_t nkwelts); +static int codegen_call_helper(struct compiler *c, location loc, + int n, asdl_expr_seq *args, + asdl_keyword_seq *keywords); +static int codegen_try_except(struct compiler *, stmt_ty); +static int codegen_try_star_except(struct compiler *, stmt_ty); static int compiler_sync_comprehension_generator( struct compiler *c, location loc, @@ -342,12 +342,12 @@ static int compiler_async_comprehension_generator( expr_ty elt, expr_ty val, int type, int iter_on_stack); -static int compiler_pattern(struct compiler *, pattern_ty, pattern_context *); -static int compiler_match(struct compiler *, stmt_ty); -static int compiler_pattern_subpattern(struct compiler *, - pattern_ty, pattern_context *); -static int compiler_make_closure(struct compiler *c, location loc, - PyCodeObject *co, Py_ssize_t flags); +static int codegen_pattern(struct compiler *, pattern_ty, pattern_context *); +static int codegen_match(struct compiler *, stmt_ty); +static int codegen_pattern_subpattern(struct compiler *, + pattern_ty, pattern_context *); +static int codegen_make_closure(struct compiler *c, location loc, + PyCodeObject *co, Py_ssize_t flags); static PyCodeObject *optimize_and_assemble(struct compiler *, int addNone); @@ -1006,7 +1006,7 @@ codegen_addop_j(instr_sequence *seq, location loc, RETURN_IF_ERROR(codegen_addop_j(INSTR_SEQUENCE(C), (LOC), (OP), (O))) #define ADDOP_COMPARE(C, LOC, CMP) \ - RETURN_IF_ERROR(compiler_addcompare((C), (LOC), (cmpop_ty)(CMP))) + RETURN_IF_ERROR(codegen_addcompare((C), (LOC), (cmpop_ty)(CMP))) #define ADDOP_BINARY(C, LOC, BINOP) \ RETURN_IF_ERROR(addop_binary((C), (LOC), (BINOP), false)) @@ -1015,13 +1015,13 @@ codegen_addop_j(instr_sequence *seq, location loc, RETURN_IF_ERROR(addop_binary((C), (LOC), (BINOP), true)) #define ADD_YIELD_FROM(C, LOC, await) \ - RETURN_IF_ERROR(compiler_add_yield_from((C), (LOC), (await))) + RETURN_IF_ERROR(codegen_add_yield_from((C), (LOC), (await))) #define POP_EXCEPT_AND_RERAISE(C, LOC) \ - RETURN_IF_ERROR(compiler_pop_except_and_reraise((C), (LOC))) + RETURN_IF_ERROR(codegen_pop_except_and_reraise((C), (LOC))) #define ADDOP_YIELD(C, LOC) \ - RETURN_IF_ERROR(addop_yield((C), (LOC))) + RETURN_IF_ERROR(codegen_addop_yield((C), (LOC))) /* VISIT and VISIT_SEQ takes an ASDL type as their second argument. They use the ASDL name to synthesize the name of the C type and the visit function. @@ -1057,7 +1057,8 @@ codegen_addop_j(instr_sequence *seq, location loc, static int compiler_enter_scope(struct compiler *c, identifier name, int scope_type, - void *key, int lineno, PyObject *private) + void *key, int lineno, PyObject *private, + _PyCompile_CodeUnitMetadata *umd) { location loc = LOCATION(lineno, lineno, 0, 0); @@ -1069,9 +1070,14 @@ compiler_enter_scope(struct compiler *c, identifier name, int scope_type, return ERROR; } u->u_scope_type = scope_type; - u->u_metadata.u_argcount = 0; - u->u_metadata.u_posonlyargcount = 0; - u->u_metadata.u_kwonlyargcount = 0; + if (umd != NULL) { + u->u_metadata = *umd; + } + else { + u->u_metadata.u_argcount = 0; + u->u_metadata.u_posonlyargcount = 0; + u->u_metadata.u_kwonlyargcount = 0; + } u->u_ste = _PySymtable_Lookup(c->c_st, key); if (!u->u_ste) { compiler_unit_free(u); @@ -1257,7 +1263,7 @@ compiler_pop_fblock(struct compiler *c, enum fblocktype t, jump_target_label blo } static int -compiler_call_exit_with_nones(struct compiler *c, location loc) +codegen_call_exit_with_nones(struct compiler *c, location loc) { ADDOP_LOAD_CONST(c, loc, Py_None); ADDOP_LOAD_CONST(c, loc, Py_None); @@ -1267,7 +1273,7 @@ compiler_call_exit_with_nones(struct compiler *c, location loc) } static int -compiler_add_yield_from(struct compiler *c, location loc, int await) +codegen_add_yield_from(struct compiler *c, location loc, int await) { NEW_JUMP_TARGET_LABEL(c, send); NEW_JUMP_TARGET_LABEL(c, fail); @@ -1292,7 +1298,7 @@ compiler_add_yield_from(struct compiler *c, location loc, int await) } static int -compiler_pop_except_and_reraise(struct compiler *c, location loc) +codegen_pop_except_and_reraise(struct compiler *c, location loc) { /* Stack contents * [exc_info, lasti, exc] COPY 3 @@ -1374,7 +1380,7 @@ compiler_unwind_fblock(struct compiler *c, location *ploc, ADDOP_I(c, *ploc, SWAP, 3); ADDOP_I(c, *ploc, SWAP, 2); } - RETURN_IF_ERROR(compiler_call_exit_with_nones(c, *ploc)); + RETURN_IF_ERROR(codegen_call_exit_with_nones(c, *ploc)); if (info->fb_type == ASYNC_WITH) { ADDOP_I(c, *ploc, GET_AWAITABLE, 2); ADDOP_LOAD_CONST(c, *ploc, Py_None); @@ -1441,14 +1447,19 @@ compiler_unwind_fblock_stack(struct compiler *c, location *ploc, } static int -compiler_setup_annotations_scope(struct compiler *c, location loc, - void *key, PyObject *name) +codegen_setup_annotations_scope(struct compiler *c, location loc, + void *key, PyObject *name) { - if (compiler_enter_scope(c, name, COMPILER_SCOPE_ANNOTATIONS, - key, loc.lineno, NULL) == -1) { - return ERROR; - } - c->u->u_metadata.u_posonlyargcount = 1; + _PyCompile_CodeUnitMetadata umd = { + .u_posonlyargcount = 1, + }; + RETURN_IF_ERROR( + compiler_enter_scope(c, name, COMPILER_SCOPE_ANNOTATIONS, + key, loc.lineno, NULL, &umd)); + assert(c->u->u_metadata.u_posonlyargcount == 1); + assert(c->u->u_metadata.u_argcount == 0); + assert(c->u->u_metadata.u_kwonlyargcount == 0); + // if .format != 1: raise NotImplementedError _Py_DECLARE_STR(format, ".format"); ADDOP_I(c, loc, LOAD_FAST, 0); @@ -1463,8 +1474,8 @@ compiler_setup_annotations_scope(struct compiler *c, location loc, } static int -compiler_leave_annotations_scope(struct compiler *c, location loc, - Py_ssize_t annotations_len) +codegen_leave_annotations_scope(struct compiler *c, location loc, + Py_ssize_t annotations_len) { ADDOP_I(c, loc, BUILD_MAP, annotations_len); ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); @@ -1473,7 +1484,7 @@ compiler_leave_annotations_scope(struct compiler *c, location loc, if (co == NULL) { return ERROR; } - if (compiler_make_closure(c, loc, co, 0) < 0) { + if (codegen_make_closure(c, loc, co, 0) < 0) { Py_DECREF(co); return ERROR; } @@ -1534,8 +1545,8 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) assert(ste->ste_annotation_block != NULL); PyObject *deferred_anno = Py_NewRef(c->u->u_deferred_annotations); void *key = (void *)((uintptr_t)ste->ste_id + 1); - if (compiler_setup_annotations_scope(c, loc, key, - ste->ste_annotation_block->ste_name) == -1) { + if (codegen_setup_annotations_scope(c, loc, key, + ste->ste_annotation_block->ste_name) == -1) { Py_DECREF(deferred_anno); return ERROR; } @@ -1555,7 +1566,7 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) Py_DECREF(deferred_anno); RETURN_IF_ERROR( - compiler_leave_annotations_scope(c, loc, annotations_len) + codegen_leave_annotations_scope(c, loc, annotations_len) ); RETURN_IF_ERROR( compiler_nameop(c, loc, &_Py_ID(__annotate__), Store) @@ -1613,12 +1624,12 @@ compiler_codegen(struct compiler *c, mod_ty mod) } static int -compiler_enter_anonymous_scope(struct compiler* c, mod_ty mod) +codegen_enter_anonymous_scope(struct compiler* c, mod_ty mod) { _Py_DECLARE_STR(anon_module, ""); RETURN_IF_ERROR( compiler_enter_scope(c, &_Py_STR(anon_module), COMPILER_SCOPE_MODULE, - mod, 1, NULL)); + mod, 1, NULL, NULL)); return SUCCESS; } @@ -1627,7 +1638,7 @@ compiler_mod(struct compiler *c, mod_ty mod) { PyCodeObject *co = NULL; int addNone = mod->kind != Expression_kind; - if (compiler_enter_anonymous_scope(c, mod) < 0) { + if (codegen_enter_anonymous_scope(c, mod) < 0) { return NULL; } if (compiler_codegen(c, mod) < 0) { @@ -1714,8 +1725,8 @@ compiler_lookup_arg(struct compiler *c, PyCodeObject *co, PyObject *name) } static int -compiler_make_closure(struct compiler *c, location loc, - PyCodeObject *co, Py_ssize_t flags) +codegen_make_closure(struct compiler *c, location loc, + PyCodeObject *co, Py_ssize_t flags) { if (co->co_nfreevars) { int i = PyUnstable_Code_GetFirstFree(co); @@ -1754,7 +1765,7 @@ compiler_make_closure(struct compiler *c, location loc, } static int -compiler_decorators(struct compiler *c, asdl_expr_seq* decos) +codegen_decorators(struct compiler *c, asdl_expr_seq* decos) { if (!decos) { return SUCCESS; @@ -1767,7 +1778,7 @@ compiler_decorators(struct compiler *c, asdl_expr_seq* decos) } static int -compiler_apply_decorators(struct compiler *c, asdl_expr_seq* decos) +codegen_apply_decorators(struct compiler *c, asdl_expr_seq* decos) { if (!decos) { return SUCCESS; @@ -1781,8 +1792,8 @@ compiler_apply_decorators(struct compiler *c, asdl_expr_seq* decos) } static int -compiler_kwonlydefaults(struct compiler *c, location loc, - asdl_arg_seq *kwonlyargs, asdl_expr_seq *kw_defaults) +codegen_kwonlydefaults(struct compiler *c, location loc, + asdl_arg_seq *kwonlyargs, asdl_expr_seq *kw_defaults) { /* Push a dict of keyword-only default values. @@ -1828,7 +1839,7 @@ compiler_visit_annexpr(struct compiler *c, expr_ty annotation) } static int -compiler_argannotation(struct compiler *c, identifier id, +codegen_argannotation(struct compiler *c, identifier id, expr_ty annotation, Py_ssize_t *annotations_len, location loc) { if (!annotation) { @@ -1862,14 +1873,14 @@ compiler_argannotation(struct compiler *c, identifier id, } static int -compiler_argannotations(struct compiler *c, asdl_arg_seq* args, - Py_ssize_t *annotations_len, location loc) +codegen_argannotations(struct compiler *c, asdl_arg_seq* args, + Py_ssize_t *annotations_len, location loc) { int i; for (i = 0; i < asdl_seq_LEN(args); i++) { arg_ty arg = (arg_ty)asdl_seq_GET(args, i); RETURN_IF_ERROR( - compiler_argannotation( + codegen_argannotation( c, arg->arg, arg->annotation, @@ -1880,40 +1891,40 @@ compiler_argannotations(struct compiler *c, asdl_arg_seq* args, } static int -compiler_annotations_in_scope(struct compiler *c, location loc, - arguments_ty args, expr_ty returns, - Py_ssize_t *annotations_len) +codegen_annotations_in_scope(struct compiler *c, location loc, + arguments_ty args, expr_ty returns, + Py_ssize_t *annotations_len) { RETURN_IF_ERROR( - compiler_argannotations(c, args->args, annotations_len, loc)); + codegen_argannotations(c, args->args, annotations_len, loc)); RETURN_IF_ERROR( - compiler_argannotations(c, args->posonlyargs, annotations_len, loc)); + codegen_argannotations(c, args->posonlyargs, annotations_len, loc)); if (args->vararg && args->vararg->annotation) { RETURN_IF_ERROR( - compiler_argannotation(c, args->vararg->arg, - args->vararg->annotation, annotations_len, loc)); + codegen_argannotation(c, args->vararg->arg, + args->vararg->annotation, annotations_len, loc)); } RETURN_IF_ERROR( - compiler_argannotations(c, args->kwonlyargs, annotations_len, loc)); + codegen_argannotations(c, args->kwonlyargs, annotations_len, loc)); if (args->kwarg && args->kwarg->annotation) { RETURN_IF_ERROR( - compiler_argannotation(c, args->kwarg->arg, - args->kwarg->annotation, annotations_len, loc)); + codegen_argannotation(c, args->kwarg->arg, + args->kwarg->annotation, annotations_len, loc)); } RETURN_IF_ERROR( - compiler_argannotation(c, &_Py_ID(return), returns, annotations_len, loc)); + codegen_argannotation(c, &_Py_ID(return), returns, annotations_len, loc)); return 0; } static int -compiler_annotations(struct compiler *c, location loc, - arguments_ty args, expr_ty returns) +codegen_annotations(struct compiler *c, location loc, + arguments_ty args, expr_ty returns) { /* Push arg annotation names and values. The expressions are evaluated separately from the rest of the source code. @@ -1930,15 +1941,15 @@ compiler_annotations(struct compiler *c, location loc, bool annotations_used = ste->ste_annotations_used; if (annotations_used) { - if (compiler_setup_annotations_scope(c, loc, (void *)args, - ste->ste_name) < 0) { + if (codegen_setup_annotations_scope(c, loc, (void *)args, + ste->ste_name) < 0) { Py_DECREF(ste); return ERROR; } } Py_DECREF(ste); - if (compiler_annotations_in_scope(c, loc, args, returns, &annotations_len) < 0) { + if (codegen_annotations_in_scope(c, loc, args, returns, &annotations_len) < 0) { if (annotations_used) { compiler_exit_scope(c); } @@ -1947,7 +1958,7 @@ compiler_annotations(struct compiler *c, location loc, if (annotations_used) { RETURN_IF_ERROR( - compiler_leave_annotations_scope(c, loc, annotations_len) + codegen_leave_annotations_scope(c, loc, annotations_len) ); return MAKE_FUNCTION_ANNOTATE; } @@ -1956,7 +1967,7 @@ compiler_annotations(struct compiler *c, location loc, } static int -compiler_defaults(struct compiler *c, arguments_ty args, +codegen_defaults(struct compiler *c, arguments_ty args, location loc) { VISIT_SEQ(c, expr, args->defaults); @@ -1965,18 +1976,18 @@ compiler_defaults(struct compiler *c, arguments_ty args, } static Py_ssize_t -compiler_default_arguments(struct compiler *c, location loc, - arguments_ty args) +codegen_default_arguments(struct compiler *c, location loc, + arguments_ty args) { Py_ssize_t funcflags = 0; if (args->defaults && asdl_seq_LEN(args->defaults) > 0) { - RETURN_IF_ERROR(compiler_defaults(c, args, loc)); + RETURN_IF_ERROR(codegen_defaults(c, args, loc)); funcflags |= MAKE_FUNCTION_DEFAULTS; } if (args->kwonlyargs) { - int res = compiler_kwonlydefaults(c, loc, - args->kwonlyargs, - args->kw_defaults); + int res = codegen_kwonlydefaults(c, loc, + args->kwonlyargs, + args->kw_defaults); RETURN_IF_ERROR(res); if (res > 0) { funcflags |= MAKE_FUNCTION_KWDEFAULTS; @@ -1986,7 +1997,7 @@ compiler_default_arguments(struct compiler *c, location loc, } static int -wrap_in_stopiteration_handler(struct compiler *c) +codegen_wrap_in_stopiteration_handler(struct compiler *c) { NEW_JUMP_TARGET_LABEL(c, handler); @@ -2005,13 +2016,13 @@ wrap_in_stopiteration_handler(struct compiler *c) } static int -compiler_type_param_bound_or_default(struct compiler *c, expr_ty e, - identifier name, void *key, - bool allow_starred) +codegen_type_param_bound_or_default(struct compiler *c, expr_ty e, + identifier name, void *key, + bool allow_starred) { PyObject *defaults = PyTuple_Pack(1, _PyLong_GetOne()); ADDOP_LOAD_CONST_NEW(c, LOC(e), defaults); - if (compiler_setup_annotations_scope(c, LOC(e), key, name) == -1) { + if (codegen_setup_annotations_scope(c, LOC(e), key, name) == -1) { return ERROR; } if (allow_starred && e->kind == Starred_kind) { @@ -2027,7 +2038,7 @@ compiler_type_param_bound_or_default(struct compiler *c, expr_ty e, if (co == NULL) { return ERROR; } - if (compiler_make_closure(c, LOC(e), co, MAKE_FUNCTION_DEFAULTS) < 0) { + if (codegen_make_closure(c, LOC(e), co, MAKE_FUNCTION_DEFAULTS) < 0) { Py_DECREF(co); return ERROR; } @@ -2036,7 +2047,7 @@ compiler_type_param_bound_or_default(struct compiler *c, expr_ty e, } static int -compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) +codegen_type_params(struct compiler *c, asdl_type_param_seq *type_params) { if (!type_params) { return SUCCESS; @@ -2052,10 +2063,9 @@ compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) ADDOP_LOAD_CONST(c, loc, typeparam->v.TypeVar.name); if (typeparam->v.TypeVar.bound) { expr_ty bound = typeparam->v.TypeVar.bound; - if (compiler_type_param_bound_or_default(c, bound, typeparam->v.TypeVar.name, - (void *)typeparam, false) < 0) { - return ERROR; - } + RETURN_IF_ERROR( + codegen_type_param_bound_or_default(c, bound, typeparam->v.TypeVar.name, + (void *)typeparam, false)); int intrinsic = bound->kind == Tuple_kind ? INTRINSIC_TYPEVAR_WITH_CONSTRAINTS @@ -2068,10 +2078,9 @@ compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) if (typeparam->v.TypeVar.default_value) { seen_default = true; expr_ty default_ = typeparam->v.TypeVar.default_value; - if (compiler_type_param_bound_or_default(c, default_, typeparam->v.TypeVar.name, - (void *)((uintptr_t)typeparam + 1), false) < 0) { - return ERROR; - } + RETURN_IF_ERROR( + codegen_type_param_bound_or_default(c, default_, typeparam->v.TypeVar.name, + (void *)((uintptr_t)typeparam + 1), false)); ADDOP_I(c, loc, CALL_INTRINSIC_2, INTRINSIC_SET_TYPEPARAM_DEFAULT); } else if (seen_default) { @@ -2087,10 +2096,9 @@ compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_TYPEVARTUPLE); if (typeparam->v.TypeVarTuple.default_value) { expr_ty default_ = typeparam->v.TypeVarTuple.default_value; - if (compiler_type_param_bound_or_default(c, default_, typeparam->v.TypeVarTuple.name, - (void *)typeparam, true) < 0) { - return ERROR; - } + RETURN_IF_ERROR( + codegen_type_param_bound_or_default(c, default_, typeparam->v.TypeVarTuple.name, + (void *)typeparam, true)); ADDOP_I(c, loc, CALL_INTRINSIC_2, INTRINSIC_SET_TYPEPARAM_DEFAULT); seen_default = true; } @@ -2107,10 +2115,9 @@ compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_PARAMSPEC); if (typeparam->v.ParamSpec.default_value) { expr_ty default_ = typeparam->v.ParamSpec.default_value; - if (compiler_type_param_bound_or_default(c, default_, typeparam->v.ParamSpec.name, - (void *)typeparam, false) < 0) { - return ERROR; - } + RETURN_IF_ERROR( + codegen_type_param_bound_or_default(c, default_, typeparam->v.ParamSpec.name, + (void *)typeparam, false)); ADDOP_I(c, loc, CALL_INTRINSIC_2, INTRINSIC_SET_TYPEPARAM_DEFAULT); seen_default = true; } @@ -2155,8 +2162,13 @@ compiler_function_body(struct compiler *c, stmt_ty s, int is_async, Py_ssize_t f scope_type = COMPILER_SCOPE_FUNCTION; } + _PyCompile_CodeUnitMetadata umd = { + .u_argcount = asdl_seq_LEN(args->args), + .u_posonlyargcount = asdl_seq_LEN(args->posonlyargs), + .u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs), + }; RETURN_IF_ERROR( - compiler_enter_scope(c, name, scope_type, (void *)s, firstlineno, NULL)); + compiler_enter_scope(c, name, scope_type, (void *)s, firstlineno, NULL, &umd)); Py_ssize_t first_instr = 0; PyObject *docstring = _PyAST_GetDocString(body); @@ -2174,23 +2186,20 @@ compiler_function_body(struct compiler *c, stmt_ty s, int is_async, Py_ssize_t f docstring = NULL; } } - if (compiler_add_const(c, docstring ? docstring : Py_None) < 0) { - Py_XDECREF(docstring); - compiler_exit_scope(c); - return ERROR; - } - Py_CLEAR(docstring); + Py_ssize_t idx = compiler_add_const(c, docstring ? docstring : Py_None); + Py_XDECREF(docstring); + RETURN_IF_ERROR_IN_SCOPE(c, idx < 0 ? ERROR : SUCCESS); - c->u->u_metadata.u_argcount = asdl_seq_LEN(args->args); - c->u->u_metadata.u_posonlyargcount = asdl_seq_LEN(args->posonlyargs); - c->u->u_metadata.u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs); + assert(c->u->u_metadata.u_argcount == asdl_seq_LEN(args->args)); + assert(c->u->u_metadata.u_posonlyargcount == asdl_seq_LEN(args->posonlyargs)); + assert(c->u->u_metadata.u_kwonlyargcount == asdl_seq_LEN(args->kwonlyargs)); NEW_JUMP_TARGET_LABEL(c, start); USE_LABEL(c, start); PySTEntryObject *ste = SYMTABLE_ENTRY(c); bool add_stopiteration_handler = ste->ste_coroutine || ste->ste_generator; if (add_stopiteration_handler) { - /* wrap_in_stopiteration_handler will push a block, so we need to account for that */ + /* codegen_wrap_in_stopiteration_handler will push a block, so we need to account for that */ RETURN_IF_ERROR( compiler_push_fblock(c, NO_LOCATION, STOP_ITERATION, start, NO_LABEL, NULL)); @@ -2200,10 +2209,7 @@ compiler_function_body(struct compiler *c, stmt_ty s, int is_async, Py_ssize_t f VISIT_IN_SCOPE(c, stmt, (stmt_ty)asdl_seq_GET(body, i)); } if (add_stopiteration_handler) { - if (wrap_in_stopiteration_handler(c) < 0) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE(c, codegen_wrap_in_stopiteration_handler(c)); compiler_pop_fblock(c, STOP_ITERATION, start); } PyCodeObject *co = optimize_and_assemble(c, 1); @@ -2212,17 +2218,13 @@ compiler_function_body(struct compiler *c, stmt_ty s, int is_async, Py_ssize_t f Py_XDECREF(co); return ERROR; } - location loc = LOC(s); - if (compiler_make_closure(c, loc, co, funcflags) < 0) { - Py_DECREF(co); - return ERROR; - } + int ret = codegen_make_closure(c, LOC(s), co, funcflags); Py_DECREF(co); - return SUCCESS; + return ret; } static int -compiler_function(struct compiler *c, stmt_ty s, int is_async) +codegen_function(struct compiler *c, stmt_ty s, int is_async) { arguments_ty args; expr_ty returns; @@ -2250,7 +2252,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) type_params = s->v.FunctionDef.type_params; } - RETURN_IF_ERROR(compiler_decorators(c, decos)); + RETURN_IF_ERROR(codegen_decorators(c, decos)); firstlineno = s->lineno; if (asdl_seq_LEN(decos)) { @@ -2261,7 +2263,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) int is_generic = asdl_seq_LEN(type_params) > 0; - funcflags = compiler_default_arguments(c, loc, args); + funcflags = codegen_default_arguments(c, loc, args); if (funcflags == -1) { return ERROR; } @@ -2282,19 +2284,22 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) if (!type_params_name) { return ERROR; } + _PyCompile_CodeUnitMetadata umd = { + .u_argcount = num_typeparam_args, + }; if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS, - (void *)type_params, firstlineno, NULL) == -1) { + (void *)type_params, firstlineno, NULL, &umd) == -1) { Py_DECREF(type_params_name); return ERROR; } Py_DECREF(type_params_name); - RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params)); + RETURN_IF_ERROR_IN_SCOPE(c, codegen_type_params(c, type_params)); for (int i = 0; i < num_typeparam_args; i++) { ADDOP_I_IN_SCOPE(c, loc, LOAD_FAST, i); } } - int annotations_flag = compiler_annotations(c, loc, args, returns); + int annotations_flag = codegen_annotations(c, loc, args, returns); if (annotations_flag < 0) { if (is_generic) { compiler_exit_scope(c); @@ -2314,13 +2319,13 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) ADDOP_I_IN_SCOPE(c, loc, SWAP, 2); ADDOP_I_IN_SCOPE(c, loc, CALL_INTRINSIC_2, INTRINSIC_SET_FUNCTION_TYPE_PARAMS); - c->u->u_metadata.u_argcount = num_typeparam_args; + assert(c->u->u_metadata.u_argcount == num_typeparam_args); PyCodeObject *co = optimize_and_assemble(c, 0); compiler_exit_scope(c); if (co == NULL) { return ERROR; } - if (compiler_make_closure(c, loc, co, 0) < 0) { + if (codegen_make_closure(c, loc, co, 0) < 0) { Py_DECREF(co); return ERROR; } @@ -2335,17 +2340,17 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) } } - RETURN_IF_ERROR(compiler_apply_decorators(c, decos)); + RETURN_IF_ERROR(codegen_apply_decorators(c, decos)); return compiler_nameop(c, loc, name, Store); } static int -compiler_set_type_params_in_class(struct compiler *c, location loc) +codegen_set_type_params_in_class(struct compiler *c, location loc) { _Py_DECLARE_STR(type_params, ".type_params"); RETURN_IF_ERROR(compiler_nameop(c, loc, &_Py_STR(type_params), Load)); RETURN_IF_ERROR(compiler_nameop(c, loc, &_Py_ID(__type_params__), Store)); - return 1; + return SUCCESS; } static int @@ -2359,42 +2364,27 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) is the class name is the positional arguments and *varargs argument is the keyword arguments and **kwds argument - This borrows from compiler_call. + This borrows from codegen_call. */ /* 1. compile the class body into a code object */ RETURN_IF_ERROR( compiler_enter_scope(c, s->v.ClassDef.name, COMPILER_SCOPE_CLASS, - (void *)s, firstlineno, s->v.ClassDef.name)); + (void *)s, firstlineno, s->v.ClassDef.name, NULL)); location loc = LOCATION(firstlineno, firstlineno, 0, 0); /* load (global) __name__ ... */ - if (compiler_nameop(c, loc, &_Py_ID(__name__), Load) < 0) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_ID(__name__), Load)); /* ... and store it as __module__ */ - if (compiler_nameop(c, loc, &_Py_ID(__module__), Store) < 0) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_ID(__module__), Store)); assert(c->u->u_metadata.u_qualname); ADDOP_LOAD_CONST(c, loc, c->u->u_metadata.u_qualname); - if (compiler_nameop(c, loc, &_Py_ID(__qualname__), Store) < 0) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_ID(__qualname__), Store)); ADDOP_LOAD_CONST_NEW(c, loc, PyLong_FromLong(c->u->u_metadata.u_firstlineno)); - if (compiler_nameop(c, loc, &_Py_ID(__firstlineno__), Store) < 0) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_ID(__firstlineno__), Store)); asdl_type_param_seq *type_params = s->v.ClassDef.type_params; if (asdl_seq_LEN(type_params) > 0) { - if (!compiler_set_type_params_in_class(c, loc)) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE(c, codegen_set_type_params_in_class(c, loc)); } if (SYMTABLE_ENTRY(c)->ste_needs_classdict) { ADDOP(c, loc, LOAD_LOCALS); @@ -2405,10 +2395,7 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) ADDOP_N_IN_SCOPE(c, loc, STORE_DEREF, &_Py_ID(__classdict__), cellvars); } /* compile the body proper */ - if (compiler_body(c, loc, s->v.ClassDef.body) < 0) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE(c, compiler_body(c, loc, s->v.ClassDef.body)); assert(c->u->u_static_attributes); PyObject *static_attributes = PySequence_Tuple(c->u->u_static_attributes); if (static_attributes == NULL) { @@ -2417,39 +2404,27 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) } ADDOP_LOAD_CONST(c, NO_LOCATION, static_attributes); Py_CLEAR(static_attributes); - if (compiler_nameop(c, NO_LOCATION, &_Py_ID(__static_attributes__), Store) < 0) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE( + c, compiler_nameop(c, NO_LOCATION, &_Py_ID(__static_attributes__), Store)); /* The following code is artificial */ /* Set __classdictcell__ if necessary */ if (SYMTABLE_ENTRY(c)->ste_needs_classdict) { /* Store __classdictcell__ into class namespace */ int i = dict_lookup_arg(c->u->u_metadata.u_cellvars, &_Py_ID(__classdict__)); - if (i < 0) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE(c, i); ADDOP_I(c, NO_LOCATION, LOAD_CLOSURE, i); - if (compiler_nameop(c, NO_LOCATION, &_Py_ID(__classdictcell__), Store) < 0) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE( + c, compiler_nameop(c, NO_LOCATION, &_Py_ID(__classdictcell__), Store)); } /* Return __classcell__ if it is referenced, otherwise return None */ if (SYMTABLE_ENTRY(c)->ste_needs_class_closure) { /* Store __classcell__ into class namespace & return it */ int i = dict_lookup_arg(c->u->u_metadata.u_cellvars, &_Py_ID(__class__)); - if (i < 0) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE(c, i); ADDOP_I(c, NO_LOCATION, LOAD_CLOSURE, i); ADDOP_I(c, NO_LOCATION, COPY, 1); - if (compiler_nameop(c, NO_LOCATION, &_Py_ID(__classcell__), Store) < 0) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE( + c, compiler_nameop(c, NO_LOCATION, &_Py_ID(__classcell__), Store)); } else { /* No methods referenced __class__, so just return None */ @@ -2474,7 +2449,7 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) ADDOP(c, loc, PUSH_NULL); /* 3. load a function (or closure) made from the code object */ - if (compiler_make_closure(c, loc, co, 0) < 0) { + if (codegen_make_closure(c, loc, co, 0) < 0) { Py_DECREF(co); return ERROR; } @@ -2487,11 +2462,11 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) } static int -compiler_class(struct compiler *c, stmt_ty s) +codegen_class(struct compiler *c, stmt_ty s) { asdl_expr_seq *decos = s->v.ClassDef.decorator_list; - RETURN_IF_ERROR(compiler_decorators(c, decos)); + RETURN_IF_ERROR(codegen_decorators(c, decos)); int firstlineno = s->lineno; if (asdl_seq_LEN(decos)) { @@ -2508,12 +2483,12 @@ compiler_class(struct compiler *c, stmt_ty s) return ERROR; } if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS, - (void *)type_params, firstlineno, s->v.ClassDef.name) == -1) { + (void *)type_params, firstlineno, s->v.ClassDef.name, NULL) == -1) { Py_DECREF(type_params_name); return ERROR; } Py_DECREF(type_params_name); - RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params)); + RETURN_IF_ERROR_IN_SCOPE(c, codegen_type_params(c, type_params)); _Py_DECLARE_STR(type_params, ".type_params"); RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_STR(type_params), Store)); } @@ -2553,9 +2528,9 @@ compiler_class(struct compiler *c, stmt_ty s) return ERROR; } asdl_seq_SET(bases, original_len, name_node); - RETURN_IF_ERROR_IN_SCOPE(c, compiler_call_helper(c, loc, 2, - bases, - s->v.ClassDef.keywords)); + RETURN_IF_ERROR_IN_SCOPE(c, codegen_call_helper(c, loc, 2, + bases, + s->v.ClassDef.keywords)); PyCodeObject *co = optimize_and_assemble(c, 0); @@ -2563,7 +2538,7 @@ compiler_class(struct compiler *c, stmt_ty s) if (co == NULL) { return ERROR; } - if (compiler_make_closure(c, loc, co, 0) < 0) { + if (codegen_make_closure(c, loc, co, 0) < 0) { Py_DECREF(co); return ERROR; } @@ -2571,13 +2546,13 @@ compiler_class(struct compiler *c, stmt_ty s) ADDOP(c, loc, PUSH_NULL); ADDOP_I(c, loc, CALL, 0); } else { - RETURN_IF_ERROR(compiler_call_helper(c, loc, 2, + RETURN_IF_ERROR(codegen_call_helper(c, loc, 2, s->v.ClassDef.bases, s->v.ClassDef.keywords)); } /* 6. apply decorators */ - RETURN_IF_ERROR(compiler_apply_decorators(c, decos)); + RETURN_IF_ERROR(codegen_apply_decorators(c, decos)); /* 7. store into */ RETURN_IF_ERROR(compiler_nameop(c, loc, s->v.ClassDef.name, Store)); @@ -2585,14 +2560,14 @@ compiler_class(struct compiler *c, stmt_ty s) } static int -compiler_typealias_body(struct compiler *c, stmt_ty s) +codegen_typealias_body(struct compiler *c, stmt_ty s) { location loc = LOC(s); PyObject *name = s->v.TypeAlias.name->v.Name.id; PyObject *defaults = PyTuple_Pack(1, _PyLong_GetOne()); ADDOP_LOAD_CONST_NEW(c, loc, defaults); RETURN_IF_ERROR( - compiler_setup_annotations_scope(c, LOC(s), s, name)); + codegen_setup_annotations_scope(c, LOC(s), s, name)); /* Make None the first constant, so the evaluate function can't have a docstring. */ RETURN_IF_ERROR(compiler_add_const(c, Py_None)); @@ -2603,7 +2578,7 @@ compiler_typealias_body(struct compiler *c, stmt_ty s) if (co == NULL) { return ERROR; } - if (compiler_make_closure(c, loc, co, MAKE_FUNCTION_DEFAULTS) < 0) { + if (codegen_make_closure(c, loc, co, MAKE_FUNCTION_DEFAULTS) < 0) { Py_DECREF(co); return ERROR; } @@ -2614,7 +2589,7 @@ compiler_typealias_body(struct compiler *c, stmt_ty s) } static int -compiler_typealias(struct compiler *c, stmt_ty s) +codegen_typealias(struct compiler *c, stmt_ty s) { location loc = LOC(s); asdl_type_param_seq *type_params = s->v.TypeAlias.type_params; @@ -2627,20 +2602,20 @@ compiler_typealias(struct compiler *c, stmt_ty s) return ERROR; } if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS, - (void *)type_params, loc.lineno, NULL) == -1) { + (void *)type_params, loc.lineno, NULL, NULL) == -1) { Py_DECREF(type_params_name); return ERROR; } Py_DECREF(type_params_name); ADDOP_LOAD_CONST_IN_SCOPE(c, loc, name); - RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params)); + RETURN_IF_ERROR_IN_SCOPE(c, codegen_type_params(c, type_params)); } else { ADDOP_LOAD_CONST(c, loc, name); ADDOP_LOAD_CONST(c, loc, Py_None); } - if (compiler_typealias_body(c, s) < 0) { + if (codegen_typealias_body(c, s) < 0) { if (is_generic) { compiler_exit_scope(c); } @@ -2653,7 +2628,7 @@ compiler_typealias(struct compiler *c, stmt_ty s) if (co == NULL) { return ERROR; } - if (compiler_make_closure(c, loc, co, 0) < 0) { + if (codegen_make_closure(c, loc, co, 0) < 0) { Py_DECREF(co); return ERROR; } @@ -2686,7 +2661,7 @@ static PyTypeObject * infer_type(expr_ty e); Emit a warning if any operand is a constant except named singletons. */ static int -check_compare(struct compiler *c, expr_ty e) +codegen_check_compare(struct compiler *c, expr_ty e) { Py_ssize_t i, n; bool left = check_is_arg(e->v.Compare.left); @@ -2714,7 +2689,7 @@ check_compare(struct compiler *c, expr_ty e) } static int -compiler_addcompare(struct compiler *c, location loc, cmpop_ty op) +codegen_addcompare(struct compiler *c, location loc, cmpop_ty op) { int cmp; switch (op) { @@ -2759,16 +2734,14 @@ compiler_addcompare(struct compiler *c, location loc, cmpop_ty op) return SUCCESS; } - - static int -compiler_jump_if(struct compiler *c, location loc, - expr_ty e, jump_target_label next, int cond) +codegen_jump_if(struct compiler *c, location loc, + expr_ty e, jump_target_label next, int cond) { switch (e->kind) { case UnaryOp_kind: if (e->v.UnaryOp.op == Not) { - return compiler_jump_if(c, loc, e->v.UnaryOp.operand, next, !cond); + return codegen_jump_if(c, loc, e->v.UnaryOp.operand, next, !cond); } /* fallback to general implementation */ break; @@ -2784,10 +2757,10 @@ compiler_jump_if(struct compiler *c, location loc, } for (i = 0; i < n; ++i) { RETURN_IF_ERROR( - compiler_jump_if(c, loc, (expr_ty)asdl_seq_GET(s, i), next2, cond2)); + codegen_jump_if(c, loc, (expr_ty)asdl_seq_GET(s, i), next2, cond2)); } RETURN_IF_ERROR( - compiler_jump_if(c, loc, (expr_ty)asdl_seq_GET(s, n), next, cond)); + codegen_jump_if(c, loc, (expr_ty)asdl_seq_GET(s, n), next, cond)); if (!SAME_LABEL(next2, next)) { USE_LABEL(c, next2); } @@ -2797,14 +2770,14 @@ compiler_jump_if(struct compiler *c, location loc, NEW_JUMP_TARGET_LABEL(c, end); NEW_JUMP_TARGET_LABEL(c, next2); RETURN_IF_ERROR( - compiler_jump_if(c, loc, e->v.IfExp.test, next2, 0)); + codegen_jump_if(c, loc, e->v.IfExp.test, next2, 0)); RETURN_IF_ERROR( - compiler_jump_if(c, loc, e->v.IfExp.body, next, cond)); + codegen_jump_if(c, loc, e->v.IfExp.body, next, cond)); ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); USE_LABEL(c, next2); RETURN_IF_ERROR( - compiler_jump_if(c, loc, e->v.IfExp.orelse, next, cond)); + codegen_jump_if(c, loc, e->v.IfExp.orelse, next, cond)); USE_LABEL(c, end); return SUCCESS; @@ -2812,7 +2785,7 @@ compiler_jump_if(struct compiler *c, location loc, case Compare_kind: { Py_ssize_t n = asdl_seq_LEN(e->v.Compare.ops) - 1; if (n > 0) { - RETURN_IF_ERROR(check_compare(c, e)); + RETURN_IF_ERROR(codegen_check_compare(c, e)); NEW_JUMP_TARGET_LABEL(c, cleanup); VISIT(c, expr, e->v.Compare.left); for (Py_ssize_t i = 0; i < n; i++) { @@ -2856,14 +2829,14 @@ compiler_jump_if(struct compiler *c, location loc, } static int -compiler_ifexp(struct compiler *c, expr_ty e) +codegen_ifexp(struct compiler *c, expr_ty e) { assert(e->kind == IfExp_kind); NEW_JUMP_TARGET_LABEL(c, end); NEW_JUMP_TARGET_LABEL(c, next); RETURN_IF_ERROR( - compiler_jump_if(c, LOC(e), e->v.IfExp.test, next, 0)); + codegen_jump_if(c, LOC(e), e->v.IfExp.test, next, 0)); VISIT(c, expr, e->v.IfExp.body); ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); @@ -2876,7 +2849,7 @@ compiler_ifexp(struct compiler *c, expr_ty e) } static int -compiler_lambda(struct compiler *c, expr_ty e) +codegen_lambda(struct compiler *c, expr_ty e) { PyCodeObject *co; Py_ssize_t funcflags; @@ -2884,23 +2857,28 @@ compiler_lambda(struct compiler *c, expr_ty e) assert(e->kind == Lambda_kind); location loc = LOC(e); - funcflags = compiler_default_arguments(c, loc, args); + funcflags = codegen_default_arguments(c, loc, args); if (funcflags == -1) { return ERROR; } + _PyCompile_CodeUnitMetadata umd = { + .u_argcount = asdl_seq_LEN(args->args), + .u_posonlyargcount = asdl_seq_LEN(args->posonlyargs), + .u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs), + }; _Py_DECLARE_STR(anon_lambda, ""); RETURN_IF_ERROR( compiler_enter_scope(c, &_Py_STR(anon_lambda), COMPILER_SCOPE_LAMBDA, - (void *)e, e->lineno, NULL)); + (void *)e, e->lineno, NULL, &umd)); /* Make None the first constant, so the lambda can't have a docstring. */ RETURN_IF_ERROR(compiler_add_const(c, Py_None)); - c->u->u_metadata.u_argcount = asdl_seq_LEN(args->args); - c->u->u_metadata.u_posonlyargcount = asdl_seq_LEN(args->posonlyargs); - c->u->u_metadata.u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs); + assert(c->u->u_metadata.u_argcount == asdl_seq_LEN(args->args)); + assert(c->u->u_metadata.u_posonlyargcount == asdl_seq_LEN(args->posonlyargs)); + assert(c->u->u_metadata.u_kwonlyargcount == asdl_seq_LEN(args->kwonlyargs)); VISIT_IN_SCOPE(c, expr, e->v.Lambda.body); if (SYMTABLE_ENTRY(c)->ste_generator) { co = optimize_and_assemble(c, 0); @@ -2915,7 +2893,7 @@ compiler_lambda(struct compiler *c, expr_ty e) return ERROR; } - if (compiler_make_closure(c, loc, co, funcflags) < 0) { + if (codegen_make_closure(c, loc, co, funcflags) < 0) { Py_DECREF(co); return ERROR; } @@ -2925,7 +2903,7 @@ compiler_lambda(struct compiler *c, expr_ty e) } static int -compiler_if(struct compiler *c, stmt_ty s) +codegen_if(struct compiler *c, stmt_ty s) { jump_target_label next; assert(s->kind == If_kind); @@ -2938,7 +2916,7 @@ compiler_if(struct compiler *c, stmt_ty s) next = end; } RETURN_IF_ERROR( - compiler_jump_if(c, LOC(s), s->v.If.test, next, 0)); + codegen_jump_if(c, LOC(s), s->v.If.test, next, 0)); VISIT_SEQ(c, stmt, s->v.If.body); if (asdl_seq_LEN(s->v.If.orelse)) { @@ -2953,7 +2931,7 @@ compiler_if(struct compiler *c, stmt_ty s) } static int -compiler_for(struct compiler *c, stmt_ty s) +codegen_for(struct compiler *c, stmt_ty s) { location loc = LOC(s); NEW_JUMP_TARGET_LABEL(c, start); @@ -3000,7 +2978,7 @@ compiler_for(struct compiler *c, stmt_ty s) static int -compiler_async_for(struct compiler *c, stmt_ty s) +codegen_async_for(struct compiler *c, stmt_ty s) { location loc = LOC(s); @@ -3045,7 +3023,7 @@ compiler_async_for(struct compiler *c, stmt_ty s) } static int -compiler_while(struct compiler *c, stmt_ty s) +codegen_while(struct compiler *c, stmt_ty s) { NEW_JUMP_TARGET_LABEL(c, loop); NEW_JUMP_TARGET_LABEL(c, end); @@ -3054,7 +3032,7 @@ compiler_while(struct compiler *c, stmt_ty s) USE_LABEL(c, loop); RETURN_IF_ERROR(compiler_push_fblock(c, LOC(s), WHILE_LOOP, loop, end, NULL)); - RETURN_IF_ERROR(compiler_jump_if(c, LOC(s), s->v.While.test, anchor, 0)); + RETURN_IF_ERROR(codegen_jump_if(c, LOC(s), s->v.While.test, anchor, 0)); VISIT_SEQ(c, stmt, s->v.While.body); ADDOP_JUMP(c, NO_LOCATION, JUMP, loop); @@ -3071,7 +3049,7 @@ compiler_while(struct compiler *c, stmt_ty s) } static int -compiler_return(struct compiler *c, stmt_ty s) +codegen_return(struct compiler *c, stmt_ty s) { location loc = LOC(s); int preserve_tos = ((s->v.Return.value != NULL) && @@ -3112,7 +3090,7 @@ compiler_return(struct compiler *c, stmt_ty s) } static int -compiler_break(struct compiler *c, location loc) +codegen_break(struct compiler *c, location loc) { struct fblockinfo *loop = NULL; location origin_loc = loc; @@ -3128,7 +3106,7 @@ compiler_break(struct compiler *c, location loc) } static int -compiler_continue(struct compiler *c, location loc) +codegen_continue(struct compiler *c, location loc) { struct fblockinfo *loop = NULL; location origin_loc = loc; @@ -3173,7 +3151,7 @@ compiler_continue(struct compiler *c, location loc) */ static int -compiler_try_finally(struct compiler *c, stmt_ty s) +codegen_try_finally(struct compiler *c, stmt_ty s) { location loc = LOC(s); @@ -3191,7 +3169,7 @@ compiler_try_finally(struct compiler *c, stmt_ty s) s->v.Try.finalbody)); if (s->v.Try.handlers && asdl_seq_LEN(s->v.Try.handlers)) { - RETURN_IF_ERROR(compiler_try_except(c, s)); + RETURN_IF_ERROR(codegen_try_except(c, s)); } else { VISIT_SEQ(c, stmt, s->v.Try.body); @@ -3224,7 +3202,7 @@ compiler_try_finally(struct compiler *c, stmt_ty s) } static int -compiler_try_star_finally(struct compiler *c, stmt_ty s) +codegen_try_star_finally(struct compiler *c, stmt_ty s) { location loc = LOC(s); @@ -3241,7 +3219,7 @@ compiler_try_star_finally(struct compiler *c, stmt_ty s) s->v.TryStar.finalbody)); if (s->v.TryStar.handlers && asdl_seq_LEN(s->v.TryStar.handlers)) { - RETURN_IF_ERROR(compiler_try_star_except(c, s)); + RETURN_IF_ERROR(codegen_try_star_except(c, s)); } else { VISIT_SEQ(c, stmt, s->v.TryStar.body); @@ -3304,7 +3282,7 @@ compiler_try_star_finally(struct compiler *c, stmt_ty s) Of course, parts are not generated if Vi or Ei is not present. */ static int -compiler_try_except(struct compiler *c, stmt_ty s) +codegen_try_except(struct compiler *c, stmt_ty s) { location loc = LOC(s); Py_ssize_t i, n; @@ -3486,7 +3464,7 @@ compiler_try_except(struct compiler *c, stmt_ty s) [] L0: */ static int -compiler_try_star_except(struct compiler *c, stmt_ty s) +codegen_try_star_except(struct compiler *c, stmt_ty s) { location loc = LOC(s); @@ -3653,27 +3631,27 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) } static int -compiler_try(struct compiler *c, stmt_ty s) { +codegen_try(struct compiler *c, stmt_ty s) { if (s->v.Try.finalbody && asdl_seq_LEN(s->v.Try.finalbody)) - return compiler_try_finally(c, s); + return codegen_try_finally(c, s); else - return compiler_try_except(c, s); + return codegen_try_except(c, s); } static int -compiler_try_star(struct compiler *c, stmt_ty s) +codegen_try_star(struct compiler *c, stmt_ty s) { if (s->v.TryStar.finalbody && asdl_seq_LEN(s->v.TryStar.finalbody)) { - return compiler_try_star_finally(c, s); + return codegen_try_star_finally(c, s); } else { - return compiler_try_star_except(c, s); + return codegen_try_star_except(c, s); } } static int -compiler_import_as(struct compiler *c, location loc, - identifier name, identifier asname) +codegen_import_as(struct compiler *c, location loc, + identifier name, identifier asname) { /* The IMPORT_NAME opcode was already generated. This function merely needs to bind the result to a name. @@ -3714,7 +3692,7 @@ compiler_import_as(struct compiler *c, location loc, } static int -compiler_import(struct compiler *c, stmt_ty s) +codegen_import(struct compiler *c, stmt_ty s) { location loc = LOC(s); /* The Import node stores a module name like a.b.c as a single @@ -3736,7 +3714,7 @@ compiler_import(struct compiler *c, stmt_ty s) ADDOP_NAME(c, loc, IMPORT_NAME, alias->name, names); if (alias->asname) { - r = compiler_import_as(c, loc, alias->name, alias->asname); + r = codegen_import_as(c, loc, alias->name, alias->asname); RETURN_IF_ERROR(r); } else { @@ -3760,7 +3738,7 @@ compiler_import(struct compiler *c, stmt_ty s) } static int -compiler_from_import(struct compiler *c, stmt_ty s) +codegen_from_import(struct compiler *c, stmt_ty s) { Py_ssize_t n = asdl_seq_LEN(s->v.ImportFrom.names); @@ -3828,7 +3806,7 @@ compiler_assert(struct compiler *c, stmt_ty s) return SUCCESS; } NEW_JUMP_TARGET_LABEL(c, end); - RETURN_IF_ERROR(compiler_jump_if(c, LOC(s), s->v.Assert.test, end, 1)); + RETURN_IF_ERROR(codegen_jump_if(c, LOC(s), s->v.Assert.test, end, 1)); ADDOP_I(c, LOC(s), LOAD_COMMON_CONSTANT, CONSTANT_ASSERTIONERROR); if (s->v.Assert.msg) { VISIT(c, expr, s->v.Assert.msg); @@ -3867,13 +3845,13 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) switch (s->kind) { case FunctionDef_kind: - return compiler_function(c, s, 0); + return codegen_function(c, s, 0); case ClassDef_kind: - return compiler_class(c, s); + return codegen_class(c, s); case TypeAlias_kind: - return compiler_typealias(c, s); + return codegen_typealias(c, s); case Return_kind: - return compiler_return(c, s); + return codegen_return(c, s); case Delete_kind: VISIT_SEQ(c, expr, s->v.Delete.targets) break; @@ -3891,17 +3869,17 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) break; } case AugAssign_kind: - return compiler_augassign(c, s); + return codegen_augassign(c, s); case AnnAssign_kind: return compiler_annassign(c, s); case For_kind: - return compiler_for(c, s); + return codegen_for(c, s); case While_kind: - return compiler_while(c, s); + return codegen_while(c, s); case If_kind: - return compiler_if(c, s); + return codegen_if(c, s); case Match_kind: - return compiler_match(c, s); + return codegen_match(c, s); case Raise_kind: { Py_ssize_t n = 0; @@ -3917,15 +3895,15 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) break; } case Try_kind: - return compiler_try(c, s); + return codegen_try(c, s); case TryStar_kind: - return compiler_try_star(c, s); + return codegen_try_star(c, s); case Assert_kind: return compiler_assert(c, s); case Import_kind: - return compiler_import(c, s); + return codegen_import(c, s); case ImportFrom_kind: - return compiler_from_import(c, s); + return codegen_from_import(c, s); case Global_kind: case Nonlocal_kind: break; @@ -3940,20 +3918,20 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) } case Break_kind: { - return compiler_break(c, LOC(s)); + return codegen_break(c, LOC(s)); } case Continue_kind: { - return compiler_continue(c, LOC(s)); + return codegen_continue(c, LOC(s)); } case With_kind: - return compiler_with(c, s, 0); + return codegen_with(c, s, 0); case AsyncFunctionDef_kind: - return compiler_function(c, s, 1); + return codegen_function(c, s, 1); case AsyncWith_kind: - return compiler_async_with(c, s, 0); + return codegen_async_with(c, s, 0); case AsyncFor_kind: - return compiler_async_for(c, s); + return codegen_async_for(c, s); } return SUCCESS; @@ -4030,7 +4008,7 @@ addop_binary(struct compiler *c, location loc, operator_ty binop, static int -addop_yield(struct compiler *c, location loc) { +codegen_addop_yield(struct compiler *c, location loc) { PySTEntryObject *ste = SYMTABLE_ENTRY(c); if (ste->ste_generator && ste->ste_coroutine) { ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_ASYNC_GEN_WRAP); @@ -4041,7 +4019,7 @@ addop_yield(struct compiler *c, location loc) { } static int -compiler_load_classdict_freevar(struct compiler *c, location loc) +codegen_load_classdict_freevar(struct compiler *c, location loc) { ADDOP_N(c, loc, LOAD_DEREF, &_Py_ID(__classdict__), freevars); return SUCCESS; @@ -4126,7 +4104,7 @@ compiler_nameop(struct compiler *c, location loc, else if (SYMTABLE_ENTRY(c)->ste_can_see_class_scope) { op = LOAD_FROM_DICT_OR_DEREF; // First load the classdict - if (compiler_load_classdict_freevar(c, loc) < 0) { + if (codegen_load_classdict_freevar(c, loc) < 0) { goto error; } } @@ -4152,7 +4130,7 @@ compiler_nameop(struct compiler *c, location loc, if (SYMTABLE_ENTRY(c)->ste_can_see_class_scope && scope == GLOBAL_IMPLICIT) { op = LOAD_FROM_DICT_OR_GLOBALS; // First load the classdict - if (compiler_load_classdict_freevar(c, loc) < 0) { + if (codegen_load_classdict_freevar(c, loc) < 0) { goto error; } } else { @@ -4195,7 +4173,7 @@ compiler_nameop(struct compiler *c, location loc, } static int -compiler_boolop(struct compiler *c, expr_ty e) +codegen_boolop(struct compiler *c, expr_ty e) { int jumpi; Py_ssize_t i, n; @@ -4350,7 +4328,7 @@ assignment_helper(struct compiler *c, location loc, asdl_expr_seq *elts) } static int -compiler_list(struct compiler *c, expr_ty e) +codegen_list(struct compiler *c, expr_ty e) { location loc = LOC(e); asdl_expr_seq *elts = e->v.List.elts; @@ -4368,7 +4346,7 @@ compiler_list(struct compiler *c, expr_ty e) } static int -compiler_tuple(struct compiler *c, expr_ty e) +codegen_tuple(struct compiler *c, expr_ty e) { location loc = LOC(e); asdl_expr_seq *elts = e->v.Tuple.elts; @@ -4386,7 +4364,7 @@ compiler_tuple(struct compiler *c, expr_ty e) } static int -compiler_set(struct compiler *c, expr_ty e) +codegen_set(struct compiler *c, expr_ty e) { location loc = LOC(e); return starunpack_helper(c, loc, e->v.Set.elts, 0, @@ -4406,7 +4384,7 @@ are_all_items_const(asdl_expr_seq *seq, Py_ssize_t begin, Py_ssize_t end) } static int -compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end) +codegen_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end) { Py_ssize_t i, n = end - begin; int big = n*2 > STACK_USE_GUIDELINE; @@ -4428,7 +4406,7 @@ compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end } static int -compiler_dict(struct compiler *c, expr_ty e) +codegen_dict(struct compiler *c, expr_ty e) { location loc = LOC(e); Py_ssize_t i, n, elements; @@ -4441,7 +4419,7 @@ compiler_dict(struct compiler *c, expr_ty e) is_unpacking = (expr_ty)asdl_seq_GET(e->v.Dict.keys, i) == NULL; if (is_unpacking) { if (elements) { - RETURN_IF_ERROR(compiler_subdict(c, e, i - elements, i)); + RETURN_IF_ERROR(codegen_subdict(c, e, i - elements, i)); if (have_dict) { ADDOP_I(c, loc, DICT_UPDATE, 1); } @@ -4457,7 +4435,7 @@ compiler_dict(struct compiler *c, expr_ty e) } else { if (elements*2 > STACK_USE_GUIDELINE) { - RETURN_IF_ERROR(compiler_subdict(c, e, i - elements, i + 1)); + RETURN_IF_ERROR(codegen_subdict(c, e, i - elements, i + 1)); if (have_dict) { ADDOP_I(c, loc, DICT_UPDATE, 1); } @@ -4470,7 +4448,7 @@ compiler_dict(struct compiler *c, expr_ty e) } } if (elements) { - RETURN_IF_ERROR(compiler_subdict(c, e, n - elements, n)); + RETURN_IF_ERROR(codegen_subdict(c, e, n - elements, n)); if (have_dict) { ADDOP_I(c, loc, DICT_UPDATE, 1); } @@ -4483,12 +4461,12 @@ compiler_dict(struct compiler *c, expr_ty e) } static int -compiler_compare(struct compiler *c, expr_ty e) +codegen_compare(struct compiler *c, expr_ty e) { location loc = LOC(e); Py_ssize_t i, n; - RETURN_IF_ERROR(check_compare(c, e)); + RETURN_IF_ERROR(codegen_check_compare(c, e)); VISIT(c, expr, e->v.Compare.left); assert(asdl_seq_LEN(e->v.Compare.ops) > 0); n = asdl_seq_LEN(e->v.Compare.ops) - 1; @@ -4838,7 +4816,7 @@ maybe_optimize_method_call(struct compiler *c, expr_ty e) if (kwdsl) { VISIT_SEQ(c, keyword, kwds); RETURN_IF_ERROR( - compiler_call_simple_kw_helper(c, loc, kwds, kwdsl)); + codegen_call_simple_kw_helper(c, loc, kwds, kwdsl)); loc = update_start_location_to_match_attr(c, LOC(e), meth); ADDOP_I(c, loc, CALL_KW, argsl + kwdsl); } @@ -4850,7 +4828,7 @@ maybe_optimize_method_call(struct compiler *c, expr_ty e) } static int -validate_keywords(struct compiler *c, asdl_keyword_seq *keywords) +codegen_validate_keywords(struct compiler *c, asdl_keyword_seq *keywords) { Py_ssize_t nkeywords = asdl_seq_LEN(keywords); for (Py_ssize_t i = 0; i < nkeywords; i++) { @@ -4870,9 +4848,9 @@ validate_keywords(struct compiler *c, asdl_keyword_seq *keywords) } static int -compiler_call(struct compiler *c, expr_ty e) +codegen_call(struct compiler *c, expr_ty e) { - RETURN_IF_ERROR(validate_keywords(c, e->v.Call.keywords)); + RETURN_IF_ERROR(codegen_validate_keywords(c, e->v.Call.keywords)); int ret = maybe_optimize_method_call(c, e); if (ret < 0) { return ERROR; @@ -4885,13 +4863,13 @@ compiler_call(struct compiler *c, expr_ty e) location loc = LOC(e->v.Call.func); ADDOP(c, loc, PUSH_NULL); loc = LOC(e); - return compiler_call_helper(c, loc, 0, - e->v.Call.args, - e->v.Call.keywords); + return codegen_call_helper(c, loc, 0, + e->v.Call.args, + e->v.Call.keywords); } static int -compiler_joined_str(struct compiler *c, expr_ty e) +codegen_joined_str(struct compiler *c, expr_ty e) { location loc = LOC(e); Py_ssize_t value_count = asdl_seq_LEN(e->v.JoinedStr.values); @@ -4921,7 +4899,7 @@ compiler_joined_str(struct compiler *c, expr_ty e) /* Used to implement f-strings. Format a single value. */ static int -compiler_formatted_value(struct compiler *c, expr_ty e) +codegen_formatted_value(struct compiler *c, expr_ty e) { /* Our oparg encodes 2 pieces of information: the conversion character, and whether or not a format_spec was provided. @@ -4967,9 +4945,9 @@ compiler_formatted_value(struct compiler *c, expr_ty e) } static int -compiler_subkwargs(struct compiler *c, location loc, - asdl_keyword_seq *keywords, - Py_ssize_t begin, Py_ssize_t end) +codegen_subkwargs(struct compiler *c, location loc, + asdl_keyword_seq *keywords, + Py_ssize_t begin, Py_ssize_t end) { Py_ssize_t i, n = end - begin; keyword_ty kw; @@ -4992,12 +4970,12 @@ compiler_subkwargs(struct compiler *c, location loc, return SUCCESS; } -/* Used by compiler_call_helper and maybe_optimize_method_call to emit +/* Used by codegen_call_helper and maybe_optimize_method_call to emit * a tuple of keyword names before CALL. */ static int -compiler_call_simple_kw_helper(struct compiler *c, location loc, - asdl_keyword_seq *keywords, Py_ssize_t nkwelts) +codegen_call_simple_kw_helper(struct compiler *c, location loc, + asdl_keyword_seq *keywords, Py_ssize_t nkwelts) { PyObject *names; names = PyTuple_New(nkwelts); @@ -5013,16 +4991,16 @@ compiler_call_simple_kw_helper(struct compiler *c, location loc, } -/* shared code between compiler_call and compiler_class */ +/* shared code between codegen_call and codegen_class */ static int -compiler_call_helper(struct compiler *c, location loc, - int n, /* Args already pushed */ - asdl_expr_seq *args, - asdl_keyword_seq *keywords) +codegen_call_helper(struct compiler *c, location loc, + int n, /* Args already pushed */ + asdl_expr_seq *args, + asdl_keyword_seq *keywords) { Py_ssize_t i, nseen, nelts, nkwelts; - RETURN_IF_ERROR(validate_keywords(c, keywords)); + RETURN_IF_ERROR(codegen_validate_keywords(c, keywords)); nelts = asdl_seq_LEN(args); nkwelts = asdl_seq_LEN(keywords); @@ -5052,7 +5030,7 @@ compiler_call_helper(struct compiler *c, location loc, if (nkwelts) { VISIT_SEQ(c, keyword, keywords); RETURN_IF_ERROR( - compiler_call_simple_kw_helper(c, loc, keywords, nkwelts)); + codegen_call_simple_kw_helper(c, loc, keywords, nkwelts)); ADDOP_I(c, loc, CALL_KW, n + nelts + nkwelts); } else { @@ -5081,7 +5059,7 @@ compiler_call_helper(struct compiler *c, location loc, if (kw->arg == NULL) { /* A keyword argument unpacking. */ if (nseen) { - RETURN_IF_ERROR(compiler_subkwargs(c, loc, keywords, i - nseen, i)); + RETURN_IF_ERROR(codegen_subkwargs(c, loc, keywords, i - nseen, i)); if (have_dict) { ADDOP_I(c, loc, DICT_MERGE, 1); } @@ -5101,7 +5079,7 @@ compiler_call_helper(struct compiler *c, location loc, } if (nseen) { /* Pack up any trailing keyword arguments. */ - RETURN_IF_ERROR(compiler_subkwargs(c, loc, keywords, nkwelts - nseen, nkwelts)); + RETURN_IF_ERROR(codegen_subkwargs(c, loc, keywords, nkwelts - nseen, nkwelts)); if (have_dict) { ADDOP_I(c, loc, DICT_MERGE, 1); } @@ -5211,7 +5189,7 @@ compiler_sync_comprehension_generator(struct compiler *c, location loc, Py_ssize_t n = asdl_seq_LEN(gen->ifs); for (Py_ssize_t i = 0; i < n; i++) { expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i); - RETURN_IF_ERROR(compiler_jump_if(c, loc, e, if_cleanup, 0)); + RETURN_IF_ERROR(codegen_jump_if(c, loc, e, if_cleanup, 0)); } if (++gen_index < asdl_seq_LEN(generators)) { @@ -5315,7 +5293,7 @@ compiler_async_comprehension_generator(struct compiler *c, location loc, Py_ssize_t n = asdl_seq_LEN(gen->ifs); for (Py_ssize_t i = 0; i < n; i++) { expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i); - RETURN_IF_ERROR(compiler_jump_if(c, loc, e, if_cleanup, 0)); + RETURN_IF_ERROR(codegen_jump_if(c, loc, e, if_cleanup, 0)); } depth++; @@ -5674,7 +5652,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, } else { if (compiler_enter_scope(c, name, COMPILER_SCOPE_COMPREHENSION, - (void *)e, e->lineno, NULL) < 0) { + (void *)e, e->lineno, NULL, NULL) < 0) { goto error; } } @@ -5726,7 +5704,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, ADDOP(c, LOC(e), RETURN_VALUE); } if (type == COMP_GENEXP) { - if (wrap_in_stopiteration_handler(c) < 0) { + if (codegen_wrap_in_stopiteration_handler(c) < 0) { goto error_in_scope; } } @@ -5738,7 +5716,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, } loc = LOC(e); - if (compiler_make_closure(c, loc, co, 0) < 0) { + if (codegen_make_closure(c, loc, co, 0) < 0) { goto error; } Py_CLEAR(co); @@ -5820,7 +5798,7 @@ compiler_visit_keyword(struct compiler *c, keyword_ty k) static int -compiler_with_except_finish(struct compiler *c, jump_target_label cleanup) { +codegen_with_except_finish(struct compiler *c, jump_target_label cleanup) { NEW_JUMP_TARGET_LABEL(c, suppress); ADDOP(c, NO_LOCATION, TO_BOOL); ADDOP_JUMP(c, NO_LOCATION, POP_JUMP_IF_TRUE, suppress); @@ -5868,7 +5846,7 @@ compiler_with_except_finish(struct compiler *c, jump_target_label cleanup) { raise */ static int -compiler_async_with(struct compiler *c, stmt_ty s, int pos) +codegen_async_with(struct compiler *c, stmt_ty s, int pos) { location loc = LOC(s); withitem_ty item = asdl_seq_GET(s->v.AsyncWith.items, pos); @@ -5913,7 +5891,7 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) VISIT_SEQ(c, stmt, s->v.AsyncWith.body) } else { - RETURN_IF_ERROR(compiler_async_with(c, s, pos)); + RETURN_IF_ERROR(codegen_async_with(c, s, pos)); } compiler_pop_fblock(c, ASYNC_WITH, block); @@ -5924,7 +5902,7 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) /* For successful outcome: * call __exit__(None, None, None) */ - RETURN_IF_ERROR(compiler_call_exit_with_nones(c, loc)); + RETURN_IF_ERROR(codegen_call_exit_with_nones(c, loc)); ADDOP_I(c, loc, GET_AWAITABLE, 2); ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); @@ -5942,7 +5920,7 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) ADDOP_I(c, loc, GET_AWAITABLE, 2); ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); - RETURN_IF_ERROR(compiler_with_except_finish(c, cleanup)); + RETURN_IF_ERROR(codegen_with_except_finish(c, cleanup)); USE_LABEL(c, exit); return SUCCESS; @@ -5971,7 +5949,7 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) */ static int -compiler_with(struct compiler *c, stmt_ty s, int pos) +codegen_with(struct compiler *c, stmt_ty s, int pos) { withitem_ty item = asdl_seq_GET(s->v.With.items, pos); @@ -6012,7 +5990,7 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) VISIT_SEQ(c, stmt, s->v.With.body) } else { - RETURN_IF_ERROR(compiler_with(c, s, pos)); + RETURN_IF_ERROR(codegen_with(c, s, pos)); } ADDOP(c, NO_LOCATION, POP_BLOCK); @@ -6023,7 +6001,7 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) /* For successful outcome: * call __exit__(None, None, None) */ - RETURN_IF_ERROR(compiler_call_exit_with_nones(c, loc)); + RETURN_IF_ERROR(codegen_call_exit_with_nones(c, loc)); ADDOP(c, loc, POP_TOP); ADDOP_JUMP(c, loc, JUMP, exit); @@ -6033,7 +6011,7 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup); ADDOP(c, loc, PUSH_EXC_INFO); ADDOP(c, loc, WITH_EXCEPT_START); - RETURN_IF_ERROR(compiler_with_except_finish(c, cleanup)); + RETURN_IF_ERROR(codegen_with_except_finish(c, cleanup)); USE_LABEL(c, exit); return SUCCESS; @@ -6050,7 +6028,7 @@ compiler_visit_expr(struct compiler *c, expr_ty e) VISIT(c, expr, e->v.NamedExpr.target); break; case BoolOp_kind: - return compiler_boolop(c, e); + return codegen_boolop(c, e); case BinOp_kind: VISIT(c, expr, e->v.BinOp.left); VISIT(c, expr, e->v.BinOp.right); @@ -6070,13 +6048,13 @@ compiler_visit_expr(struct compiler *c, expr_ty e) } break; case Lambda_kind: - return compiler_lambda(c, e); + return codegen_lambda(c, e); case IfExp_kind: - return compiler_ifexp(c, e); + return codegen_ifexp(c, e); case Dict_kind: - return compiler_dict(c, e); + return codegen_dict(c, e); case Set_kind: - return compiler_set(c, e); + return codegen_set(c, e); case GeneratorExp_kind: return compiler_genexp(c, e); case ListComp_kind: @@ -6121,16 +6099,16 @@ compiler_visit_expr(struct compiler *c, expr_ty e) ADD_YIELD_FROM(c, loc, 1); break; case Compare_kind: - return compiler_compare(c, e); + return codegen_compare(c, e); case Call_kind: - return compiler_call(c, e); + return codegen_call(c, e); case Constant_kind: ADDOP_LOAD_CONST(c, loc, e->v.Constant.value); break; case JoinedStr_kind: - return compiler_joined_str(c, e); + return codegen_joined_str(c, e); case FormattedValue_kind: - return compiler_formatted_value(c, e); + return codegen_formatted_value(c, e); /* The following exprs can be assignment targets. */ case Attribute_kind: if (e->v.Attribute.ctx == Load) { @@ -6163,12 +6141,12 @@ compiler_visit_expr(struct compiler *c, expr_ty e) } break; case Subscript_kind: - return compiler_subscript(c, e); + return codegen_subscript(c, e); case Starred_kind: switch (e->v.Starred.ctx) { case Store: /* In all legitimate cases, the Starred node was already replaced - * by compiler_list/compiler_tuple. XXX: is that okay? */ + * by codegen_list/codegen_tuple. XXX: is that okay? */ return compiler_error(c, loc, "starred assignment target must be in a list or tuple"); default: @@ -6178,7 +6156,7 @@ compiler_visit_expr(struct compiler *c, expr_ty e) break; case Slice_kind: { - int n = compiler_slice(c, e); + int n = codegen_slice(c, e); RETURN_IF_ERROR(n); ADDOP_I(c, loc, BUILD_SLICE, n); break; @@ -6187,9 +6165,9 @@ compiler_visit_expr(struct compiler *c, expr_ty e) return compiler_nameop(c, loc, e->v.Name.id, e->v.Name.ctx); /* child nodes of List and Tuple will have expr_context set */ case List_kind: - return compiler_list(c, e); + return codegen_list(c, e); case Tuple_kind: - return compiler_tuple(c, e); + return codegen_tuple(c, e); } return SUCCESS; } @@ -6202,7 +6180,7 @@ is_two_element_slice(expr_ty s) } static int -compiler_augassign(struct compiler *c, stmt_ty s) +codegen_augassign(struct compiler *c, stmt_ty s) { assert(s->kind == AugAssign_kind); expr_ty e = s->v.AugAssign.target; @@ -6219,7 +6197,7 @@ compiler_augassign(struct compiler *c, stmt_ty s) case Subscript_kind: VISIT(c, expr, e->v.Subscript.value); if (is_two_element_slice(e->v.Subscript.slice)) { - RETURN_IF_ERROR(compiler_slice(c, e->v.Subscript.slice)); + RETURN_IF_ERROR(codegen_slice(c, e->v.Subscript.slice)); ADDOP_I(c, loc, COPY, 3); ADDOP_I(c, loc, COPY, 3); ADDOP_I(c, loc, COPY, 3); @@ -6277,7 +6255,7 @@ compiler_augassign(struct compiler *c, stmt_ty s) } static int -check_ann_expr(struct compiler *c, expr_ty e) +codegen_check_ann_expr(struct compiler *c, expr_ty e) { VISIT(c, expr, e); ADDOP(c, LOC(e), POP_TOP); @@ -6285,7 +6263,7 @@ check_ann_expr(struct compiler *c, expr_ty e) } static int -check_annotation(struct compiler *c, stmt_ty s) +compiler_check_annotation(struct compiler *c, stmt_ty s) { /* Annotations of complex targets does not produce anything under annotations future */ @@ -6296,24 +6274,24 @@ check_annotation(struct compiler *c, stmt_ty s) /* Annotations are only evaluated in a module or class. */ if (c->u->u_scope_type == COMPILER_SCOPE_MODULE || c->u->u_scope_type == COMPILER_SCOPE_CLASS) { - return check_ann_expr(c, s->v.AnnAssign.annotation); + return codegen_check_ann_expr(c, s->v.AnnAssign.annotation); } return SUCCESS; } static int -check_ann_subscr(struct compiler *c, expr_ty e) +codegen_check_ann_subscr(struct compiler *c, expr_ty e) { /* We check that everything in a subscript is defined at runtime. */ switch (e->kind) { case Slice_kind: - if (e->v.Slice.lower && check_ann_expr(c, e->v.Slice.lower) < 0) { + if (e->v.Slice.lower && codegen_check_ann_expr(c, e->v.Slice.lower) < 0) { return ERROR; } - if (e->v.Slice.upper && check_ann_expr(c, e->v.Slice.upper) < 0) { + if (e->v.Slice.upper && codegen_check_ann_expr(c, e->v.Slice.upper) < 0) { return ERROR; } - if (e->v.Slice.step && check_ann_expr(c, e->v.Slice.step) < 0) { + if (e->v.Slice.step && codegen_check_ann_expr(c, e->v.Slice.step) < 0) { return ERROR; } return SUCCESS; @@ -6322,12 +6300,12 @@ check_ann_subscr(struct compiler *c, expr_ty e) asdl_expr_seq *elts = e->v.Tuple.elts; Py_ssize_t i, n = asdl_seq_LEN(elts); for (i = 0; i < n; i++) { - RETURN_IF_ERROR(check_ann_subscr(c, asdl_seq_GET(elts, i))); + RETURN_IF_ERROR(codegen_check_ann_subscr(c, asdl_seq_GET(elts, i))); } return SUCCESS; } default: - return check_ann_expr(c, e); + return codegen_check_ann_expr(c, e); } } @@ -6380,14 +6358,14 @@ compiler_annassign(struct compiler *c, stmt_ty s) break; case Attribute_kind: if (!s->v.AnnAssign.value && - check_ann_expr(c, targ->v.Attribute.value) < 0) { + codegen_check_ann_expr(c, targ->v.Attribute.value) < 0) { return ERROR; } break; case Subscript_kind: if (!s->v.AnnAssign.value && - (check_ann_expr(c, targ->v.Subscript.value) < 0 || - check_ann_subscr(c, targ->v.Subscript.slice) < 0)) { + (codegen_check_ann_expr(c, targ->v.Subscript.value) < 0 || + codegen_check_ann_subscr(c, targ->v.Subscript.slice) < 0)) { return ERROR; } break; @@ -6398,7 +6376,7 @@ compiler_annassign(struct compiler *c, stmt_ty s) return ERROR; } /* Annotation is evaluated last. */ - if (future_annotations && !s->v.AnnAssign.simple && check_annotation(c, s) < 0) { + if (future_annotations && !s->v.AnnAssign.simple && compiler_check_annotation(c, s) < 0) { return ERROR; } return SUCCESS; @@ -6470,7 +6448,7 @@ compiler_warn(struct compiler *c, location loc, } static int -compiler_subscript(struct compiler *c, expr_ty e) +codegen_subscript(struct compiler *c, expr_ty e) { location loc = LOC(e); expr_context_ty ctx = e->v.Subscript.ctx; @@ -6483,7 +6461,7 @@ compiler_subscript(struct compiler *c, expr_ty e) VISIT(c, expr, e->v.Subscript.value); if (is_two_element_slice(e->v.Subscript.slice) && ctx != Del) { - RETURN_IF_ERROR(compiler_slice(c, e->v.Subscript.slice)); + RETURN_IF_ERROR(codegen_slice(c, e->v.Subscript.slice)); if (ctx == Load) { ADDOP(c, loc, BINARY_SLICE); } @@ -6508,7 +6486,7 @@ compiler_subscript(struct compiler *c, expr_ty e) /* Returns the number of the values emitted, * thus are needed to build the slice, or -1 if there is an error. */ static int -compiler_slice(struct compiler *c, expr_ty s) +codegen_slice(struct compiler *c, expr_ty s) { int n = 2; assert(s->kind == Slice_kind); @@ -6538,9 +6516,9 @@ compiler_slice(struct compiler *c, expr_ty s) // PEP 634: Structural Pattern Matching -// To keep things simple, all compiler_pattern_* and pattern_helper_* routines -// follow the convention of consuming TOS (the subject for the given pattern) -// and calling jump_to_fail_pop on failure (no match). +// To keep things simple, all codegen_pattern_* routines follow the convention +// of consuming TOS (the subject for the given pattern) and calling +// jump_to_fail_pop on failure (no match). // When calling into these routines, it's important that pc->on_top be kept // updated to reflect the current number of items that we are using on the top @@ -6626,7 +6604,7 @@ compiler_error_duplicate_store(struct compiler *c, location loc, identifier n) // Duplicate the effect of 3.10's ROT_* instructions using SWAPs. static int -pattern_helper_rotate(struct compiler *c, location loc, Py_ssize_t count) +codegen_pattern_helper_rotate(struct compiler *c, location loc, Py_ssize_t count) { while (1 < count) { ADDOP_I(c, loc, SWAP, count--); @@ -6635,8 +6613,8 @@ pattern_helper_rotate(struct compiler *c, location loc, Py_ssize_t count) } static int -pattern_helper_store_name(struct compiler *c, location loc, - identifier n, pattern_context *pc) +codegen_pattern_helper_store_name(struct compiler *c, location loc, + identifier n, pattern_context *pc) { if (n == NULL) { ADDOP(c, loc, POP_TOP); @@ -6650,15 +6628,15 @@ pattern_helper_store_name(struct compiler *c, location loc, } // Rotate this object underneath any items we need to preserve: Py_ssize_t rotations = pc->on_top + PyList_GET_SIZE(pc->stores) + 1; - RETURN_IF_ERROR(pattern_helper_rotate(c, loc, rotations)); + RETURN_IF_ERROR(codegen_pattern_helper_rotate(c, loc, rotations)); RETURN_IF_ERROR(PyList_Append(pc->stores, n)); return SUCCESS; } static int -pattern_unpack_helper(struct compiler *c, location loc, - asdl_pattern_seq *elts) +codegen_pattern_unpack_helper(struct compiler *c, location loc, + asdl_pattern_seq *elts) { Py_ssize_t n = asdl_seq_LEN(elts); int seen_star = 0; @@ -6690,7 +6668,7 @@ pattern_helper_sequence_unpack(struct compiler *c, location loc, asdl_pattern_seq *patterns, Py_ssize_t star, pattern_context *pc) { - RETURN_IF_ERROR(pattern_unpack_helper(c, loc, patterns)); + RETURN_IF_ERROR(codegen_pattern_unpack_helper(c, loc, patterns)); Py_ssize_t size = asdl_seq_LEN(patterns); // We've now got a bunch of new subjects on the stack. They need to remain // there after each subpattern match: @@ -6699,7 +6677,7 @@ pattern_helper_sequence_unpack(struct compiler *c, location loc, // One less item to keep track of each time we loop through: pc->on_top--; pattern_ty pattern = asdl_seq_GET(patterns, i); - RETURN_IF_ERROR(compiler_pattern_subpattern(c, pattern, pc)); + RETURN_IF_ERROR(codegen_pattern_subpattern(c, pattern, pc)); } return SUCCESS; } @@ -6736,7 +6714,7 @@ pattern_helper_sequence_subscr(struct compiler *c, location loc, ADDOP_BINARY(c, loc, Sub); } ADDOP(c, loc, BINARY_SUBSCR); - RETURN_IF_ERROR(compiler_pattern_subpattern(c, pattern, pc)); + RETURN_IF_ERROR(codegen_pattern_subpattern(c, pattern, pc)); } // Pop the subject, we're done with it: pc->on_top--; @@ -6744,20 +6722,20 @@ pattern_helper_sequence_subscr(struct compiler *c, location loc, return SUCCESS; } -// Like compiler_pattern, but turn off checks for irrefutability. +// Like codegen_pattern, but turn off checks for irrefutability. static int -compiler_pattern_subpattern(struct compiler *c, +codegen_pattern_subpattern(struct compiler *c, pattern_ty p, pattern_context *pc) { int allow_irrefutable = pc->allow_irrefutable; pc->allow_irrefutable = 1; - RETURN_IF_ERROR(compiler_pattern(c, p, pc)); + RETURN_IF_ERROR(codegen_pattern(c, p, pc)); pc->allow_irrefutable = allow_irrefutable; return SUCCESS; } static int -compiler_pattern_as(struct compiler *c, pattern_ty p, pattern_context *pc) +codegen_pattern_as(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchAs_kind); if (p->v.MatchAs.pattern == NULL) { @@ -6770,24 +6748,24 @@ compiler_pattern_as(struct compiler *c, pattern_ty p, pattern_context *pc) const char *e = "wildcard makes remaining patterns unreachable"; return compiler_error(c, LOC(p), e); } - return pattern_helper_store_name(c, LOC(p), p->v.MatchAs.name, pc); + return codegen_pattern_helper_store_name(c, LOC(p), p->v.MatchAs.name, pc); } // Need to make a copy for (possibly) storing later: pc->on_top++; ADDOP_I(c, LOC(p), COPY, 1); - RETURN_IF_ERROR(compiler_pattern(c, p->v.MatchAs.pattern, pc)); + RETURN_IF_ERROR(codegen_pattern(c, p->v.MatchAs.pattern, pc)); // Success! Store it: pc->on_top--; - RETURN_IF_ERROR(pattern_helper_store_name(c, LOC(p), p->v.MatchAs.name, pc)); + RETURN_IF_ERROR(codegen_pattern_helper_store_name(c, LOC(p), p->v.MatchAs.name, pc)); return SUCCESS; } static int -compiler_pattern_star(struct compiler *c, pattern_ty p, pattern_context *pc) +codegen_pattern_star(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchStar_kind); RETURN_IF_ERROR( - pattern_helper_store_name(c, LOC(p), p->v.MatchStar.name, pc)); + codegen_pattern_helper_store_name(c, LOC(p), p->v.MatchStar.name, pc)); return SUCCESS; } @@ -6812,7 +6790,7 @@ validate_kwd_attrs(struct compiler *c, asdl_identifier_seq *attrs, asdl_pattern_ } static int -compiler_pattern_class(struct compiler *c, pattern_ty p, pattern_context *pc) +codegen_pattern_class(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchClass_kind); asdl_pattern_seq *patterns = p->v.MatchClass.patterns; @@ -6869,15 +6847,15 @@ compiler_pattern_class(struct compiler *c, pattern_ty p, pattern_context *pc) ADDOP(c, LOC(p), POP_TOP); continue; } - RETURN_IF_ERROR(compiler_pattern_subpattern(c, pattern, pc)); + RETURN_IF_ERROR(codegen_pattern_subpattern(c, pattern, pc)); } // Success! Pop the tuple of attributes: return SUCCESS; } static int -compiler_pattern_mapping(struct compiler *c, pattern_ty p, - pattern_context *pc) +codegen_pattern_mapping(struct compiler *c, pattern_ty p, + pattern_context *pc) { assert(p->kind == MatchMapping_kind); asdl_expr_seq *keys = p->v.MatchMapping.keys; @@ -6976,7 +6954,7 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, for (Py_ssize_t i = 0; i < size; i++) { pc->on_top--; pattern_ty pattern = asdl_seq_GET(patterns, i); - RETURN_IF_ERROR(compiler_pattern_subpattern(c, pattern, pc)); + RETURN_IF_ERROR(codegen_pattern_subpattern(c, pattern, pc)); } // If we get this far, it's a match! Whatever happens next should consume // the tuple of keys and the subject: @@ -6997,7 +6975,7 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, ADDOP_I(c, LOC(p), SWAP, 2); // [copy, keys..., copy, key] ADDOP(c, LOC(p), DELETE_SUBSCR); // [copy, keys...] } - RETURN_IF_ERROR(pattern_helper_store_name(c, LOC(p), star_target, pc)); + RETURN_IF_ERROR(codegen_pattern_helper_store_name(c, LOC(p), star_target, pc)); } else { ADDOP(c, LOC(p), POP_TOP); // Tuple of keys. @@ -7011,7 +6989,7 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, } static int -compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) +codegen_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchOr_kind); NEW_JUMP_TARGET_LABEL(c, end); @@ -7038,7 +7016,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) pc->fail_pop_size = 0; pc->on_top = 0; if (codegen_addop_i(INSTR_SEQUENCE(c), COPY, 1, LOC(alt)) < 0 || - compiler_pattern(c, alt, pc) < 0) { + codegen_pattern(c, alt, pc) < 0) { goto error; } // Success! @@ -7092,7 +7070,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) // Do the same thing to the stack, using several // rotations: while (rotations--) { - if (pattern_helper_rotate(c, LOC(alt), icontrol + 1) < 0) { + if (codegen_pattern_helper_rotate(c, LOC(alt), icontrol + 1) < 0) { goto error; } } @@ -7128,7 +7106,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) Py_ssize_t nrots = nstores + 1 + pc->on_top + PyList_GET_SIZE(pc->stores); for (Py_ssize_t i = 0; i < nstores; i++) { // Rotate this capture to its proper place on the stack: - if (pattern_helper_rotate(c, LOC(p), nrots) < 0) { + if (codegen_pattern_helper_rotate(c, LOC(p), nrots) < 0) { goto error; } // Update the list of previous stores with this new name, checking for @@ -7163,8 +7141,8 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) static int -compiler_pattern_sequence(struct compiler *c, pattern_ty p, - pattern_context *pc) +codegen_pattern_sequence(struct compiler *c, pattern_ty p, + pattern_context *pc) { assert(p->kind == MatchSequence_kind); asdl_pattern_seq *patterns = p->v.MatchSequence.patterns; @@ -7221,7 +7199,7 @@ compiler_pattern_sequence(struct compiler *c, pattern_ty p, } static int -compiler_pattern_value(struct compiler *c, pattern_ty p, pattern_context *pc) +codegen_pattern_value(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchValue_kind); expr_ty value = p->v.MatchValue.value; @@ -7237,7 +7215,7 @@ compiler_pattern_value(struct compiler *c, pattern_ty p, pattern_context *pc) } static int -compiler_pattern_singleton(struct compiler *c, pattern_ty p, pattern_context *pc) +codegen_pattern_singleton(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchSingleton_kind); ADDOP_LOAD_CONST(c, LOC(p), p->v.MatchSingleton.value); @@ -7247,25 +7225,25 @@ compiler_pattern_singleton(struct compiler *c, pattern_ty p, pattern_context *pc } static int -compiler_pattern(struct compiler *c, pattern_ty p, pattern_context *pc) +codegen_pattern(struct compiler *c, pattern_ty p, pattern_context *pc) { switch (p->kind) { case MatchValue_kind: - return compiler_pattern_value(c, p, pc); + return codegen_pattern_value(c, p, pc); case MatchSingleton_kind: - return compiler_pattern_singleton(c, p, pc); + return codegen_pattern_singleton(c, p, pc); case MatchSequence_kind: - return compiler_pattern_sequence(c, p, pc); + return codegen_pattern_sequence(c, p, pc); case MatchMapping_kind: - return compiler_pattern_mapping(c, p, pc); + return codegen_pattern_mapping(c, p, pc); case MatchClass_kind: - return compiler_pattern_class(c, p, pc); + return codegen_pattern_class(c, p, pc); case MatchStar_kind: - return compiler_pattern_star(c, p, pc); + return codegen_pattern_star(c, p, pc); case MatchAs_kind: - return compiler_pattern_as(c, p, pc); + return codegen_pattern_as(c, p, pc); case MatchOr_kind: - return compiler_pattern_or(c, p, pc); + return codegen_pattern_or(c, p, pc); } // AST validator shouldn't let this happen, but if it does, // just fail, don't crash out of the interpreter @@ -7274,7 +7252,7 @@ compiler_pattern(struct compiler *c, pattern_ty p, pattern_context *pc) } static int -compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) +codegen_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) { VISIT(c, expr, s->v.Match.subject); NEW_JUMP_TARGET_LABEL(c, end); @@ -7298,7 +7276,7 @@ compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) pc->fail_pop_size = 0; pc->on_top = 0; // NOTE: Can't use returning macros here (they'll leak pc->stores)! - if (compiler_pattern(c, m->pattern, pc) < 0) { + if (codegen_pattern(c, m->pattern, pc) < 0) { Py_DECREF(pc->stores); return ERROR; } @@ -7316,14 +7294,14 @@ compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) // NOTE: Returning macros are safe again. if (m->guard) { RETURN_IF_ERROR(ensure_fail_pop(c, pc, 0)); - RETURN_IF_ERROR(compiler_jump_if(c, LOC(m->pattern), m->guard, pc->fail_pop[0], 0)); + RETURN_IF_ERROR(codegen_jump_if(c, LOC(m->pattern), m->guard, pc->fail_pop[0], 0)); } // Success! Pop the subject off, we're done with it: if (i != cases - has_default - 1) { ADDOP(c, LOC(m->pattern), POP_TOP); } VISIT_SEQ(c, stmt, m->body); - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP, end); // If the pattern fails to match, we want the line number of the // cleanup to be associated with the failed pattern, not the last line // of the body @@ -7342,7 +7320,7 @@ compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) ADDOP(c, LOC(m->pattern), NOP); } if (m->guard) { - RETURN_IF_ERROR(compiler_jump_if(c, LOC(m->pattern), m->guard, end, 0)); + RETURN_IF_ERROR(codegen_jump_if(c, LOC(m->pattern), m->guard, end, 0)); } VISIT_SEQ(c, stmt, m->body); } @@ -7351,11 +7329,11 @@ compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) } static int -compiler_match(struct compiler *c, stmt_ty s) +codegen_match(struct compiler *c, stmt_ty s) { pattern_context pc; pc.fail_pop = NULL; - int result = compiler_match_inner(c, s, &pc); + int result = codegen_match_inner(c, s, &pc); PyMem_Free(pc.fail_pop); return result; } @@ -7678,7 +7656,7 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, return NULL; } - if (compiler_enter_anonymous_scope(c, mod) < 0) { + if (codegen_enter_anonymous_scope(c, mod) < 0) { return NULL; } if (compiler_codegen(c, mod) < 0) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index f266c0efbe3279..6d12bf01eacf96 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1031,7 +1031,7 @@ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem = ht->_spec_cache.getitem; - new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2); + new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2, frame); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); new_frame->localsplus[0] = container; @@ -1319,6 +1319,7 @@ tstate->exc_info = &gen->gi_exc_state; assert(1 + INLINE_CACHE_ENTRIES_SEND + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); + gen_frame->previous = frame; stack_pointer[-1].bits = (uintptr_t)gen_frame; break; } @@ -2552,7 +2553,7 @@ } STAT_INC(LOAD_ATTR, hit); Py_INCREF(fget); - new_frame = _PyFrame_PushUnchecked(tstate, f, 1); + new_frame = _PyFrame_PushUnchecked(tstate, f, 1, frame); new_frame->localsplus[0] = owner; stack_pointer[-1].bits = (uintptr_t)new_frame; break; @@ -3317,6 +3318,7 @@ gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; + gen_frame->previous = frame; // oparg is the return offset from the next instruction. frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg); stack_pointer[0].bits = (uintptr_t)gen_frame; @@ -3605,7 +3607,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, total_args, NULL, frame ); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. @@ -3839,7 +3841,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -3865,7 +3867,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -3891,7 +3893,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -3917,7 +3919,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -3943,7 +3945,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -3968,7 +3970,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -3989,7 +3991,7 @@ stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -4088,7 +4090,85 @@ break; } - /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + case _CHECK_AND_ALLOCATE_OBJECT: { + _PyStackRef *args; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef self; + _PyStackRef init; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t type_version = (uint32_t)CURRENT_OPERAND(); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyType_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyTypeObject *tp = (PyTypeObject *)callable_o; + if (tp->tp_version_tag != type_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; + PyFunctionObject *init_func = (PyFunctionObject *)cls->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init_func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(CALL, hit); + self = PyStackRef_FromPyObjectSteal(_PyType_NewManagedObject(tp)); + if (PyStackRef_IsNull(self)) { + JUMP_TO_ERROR(); + } + PyStackRef_CLOSE(callable); + init = PyStackRef_FromPyObjectNew(init_func); + stack_pointer[-1 - oparg] = init; + stack_pointer[-2 - oparg] = self; + break; + } + + case _CREATE_INIT_FRAME: { + _PyStackRef *args; + _PyStackRef init; + _PyStackRef self; + _PyInterpreterFrame *init_frame; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + init = stack_pointer[-1 - oparg]; + self = stack_pointer[-2 - oparg]; + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); + /* Push self onto stack of shim */ + shim->localsplus[0] = PyStackRef_DUP(self); + PyFunctionObject *init_func = (PyFunctionObject *)PyStackRef_AsPyObjectSteal(init); + args[-1] = self; + init_frame = _PyEvalFramePushAndInit( + tstate, init_func, NULL, args-1, oparg+1, NULL, shim); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (init_frame == NULL) { + _PyEval_FrameClearAndPop(tstate, shim); + JUMP_TO_ERROR(); + } + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + /* Account for pushing the extra frame. + * We don't check recursion depth here, + * as it will be checked after start_frame */ + tstate->py_recursion_remaining--; + stack_pointer[0].bits = (uintptr_t)init_frame; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _EXIT_INIT_CHECK: { _PyStackRef should_be_none; @@ -4706,7 +4786,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, positional_args, kwnames_o + args, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 8e76cfa649fcf8..5ada643e7d81d7 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -504,7 +504,7 @@ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem = ht->_spec_cache.getitem; - new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2); + new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2, frame); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); new_frame->localsplus[0] = container; @@ -517,7 +517,7 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -888,7 +888,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, total_args, NULL, frame ); // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); @@ -975,60 +975,79 @@ _PyStackRef callable; _PyStackRef null; _PyStackRef *args; + _PyStackRef self; + _PyStackRef init; + _PyInterpreterFrame *init_frame; + _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ - /* Skip 2 cache entries */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_AND_ALLOCATE_OBJECT args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - /* This instruction does the following: - * 1. Creates the object (by calling ``object.__new__``) - * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) - * 3. Pushes the frame for ``__init__`` to the frame stack - * */ - _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(!PyType_Check(callable_o), CALL); - PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); - assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; - PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; - PyCodeObject *code = (PyCodeObject *)init->func_code; - DEOPT_IF(code->co_argcount != oparg+1, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); - STAT_INC(CALL, hit); - PyObject *self = _PyType_NewManagedObject(tp); - if (self == NULL) { - goto error; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL); + PyTypeObject *tp = (PyTypeObject *)callable_o; + DEOPT_IF(tp->tp_version_tag != type_version, CALL); + assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; + PyFunctionObject *init_func = (PyFunctionObject *)cls->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init_func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + STAT_INC(CALL, hit); + self = PyStackRef_FromPyObjectSteal(_PyType_NewManagedObject(tp)); + if (PyStackRef_IsNull(self)) { + goto error; + } + PyStackRef_CLOSE(callable); + init = PyStackRef_FromPyObjectNew(init_func); + stack_pointer[-1 - oparg] = init; + } + // _CREATE_INIT_FRAME + { + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); + /* Push self onto stack of shim */ + shim->localsplus[0] = PyStackRef_DUP(self); + PyFunctionObject *init_func = (PyFunctionObject *)PyStackRef_AsPyObjectSteal(init); + args[-1] = self; + init_frame = _PyEvalFramePushAndInit( + tstate, init_func, NULL, args-1, oparg+1, NULL, shim); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (init_frame == NULL) { + _PyEval_FrameClearAndPop(tstate, shim); + goto error; + } + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + /* Account for pushing the extra frame. + * We don't check recursion depth here, + * as it will be checked after start_frame */ + tstate->py_recursion_remaining--; } - PyStackRef_CLOSE(callable); - _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1); - assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); - /* Push self onto stack of shim */ - Py_INCREF(self); - shim->localsplus[0] = PyStackRef_FromPyObjectSteal(self); - Py_INCREF(init); - _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); - /* Copy self followed by args to __init__ frame */ - init_frame->localsplus[0] = PyStackRef_FromPyObjectSteal(self); - for (int i = 0; i < oparg; i++) { - init_frame->localsplus[i+1] = args[i]; + // _PUSH_FRAME + new_frame = init_frame; + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } - frame->return_offset = (uint16_t)(next_instr - this_instr); - STACK_SHRINK(oparg+2); - _PyFrame_SetStackPointer(frame, stack_pointer); - /* Link frames */ - init_frame->previous = shim; - shim->previous = frame; - frame = tstate->current_frame = init_frame; - CALL_STAT_INC(inlined_py_calls); - /* Account for pushing the extra frame. - * We don't check recursion depth here, - * as it will be checked after start_frame */ - tstate->py_recursion_remaining--; - goto start_frame; + DISPATCH(); } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { @@ -1099,7 +1118,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -1123,7 +1142,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -1194,7 +1213,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, total_args, NULL, frame ); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. @@ -1219,7 +1238,7 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -1595,7 +1614,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, - nargs, callargs, kwargs); + nargs, callargs, kwargs, frame); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { @@ -1742,7 +1761,7 @@ _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); DISPATCH_SAME_OPARG(); } - STAT_INC(CALL, deferred); + STAT_INC(CALL_KW, deferred); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } @@ -1781,7 +1800,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, positional_args, kwnames_o + args, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); // Manipulate stack directly since we leave using DISPATCH_INLINED(). @@ -1916,7 +1935,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, positional_args, kwnames_o + args, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, @@ -1942,7 +1961,7 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -2086,7 +2105,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, positional_args, kwnames_o + args, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, @@ -2112,7 +2131,7 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -2635,7 +2654,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -2659,7 +2678,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -2710,7 +2729,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, total_args, NULL, frame ); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. @@ -2735,7 +2754,7 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -3667,6 +3686,7 @@ gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; + gen_frame->previous = frame; // oparg is the return offset from the next instruction. frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg); } @@ -3677,7 +3697,7 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -4090,7 +4110,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, total_args, NULL, frame ); // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); @@ -4944,7 +4964,7 @@ STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); Py_INCREF(f); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2, frame); // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; @@ -5274,7 +5294,7 @@ DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(fget); - new_frame = _PyFrame_PushUnchecked(tstate, f, 1); + new_frame = _PyFrame_PushUnchecked(tstate, f, 1, frame); new_frame->localsplus[0] = owner; } // _SAVE_RETURN_OFFSET @@ -5294,7 +5314,7 @@ stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -6551,6 +6571,8 @@ tstate->exc_info = &gen->gi_exc_state; assert(next_instr - this_instr + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + assert(gen_frame->previous == NULL); + gen_frame->previous = frame; DISPATCH_INLINED(gen_frame); } if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) { @@ -6611,6 +6633,7 @@ tstate->exc_info = &gen->gi_exc_state; assert(1 + INLINE_CACHE_ENTRIES_SEND + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); + gen_frame->previous = frame; } // _PUSH_FRAME new_frame = gen_frame; @@ -6621,7 +6644,7 @@ stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -7651,7 +7674,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, - nargs, callargs, kwargs); + nargs, callargs, kwargs, frame); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 97e4c642225786..6e46d9bed11fbc 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -627,7 +627,7 @@ dummy_func(void) { ctx->done = true; } - op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _Py_UOpsAbstractFrame*)) { + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _Py_UOpsAbstractFrame *)) { (void)callable; (void)self_or_null; (void)args; @@ -636,6 +636,23 @@ dummy_func(void) { ctx->done = true; } + op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, null, args[oparg] -- self, init, args[oparg])) { + (void)type_version; + (void)callable; + (void)null; + (void)args; + self = sym_new_not_null(ctx); + init = sym_new_not_null(ctx); + } + + op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame: _Py_UOpsAbstractFrame *)) { + (void)self; + (void)init; + (void)args; + init_frame = NULL; + ctx->done = true; + } + op(_RETURN_VALUE, (retval -- res)) { SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 3ec2e6985d62f2..e5be9d0e3b5ee7 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1866,7 +1866,46 @@ break; } - /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 */ + case _CHECK_AND_ALLOCATE_OBJECT: { + _Py_UopsSymbol **args; + _Py_UopsSymbol *null; + _Py_UopsSymbol *callable; + _Py_UopsSymbol *self; + _Py_UopsSymbol *init; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-oparg]; + uint32_t type_version = (uint32_t)this_instr->operand; + (void)type_version; + (void)callable; + (void)null; + (void)args; + self = sym_new_not_null(ctx); + init = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = self; + stack_pointer[-1 - oparg] = init; + break; + } + + case _CREATE_INIT_FRAME: { + _Py_UopsSymbol **args; + _Py_UopsSymbol *init; + _Py_UopsSymbol *self; + _Py_UOpsAbstractFrame *init_frame; + args = &stack_pointer[-oparg]; + init = stack_pointer[-1 - oparg]; + self = stack_pointer[-2 - oparg]; + (void)self; + (void)init; + (void)args; + init_frame = NULL; + ctx->done = true; + stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)init_frame; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _EXIT_INIT_CHECK: { stack_pointer += -1; diff --git a/Python/specialize.c b/Python/specialize.c index 4fec5a36fc91c7..b3a2e07c3bbcb8 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1809,10 +1809,6 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) return -1; } if (init != NULL) { - if (((PyCodeObject *)init->func_code)->co_argcount != nargs+1) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); - return -1; - } _PyCallCache *cache = (_PyCallCache *)(instr + 1); write_u32(cache->func_version, tp->tp_version_tag); _Py_SET_OPCODE(*instr, CALL_ALLOC_AND_ENTER_INIT); @@ -2654,7 +2650,7 @@ static const PyBytesObject no_location = { .ob_sval = { NO_LOC_4 } }; -const struct _PyCode_DEF(8) _Py_InitCleanup = { +const struct _PyCode8 _Py_InitCleanup = { _PyVarObject_HEAD_INIT(&PyCode_Type, 3), .co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty), .co_names = (PyObject *)&_Py_SINGLETON(tuple_empty), diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 63b640e465ac6b..b91b11763fb56c 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -742,3 +742,7 @@ Modules/clinic/md5module.c.h _md5_md5 _keywords - Modules/clinic/grpmodule.c.h grp_getgrgid _keywords - Modules/clinic/grpmodule.c.h grp_getgrnam _keywords - Objects/object.c - constants static PyObject*[] + + +## False positives +Python/specialize.c - _Py_InitCleanup -