diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9587b3996a9ac2a..7933d3195505767 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -21,6 +21,7 @@ configure* @erlend-aasland @corona10 **/*context* @1st1 **/*genobject* @markshannon **/*hamt* @1st1 +**/*jit* @brandtbucher Objects/set* @rhettinger Objects/dict* @methane @markshannon Objects/typevarobject.c @JelleZijlstra @@ -37,11 +38,37 @@ Python/ast_opt.c @isidentical Python/bytecodes.c @markshannon @gvanrossum Python/optimizer*.c @markshannon @gvanrossum Lib/test/test_patma.py @brandtbucher -Lib/test/test_peepholer.py @brandtbucher Lib/test/test_type_*.py @JelleZijlstra Lib/test/test_capi/test_misc.py @markshannon @gvanrossum Tools/c-analyzer/ @ericsnowcurrently +# dbm +**/*dbm* @corona10 @erlend-aasland @serhiy-storchaka + +# runtime state/lifecycle +**/*pylifecycle* @ericsnowcurrently +**/*pystate* @ericsnowcurrently +**/*preconfig* @ericsnowcurrently +**/*initconfig* @ericsnowcurrently +**/*pathconfig* @ericsnowcurrently +**/*sysmodule* @ericsnowcurrently +**/*bltinmodule* @ericsnowcurrently +**/*gil* @ericsnowcurrently +Include/internal/pycore_runtime.h @ericsnowcurrently +Include/internal/pycore_interp.h @ericsnowcurrently +Include/internal/pycore_tstate.h @ericsnowcurrently +Include/internal/pycore_*_state.h @ericsnowcurrently +Include/internal/pycore_*_init.h @ericsnowcurrently +Include/internal/pycore_atexit.h @ericsnowcurrently +Include/internal/pycore_freelist.h @ericsnowcurrently +Include/internal/pycore_global_objects.h @ericsnowcurrently +Include/internal/pycore_obmalloc.h @ericsnowcurrently +Include/internal/pycore_pymem.h @ericsnowcurrently +Modules/main.c @ericsnowcurrently +Programs/_bootstrap_python.c @ericsnowcurrently +Programs/python.c @ericsnowcurrently +Tools/build/generate_global_objects.py @ericsnowcurrently + # Exceptions Lib/traceback.py @iritkatriel Lib/test/test_except*.py @iritkatriel @@ -76,7 +103,20 @@ Modules/_hacl/** @gpshead # Import (including importlib). **/*import* @brettcannon @ericsnowcurrently @ncoghlan @warsaw /Python/import.c @kumaraditya303 -**/*importlib/resources/* @jaraco @warsaw @FFY00 +Python/dynload_*.c @ericsnowcurrently +**/*freeze* @ericsnowcurrently +**/*frozen* @ericsnowcurrently +**/*modsupport* @ericsnowcurrently +**/*modulefinder* @ericsnowcurrently +**/*moduleobject* @ericsnowcurrently +**/*multiphase* @ericsnowcurrently +**/*pkgutil* @ericsnowcurrently +**/*pythonrun* @ericsnowcurrently +**/*runpy* @ericsnowcurrently +**/*singlephase* @ericsnowcurrently +Lib/test/test_module/ @ericsnowcurrently +Doc/c-api/module.rst @ericsnowcurrently +**/*importlib/resources/* @jaraco @warsaw @FFY00 **/importlib/metadata/* @jaraco @warsaw # Dates and times @@ -190,11 +230,13 @@ Doc/c-api/stable.rst @encukou **/*zipfile/_path/* @jaraco # Argument Clinic -/Tools/clinic/** @erlend-aasland @AlexWaygood -/Lib/test/test_clinic.py @erlend-aasland @AlexWaygood +/Tools/clinic/** @erlend-aasland +/Lib/test/test_clinic.py @erlend-aasland Doc/howto/clinic.rst @erlend-aasland # Subinterpreters +**/*interpreteridobject.* @ericsnowcurrently +**/*crossinterp* @ericsnowcurrently Lib/test/support/interpreters/ @ericsnowcurrently Modules/_xx*interp*module.c @ericsnowcurrently Lib/test/test_interpreters/ @ericsnowcurrently diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cc5ecc09fbc5923..0a2f6da50ed8a09 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -132,7 +132,7 @@ jobs: with: python-version: '3.x' - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: config.cache key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }} @@ -250,7 +250,7 @@ jobs: strategy: fail-fast: false matrix: - openssl_ver: [1.1.1w, 3.0.11, 3.1.3] + openssl_ver: [1.1.1w, 3.0.13, 3.1.5, 3.2.1] env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl @@ -259,7 +259,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: config.cache key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} @@ -274,7 +274,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -304,7 +304,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' env: - OPENSSL_VER: 3.0.11 + OPENSSL_VER: 3.0.13 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v4 @@ -319,7 +319,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -342,7 +342,7 @@ jobs: - name: Bind mount sources read-only run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.CPYTHON_BUILDDIR }}/config.cache key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} @@ -375,7 +375,7 @@ jobs: ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -r ${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt - name: 'Restore Hypothesis database' id: cache-hypothesis-database - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./hypothesis key: hypothesis-database-${{ github.head_ref || github.run_id }} @@ -415,13 +415,13 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 3.0.11 + OPENSSL_VER: 3.0.13 PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: - uses: actions/checkout@v4 - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: config.cache key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} @@ -440,7 +440,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml new file mode 100644 index 000000000000000..69648d87947ad66 --- /dev/null +++ b/.github/workflows/jit.yml @@ -0,0 +1,127 @@ +name: JIT +on: + pull_request: + paths: + - '**jit**' + - 'Python/bytecodes.c' + push: + paths: + - '**jit**' + - 'Python/bytecodes.c' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + jit: + name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + target: + - i686-pc-windows-msvc/msvc + - x86_64-pc-windows-msvc/msvc + - x86_64-apple-darwin/clang + - aarch64-apple-darwin/clang + - x86_64-unknown-linux-gnu/gcc + - x86_64-unknown-linux-gnu/clang + - aarch64-unknown-linux-gnu/gcc + - aarch64-unknown-linux-gnu/clang + debug: + - true + - false + llvm: + - 16 + include: + - target: i686-pc-windows-msvc/msvc + architecture: Win32 + runner: windows-latest + compiler: msvc + - target: x86_64-pc-windows-msvc/msvc + architecture: x64 + runner: windows-latest + compiler: msvc + - target: x86_64-apple-darwin/clang + architecture: x86_64 + runner: macos-13 + compiler: clang + - target: aarch64-apple-darwin/clang + architecture: aarch64 + runner: macos-14 + compiler: clang + - target: x86_64-unknown-linux-gnu/gcc + architecture: x86_64 + runner: ubuntu-latest + compiler: gcc + - target: x86_64-unknown-linux-gnu/clang + architecture: x86_64 + runner: ubuntu-latest + compiler: clang + - target: aarch64-unknown-linux-gnu/gcc + architecture: aarch64 + runner: ubuntu-latest + compiler: gcc + # These fail because of emulation, not because of the JIT: + exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv + - target: aarch64-unknown-linux-gnu/clang + architecture: aarch64 + runner: ubuntu-latest + compiler: clang + # These fail because of emulation, not because of the JIT: + exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv + env: + CC: ${{ matrix.compiler }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Windows + if: runner.os == 'Windows' + run: | + choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} + ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }} + ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 + + - name: macOS + if: runner.os == 'macOS' + run: | + brew install llvm@${{ matrix.llvm }} + export SDKROOT="$(xcrun --show-sdk-path)" + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + make all --jobs 4 + ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 + + - name: Native Linux + if: runner.os == 'Linux' && matrix.architecture == 'x86_64' + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + make all --jobs 4 + ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 + + - name: Emulated Linux + if: runner.os == 'Linux' && matrix.architecture != 'x86_64' + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + ./configure --prefix="$(pwd)/../build" + make install --jobs 4 + make clean --jobs 4 + export HOST=${{ matrix.architecture }}-linux-gnu + sudo apt install --yes "gcc-$HOST" qemu-user + ${{ !matrix.debug && matrix.compiler == 'clang' && './configure --enable-optimizations' || '' }} + ${{ !matrix.debug && matrix.compiler == 'clang' && 'make profile-run-stamp --jobs 4' || '' }} + export CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" + export CPP="$CC --preprocess" + export HOSTRUNNER=qemu-${{ matrix.architecture }} + export QEMU_LD_PREFIX="/usr/$HOST" + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes + make all --jobs 4 + ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 11928e72b9b43a1..b766785de405d2e 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -12,6 +12,7 @@ on: - "Tools/build/generate_sbom.py" - "Tools/cases_generator/**" - "Tools/clinic/**" + - "Tools/jit/**" - "Tools/peg_generator/**" - "Tools/requirements-dev.txt" - "Tools/wasm/**" @@ -38,6 +39,7 @@ jobs: "Tools/build/", "Tools/cases_generator", "Tools/clinic", + "Tools/jit", "Tools/peg_generator", "Tools/wasm", ] diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml index 080204bcfd3b94f..ff5cbdf3eda749a 100644 --- a/.github/workflows/require-pr-label.yml +++ b/.github/workflows/require-pr-label.yml @@ -11,6 +11,7 @@ permissions: jobs: label: name: DO-NOT-MERGE / unresolved review + if: github.repository_owner == 'python' runs-on: ubuntu-latest timeout-minutes: 10 diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index e534751ee1011da..cea8f93d67b29c1 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -89,7 +89,7 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout@v4 - - uses: actions/cache@v3 + - uses: actions/cache@v4 with: path: ~/.cache/pip key: ubuntu-doc-${{ hashFiles('Doc/requirements.txt') }} diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index c24b6e963ddfd67..cad619b78ce5f2c 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -12,20 +12,27 @@ on: jobs: build_macos: name: 'build and test' - runs-on: macos-latest timeout-minutes: 60 env: HOMEBREW_NO_ANALYTICS: 1 HOMEBREW_NO_AUTO_UPDATE: 1 HOMEBREW_NO_INSTALL_CLEANUP: 1 PYTHONSTRICTEXTENSIONBUILD: 1 + strategy: + fail-fast: false + matrix: + os: [ + "macos-14", # M1 + "macos-13", # Intel + ] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ inputs.config_hash }} + key: ${{ github.job }}-${{ matrix.os }}-${{ inputs.config_hash }} - name: Install Homebrew dependencies run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk - name: Configure CPython diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index c2194280c0a50f4..0cbad57f0c6572e 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -14,7 +14,7 @@ jobs: timeout-minutes: 60 runs-on: ubuntu-20.04 env: - OPENSSL_VER: 3.0.11 + OPENSSL_VER: 3.0.13 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v4 @@ -29,7 +29,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -53,7 +53,7 @@ jobs: - name: Bind mount sources read-only run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.CPYTHON_BUILDDIR }}/config.cache key: ${{ github.job }}-${{ runner.os }}-${{ inputs.config_hash }} diff --git a/.gitignore b/.gitignore index c424a894c2a6e09..6ed7197e3ab6269 100644 --- a/.gitignore +++ b/.gitignore @@ -126,6 +126,7 @@ Tools/unicode/data/ # hendrikmuhs/ccache-action@v1 /.ccache /cross-build/ +/jit_stencils.h /platform /profile-clean-stamp /profile-run-stamp @@ -158,5 +159,5 @@ Python/frozen_modules/MANIFEST /python !/Python/ -# main branch only: ABI files are not checked/maintained +# main branch only: ABI files are not checked/maintained. Doc/data/python*.abi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 19033ce243d9d35..69d85238985150c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.7 + rev: v0.2.0 hooks: - id: ruff name: Run Ruff on Lib/test/ diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 5082b0cb6ad3f3e..11c12e685fcace4 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -22,12 +22,13 @@ bound into a function. .. c:var:: PyTypeObject PyCode_Type This is an instance of :c:type:`PyTypeObject` representing the Python - :class:`code` type. + :ref:`code object `. .. c:function:: int PyCode_Check(PyObject *co) - Return true if *co* is a :class:`code` object. This function always succeeds. + Return true if *co* is a :ref:`code object `. + This function always succeeds. .. c:function:: int PyCode_GetNumFree(PyCodeObject *co) diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index c5350123dfdfdcd..4aaf3905e81c8a6 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -48,6 +48,42 @@ The return value (*rv*) for these functions should be interpreted as follows: The following functions provide locale-independent string to number conversions. +.. c:function:: unsigned long PyOS_strtoul(const char *str, char **ptr, int base) + + Convert the initial part of the string in ``str`` to an :c:expr:`unsigned + long` value according to the given ``base``, which must be between ``2`` and + ``36`` inclusive, or be the special value ``0``. + + Leading white space and case of characters are ignored. If ``base`` is zero + it looks for a leading ``0b``, ``0o`` or ``0x`` to tell which base. If + these are absent it defaults to ``10``. Base must be 0 or between 2 and 36 + (inclusive). If ``ptr`` is non-``NULL`` it will contain a pointer to the + end of the scan. + + If the converted value falls out of range of corresponding return type, + range error occurs (:c:data:`errno` is set to :c:macro:`!ERANGE`) and + :c:macro:`!ULONG_MAX` is returned. If no conversion can be performed, ``0`` + is returned. + + See also the Unix man page :manpage:`strtoul(3)`. + + .. versionadded:: 3.2 + + +.. c:function:: long PyOS_strtol(const char *str, char **ptr, int base) + + Convert the initial part of the string in ``str`` to an :c:expr:`long` value + according to the given ``base``, which must be between ``2`` and ``36`` + inclusive, or be the special value ``0``. + + Same as :c:func:`PyOS_strtoul`, but return a :c:expr:`long` value instead + and :c:macro:`LONG_MAX` on overflows. + + See also the Unix man page :manpage:`strtol(3)`. + + .. versionadded:: 3.2 + + .. c:function:: double PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception) Convert a string ``s`` to a :c:expr:`double`, raising a Python diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 8471c98d0448721..03f3d28187bfe9a 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -174,6 +174,26 @@ Dictionary Objects .. versionadded:: 3.4 +.. c:function:: int PyDict_SetDefaultRef(PyObject *p, PyObject *key, PyObject *default_value, PyObject **result) + + Inserts *default_value* into the dictionary *p* with a key of *key* if the + key is not already present in the dictionary. If *result* is not ``NULL``, + then *\*result* is set to a :term:`strong reference` to either + *default_value*, if the key was not present, or the existing value, if *key* + was already present in the dictionary. + Returns ``1`` if the key was present and *default_value* was not inserted, + or ``0`` if the key was not present and *default_value* was inserted. + On failure, returns ``-1``, sets an exception, and sets ``*result`` + to ``NULL``. + + For clarity: if you have a strong reference to *default_value* before + calling this function, then after it returns, you hold a strong reference + to both *default_value* and *\*result* (if it's not ``NULL``). + These may refer to the same object: in that case you hold two separate + references to it. + .. versionadded:: 3.13 + + .. c:function:: int PyDict_Pop(PyObject *p, PyObject *key, PyObject **result) Remove *key* from dictionary *p* and optionally return the removed value. diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index b36c800e00444ac..d3a78c588454e8e 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -65,8 +65,14 @@ the :mod:`io` APIs instead. Overrides the normal behavior of :func:`io.open_code` to pass its parameter through the provided handler. - The handler is a function of type :c:expr:`PyObject *(\*)(PyObject *path, - void *userData)`, where *path* is guaranteed to be :c:type:`PyUnicodeObject`. + The *handler* is a function of type: + + .. c:namespace:: NULL + .. c:type:: PyObject * (*Py_OpenCodeHookFunction)(PyObject *, void *) + + Equivalent of :c:expr:`PyObject *(\*)(PyObject *path, + void *userData)`, where *path* is guaranteed to be + :c:type:`PyUnicodeObject`. The *userData* pointer is passed into the hook function. Since hook functions may be called from different runtimes, this pointer should not diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 51c20b202f091c5..7c74e9e88678dc0 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -13,20 +13,8 @@ Importing Modules single: __all__ (package variable) single: modules (in module sys) - This is a simplified interface to :c:func:`PyImport_ImportModuleEx` below, - leaving the *globals* and *locals* arguments set to ``NULL`` and *level* set - to 0. When the *name* - argument contains a dot (when it specifies a submodule of a package), the - *fromlist* argument is set to the list ``['*']`` so that the return value is the - named module rather than the top-level package containing it as would otherwise - be the case. (Unfortunately, this has an additional side effect when *name* in - fact specifies a subpackage instead of a submodule: the submodules specified in - the package's ``__all__`` variable are loaded.) Return a new reference to the - imported module, or ``NULL`` with an exception set on failure. A failing - import of a module doesn't leave the module in :data:`sys.modules`. - - This function always uses absolute imports. - + This is a wrapper around :c:func:`PyImport_Import()` which takes a + :c:expr:`const char *` as an argument instead of a :c:expr:`PyObject *`. .. c:function:: PyObject* PyImport_ImportModuleNoBlock(const char *name) diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 4dbca92b18b5cd2..dcda1071a58f35b 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -148,7 +148,7 @@ complete listing. worse performances (due to increased code size for example). The compiler is usually smarter than the developer for the cost/benefit analysis. - If Python is :ref:`built in debug mode ` (if the ``Py_DEBUG`` + If Python is :ref:`built in debug mode ` (if the :c:macro:`Py_DEBUG` macro is defined), the :c:macro:`Py_ALWAYS_INLINE` macro does nothing. It must be specified before the function return type. Usage:: @@ -812,12 +812,14 @@ available that support tracing of reference counts, debugging the memory allocator, or low-level profiling of the main interpreter loop. Only the most frequently used builds will be described in the remainder of this section. -Compiling the interpreter with the :c:macro:`Py_DEBUG` macro defined produces +.. c:macro:: Py_DEBUG + +Compiling the interpreter with the :c:macro:`!Py_DEBUG` macro defined produces what is generally meant by :ref:`a debug build of Python `. -:c:macro:`Py_DEBUG` is enabled in the Unix build by adding +:c:macro:`!Py_DEBUG` is enabled in the Unix build by adding :option:`--with-pydebug` to the :file:`./configure` command. It is also implied by the presence of the -not-Python-specific :c:macro:`_DEBUG` macro. When :c:macro:`Py_DEBUG` is enabled +not-Python-specific :c:macro:`!_DEBUG` macro. When :c:macro:`!Py_DEBUG` is enabled in the Unix build, compiler optimization is disabled. In addition to the reference count debugging described below, extra checks are @@ -832,4 +834,3 @@ after every statement run by the interpreter.) Please refer to :file:`Misc/SpecialBuilds.txt` in the Python source distribution for more detailed information. - diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index c8b64bad702f508..53eb54d3e1021a3 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -56,13 +56,21 @@ List Objects Similar to :c:func:`PyList_Size`, but without error checking. -.. c:function:: PyObject* PyList_GetItem(PyObject *list, Py_ssize_t index) +.. c:function:: PyObject* PyList_GetItemRef(PyObject *list, Py_ssize_t index) Return the object at position *index* in the list pointed to by *list*. The position must be non-negative; indexing from the end of the list is not - supported. If *index* is out of bounds (<0 or >=len(list)), + supported. If *index* is out of bounds (:code:`<0 or >=len(list)`), return ``NULL`` and set an :exc:`IndexError` exception. + .. versionadded:: 3.13 + + +.. c:function:: PyObject* PyList_GetItem(PyObject *list, Py_ssize_t index) + + Like :c:func:`PyList_GetItemRef`, but returns a + :term:`borrowed reference` instead of a :term:`strong reference`. + .. c:function:: PyObject* PyList_GET_ITEM(PyObject *list, Py_ssize_t i) diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst index 2aa43318e7a455c..f6038032805259f 100644 --- a/Doc/c-api/memoryview.rst +++ b/Doc/c-api/memoryview.rst @@ -20,6 +20,17 @@ any other object. read/write, otherwise it may be either read-only or read/write at the discretion of the exporter. + +.. c:macro:: PyBUF_READ + + Flag to request a readonly buffer. + + +.. c:macro:: PyBUF_WRITE + + Flag to request a writable buffer. + + .. c:function:: PyObject *PyMemoryView_FromMemory(char *mem, Py_ssize_t size, int flags) Create a memoryview object using *mem* as the underlying buffer. @@ -41,6 +52,8 @@ any other object. original memory. Otherwise, a copy is made and the memoryview points to a new bytes object. + *buffertype* can be one of :c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE`. + .. c:function:: int PyMemoryView_Check(PyObject *obj) diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 8a179690d048e39..12476412799a4f1 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -19,6 +19,14 @@ Object Protocol to NotImplemented and return it). +.. c:macro:: Py_PRINT_RAW + + Flag to be used with multiple functions that print the object (like + :c:func:`PyObject_Print` and :c:func:`PyFile_WriteObject`). + If passed, these function would use the :func:`str` of the object + instead of the :func:`repr`. + + .. c:function:: int PyObject_Print(PyObject *o, FILE *fp, int flags) Print an object *o*, on file *fp*. Returns ``-1`` on error. The flags argument @@ -221,12 +229,8 @@ Object Protocol .. c:function:: int PyObject_RichCompareBool(PyObject *o1, PyObject *o2, int opid) Compare the values of *o1* and *o2* using the operation specified by *opid*, - which must be one of :c:macro:`Py_LT`, :c:macro:`Py_LE`, :c:macro:`Py_EQ`, - :c:macro:`Py_NE`, :c:macro:`Py_GT`, or :c:macro:`Py_GE`, corresponding to ``<``, - ``<=``, ``==``, ``!=``, ``>``, or ``>=`` respectively. Returns ``-1`` on error, - ``0`` if the result is false, ``1`` otherwise. This is the equivalent of the - Python expression ``o1 op o2``, where ``op`` is the operator corresponding to - *opid*. + like :c:func:`PyObject_RichCompare`, but returns ``-1`` on error, ``0`` if + the result is false, ``1`` otherwise. .. note:: If *o1* and *o2* are the same object, :c:func:`PyObject_RichCompareBool` diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 86c779472fd244e..0032da9659636c5 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -551,11 +551,11 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: from ``PyObject``. Can only be used as part of :c:member:`Py_tp_members ` - :c:type:`slot ` when creating a class using negative + :c:type:`slot ` when creating a class using negative :c:member:`~PyType_Spec.basicsize`. It is mandatory in that case. - This flag is only used in :c:type:`PyTypeSlot`. + This flag is only used in :c:type:`PyType_Slot`. When setting :c:member:`~PyTypeObject.tp_members` during class creation, Python clears it and sets :c:member:`PyMemberDef.offset` to the offset from the ``PyObject`` struct. @@ -693,7 +693,8 @@ Defining Getters and Setters .. c:member:: setter set - Optional C function to set or delete the attribute, if omitted the attribute is readonly. + Optional C function to set or delete the attribute. + If ``NULL``, the attribute is read-only. .. c:member:: const char* doc @@ -703,18 +704,18 @@ Defining Getters and Setters Optional function pointer, providing additional data for getter and setter. - The ``get`` function takes one :c:expr:`PyObject*` parameter (the - instance) and a function pointer (the associated ``closure``):: +.. c:type:: PyObject *(*getter)(PyObject *, void *) - typedef PyObject *(*getter)(PyObject *, void *); + The ``get`` function takes one :c:expr:`PyObject*` parameter (the + instance) and a function pointer (the associated ``closure``): It should return a new reference on success or ``NULL`` with a set exception on failure. - ``set`` functions take two :c:expr:`PyObject*` parameters (the instance and - the value to be set) and a function pointer (the associated ``closure``):: +.. c:type:: int (*setter)(PyObject *, PyObject *, void *) - typedef int (*setter)(PyObject *, PyObject *, void *); + ``set`` functions take two :c:expr:`PyObject*` parameters (the instance and + the value to be set) and a function pointer (the associated ``closure``): In case the attribute should be deleted the second parameter is ``NULL``. Should return ``0`` on success or ``-1`` with a set exception on failure. diff --git a/Doc/conf.py b/Doc/conf.py index 458954370debe28..c2d57696aeeaa37 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -6,7 +6,9 @@ # The contents of this file are pickled, so don't put values in the namespace # that aren't pickleable (module imports are okay, they're removed automatically). -import sys, os, time +import os +import sys +import time sys.path.append(os.path.abspath('tools/extensions')) sys.path.append(os.path.abspath('includes')) @@ -55,7 +57,7 @@ # General substitutions. project = 'Python' -copyright = '2001-%s, Python Software Foundation' % time.strftime('%Y') +copyright = f"2001-{time.strftime('%Y')}, Python Software Foundation" # We look for the Include/patchlevel.h file in the current Python source tree # and replace the values accordingly. @@ -89,22 +91,32 @@ nitpick_ignore = [ # Standard C functions ('c:func', 'calloc'), + ('c:func', 'ctime'), ('c:func', 'dlopen'), ('c:func', 'exec'), ('c:func', 'fcntl'), ('c:func', 'fork'), ('c:func', 'free'), + ('c:func', 'gettimeofday'), ('c:func', 'gmtime'), + ('c:func', 'grantpt'), + ('c:func', 'localeconv'), ('c:func', 'localtime'), ('c:func', 'main'), ('c:func', 'malloc'), + ('c:func', 'mktime'), + ('c:func', 'posix_openpt'), ('c:func', 'printf'), + ('c:func', 'ptsname'), + ('c:func', 'ptsname_r'), ('c:func', 'realloc'), ('c:func', 'snprintf'), ('c:func', 'sprintf'), ('c:func', 'stat'), + ('c:func', 'strftime'), ('c:func', 'system'), ('c:func', 'time'), + ('c:func', 'unlockpt'), ('c:func', 'vsnprintf'), # Standard C types ('c:type', 'FILE'), @@ -292,6 +304,9 @@ 'root_include_title': False # We use the version switcher instead. } +if os.getenv("READTHEDOCS"): + html_theme_options["hosted_on"] = 'Read the Docs' + # Override stylesheet fingerprinting for Windows CHM htmlhelp to fix GH-91207 # https://github.com/python/cpython/issues/91207 if any('htmlhelp' in arg for arg in sys.argv): @@ -300,7 +315,7 @@ print("It may be removed in the future\n") # Short title used e.g. for HTML tags. -html_short_title = '%s Documentation' % release +html_short_title = f'{release} Documentation' # Deployment preview information # (See .readthedocs.yml and https://docs.readthedocs.io/en/stable/reference/environment-variables.html) @@ -349,12 +364,9 @@ latex_engine = 'xelatex' -# Get LaTeX to handle Unicode correctly latex_elements = { -} - -# Additional stuff for the LaTeX preamble. -latex_elements['preamble'] = r''' + # For the LaTeX preamble. + 'preamble': r''' \authoraddress{ \sphinxstrong{Python Software Foundation}\\ Email: \sphinxemail{docs@python.org} @@ -362,13 +374,12 @@ \let\Verbatim=\OriginalVerbatim \let\endVerbatim=\endOriginalVerbatim \setcounter{tocdepth}{2} -''' - -# The paper size ('letter' or 'a4'). -latex_elements['papersize'] = 'a4' - -# The font size ('10pt', '11pt' or '12pt'). -latex_elements['pointsize'] = '10pt' +''', + # The paper size ('letter' or 'a4'). + 'papersize': 'a4', + # The font size ('10pt', '11pt' or '12pt'). + 'pointsize': '10pt', +} # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). @@ -431,9 +442,9 @@ # Regexes to find C items in the source files. coverage_c_regexes = { - 'cfunction': (r'^PyAPI_FUNC\(.*\)\s+([^_][\w_]+)'), - 'data': (r'^PyAPI_DATA\(.*\)\s+([^_][\w_]+)'), - 'macro': (r'^#define ([^_][\w_]+)\(.*\)[\s|\\]'), + 'cfunction': r'^PyAPI_FUNC\(.*\)\s+([^_][\w_]+)', + 'data': r'^PyAPI_DATA\(.*\)\s+([^_][\w_]+)', + 'macro': r'^#define ([^_][\w_]+)\(.*\)[\s|\\]', } # The coverage checker will ignore all C items whose names match these regexes diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index f719ce153b239a9..62a96146d605ff7 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -1133,6 +1133,10 @@ PyList_GetItem:PyObject*::0: PyList_GetItem:PyObject*:list:0: PyList_GetItem:Py_ssize_t:index:: +PyList_GetItemRef:PyObject*::+1: +PyList_GetItemRef:PyObject*:list:0: +PyList_GetItemRef:Py_ssize_t:index:: + PyList_GetSlice:PyObject*::+1: PyList_GetSlice:PyObject*:list:0: PyList_GetSlice:Py_ssize_t:low:: diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 811b1bd84d24174..def1903204add7a 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -220,6 +220,7 @@ var,PyExc_GeneratorExit,3.2,, var,PyExc_IOError,3.2,, var,PyExc_ImportError,3.2,, var,PyExc_ImportWarning,3.2,, +var,PyExc_IncompleteInputError,3.13,, var,PyExc_IndentationError,3.2,, var,PyExc_IndexError,3.2,, var,PyExc_InterruptedError,3.7,, @@ -335,6 +336,7 @@ var,PyListRevIter_Type,3.2,, function,PyList_Append,3.2,, function,PyList_AsTuple,3.2,, function,PyList_GetItem,3.2,, +function,PyList_GetItemRef,3.13,, function,PyList_GetSlice,3.2,, function,PyList_Insert,3.2,, function,PyList_New,3.2,, diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index 476a43d9c288f18..e2f8004c7e3aea5 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -405,22 +405,37 @@ lists. When in doubt, use a mutex! Can't we get rid of the Global Interpreter Lock? ------------------------------------------------ -.. XXX link to dbeazley's talk about GIL? - The :term:`global interpreter lock` (GIL) is often seen as a hindrance to Python's deployment on high-end multiprocessor server machines, because a multi-threaded Python program effectively only uses one CPU, due to the insistence that (almost) all Python code can only run while the GIL is held. -Back in the days of Python 1.5, Greg Stein actually implemented a comprehensive +With the approval of :pep:`703` work is now underway to remove the GIL from the +CPython implementation of Python. Initially it will be implemented as an +optional compiler flag when building the interpreter, and so separate +builds will be available with and without the GIL. Long-term, the hope is +to settle on a single build, once the performance implications of removing the +GIL are fully understood. Python 3.13 is likely to be the first release +containing this work, although it may not be completely functional in this +release. + +The current work to remove the GIL is based on a +`fork of Python 3.9 with the GIL removed <https://github.com/colesbury/nogil>`_ +by Sam Gross. +Prior to that, +in the days of Python 1.5, Greg Stein actually implemented a comprehensive patch set (the "free threading" patches) that removed the GIL and replaced it -with fine-grained locking. Adam Olsen recently did a similar experiment +with fine-grained locking. Adam Olsen did a similar experiment in his `python-safethread <https://code.google.com/archive/p/python-safethread>`_ -project. Unfortunately, both experiments exhibited a sharp drop in single-thread +project. Unfortunately, both of these earlier experiments exhibited a sharp +drop in single-thread performance (at least 30% slower), due to the amount of fine-grained locking -necessary to compensate for the removal of the GIL. +necessary to compensate for the removal of the GIL. The Python 3.9 fork +is the first attempt at removing the GIL with an acceptable performance +impact. -This doesn't mean that you can't make good use of Python on multi-CPU machines! +The presence of the GIL in current Python releases +doesn't mean that you can't make good use of Python on multi-CPU machines! You just have to be creative with dividing the work up between multiple *processes* rather than multiple *threads*. The :class:`~concurrent.futures.ProcessPoolExecutor` class in the new @@ -434,22 +449,13 @@ thread of execution is in the C code and allow other threads to get some work done. Some standard library modules such as :mod:`zlib` and :mod:`hashlib` already do this. -It has been suggested that the GIL should be a per-interpreter-state lock rather -than truly global; interpreters then wouldn't be able to share objects. -Unfortunately, this isn't likely to happen either. It would be a tremendous -amount of work, because many object implementations currently have global state. -For example, small integers and short strings are cached; these caches would -have to be moved to the interpreter state. Other object types have their own -free list; these free lists would have to be moved to the interpreter state. -And so on. - -And I doubt that it can even be done in finite time, because the same problem -exists for 3rd party extensions. It is likely that 3rd party extensions are -being written at a faster rate than you can convert them to store all their -global state in the interpreter state. - -And finally, once you have multiple interpreters not sharing any state, what -have you gained over running each interpreter in a separate process? +An alternative approach to reducing the impact of the GIL is +to make the GIL a per-interpreter-state lock rather than truly global. +This was :ref:`first implemented in Python 3.12 <whatsnew312-pep684>` and is +available in the C API. A Python interface to it is expected in Python 3.13. +The main limitation to it at the moment is likely to be 3rd party extension +modules, since these must be written with multiple interpreters in mind in +order to be usable, so many older extension modules will not be usable. Input and Output diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 098bfffb104ef69..f656e32514c7178 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -341,7 +341,7 @@ Glossary docstring A string literal which appears as the first expression in a class, function or module. While ignored when the suite is executed, it is - recognized by the compiler and put into the :attr:`__doc__` attribute + recognized by the compiler and put into the :attr:`!__doc__` attribute of the enclosing class, function or module. Since it is available via introspection, it is the canonical place for documentation of the object. @@ -1104,10 +1104,12 @@ Glossary The :class:`collections.abc.Sequence` abstract base class defines a much richer interface that goes beyond just :meth:`~object.__getitem__` and :meth:`~object.__len__`, adding - :meth:`count`, :meth:`index`, :meth:`~object.__contains__`, and + :meth:`!count`, :meth:`!index`, :meth:`~object.__contains__`, and :meth:`~object.__reversed__`. Types that implement this expanded interface can be registered explicitly using - :func:`~abc.ABCMeta.register`. + :func:`~abc.ABCMeta.register`. For more documentation on sequence + methods generally, see + :ref:`Common Sequence Operations <typesseq-common>`. set comprehension A compact way to process all or part of the elements in an iterable and diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 1e9ac9b6761b647..30be15230fc088e 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -497,13 +497,30 @@ the :meth:`~Enum.__repr__` omits the inherited class' name. For example:: >>> Creature.DOG <Creature.DOG: size='medium', legs=4> -Use the :func:`!dataclass` argument ``repr=False`` +Use the :func:`~dataclasses.dataclass` argument ``repr=False`` to use the standard :func:`repr`. .. versionchanged:: 3.12 Only the dataclass fields are shown in the value area, not the dataclass' name. +.. note:: + + Adding :func:`~dataclasses.dataclass` decorator to :class:`Enum` + and its subclasses is not supported. It will not raise any errors, + but it will produce very strange results at runtime, such as members + being equal to each other:: + + >>> @dataclass # don't do this: it does not make any sense + ... class Color(Enum): + ... RED = 1 + ... BLUE = 2 + ... + >>> Color.RED is Color.BLUE + False + >>> Color.RED == Color.BLUE # problem is here: they should not be equal + True + Pickling -------- diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index ea494f2fdbbce42..80147e31fcbae11 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -1933,30 +1933,28 @@ This dictionary is passed to :func:`~config.dictConfig` to put the configuration LOGGING = { 'version': 1, - 'disable_existing_loggers': True, + 'disable_existing_loggers': False, 'formatters': { 'verbose': { - 'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s' + 'format': '{levelname} {asctime} {module} {process:d} {thread:d} {message}', + 'style': '{', }, 'simple': { - 'format': '%(levelname)s %(message)s' + 'format': '{levelname} {message}', + 'style': '{', }, }, 'filters': { 'special': { '()': 'project.logging.SpecialFilter', 'foo': 'bar', - } + }, }, 'handlers': { - 'null': { - 'level':'DEBUG', - 'class':'django.utils.log.NullHandler', - }, - 'console':{ - 'level':'DEBUG', - 'class':'logging.StreamHandler', - 'formatter': 'simple' + 'console': { + 'level': 'INFO', + 'class': 'logging.StreamHandler', + 'formatter': 'simple', }, 'mail_admins': { 'level': 'ERROR', @@ -1966,9 +1964,8 @@ This dictionary is passed to :func:`~config.dictConfig` to put the configuration }, 'loggers': { 'django': { - 'handlers':['null'], + 'handlers': ['console'], 'propagate': True, - 'level':'INFO', }, 'django.request': { 'handlers': ['mail_admins'], diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index f164b461c93b9cd..347330e98dd00c9 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -520,7 +520,7 @@ custom handlers) are the following configuration methods: * The :meth:`~Handler.setLevel` method, just as in logger objects, specifies the lowest severity that will be dispatched to the appropriate destination. Why - are there two :func:`setLevel` methods? The level set in the logger + are there two :meth:`~Handler.setLevel` methods? The level set in the logger determines which severity of messages it will pass to its handlers. The level set in each handler determines which messages that handler will send on. @@ -774,29 +774,29 @@ What happens if no configuration is provided If no logging configuration is provided, it is possible to have a situation where a logging event needs to be output, but no handlers can be found to -output the event. The behaviour of the logging package in these -circumstances is dependent on the Python version. +output the event. -For versions of Python prior to 3.2, the behaviour is as follows: +The event is output using a 'handler of last resort', stored in +:data:`lastResort`. This internal handler is not associated with any +logger, and acts like a :class:`~logging.StreamHandler` which writes the +event description message to the current value of ``sys.stderr`` (therefore +respecting any redirections which may be in effect). No formatting is +done on the message - just the bare event description message is printed. +The handler's level is set to ``WARNING``, so all events at this and +greater severities will be output. -* If *logging.raiseExceptions* is ``False`` (production mode), the event is - silently dropped. +.. versionchanged:: 3.2 -* If *logging.raiseExceptions* is ``True`` (development mode), a message - 'No handlers could be found for logger X.Y.Z' is printed once. + For versions of Python prior to 3.2, the behaviour is as follows: -In Python 3.2 and later, the behaviour is as follows: + * If :data:`raiseExceptions` is ``False`` (production mode), the event is + silently dropped. -* The event is output using a 'handler of last resort', stored in - ``logging.lastResort``. This internal handler is not associated with any - logger, and acts like a :class:`~logging.StreamHandler` which writes the - event description message to the current value of ``sys.stderr`` (therefore - respecting any redirections which may be in effect). No formatting is - done on the message - just the bare event description message is printed. - The handler's level is set to ``WARNING``, so all events at this and - greater severities will be output. + * If :data:`raiseExceptions` is ``True`` (development mode), a message + 'No handlers could be found for logger X.Y.Z' is printed once. -To obtain the pre-3.2 behaviour, ``logging.lastResort`` can be set to ``None``. + To obtain the pre-3.2 behaviour, + :data:`lastResort` can be set to ``None``. .. _library-config: @@ -998,7 +998,7 @@ Logged messages are formatted for presentation through instances of the use with the % operator and a dictionary. For formatting multiple messages in a batch, instances of -:class:`~handlers.BufferingFormatter` can be used. In addition to the format +:class:`BufferingFormatter` can be used. In addition to the format string (which is applied to each message in the batch), there is provision for header and trailer format strings. @@ -1034,7 +1034,8 @@ checks to see if a module-level variable, :data:`raiseExceptions`, is set. If set, a traceback is printed to :data:`sys.stderr`. If not set, the exception is swallowed. -.. note:: The default value of :data:`raiseExceptions` is ``True``. This is +.. note:: + The default value of :data:`raiseExceptions` is ``True``. This is because during development, you typically want to be notified of any exceptions that occur. It's advised that you set :data:`raiseExceptions` to ``False`` for production usage. @@ -1072,7 +1073,7 @@ You can write code like this:: expensive_func2()) so that if the logger's threshold is set above ``DEBUG``, the calls to -:func:`expensive_func1` and :func:`expensive_func2` are never made. +``expensive_func1`` and ``expensive_func2`` are never made. .. note:: In some cases, :meth:`~Logger.isEnabledFor` can itself be more expensive than you'd like (e.g. for deeply nested loggers where an explicit diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index d261e4a4f338a56..762f8b4695b3dd9 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -8,20 +8,68 @@ -------------- -:mod:`__future__` is a real module, and serves three purposes: +Imports of the form ``from __future__ import feature`` are called +:ref:`future statements <future>`. These are special-cased by the Python compiler +to allow the use of new Python features in modules containing the future statement +before the release in which the feature becomes standard. + +While these future statements are given additional special meaning by the +Python compiler, they are still executed like any other import statement and +the :mod:`__future__` exists and is handled by the import system the same way +any other Python module would be. This design serves three purposes: * To avoid confusing existing tools that analyze import statements and expect to find the modules they're importing. -* To ensure that :ref:`future statements <future>` run under releases prior to - 2.1 at least yield runtime exceptions (the import of :mod:`__future__` will - fail, because there was no module of that name prior to 2.1). - * To document when incompatible changes were introduced, and when they will be --- or were --- made mandatory. This is a form of executable documentation, and can be inspected programmatically via importing :mod:`__future__` and examining its contents. +* To ensure that :ref:`future statements <future>` run under releases prior to + Python 2.1 at least yield runtime exceptions (the import of :mod:`__future__` + will fail, because there was no module of that name prior to 2.1). + +Module Contents +--------------- + +No feature description will ever be deleted from :mod:`__future__`. Since its +introduction in Python 2.1 the following features have found their way into the +language using this mechanism: + ++------------------+-------------+--------------+---------------------------------------------+ +| feature | optional in | mandatory in | effect | ++==================+=============+==============+=============================================+ +| nested_scopes | 2.1.0b1 | 2.2 | :pep:`227`: | +| | | | *Statically Nested Scopes* | ++------------------+-------------+--------------+---------------------------------------------+ +| generators | 2.2.0a1 | 2.3 | :pep:`255`: | +| | | | *Simple Generators* | ++------------------+-------------+--------------+---------------------------------------------+ +| division | 2.2.0a2 | 3.0 | :pep:`238`: | +| | | | *Changing the Division Operator* | ++------------------+-------------+--------------+---------------------------------------------+ +| absolute_import | 2.5.0a1 | 3.0 | :pep:`328`: | +| | | | *Imports: Multi-Line and Absolute/Relative* | ++------------------+-------------+--------------+---------------------------------------------+ +| with_statement | 2.5.0a1 | 2.6 | :pep:`343`: | +| | | | *The "with" Statement* | ++------------------+-------------+--------------+---------------------------------------------+ +| print_function | 2.6.0a2 | 3.0 | :pep:`3105`: | +| | | | *Make print a function* | ++------------------+-------------+--------------+---------------------------------------------+ +| unicode_literals | 2.6.0a2 | 3.0 | :pep:`3112`: | +| | | | *Bytes literals in Python 3000* | ++------------------+-------------+--------------+---------------------------------------------+ +| generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | +| | | | *StopIteration handling inside generators* | ++------------------+-------------+--------------+---------------------------------------------+ +| annotations | 3.7.0b1 | TBD [1]_ | :pep:`563`: | +| | | | *Postponed evaluation of annotations* | ++------------------+-------------+--------------+---------------------------------------------+ + +.. XXX Adding a new entry? Remember to update simple_stmts.rst, too. + .. _future-classes: .. class:: _Feature @@ -65,43 +113,6 @@ dynamically compiled code. This flag is stored in the :attr:`_Feature.compiler_flag` attribute on :class:`_Feature` instances. -No feature description will ever be deleted from :mod:`__future__`. Since its -introduction in Python 2.1 the following features have found their way into the -language using this mechanism: - -+------------------+-------------+--------------+---------------------------------------------+ -| feature | optional in | mandatory in | effect | -+==================+=============+==============+=============================================+ -| nested_scopes | 2.1.0b1 | 2.2 | :pep:`227`: | -| | | | *Statically Nested Scopes* | -+------------------+-------------+--------------+---------------------------------------------+ -| generators | 2.2.0a1 | 2.3 | :pep:`255`: | -| | | | *Simple Generators* | -+------------------+-------------+--------------+---------------------------------------------+ -| division | 2.2.0a2 | 3.0 | :pep:`238`: | -| | | | *Changing the Division Operator* | -+------------------+-------------+--------------+---------------------------------------------+ -| absolute_import | 2.5.0a1 | 3.0 | :pep:`328`: | -| | | | *Imports: Multi-Line and Absolute/Relative* | -+------------------+-------------+--------------+---------------------------------------------+ -| with_statement | 2.5.0a1 | 2.6 | :pep:`343`: | -| | | | *The "with" Statement* | -+------------------+-------------+--------------+---------------------------------------------+ -| print_function | 2.6.0a2 | 3.0 | :pep:`3105`: | -| | | | *Make print a function* | -+------------------+-------------+--------------+---------------------------------------------+ -| unicode_literals | 2.6.0a2 | 3.0 | :pep:`3112`: | -| | | | *Bytes literals in Python 3000* | -+------------------+-------------+--------------+---------------------------------------------+ -| generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | -| | | | *StopIteration handling inside generators* | -+------------------+-------------+--------------+---------------------------------------------+ -| annotations | 3.7.0b1 | TBD [1]_ | :pep:`563`: | -| | | | *Postponed evaluation of annotations* | -+------------------+-------------+--------------+---------------------------------------------+ - -.. XXX Adding a new entry? Remember to update simple_stmts.rst, too. - .. [1] ``from __future__ import annotations`` was previously scheduled to become mandatory in Python 3.10, but the Python Steering Council @@ -115,3 +126,6 @@ language using this mechanism: :ref:`future` How the compiler treats future imports. + + :pep:`236` - Back to the __future__ + The original proposal for the __future__ mechanism. diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 1395d457f874b0c..952643a46416d21 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -777,6 +777,8 @@ The add_argument() method * dest_ - The name of the attribute to be added to the object returned by :meth:`parse_args`. + * deprecated_ - Whether or not use of the argument is deprecated. + The following sections describe how each of these are used. @@ -1439,6 +1441,34 @@ behavior:: >>> parser.parse_args('--foo XXX'.split()) Namespace(bar='XXX') + +.. _deprecated: + +deprecated +^^^^^^^^^^ + +During a project's lifetime, some arguments may need to be removed from the +command line. Before removing them, you should inform +your users that the arguments are deprecated and will be removed. +The ``deprecated`` keyword argument of +:meth:`~ArgumentParser.add_argument`, which defaults to ``False``, +specifies if the argument is deprecated and will be removed +in the future. +For arguments, if ``deprecated`` is ``True``, then a warning will be +printed to standard error when the argument is used:: + + >>> import argparse + >>> parser = argparse.ArgumentParser(prog='snake.py') + >>> parser.add_argument('--legs', default=0, type=int, deprecated=True) + >>> parser.parse_args([]) + Namespace(legs=0) + >>> parser.parse_args(['--legs', '4']) # doctest: +SKIP + snake.py: warning: option '--legs' is deprecated + Namespace(legs=4) + +.. versionchanged:: 3.13 + + Action classes ^^^^^^^^^^^^^^ @@ -1842,7 +1872,8 @@ Sub-commands {foo,bar} additional help - Furthermore, ``add_parser`` supports an additional ``aliases`` argument, + Furthermore, :meth:`~_SubParsersAction.add_parser` supports an additional + *aliases* argument, which allows multiple strings to refer to the same subparser. This example, like ``svn``, aliases ``co`` as a shorthand for ``checkout``:: @@ -1853,6 +1884,20 @@ Sub-commands >>> parser.parse_args(['co', 'bar']) Namespace(foo='bar') + :meth:`~_SubParsersAction.add_parser` supports also an additional + *deprecated* argument, which allows to deprecate the subparser. + + >>> import argparse + >>> parser = argparse.ArgumentParser(prog='chicken.py') + >>> subparsers = parser.add_subparsers() + >>> run = subparsers.add_parser('run') + >>> fly = subparsers.add_parser('fly', deprecated=True) + >>> parser.parse_args(['fly']) # doctest: +SKIP + chicken.py: warning: command 'fly' is deprecated + Namespace() + + .. versionadded:: 3.13 + One particularly effective way of handling sub-commands is to combine the use of the :meth:`add_subparsers` method with calls to :meth:`set_defaults` so that each subparser knows which Python function it should execute. For diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 05bdf5488af143c..3cf8e2737e85dcd 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -216,8 +216,8 @@ Condition .. method:: notify(n=1) - Wake up at most *n* tasks (1 by default) waiting on this - condition. The method is no-op if no tasks are waiting. + Wake up *n* tasks (1 by default) waiting on this + condition. If fewer than *n* tasks are waiting they are all awakened. The lock must be acquired before this method is called and released shortly after. If called with an *unlocked* lock @@ -257,12 +257,18 @@ Condition Once awakened, the Condition re-acquires its lock and this method returns ``True``. + Note that a task *may* return from this call spuriously, + which is why the caller should always re-check the state + and be prepared to :meth:`wait` again. For this reason, you may + prefer to use :meth:`wait_for` instead. + .. coroutinemethod:: wait_for(predicate) Wait until a predicate becomes *true*. The predicate must be a callable which result will be - interpreted as a boolean value. The final value is the + interpreted as a boolean value. The method will repeatedly + :meth:`wait` until the predicate evaluates to *true*. The final value is the return value. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 797065c8ccf8941..24bd36e6431b4f4 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -828,23 +828,22 @@ Waiting Primitives *return_when* indicates when this function should return. It must be one of the following constants: - .. tabularcolumns:: |l|L| - - +-----------------------------+----------------------------------------+ - | Constant | Description | - +=============================+========================================+ - | :const:`FIRST_COMPLETED` | The function will return when any | - | | future finishes or is cancelled. | - +-----------------------------+----------------------------------------+ - | :const:`FIRST_EXCEPTION` | The function will return when any | - | | future finishes by raising an | - | | exception. If no future raises an | - | | exception then it is equivalent to | - | | :const:`ALL_COMPLETED`. | - +-----------------------------+----------------------------------------+ - | :const:`ALL_COMPLETED` | The function will return when all | - | | futures finish or are cancelled. | - +-----------------------------+----------------------------------------+ + .. list-table:: + :header-rows: 1 + + * - Constant + - Description + + * - .. data:: FIRST_COMPLETED + - The function will return when any future finishes or is cancelled. + + * - .. data:: FIRST_EXCEPTION + - The function will return when any future finishes by raising an + exception. If no future raises an exception + then it is equivalent to :const:`ALL_COMPLETED`. + + * - .. data:: ALL_COMPLETED + - The function will return when all futures finish or are cancelled. Unlike :func:`~asyncio.wait_for`, ``wait()`` does not cancel the futures when a timeout occurs. diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst index 3dbef69580d9b30..43a8bd2d7cd1339 100644 --- a/Doc/library/atexit.rst +++ b/Doc/library/atexit.rst @@ -4,8 +4,8 @@ .. module:: atexit :synopsis: Register and execute cleanup functions. -.. moduleauthor:: Skip Montanaro <skip@pobox.com> -.. sectionauthor:: Skip Montanaro <skip@pobox.com> +.. moduleauthor:: Skip Montanaro <skip.montanaro@gmail.com> +.. sectionauthor:: Skip Montanaro <skip.montanaro@gmail.com> -------------- diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 6586f539a8da4fc..e699a7284ac8023 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -512,7 +512,7 @@ to interactively print a calendar. python -m calendar [-h] [-L LOCALE] [-e ENCODING] [-t {text,html}] [-w WIDTH] [-l LINES] [-s SPACING] [-m MONTHS] [-c CSS] - [year] [month] + [-f FIRST_WEEKDAY] [year] [month] For example, to print a calendar for the year 2000: @@ -586,10 +586,17 @@ The following options are accepted: or as an HTML document. +.. option:: --first-weekday FIRST_WEEKDAY, -f FIRST_WEEKDAY + + The weekday to start each week. + Must be a number between 0 (Monday) and 6 (Sunday). + Defaults to 0. + + .. versionadded:: 3.13 + .. option:: year The year to print the calendar for. - Must be a number between 1 and 9999. Defaults to the current year. diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 582bb18f752bd59..7bcaba60c6ddbd9 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -136,8 +136,8 @@ ABC Inherits from Abstract Methods Mi :class:`Collection` ``__len__`` ``index``, and ``count`` :class:`MutableSequence` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods and - ``__setitem__``, ``append``, ``reverse``, ``extend``, ``pop``, - ``__delitem__``, ``remove``, and ``__iadd__`` + ``__setitem__``, ``append``, ``clear``, ``reverse``, ``extend``, + ``__delitem__``, ``pop``, ``remove``, and ``__iadd__`` ``__len__``, ``insert`` diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index 800c7f6739d8a36..d3c7a40aa9d390d 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -275,7 +275,8 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. .. versionchanged:: 3.3 When one of the worker processes terminates abruptly, a - :exc:`BrokenProcessPool` error is now raised. Previously, behaviour + :exc:`~concurrent.futures.process.BrokenProcessPool` error is now raised. + Previously, behaviour was undefined but operations on the executor or its futures would often freeze or deadlock. @@ -493,23 +494,22 @@ Module Functions *return_when* indicates when this function should return. It must be one of the following constants: - .. tabularcolumns:: |l|L| - - +-----------------------------+----------------------------------------+ - | Constant | Description | - +=============================+========================================+ - | :const:`FIRST_COMPLETED` | The function will return when any | - | | future finishes or is cancelled. | - +-----------------------------+----------------------------------------+ - | :const:`FIRST_EXCEPTION` | The function will return when any | - | | future finishes by raising an | - | | exception. If no future raises an | - | | exception then it is equivalent to | - | | :const:`ALL_COMPLETED`. | - +-----------------------------+----------------------------------------+ - | :const:`ALL_COMPLETED` | The function will return when all | - | | futures finish or are cancelled. | - +-----------------------------+----------------------------------------+ + .. list-table:: + :header-rows: 1 + + * - Constant + - Description + + * - .. data:: FIRST_COMPLETED + - The function will return when any future finishes or is cancelled. + + * - .. data:: FIRST_EXCEPTION + - The function will return when any future finishes by raising an + exception. If no future raises an exception + then it is equivalent to :const:`ALL_COMPLETED`. + + * - .. data:: ALL_COMPLETED + - The function will return when all futures finish or are cancelled. .. function:: as_completed(fs, timeout=None) @@ -570,7 +570,8 @@ Exception classes .. exception:: BrokenThreadPool Derived from :exc:`~concurrent.futures.BrokenExecutor`, this exception - class is raised when one of the workers of a :class:`ThreadPoolExecutor` + class is raised when one of the workers + of a :class:`~concurrent.futures.ThreadPoolExecutor` has failed initializing. .. versionadded:: 3.7 @@ -581,7 +582,8 @@ Exception classes Derived from :exc:`~concurrent.futures.BrokenExecutor` (formerly :exc:`RuntimeError`), this exception class is raised when one of the - workers of a :class:`ProcessPoolExecutor` has terminated in a non-clean + workers of a :class:`~concurrent.futures.ProcessPoolExecutor` + has terminated in a non-clean fashion (for example, if it was killed from the outside). .. versionadded:: 3.3 diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index b73373bc2363fb0..73e53aec9cbf1c1 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -185,7 +185,7 @@ Functions and classes provided: .. note:: Most types managing resources support the :term:`context manager` protocol, - which closes *thing* on leaving the :keyword:`with` statment. + which closes *thing* on leaving the :keyword:`with` statement. As such, :func:`!closing` is most useful for third party types that don't support context managers. This example is purely for illustration purposes, diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 7a5589e68b30525..4ee7820585d3a21 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -4,7 +4,7 @@ .. module:: csv :synopsis: Write and read tabular data to and from delimited files. -.. sectionauthor:: Skip Montanaro <skip@pobox.com> +.. sectionauthor:: Skip Montanaro <skip.montanaro@gmail.com> **Source code:** :source:`Lib/csv.py` @@ -88,7 +88,7 @@ The :mod:`csv` module defines the following functions: Return a writer object responsible for converting the user's data into delimited strings on the given file-like object. *csvfile* can be any object with a - :func:`write` method. If *csvfile* is a file object, it should be opened with + :meth:`~io.TextIOBase.write` method. If *csvfile* is a file object, it should be opened with ``newline=''`` [1]_. An optional *dialect* parameter can be given which is used to define a set of parameters specific to a particular CSV dialect. It may be an instance of a subclass of the @@ -197,10 +197,10 @@ The :mod:`csv` module defines the following classes: Create an object which operates like a regular writer but maps dictionaries onto output rows. The *fieldnames* parameter is a :mod:`sequence <collections.abc>` of keys that identify the order in which values in the - dictionary passed to the :meth:`writerow` method are written to file + dictionary passed to the :meth:`~csvwriter.writerow` method are written to file *f*. The optional *restval* parameter specifies the value to be written if the dictionary is missing a key in *fieldnames*. If the - dictionary passed to the :meth:`writerow` method contains a key not found in + dictionary passed to the :meth:`~csvwriter.writerow` method contains a key not found in *fieldnames*, the optional *extrasaction* parameter indicates what action to take. If it is set to ``'raise'``, the default value, a :exc:`ValueError` @@ -244,7 +244,6 @@ The :mod:`csv` module defines the following classes: with open('students.csv', 'w', newline='') as csvfile: writer = csv.writer(csvfile, dialect='unix') - ^^^^^^^^^^^^^^ .. class:: excel() @@ -351,6 +350,8 @@ The :mod:`csv` module defines the following constants: Instructs :class:`reader` objects to interpret an empty (unquoted) field as None and to otherwise behave as :data:`QUOTE_ALL`. + .. versionadded:: 3.12 + .. data:: QUOTE_STRINGS Instructs :class:`writer` objects to always place quotes around fields @@ -360,6 +361,8 @@ The :mod:`csv` module defines the following constants: Instructs :class:`reader` objects to interpret an empty (unquoted) string as ``None`` and to otherwise behave as :data:`QUOTE_NONNUMERIC`. + .. versionadded:: 3.12 + The :mod:`csv` module defines the following exception: @@ -374,8 +377,8 @@ Dialects and Formatting Parameters To make it easier to specify the format of input and output records, specific formatting parameters are grouped together into dialects. A dialect is a -subclass of the :class:`Dialect` class having a set of specific methods and a -single :meth:`validate` method. When creating :class:`reader` or +subclass of the :class:`Dialect` class containing various attributes +describing the format of the CSV file. When creating :class:`reader` or :class:`writer` objects, the programmer can specify a string or a subclass of the :class:`Dialect` class as the dialect parameter. In addition to, or instead of, the *dialect* parameter, the programmer can also specify individual @@ -492,9 +495,9 @@ DictReader objects have the following public attribute: Writer Objects -------------- -:class:`Writer` objects (:class:`DictWriter` instances and objects returned by +:class:`writer` objects (:class:`DictWriter` instances and objects returned by the :func:`writer` function) have the following public methods. A *row* must be -an iterable of strings or numbers for :class:`Writer` objects and a dictionary +an iterable of strings or numbers for :class:`writer` objects and a dictionary mapping fieldnames to strings or numbers (by passing them through :func:`str` first) for :class:`DictWriter` objects. Note that complex numbers are written out surrounded by parens. This may cause some problems for other programs which diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index cde147d1cbb501b..4ada69d63abada1 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -141,7 +141,7 @@ Module contents then :func:`dataclass` *may* add an implicit :meth:`~object.__hash__` method. Although not recommended, you can force :func:`dataclass` to create a :meth:`~object.__hash__` method with ``unsafe_hash=True``. This might be the case - if your class is logically immutable but can nonetheless be mutated. + if your class is logically immutable but can still be mutated. This is a specialized use case and should be considered carefully. Here are the rules governing implicit creation of a :meth:`~object.__hash__` @@ -538,8 +538,8 @@ that has to be called, it is common to call this method in a class Rectangle: def __init__(self, height, width): - self.height = height - self.width = width + self.height = height + self.width = width @dataclass class Square(Rectangle): diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index b36f8c19cd6040b..930af6cbbe9e8d1 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -14,7 +14,7 @@ .. XXX what order should the types be discussed in? -The :mod:`datetime` module supplies classes for manipulating dates and times. +The :mod:`!datetime` module supplies classes for manipulating dates and times. While date and time arithmetic is supported, the focus of the implementation is on efficient attribute extraction for output formatting and manipulation. @@ -70,7 +70,7 @@ These :class:`tzinfo` objects capture information about the offset from UTC time, the time zone name, and whether daylight saving time is in effect. Only one concrete :class:`tzinfo` class, the :class:`timezone` class, is -supplied by the :mod:`datetime` module. The :class:`timezone` class can +supplied by the :mod:`!datetime` module. The :class:`timezone` class can represent simple timezones with fixed offsets from UTC, such as UTC itself or North American EST and EDT timezones. Supporting timezones at deeper levels of detail is up to the application. The rules for time adjustment across the @@ -80,7 +80,7 @@ standard suitable for every application aside from UTC. Constants --------- -The :mod:`datetime` module exports the following constants: +The :mod:`!datetime` module exports the following constants: .. data:: MINYEAR @@ -130,8 +130,8 @@ Available Types .. class:: timedelta :noindex: - A duration expressing the difference between two :class:`date`, :class:`.time`, - or :class:`.datetime` instances to microsecond resolution. + A duration expressing the difference between two :class:`.datetime` + or :class:`date` instances to microsecond resolution. .. class:: tzinfo @@ -203,7 +203,7 @@ objects. -------------------------- A :class:`timedelta` object represents a duration, the difference between two -dates or times. +:class:`.datetime` or :class:`date` instances. .. class:: timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0) @@ -400,30 +400,7 @@ objects (see below). the :func:`divmod` function. True division and multiplication of a :class:`timedelta` object by a :class:`float` object are now supported. - -Comparisons of :class:`timedelta` objects are supported, with some caveats. - -The comparisons ``==`` or ``!=`` *always* return a :class:`bool`, no matter -the type of the compared object:: - - >>> from datetime import timedelta - >>> delta1 = timedelta(seconds=57) - >>> delta2 = timedelta(hours=25, seconds=2) - >>> delta2 != delta1 - True - >>> delta2 == 5 - False - -For all other comparisons (such as ``<`` and ``>``), when a :class:`timedelta` -object is compared to an object of a different type, :exc:`TypeError` -is raised:: - - >>> delta2 > delta1 - True - >>> delta2 > 5 - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - TypeError: '>' not supported between instances of 'datetime.timedelta' and 'int' +:class:`timedelta` objects support equality and order comparisons. In Boolean contexts, a :class:`timedelta` object is considered to be true if and only if it isn't equal to ``timedelta(0)``. @@ -536,7 +513,15 @@ Other constructors, all class methods: .. classmethod:: date.fromisoformat(date_string) Return a :class:`date` corresponding to a *date_string* given in any valid - ISO 8601 format, except ordinal dates (e.g. ``YYYY-DDD``):: + ISO 8601 format, with the following exceptions: + + 1. Reduced precision dates are not currently supported (``YYYY-MM``, + ``YYYY``). + 2. Extended date representations are not currently supported + (``±YYYYYY-MM-DD``). + 3. Ordinal dates are not currently supported (``YYYY-OOO``). + + Examples:: >>> from datetime import date >>> date.fromisoformat('2019-12-04') @@ -606,8 +591,13 @@ Supported operations: +-------------------------------+----------------------------------------------+ | ``timedelta = date1 - date2`` | \(3) | +-------------------------------+----------------------------------------------+ -| ``date1 < date2`` | *date1* is considered less than *date2* when | -| | *date1* precedes *date2* in time. (4) | +| | ``date1 == date2`` | Equality comparison. (4) | +| | ``date1 != date2`` | | ++-------------------------------+----------------------------------------------+ +| | ``date1 < date2`` | Order comparison. (5) | +| | ``date1 > date2`` | | +| | ``date1 <= date2`` | | +| | ``date1 >= date2`` | | +-------------------------------+----------------------------------------------+ Notes: @@ -627,15 +617,12 @@ Notes: timedelta.microseconds are 0, and date2 + timedelta == date1 after. (4) + :class:`date` objects are equal if they represent the same date. + +(5) + *date1* is considered less than *date2* when *date1* precedes *date2* in time. In other words, ``date1 < date2`` if and only if ``date1.toordinal() < - date2.toordinal()``. Date comparison raises :exc:`TypeError` if - the other comparand isn't also a :class:`date` object. However, - ``NotImplemented`` is returned instead if the other comparand has a - :meth:`timetuple` attribute. This hook gives other kinds of date objects a - chance at implementing mixed-type comparison. If not, when a :class:`date` - object is compared to an object of a different type, :exc:`TypeError` is raised - unless the comparison is ``==`` or ``!=``. The latter cases return - :const:`False` or :const:`True`, respectively. + date2.toordinal()``. In Boolean contexts, all :class:`date` objects are considered to be true. @@ -1017,8 +1004,12 @@ Other constructors, all class methods: 1. Time zone offsets may have fractional seconds. 2. The ``T`` separator may be replaced by any single unicode character. - 3. Ordinal dates are not currently supported. - 4. Fractional hours and minutes are not supported. + 3. Fractional hours and minutes are not supported. + 4. Reduced precision dates are not currently supported (``YYYY-MM``, + ``YYYY``). + 5. Extended date representations are not currently supported + (``±YYYYYY-MM-DD``). + 6. Ordinal dates are not currently supported (``YYYY-OOO``). Examples:: @@ -1158,8 +1149,13 @@ Supported operations: +---------------------------------------+--------------------------------+ | ``timedelta = datetime1 - datetime2`` | \(3) | +---------------------------------------+--------------------------------+ -| ``datetime1 < datetime2`` | Compares :class:`.datetime` to | -| | :class:`.datetime`. (4) | +| | ``datetime1 == datetime2`` | Equality comparison. (4) | +| | ``datetime1 != datetime2`` | | ++---------------------------------------+--------------------------------+ +| | ``datetime1 < datetime2`` | Order comparison. (5) | +| | ``datetime1 > datetime2`` | | +| | ``datetime1 <= datetime2`` | | +| | ``datetime1 >= datetime2`` | | +---------------------------------------+--------------------------------+ (1) @@ -1187,39 +1183,40 @@ Supported operations: are done in this case. If both are aware and have different :attr:`~.datetime.tzinfo` attributes, ``a-b`` acts - as if *a* and *b* were first converted to naive UTC datetimes first. The + as if *a* and *b* were first converted to naive UTC datetimes. The result is ``(a.replace(tzinfo=None) - a.utcoffset()) - (b.replace(tzinfo=None) - b.utcoffset())`` except that the implementation never overflows. (4) - *datetime1* is considered less than *datetime2* when *datetime1* precedes - *datetime2* in time. + :class:`.datetime` objects are equal if they represent the same date + and time, taking into account the time zone. - If one comparand is naive and the other is aware, :exc:`TypeError` - is raised if an order comparison is attempted. For equality - comparisons, naive instances are never equal to aware instances. + Naive and aware :class:`!datetime` objects are never equal. + :class:`!datetime` objects are never equal to :class:`date` objects + that are not also :class:`!datetime` instances, even if they represent + the same date. - If both comparands are aware, and have the same :attr:`~.datetime.tzinfo` attribute, the - common :attr:`~.datetime.tzinfo` attribute is ignored and the base datetimes are - compared. If both comparands are aware and have different :attr:`~.datetime.tzinfo` - attributes, the comparands are first adjusted by subtracting their UTC - offsets (obtained from ``self.utcoffset()``). + If both comparands are aware and have different :attr:`~.datetime.tzinfo` + attributes, the comparison acts as comparands were first converted to UTC + datetimes except that the implementation never overflows. + :class:`!datetime` instances in a repeated interval are never equal to + :class:`!datetime` instances in other time zone. - .. versionchanged:: 3.3 - Equality comparisons between aware and naive :class:`.datetime` - instances don't raise :exc:`TypeError`. +(5) + *datetime1* is considered less than *datetime2* when *datetime1* precedes + *datetime2* in time, taking into account the time zone. - .. note:: + Order comparison between naive and aware :class:`.datetime` objects, + as well as a :class:`!datetime` object and a :class:`!date` object + that is not also a :class:`!datetime` instance, raises :exc:`TypeError`. + + If both comparands are aware and have different :attr:`~.datetime.tzinfo` + attributes, the comparison acts as comparands were first converted to UTC + datetimes except that the implementation never overflows. - In order to stop comparison from falling back to the default scheme of comparing - object addresses, datetime comparison normally raises :exc:`TypeError` if the - other comparand isn't also a :class:`.datetime` object. However, - ``NotImplemented`` is returned instead if the other comparand has a - :meth:`timetuple` attribute. This hook gives other kinds of date objects a - chance at implementing mixed-type comparison. If not, when a :class:`.datetime` - object is compared to an object of a different type, :exc:`TypeError` is raised - unless the comparison is ``==`` or ``!=``. The latter cases return - :const:`False` or :const:`True`, respectively. +.. versionchanged:: 3.3 + Equality comparisons between aware and naive :class:`.datetime` + instances don't raise :exc:`TypeError`. Instance methods: @@ -1255,7 +1252,7 @@ Instance methods: ``tzinfo=None`` can be specified to create a naive datetime from an aware datetime with no conversion of date and time data. - :class:`datetime` objects are also supported by generic function + :class:`.datetime` objects are also supported by generic function :func:`copy.replace`. .. versionchanged:: 3.6 @@ -1347,22 +1344,22 @@ Instance methods: where ``yday = d.toordinal() - date(d.year, 1, 1).toordinal() + 1`` is the day number within the current year starting with ``1`` for January - 1st. The :attr:`tm_isdst` flag of the result is set according to the + 1st. The :attr:`~time.struct_time.tm_isdst` flag of the result is set according to the :meth:`dst` method: :attr:`.tzinfo` is ``None`` or :meth:`dst` returns - ``None``, :attr:`tm_isdst` is set to ``-1``; else if :meth:`dst` returns a - non-zero value, :attr:`tm_isdst` is set to ``1``; else :attr:`tm_isdst` is + ``None``, :attr:`!tm_isdst` is set to ``-1``; else if :meth:`dst` returns a + non-zero value, :attr:`!tm_isdst` is set to ``1``; else :attr:`!tm_isdst` is set to ``0``. .. method:: datetime.utctimetuple() If :class:`.datetime` instance *d* is naive, this is the same as - ``d.timetuple()`` except that :attr:`tm_isdst` is forced to 0 regardless of what + ``d.timetuple()`` except that :attr:`~.time.struct_time.tm_isdst` is forced to 0 regardless of what ``d.dst()`` returns. DST is never in effect for a UTC time. If *d* is aware, *d* is normalized to UTC time, by subtracting ``d.utcoffset()``, and a :class:`time.struct_time` for the - normalized time is returned. :attr:`tm_isdst` is forced to 0. Note + normalized time is returned. :attr:`!tm_isdst` is forced to 0. Note that an :exc:`OverflowError` may be raised if *d*.year was ``MINYEAR`` or ``MAXYEAR`` and UTC adjustment spills over a year boundary. @@ -1550,7 +1547,7 @@ Instance methods: Examples of Usage: :class:`.datetime` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Examples of working with :class:`~datetime.datetime` objects: +Examples of working with :class:`.datetime` objects: .. doctest:: @@ -1678,7 +1675,7 @@ Usage of ``KabulTz`` from above:: :class:`.time` Objects ---------------------- -A :class:`time` object represents a (local) time of day, independent of any particular +A :class:`.time` object represents a (local) time of day, independent of any particular day, and subject to adjustment via a :class:`tzinfo` object. .. class:: time(hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0) @@ -1754,24 +1751,21 @@ Instance attributes (read-only): .. versionadded:: 3.6 -:class:`.time` objects support comparison of :class:`.time` to :class:`.time`, -where *a* is considered less -than *b* when *a* precedes *b* in time. If one comparand is naive and the other -is aware, :exc:`TypeError` is raised if an order comparison is attempted. For equality -comparisons, naive instances are never equal to aware instances. +:class:`.time` objects support equality and order comparisons, +where *a* is considered less than *b* when *a* precedes *b* in time. + +Naive and aware :class:`!time` objects are never equal. +Order comparison between naive and aware :class:`!time` objects raises +:exc:`TypeError`. If both comparands are aware, and have -the same :attr:`~time.tzinfo` attribute, the common :attr:`~time.tzinfo` attribute is +the same :attr:`~.time.tzinfo` attribute, the common :attr:`!tzinfo` attribute is ignored and the base times are compared. If both comparands are aware and -have different :attr:`~time.tzinfo` attributes, the comparands are first adjusted by -subtracting their UTC offsets (obtained from ``self.utcoffset()``). In order -to stop mixed-type comparisons from falling back to the default comparison by -object address, when a :class:`.time` object is compared to an object of a -different type, :exc:`TypeError` is raised unless the comparison is ``==`` or -``!=``. The latter cases return :const:`False` or :const:`True`, respectively. +have different :attr:`!tzinfo` attributes, the comparands are first adjusted by +subtracting their UTC offsets (obtained from ``self.utcoffset()``). .. versionchanged:: 3.3 - Equality comparisons between aware and naive :class:`~datetime.time` instances + Equality comparisons between aware and naive :class:`.time` instances don't raise :exc:`TypeError`. In Boolean contexts, a :class:`.time` object is always considered to be true. @@ -1836,7 +1830,7 @@ Instance methods: ``tzinfo=None`` can be specified to create a naive :class:`.time` from an aware :class:`.time`, without conversion of the time data. - :class:`time` objects are also supported by generic function + :class:`.time` objects are also supported by generic function :func:`copy.replace`. .. versionchanged:: 3.6 @@ -1981,7 +1975,7 @@ Examples of working with a :class:`.time` object:: You need to derive a concrete subclass, and (at least) supply implementations of the standard :class:`tzinfo` methods needed by the - :class:`.datetime` methods you use. The :mod:`datetime` module provides + :class:`.datetime` methods you use. The :mod:`!datetime` module provides :class:`timezone`, a simple concrete subclass of :class:`tzinfo` which can represent timezones with fixed offset from UTC such as UTC itself or North American EST and EDT. @@ -1994,7 +1988,7 @@ Examples of working with a :class:`.time` object:: A concrete subclass of :class:`tzinfo` may need to implement the following methods. Exactly which methods are needed depends on the uses made of aware - :mod:`datetime` objects. If in doubt, simply implement all of them. + :mod:`!datetime` objects. If in doubt, simply implement all of them. .. method:: tzinfo.utcoffset(dt) @@ -2035,7 +2029,7 @@ Examples of working with a :class:`.time` object:: already been added to the UTC offset returned by :meth:`utcoffset`, so there's no need to consult :meth:`dst` unless you're interested in obtaining DST info separately. For example, :meth:`datetime.timetuple` calls its :attr:`~.datetime.tzinfo` - attribute's :meth:`dst` method to determine how the :attr:`tm_isdst` flag + attribute's :meth:`dst` method to determine how the :attr:`~time.struct_time.tm_isdst` flag should be set, and :meth:`tzinfo.fromutc` calls :meth:`dst` to account for DST changes when crossing time zones. @@ -2051,7 +2045,7 @@ Examples of working with a :class:`.time` object:: relies on this, but cannot detect violations; it's the programmer's responsibility to ensure it. If a :class:`tzinfo` subclass cannot guarantee this, it may be able to override the default implementation of - :meth:`tzinfo.fromutc` to work correctly with :meth:`astimezone` regardless. + :meth:`tzinfo.fromutc` to work correctly with :meth:`~.datetime.astimezone` regardless. Most implementations of :meth:`dst` will probably look like one of these two:: @@ -2080,7 +2074,7 @@ Examples of working with a :class:`.time` object:: .. method:: tzinfo.tzname(dt) Return the time zone name corresponding to the :class:`.datetime` object *dt*, as - a string. Nothing about string names is defined by the :mod:`datetime` module, + a string. Nothing about string names is defined by the :mod:`!datetime` module, and there's no requirement that it mean anything in particular. For example, "GMT", "UTC", "-500", "-5:00", "EDT", "US/Eastern", "America/New York" are all valid replies. Return ``None`` if a string name isn't known. Note that this is @@ -2128,7 +2122,7 @@ There is one more :class:`tzinfo` method that a subclass may wish to override: different years. An example of a time zone the default :meth:`fromutc` implementation may not handle correctly in all cases is one where the standard offset (from UTC) depends on the specific date and time passed, which can happen - for political reasons. The default implementations of :meth:`astimezone` and + for political reasons. The default implementations of :meth:`~.datetime.astimezone` and :meth:`fromutc` may not produce the result you want if the result is one of the hours straddling the moment the standard offset changes. @@ -2194,10 +2188,10 @@ hour that can't be spelled unambiguously in local wall time: the last hour of daylight time. In Eastern, that's times of the form 5:MM UTC on the day daylight time ends. The local wall clock leaps from 1:59 (daylight time) back to 1:00 (standard time) again. Local times of the form 1:MM are ambiguous. -:meth:`astimezone` mimics the local clock's behavior by mapping two adjacent UTC +:meth:`~.datetime.astimezone` mimics the local clock's behavior by mapping two adjacent UTC hours into the same local hour then. In the Eastern example, UTC times of the form 5:MM and 6:MM both map to 1:MM when converted to Eastern, but earlier times -have the :attr:`~datetime.fold` attribute set to 0 and the later times have it set to 1. +have the :attr:`~.datetime.fold` attribute set to 0 and the later times have it set to 1. For example, at the Fall back transition of 2016, we get:: >>> u0 = datetime(2016, 11, 6, 4, tzinfo=timezone.utc) @@ -2212,10 +2206,10 @@ For example, at the Fall back transition of 2016, we get:: 07:00:00 UTC = 02:00:00 EST 0 Note that the :class:`.datetime` instances that differ only by the value of the -:attr:`~datetime.fold` attribute are considered equal in comparisons. +:attr:`~.datetime.fold` attribute are considered equal in comparisons. Applications that can't bear wall-time ambiguities should explicitly check the -value of the :attr:`~datetime.fold` attribute or avoid using hybrid +value of the :attr:`~.datetime.fold` attribute or avoid using hybrid :class:`tzinfo` subclasses; there are no ambiguities when using :class:`timezone`, or any other fixed-offset :class:`tzinfo` subclass (such as a class representing only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). @@ -2223,7 +2217,7 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. seealso:: :mod:`zoneinfo` - The :mod:`datetime` module has a basic :class:`timezone` class (for + The :mod:`!datetime` module has a basic :class:`timezone` class (for handling arbitrary fixed offsets from UTC) and its :attr:`timezone.utc` attribute (a UTC timezone instance). @@ -2241,7 +2235,7 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. _datetime-timezone: :class:`timezone` Objects --------------------------- +------------------------- The :class:`timezone` class is a subclass of :class:`tzinfo`, each instance of which represents a timezone defined by a fixed offset from @@ -2316,8 +2310,8 @@ Class attributes: .. _strftime-strptime-behavior: -:meth:`strftime` and :meth:`strptime` Behavior ----------------------------------------------- +:meth:`~.datetime.strftime` and :meth:`~.datetime.strptime` Behavior +-------------------------------------------------------------------- :class:`date`, :class:`.datetime`, and :class:`.time` objects all support a ``strftime(format)`` method, to create a string representing the time under the @@ -2327,8 +2321,8 @@ Conversely, the :meth:`datetime.strptime` class method creates a :class:`.datetime` object from a string representing a date and time and a corresponding format string. -The table below provides a high-level comparison of :meth:`strftime` -versus :meth:`strptime`: +The table below provides a high-level comparison of :meth:`~.datetime.strftime` +versus :meth:`~.datetime.strptime`: +----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ | | ``strftime`` | ``strptime`` | @@ -2345,8 +2339,8 @@ versus :meth:`strptime`: .. _format-codes: -:meth:`strftime` and :meth:`strptime` Format Codes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:meth:`~.datetime.strftime` and :meth:`~.datetime.strptime` Format Codes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ These methods accept format codes that can be used to parse and format dates:: @@ -2485,13 +2479,13 @@ convenience. These parameters all correspond to ISO 8601 date values. | | naive). | -03:07:12.345216 | | +-----------+--------------------------------+------------------------+-------+ -These may not be available on all platforms when used with the :meth:`strftime` +These may not be available on all platforms when used with the :meth:`~.datetime.strftime` method. The ISO 8601 year and ISO 8601 week directives are not interchangeable -with the year and week number directives above. Calling :meth:`strptime` with +with the year and week number directives above. Calling :meth:`~.datetime.strptime` with incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. The full set of format codes supported varies across platforms, because Python -calls the platform C library's :func:`strftime` function, and platform +calls the platform C library's :c:func:`strftime` function, and platform variations are common. To see the full set of format codes supported on your platform, consult the :manpage:`strftime(3)` documentation. There are also differences between platforms in handling of unsupported format specifiers. @@ -2507,9 +2501,9 @@ Technical Detail Broadly speaking, ``d.strftime(fmt)`` acts like the :mod:`time` module's ``time.strftime(fmt, d.timetuple())`` although not all objects support a -:meth:`timetuple` method. +:meth:`~date.timetuple` method. -For the :meth:`datetime.strptime` class method, the default value is +For the :meth:`.datetime.strptime` class method, the default value is ``1900-01-01T00:00:00.000``: any components not specified in the format string will be pulled from the default value. [#]_ @@ -2522,7 +2516,7 @@ information, which are supported in ``datetime.strptime`` but are discarded by ``time.strptime``. For :class:`.time` objects, the format codes for year, month, and day should not -be used, as :class:`time` objects have no such values. If they're used anyway, +be used, as :class:`!time` objects have no such values. If they're used anyway, ``1900`` is substituted for the year, and ``1`` for the month and day. For :class:`date` objects, the format codes for hours, minutes, seconds, and @@ -2544,27 +2538,27 @@ Notes: contain non-ASCII characters. (2) - The :meth:`strptime` method can parse years in the full [1, 9999] range, but + The :meth:`~.datetime.strptime` method can parse years in the full [1, 9999] range, but years < 1000 must be zero-filled to 4-digit width. .. versionchanged:: 3.2 - In previous versions, :meth:`strftime` method was restricted to + In previous versions, :meth:`~.datetime.strftime` method was restricted to years >= 1900. .. versionchanged:: 3.3 - In version 3.2, :meth:`strftime` method was restricted to + In version 3.2, :meth:`~.datetime.strftime` method was restricted to years >= 1000. (3) - When used with the :meth:`strptime` method, the ``%p`` directive only affects + When used with the :meth:`~.datetime.strptime` method, the ``%p`` directive only affects the output hour field if the ``%I`` directive is used to parse the hour. (4) - Unlike the :mod:`time` module, the :mod:`datetime` module does not support + Unlike the :mod:`time` module, the :mod:`!datetime` module does not support leap seconds. (5) - When used with the :meth:`strptime` method, the ``%f`` directive + When used with the :meth:`~.datetime.strptime` method, the ``%f`` directive accepts from one to six digits and zero pads on the right. ``%f`` is an extension to the set of format characters in the C standard (but implemented separately in datetime objects, and therefore always @@ -2577,7 +2571,7 @@ Notes: For an aware object: ``%z`` - :meth:`utcoffset` is transformed into a string of the form + :meth:`~.datetime.utcoffset` is transformed into a string of the form ``±HHMM[SS[.ffffff]]``, where ``HH`` is a 2-digit string giving the number of UTC offset hours, ``MM`` is a 2-digit string giving the number of UTC offset minutes, ``SS`` is a 2-digit string giving the number of UTC offset @@ -2585,14 +2579,14 @@ Notes: offset microseconds. The ``ffffff`` part is omitted when the offset is a whole number of seconds and both the ``ffffff`` and the ``SS`` part is omitted when the offset is a whole number of minutes. For example, if - :meth:`utcoffset` returns ``timedelta(hours=-3, minutes=-30)``, ``%z`` is + :meth:`~.datetime.utcoffset` returns ``timedelta(hours=-3, minutes=-30)``, ``%z`` is replaced with the string ``'-0330'``. .. versionchanged:: 3.7 The UTC offset is not restricted to a whole number of minutes. .. versionchanged:: 3.7 - When the ``%z`` directive is provided to the :meth:`strptime` method, + When the ``%z`` directive is provided to the :meth:`~.datetime.strptime` method, the UTC offsets can have a colon as a separator between hours, minutes and seconds. For example, ``'+01:00:00'`` will be parsed as an offset of one hour. @@ -2603,11 +2597,11 @@ Notes: hours, minutes and seconds. ``%Z`` - In :meth:`strftime`, ``%Z`` is replaced by an empty string if - :meth:`tzname` returns ``None``; otherwise ``%Z`` is replaced by the + In :meth:`~.datetime.strftime`, ``%Z`` is replaced by an empty string if + :meth:`~.datetime.tzname` returns ``None``; otherwise ``%Z`` is replaced by the returned value, which must be a string. - :meth:`strptime` only accepts certain values for ``%Z``: + :meth:`~.datetime.strptime` only accepts certain values for ``%Z``: 1. any value in ``time.tzname`` for your machine's locale 2. the hard-coded values ``UTC`` and ``GMT`` @@ -2617,23 +2611,23 @@ Notes: invalid values. .. versionchanged:: 3.2 - When the ``%z`` directive is provided to the :meth:`strptime` method, an + When the ``%z`` directive is provided to the :meth:`~.datetime.strptime` method, an aware :class:`.datetime` object will be produced. The ``tzinfo`` of the result will be set to a :class:`timezone` instance. (7) - When used with the :meth:`strptime` method, ``%U`` and ``%W`` are only used + When used with the :meth:`~.datetime.strptime` method, ``%U`` and ``%W`` are only used in calculations when the day of the week and the calendar year (``%Y``) are specified. (8) Similar to ``%U`` and ``%W``, ``%V`` is only used in calculations when the day of the week and the ISO year (``%G``) are specified in a - :meth:`strptime` format string. Also note that ``%G`` and ``%Y`` are not + :meth:`~.datetime.strptime` format string. Also note that ``%G`` and ``%Y`` are not interchangeable. (9) - When used with the :meth:`strptime` method, the leading zero is optional + When used with the :meth:`~.datetime.strptime` method, the leading zero is optional for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%j``, ``%U``, ``%W``, and ``%V``. Format ``%y`` does require a leading zero. diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index cb95c61322582f6..9bb5e5f89509568 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -28,70 +28,85 @@ the Oracle Berkeley DB. available --- :mod:`dbm.gnu`, :mod:`dbm.ndbm` or :mod:`dbm.dumb` --- should be used to open a given file. - Returns one of the following values: ``None`` if the file can't be opened - because it's unreadable or doesn't exist; the empty string (``''``) if the - file's format can't be guessed; or a string containing the required module - name, such as ``'dbm.ndbm'`` or ``'dbm.gnu'``. + Return one of the following values: -.. versionchanged:: 3.11 - Accepts :term:`path-like object` for filename. + * ``None`` if the file can't be opened because it's unreadable or doesn't exist + * the empty string (``''``) if the file's format can't be guessed + * a string containing the required module name, such as ``'dbm.ndbm'`` or ``'dbm.gnu'`` -.. function:: open(file, flag='r', mode=0o666) + .. versionchanged:: 3.11 + *filename* accepts a :term:`path-like object`. - Open the database file *file* and return a corresponding object. +.. Substitutions for the open() flag param docs; + all submodules use the same text. - If the database file already exists, the :func:`whichdb` function is used to - determine its type and the appropriate module is used; if it does not exist, - the first module listed above that can be imported is used. +.. |flag_r| replace:: + Open existing database for reading only. - The optional *flag* argument can be: +.. |flag_w| replace:: + Open existing database for reading and writing. - +---------+-------------------------------------------+ - | Value | Meaning | - +=========+===========================================+ - | ``'r'`` | Open existing database for reading only | - | | (default) | - +---------+-------------------------------------------+ - | ``'w'`` | Open existing database for reading and | - | | writing | - +---------+-------------------------------------------+ - | ``'c'`` | Open database for reading and writing, | - | | creating it if it doesn't exist | - +---------+-------------------------------------------+ - | ``'n'`` | Always create a new, empty database, open | - | | for reading and writing | - +---------+-------------------------------------------+ +.. |flag_c| replace:: + Open database for reading and writing, creating it if it doesn't exist. - The optional *mode* argument is the Unix mode of the file, used only when the - database has to be created. It defaults to octal ``0o666`` (and will be - modified by the prevailing umask). +.. |flag_n| replace:: + Always create a new, empty database, open for reading and writing. +.. |mode_param_doc| replace:: + The Unix file access mode of the file (default: octal ``0o666``), + used only when the database has to be created. -The object returned by :func:`.open` supports the same basic functionality as -dictionaries; keys and their corresponding values can be stored, retrieved, and -deleted, and the :keyword:`in` operator and the :meth:`keys` method are -available, as well as :meth:`get` and :meth:`setdefault`. +.. |incompat_note| replace:: + The file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible + and can not be used interchangeably. -.. versionchanged:: 3.2 - :meth:`get` and :meth:`setdefault` are now available in all database modules. +.. function:: open(file, flag='r', mode=0o666) -.. versionchanged:: 3.8 - Deleting a key from a read-only database raises database module specific error - instead of :exc:`KeyError`. + Open a database and return the corresponding database object. + + :param file: + The database file to open. + + If the database file already exists, the :func:`whichdb` function is used to + determine its type and the appropriate module is used; if it does not exist, + the first submodule listed above that can be imported is used. + :type file: :term:`path-like object` + + :param str flag: + * ``'r'`` (default): |flag_r| + * ``'w'``: |flag_w| + * ``'c'``: |flag_c| + * ``'n'``: |flag_n| -.. versionchanged:: 3.11 - Accepts :term:`path-like object` for file. + :param int mode: + |mode_param_doc| -Key and values are always stored as bytes. This means that when + .. versionchanged:: 3.11 + *file* accepts a :term:`path-like object`. + +The object returned by :func:`~dbm.open` supports the same basic functionality as a +:class:`dict`; keys and their corresponding values can be stored, retrieved, and +deleted, and the :keyword:`in` operator and the :meth:`!keys` method are +available, as well as :meth:`!get` and :meth:`!setdefault` methods. + +Key and values are always stored as :class:`bytes`. This means that when strings are used they are implicitly converted to the default encoding before being stored. These objects also support being used in a :keyword:`with` statement, which will automatically close them when done. +.. versionchanged:: 3.2 + :meth:`!get` and :meth:`!setdefault` methods are now available for all + :mod:`dbm` backends. + .. versionchanged:: 3.4 Added native support for the context management protocol to the objects - returned by :func:`.open`. + returned by :func:`~dbm.open`. + +.. versionchanged:: 3.8 + Deleting a key from a read-only database raises a database module specific exception + instead of :exc:`KeyError`. The following example records some hostnames and a corresponding title, and then prints out the contents of the database:: @@ -130,27 +145,26 @@ then prints out the contents of the database:: The individual submodules are described in the following sections. -:mod:`dbm.gnu` --- GNU's reinterpretation of dbm ------------------------------------------------- +:mod:`dbm.gnu` --- GNU database manager +--------------------------------------- .. module:: dbm.gnu :platform: Unix - :synopsis: GNU's reinterpretation of dbm. + :synopsis: GNU database manager **Source code:** :source:`Lib/dbm/gnu.py` -------------- -This module is quite similar to the :mod:`dbm` module, but uses the GNU library -``gdbm`` instead to provide some additional functionality. Please note that the -file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible. +The :mod:`dbm.gnu` module provides an interface to the :abbr:`GDBM (GNU dbm)` +library, similar to the :mod:`dbm.ndbm` module, but with additional +functionality like crash tolerance. -The :mod:`dbm.gnu` module provides an interface to the GNU DBM library. -``dbm.gnu.gdbm`` objects behave like mappings (dictionaries), except that keys and -values are always converted to bytes before storing. Printing a ``gdbm`` -object doesn't print the -keys and values, and the :meth:`items` and :meth:`values` methods are not -supported. +:class:`!gdbm` objects behave similar to :term:`mappings <mapping>`, +except that keys and values are always converted to :class:`bytes` before storing, +and the :meth:`!items` and :meth:`!values` methods are not supported. + +.. note:: |incompat_note| .. exception:: error @@ -158,62 +172,52 @@ supported. raised for general mapping errors like specifying an incorrect key. -.. function:: open(filename[, flag[, mode]]) - - Open a ``gdbm`` database and return a :class:`gdbm` object. The *filename* - argument is the name of the database file. - - The optional *flag* argument can be: - - +---------+-------------------------------------------+ - | Value | Meaning | - +=========+===========================================+ - | ``'r'`` | Open existing database for reading only | - | | (default) | - +---------+-------------------------------------------+ - | ``'w'`` | Open existing database for reading and | - | | writing | - +---------+-------------------------------------------+ - | ``'c'`` | Open database for reading and writing, | - | | creating it if it doesn't exist | - +---------+-------------------------------------------+ - | ``'n'`` | Always create a new, empty database, open | - | | for reading and writing | - +---------+-------------------------------------------+ - - The following additional characters may be appended to the flag to control - how the database is opened: - - +---------+--------------------------------------------+ - | Value | Meaning | - +=========+============================================+ - | ``'f'`` | Open the database in fast mode. Writes | - | | to the database will not be synchronized. | - +---------+--------------------------------------------+ - | ``'s'`` | Synchronized mode. This will cause changes | - | | to the database to be immediately written | - | | to the file. | - +---------+--------------------------------------------+ - | ``'u'`` | Do not lock database. | - +---------+--------------------------------------------+ - - Not all flags are valid for all versions of ``gdbm``. The module constant - :const:`open_flags` is a string of supported flag characters. The exception - :exc:`error` is raised if an invalid flag is specified. - - The optional *mode* argument is the Unix mode of the file, used only when the - database has to be created. It defaults to octal ``0o666``. - - In addition to the dictionary-like methods, ``gdbm`` objects have the - following methods: +.. function:: open(filename, flag="r", mode=0o666, /) + + Open a GDBM database and return a :class:`!gdbm` object. + + :param filename: + The database file to open. + :type filename: :term:`path-like object` + + :param str flag: + * ``'r'`` (default): |flag_r| + * ``'w'``: |flag_w| + * ``'c'``: |flag_c| + * ``'n'``: |flag_n| + + The following additional characters may be appended + to control how the database is opened: + + * ``'f'``: Open the database in fast mode. + Writes to the database will not be synchronized. + * ``'s'``: Synchronized mode. + Changes to the database will be written immediately to the file. + * ``'u'``: Do not lock database. + + Not all flags are valid for all versions of GDBM. + See the :data:`open_flags` member for a list of supported flag characters. + + :param int mode: + |mode_param_doc| + + :raises error: + If an invalid *flag* argument is passed. .. versionchanged:: 3.11 - Accepts :term:`path-like object` for filename. + *filename* accepts a :term:`path-like object`. + + .. data:: open_flags + + A string of characters the *flag* parameter of :meth:`~dbm.gnu.open` supports. + + In addition to the dictionary-like methods, :class:`gdbm` objects have the + following methods and attributes: .. method:: gdbm.firstkey() It's possible to loop over every key in the database using this method and the - :meth:`nextkey` method. The traversal is ordered by ``gdbm``'s internal + :meth:`nextkey` method. The traversal is ordered by GDBM's internal hash values, and won't be sorted by the key values. This method returns the starting key. @@ -231,7 +235,7 @@ supported. .. method:: gdbm.reorganize() If you have carried out a lot of deletions and would like to shrink the space - used by the ``gdbm`` file, this routine will reorganize the database. ``gdbm`` + used by the GDBM file, this routine will reorganize the database. :class:`!gdbm` objects will not shorten the length of a database file except by using this reorganization; otherwise, deleted file space will be kept and reused as new (key, value) pairs are added. @@ -243,38 +247,40 @@ supported. .. method:: gdbm.close() - Close the ``gdbm`` database. + Close the GDBM database. .. method:: gdbm.clear() - Remove all items from the ``gdbm`` database. + Remove all items from the GDBM database. .. versionadded:: 3.13 -:mod:`dbm.ndbm` --- Interface based on ndbm -------------------------------------------- +:mod:`dbm.ndbm` --- New Database Manager +---------------------------------------- .. module:: dbm.ndbm :platform: Unix - :synopsis: The standard "database" interface, based on ndbm. + :synopsis: The New Database Manager **Source code:** :source:`Lib/dbm/ndbm.py` -------------- -The :mod:`dbm.ndbm` module provides an interface to the Unix "(n)dbm" library. -Dbm objects behave like mappings (dictionaries), except that keys and values are -always stored as bytes. Printing a ``dbm`` object doesn't print the keys and -values, and the :meth:`items` and :meth:`values` methods are not supported. +The :mod:`dbm.ndbm` module provides an interface to the +:abbr:`NDBM (New Database Manager)` library. +:class:`!ndbm` objects behave similar to :term:`mappings <mapping>`, +except that keys and values are always stored as :class:`bytes`, +and the :meth:`!items` and :meth:`!values` methods are not supported. -This module can be used with the "classic" ndbm interface or the GNU GDBM -compatibility interface. On Unix, the :program:`configure` script will attempt -to locate the appropriate header file to simplify building this module. +This module can be used with the "classic" NDBM interface or the +:abbr:`GDBM (GNU dbm)` compatibility interface. + +.. note:: |incompat_note| .. warning:: - The ndbm library shipped as part of macOS has an undocumented limitation on the + The NDBM library shipped as part of macOS has an undocumented limitation on the size of values, which can result in corrupted database files when storing values larger than this limit. Reading such corrupted files can result in a hard crash (segmentation fault). @@ -287,37 +293,28 @@ to locate the appropriate header file to simplify building this module. .. data:: library - Name of the ``ndbm`` implementation library used. + Name of the NDBM implementation library used. -.. function:: open(filename[, flag[, mode]]) +.. function:: open(filename, flag="r", mode=0o666, /) - Open a dbm database and return a ``ndbm`` object. The *filename* argument is the - name of the database file (without the :file:`.dir` or :file:`.pag` extensions). + Open an NDBM database and return an :class:`!ndbm` object. - The optional *flag* argument must be one of these values: + :param filename: + The basename of the database file + (without the :file:`.dir` or :file:`.pag` extensions). + :type filename: :term:`path-like object` - +---------+-------------------------------------------+ - | Value | Meaning | - +=========+===========================================+ - | ``'r'`` | Open existing database for reading only | - | | (default) | - +---------+-------------------------------------------+ - | ``'w'`` | Open existing database for reading and | - | | writing | - +---------+-------------------------------------------+ - | ``'c'`` | Open database for reading and writing, | - | | creating it if it doesn't exist | - +---------+-------------------------------------------+ - | ``'n'`` | Always create a new, empty database, open | - | | for reading and writing | - +---------+-------------------------------------------+ + :param str flag: + * ``'r'`` (default): |flag_r| + * ``'w'``: |flag_w| + * ``'c'``: |flag_c| + * ``'n'``: |flag_n| - The optional *mode* argument is the Unix mode of the file, used only when the - database has to be created. It defaults to octal ``0o666`` (and will be - modified by the prevailing umask). + :param int mode: + |mode_param_doc| - In addition to the dictionary-like methods, ``ndbm`` objects + In addition to the dictionary-like methods, :class:`!ndbm` objects provide the following method: .. versionchanged:: 3.11 @@ -325,11 +322,11 @@ to locate the appropriate header file to simplify building this module. .. method:: ndbm.close() - Close the ``ndbm`` database. + Close the NDBM database. .. method:: ndbm.clear() - Remove all items from the ``ndbm`` database. + Remove all items from the NDBM database. .. versionadded:: 3.13 @@ -353,13 +350,14 @@ to locate the appropriate header file to simplify building this module. -------------- -The :mod:`dbm.dumb` module provides a persistent dictionary-like interface which -is written entirely in Python. Unlike other modules such as :mod:`dbm.gnu` no -external library is required. As with other persistent mappings, the keys and -values are always stored as bytes. - -The module defines the following: +The :mod:`dbm.dumb` module provides a persistent :class:`dict`-like +interface which is written entirely in Python. +Unlike other :mod:`dbm` backends, such as :mod:`dbm.gnu`, no +external library is required. +As with other :mod:`dbm` backends, +the keys and values are always stored as :class:`bytes`. +The :mod:`!dbm.dumb` module defines the following: .. exception:: error @@ -367,34 +365,29 @@ The module defines the following: raised for general mapping errors like specifying an incorrect key. -.. function:: open(filename[, flag[, mode]]) +.. function:: open(filename, flag="c", mode=0o666) - Open a ``dumbdbm`` database and return a dumbdbm object. The *filename* argument is - the basename of the database file (without any specific extensions). When a - dumbdbm database is created, files with :file:`.dat` and :file:`.dir` extensions - are created. + Open a :mod:`!dbm.dumb` database. + The returned database object behaves similar to a :term:`mapping`, + in addition to providing :meth:`~dumbdbm.sync` and :meth:`~dumbdbm.close` + methods. - The optional *flag* argument can be: + :param filename: + The basename of the database file (without extensions). + A new database creates the following files: - +---------+-------------------------------------------+ - | Value | Meaning | - +=========+===========================================+ - | ``'r'`` | Open existing database for reading only | - | | (default) | - +---------+-------------------------------------------+ - | ``'w'`` | Open existing database for reading and | - | | writing | - +---------+-------------------------------------------+ - | ``'c'`` | Open database for reading and writing, | - | | creating it if it doesn't exist | - +---------+-------------------------------------------+ - | ``'n'`` | Always create a new, empty database, open | - | | for reading and writing | - +---------+-------------------------------------------+ + - :file:`{filename}.dat` + - :file:`{filename}.dir` + :type database: :term:`path-like object` - The optional *mode* argument is the Unix mode of the file, used only when the - database has to be created. It defaults to octal ``0o666`` (and will be modified - by the prevailing umask). + :param str flag: + * ``'r'``: |flag_r| + * ``'w'``: |flag_w| + * ``'c'`` (default): |flag_c| + * ``'n'``: |flag_n| + + :param int mode: + |mode_param_doc| .. warning:: It is possible to crash the Python interpreter when loading a database @@ -402,20 +395,18 @@ The module defines the following: Python's AST compiler. .. versionchanged:: 3.5 - :func:`.open` always creates a new database when the flag has the value - ``'n'``. + :func:`~dbm.dumb.open` always creates a new database when *flag* is ``'n'``. .. versionchanged:: 3.8 - A database opened with flags ``'r'`` is now read-only. Opening with - flags ``'r'`` and ``'w'`` no longer creates a database if it does not - exist. + A database opened read-only if *flag* is ``'r'``. + A database is not created if it does not exist if *flag* is ``'r'`` or ``'w'``. .. versionchanged:: 3.11 - Accepts :term:`path-like object` for filename. + *filename* accepts a :term:`path-like object`. In addition to the methods provided by the - :class:`collections.abc.MutableMapping` class, :class:`dumbdbm` objects - provide the following methods: + :class:`collections.abc.MutableMapping` class, + the following methods are provided: .. method:: dumbdbm.sync() @@ -424,5 +415,5 @@ The module defines the following: .. method:: dumbdbm.close() - Close the ``dumbdbm`` database. + Close the database. diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index b97d48fafab3b6c..e654760fb91c650 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -546,8 +546,8 @@ operations on it as if it was a Python list. The top of the stack corresponds to .. opcode:: END_FOR - Removes the top two values from the stack. - Equivalent to ``POP_TOP``; ``POP_TOP``. + Removes the top-of-stack item. + Equivalent to ``POP_TOP``. Used to clean up at the end of loops, hence the name. .. versionadded:: 3.12 diff --git a/Doc/library/email.mime.rst b/Doc/library/email.mime.rst index d7c0d203d191f86..dc0dd3b9eebde6e 100644 --- a/Doc/library/email.mime.rst +++ b/Doc/library/email.mime.rst @@ -28,7 +28,7 @@ make things easier. Here are the classes: -.. currentmodule:: email.mime.base +.. module:: email.mime.base .. class:: MIMEBase(_maintype, _subtype, *, policy=compat32, **_params) @@ -58,7 +58,7 @@ Here are the classes: Added *policy* keyword-only parameter. -.. currentmodule:: email.mime.nonmultipart +.. module:: email.mime.nonmultipart .. class:: MIMENonMultipart() @@ -72,7 +72,7 @@ Here are the classes: is called, a :exc:`~email.errors.MultipartConversionError` exception is raised. -.. currentmodule:: email.mime.multipart +.. module:: email.mime.multipart .. class:: MIMEMultipart(_subtype='mixed', boundary=None, _subparts=None, \ *, policy=compat32, **_params) @@ -104,7 +104,7 @@ Here are the classes: .. versionchanged:: 3.6 Added *policy* keyword-only parameter. -.. currentmodule:: email.mime.application +.. module:: email.mime.application .. class:: MIMEApplication(_data, _subtype='octet-stream', \ _encoder=email.encoders.encode_base64, \ @@ -135,7 +135,7 @@ Here are the classes: .. versionchanged:: 3.6 Added *policy* keyword-only parameter. -.. currentmodule:: email.mime.audio +.. module:: email.mime.audio .. class:: MIMEAudio(_audiodata, _subtype=None, \ _encoder=email.encoders.encode_base64, \ @@ -169,7 +169,7 @@ Here are the classes: .. versionchanged:: 3.6 Added *policy* keyword-only parameter. -.. currentmodule:: email.mime.image +.. module:: email.mime.image .. class:: MIMEImage(_imagedata, _subtype=None, \ _encoder=email.encoders.encode_base64, \ @@ -205,7 +205,7 @@ Here are the classes: .. versionchanged:: 3.6 Added *policy* keyword-only parameter. -.. currentmodule:: email.mime.message +.. module:: email.mime.message .. class:: MIMEMessage(_msg, _subtype='rfc822', *, policy=compat32) @@ -225,7 +225,7 @@ Here are the classes: .. versionchanged:: 3.6 Added *policy* keyword-only parameter. -.. currentmodule:: email.mime.text +.. module:: email.mime.text .. class:: MIMEText(_text, _subtype='plain', _charset=None, *, policy=compat32) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 07b15e23b2c10a4..30d80ce8d488ccb 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -286,6 +286,19 @@ Data Types appropriate value will be chosen for you. See :class:`auto` for the details. + .. attribute:: Enum._name_ + + Name of the member. + + .. attribute:: Enum._value_ + + Value of the member, can be set in :meth:`~object.__new__`. + + .. attribute:: Enum._order_ + + No longer used, kept for backward compatibility. + (class attribute, removed during class creation). + .. attribute:: Enum._ignore_ ``_ignore_`` is only used during creation and is removed from the @@ -337,6 +350,17 @@ Data Types >>> PowersOfThree.SECOND.value 9 + .. method:: Enum.__init__(self, \*args, \**kwds) + + By default, does nothing. If multiple values are given in the member + assignment, those values become separate arguments to ``__init__``; e.g. + + >>> from enum import Enum + >>> class Weekday(Enum): + ... MONDAY = 1, 'Mon' + + ``Weekday.__init__()`` would be called as ``Weekday.__init__(self, 1, 'Mon')`` + .. method:: Enum.__init_subclass__(cls, \**kwds) A *classmethod* that is used to further configure subsequent subclasses. @@ -364,6 +388,18 @@ Data Types >>> Build('deBUG') <Build.DEBUG: 'debug'> + .. method:: Enum.__new__(cls, \*args, \**kwds) + + By default, doesn't exist. If specified, either in the enum class + definition or in a mixin class (such as ``int``), all values given + in the member assignment will be passed; e.g. + + >>> from enum import Enum + >>> class MyIntEnum(Enum): + ... SEVENTEEN = '1a', 16 + + results in the call ``int('1a', 16)`` and a value of ``17`` for the member. + .. method:: Enum.__repr__(self) Returns the string used for *repr()* calls. By default, returns the @@ -477,9 +513,9 @@ Data Types .. class:: Flag - *Flag* members support the bitwise operators ``&`` (*AND*), ``|`` (*OR*), - ``^`` (*XOR*), and ``~`` (*INVERT*); the results of those operators are members - of the enumeration. + ``Flag`` is the same as :class:`Enum`, but its members support the bitwise + operators ``&`` (*AND*), ``|`` (*OR*), ``^`` (*XOR*), and ``~`` (*INVERT*); + the results of those operators are members of the enumeration. .. method:: __contains__(self, value) @@ -511,9 +547,7 @@ Data Types >>> list(purple) [<Color.RED: 1>, <Color.BLUE: 4>] - .. versionchanged:: 3.11 - - Aliases are no longer returned during iteration. + .. versionadded:: 3.11 .. method:: __len__(self): @@ -802,8 +836,8 @@ Supported ``_sunder_`` names - :attr:`~Enum._ignore_` -- a list of names, either as a :class:`list` or a :class:`str`, that will not be transformed into members, and will be removed from the final class -- :attr:`~Enum._order_` -- used in Python 2/3 code to ensure member order is - consistent (class attribute, removed during class creation) +- :attr:`~Enum._order_` -- no longer used, kept for backward + compatibility (class attribute, removed during class creation) - :meth:`~Enum._generate_next_value_` -- used to get an appropriate value for an enum member; may be overridden diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index f821776c2861331..3191315049ad5a4 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -16,7 +16,7 @@ equivalent, even if they have the same name. .. index:: pair: statement; raise -The built-in exceptions listed below can be generated by the interpreter or +The built-in exceptions listed in this chapter can be generated by the interpreter or built-in functions. Except where mentioned, they have an "associated value" indicating the detailed cause of the error. This may be a string or a tuple of several items of information (e.g., an error code and a string explaining the diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index e93a1e85598e3a7..2f98a272c297ae3 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -78,6 +78,9 @@ FTP objects A 2-tuple ``(host, port)`` for the socket to bind to as its source address before connecting. +.. |param_doc_encoding| replace:: + The encoding for directories and filenames (default: ``'utf-8'``). + .. class:: FTP(host='', user='', passwd='', acct='', timeout=None, \ source_address=None, *, encoding='utf-8') @@ -108,8 +111,7 @@ FTP objects :type source_address: tuple | None :param str encoding: - The *encoding* parameter specifies the encoding - for directories and filenames. + |param_doc_encoding| The :class:`FTP` class supports the :keyword:`with` statement, e.g.: @@ -447,19 +449,53 @@ FTP_TLS objects .. class:: FTP_TLS(host='', user='', passwd='', acct='', *, context=None, \ timeout=None, source_address=None, encoding='utf-8') - A :class:`FTP` subclass which adds TLS support to FTP as described in + An :class:`FTP` subclass which adds TLS support to FTP as described in :rfc:`4217`. - Connect as usual to port 21 implicitly securing the FTP control connection - before authenticating. Securing the data connection requires the user to - explicitly ask for it by calling the :meth:`prot_p` method. *context* - is a :class:`ssl.SSLContext` object which allows bundling SSL configuration - options, certificates and private keys into a single (potentially - long-lived) structure. Please read :ref:`ssl-security` for best practices. + Connect to port 21 implicitly securing the FTP control connection + before authenticating. + + .. note:: + The user must explicitly secure the data connection + by calling the :meth:`prot_p` method. + + :param str host: + The hostname to connect to. + If given, :code:`connect(host)` is implicitly called by the constructor. + + :param str user: + |param_doc_user| + If given, :code:`login(host, passwd, acct)` is implicitly called + by the constructor. + + :param str passwd: + |param_doc_passwd| + + :param str acct: + |param_doc_acct| + + :param context: + An SSL context object which allows bundling SSL configuration options, + certificates and private keys into a single, potentially long-lived, + structure. + Please read :ref:`ssl-security` for best practices. + :type context: :class:`ssl.SSLContext` + + :param timeout: + A timeout in seconds for blocking operations like :meth:`~FTP.connect` + (default: the global default timeout setting). + :type timeout: int | None + + :param source_address: + |param_doc_source_address| + :type source_address: tuple | None + + :param str encoding: + |param_doc_encoding| .. versionadded:: 3.2 .. versionchanged:: 3.3 - *source_address* parameter was added. + Added the *source_address* parameter. .. versionchanged:: 3.4 The class now supports hostname check with diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index 6e4f72c19ff4c97..19a0bbba8966bae 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -147,8 +147,9 @@ The :mod:`glob` module defines the following functions: .. seealso:: - :meth:`pathlib.PurePath.match` and :meth:`pathlib.Path.glob` methods, - which call this function to implement pattern matching and globbing. + :meth:`pathlib.PurePath.full_match` and :meth:`pathlib.Path.glob` + methods, which call this function to implement pattern matching and + globbing. .. versionadded:: 3.13 diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index 50cde09fa10a9d9..79be215a7660454 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -61,7 +61,7 @@ The module defines the following items: .. exception:: BadGzipFile - An exception raised for invalid gzip files. It inherits :exc:`OSError`. + An exception raised for invalid gzip files. It inherits from :exc:`OSError`. :exc:`EOFError` and :exc:`zlib.error` can also be raised for invalid gzip files. @@ -287,4 +287,3 @@ Command line options .. option:: -h, --help Show the help message. - diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 1f774e64b0eae33..d5c868def3b64f7 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -531,7 +531,7 @@ An :class:`IMAP4` instance has the following methods: allowed creation of such tags, and popular IMAP servers, such as Gmail, accept and produce such flags. There are non-Python programs which also create such tags. Although it is an RFC violation and IMAP clients and - servers are supposed to be strict, imaplib nonetheless continues to allow + servers are supposed to be strict, imaplib still continues to allow such tags to be created for backward compatibility reasons, and as of Python 3.6, handles them if they are sent from the server, since this improves real-world compatibility. diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 6736aa9ee2b0efc..8eb531aa4ea2487 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -466,7 +466,7 @@ I/O Base Classes .. class:: RawIOBase - Base class for raw binary streams. It inherits :class:`IOBase`. + Base class for raw binary streams. It inherits from :class:`IOBase`. Raw binary streams typically provide low-level access to an underlying OS device or API, and do not try to encapsulate it in high-level primitives @@ -519,7 +519,7 @@ I/O Base Classes .. class:: BufferedIOBase Base class for binary streams that support some kind of buffering. - It inherits :class:`IOBase`. + It inherits from :class:`IOBase`. The main difference with :class:`RawIOBase` is that methods :meth:`read`, :meth:`readinto` and :meth:`write` will try (respectively) to read as much @@ -633,7 +633,7 @@ Raw File I/O .. class:: FileIO(name, mode='r', closefd=True, opener=None) A raw binary stream representing an OS-level file containing bytes data. It - inherits :class:`RawIOBase`. + inherits from :class:`RawIOBase`. The *name* can be one of two things: @@ -696,7 +696,7 @@ than raw I/O does. .. class:: BytesIO(initial_bytes=b'') - A binary stream using an in-memory bytes buffer. It inherits + A binary stream using an in-memory bytes buffer. It inherits from :class:`BufferedIOBase`. The buffer is discarded when the :meth:`~IOBase.close` method is called. @@ -745,7 +745,7 @@ than raw I/O does. .. class:: BufferedReader(raw, buffer_size=DEFAULT_BUFFER_SIZE) A buffered binary stream providing higher-level access to a readable, non - seekable :class:`RawIOBase` raw binary stream. It inherits + seekable :class:`RawIOBase` raw binary stream. It inherits from :class:`BufferedIOBase`. When reading data from this object, a larger amount of data may be @@ -783,7 +783,7 @@ than raw I/O does. .. class:: BufferedWriter(raw, buffer_size=DEFAULT_BUFFER_SIZE) A buffered binary stream providing higher-level access to a writeable, non - seekable :class:`RawIOBase` raw binary stream. It inherits + seekable :class:`RawIOBase` raw binary stream. It inherits from :class:`BufferedIOBase`. When writing to this object, data is normally placed into an internal @@ -818,7 +818,7 @@ than raw I/O does. .. class:: BufferedRandom(raw, buffer_size=DEFAULT_BUFFER_SIZE) A buffered binary stream providing higher-level access to a seekable - :class:`RawIOBase` raw binary stream. It inherits :class:`BufferedReader` + :class:`RawIOBase` raw binary stream. It inherits from :class:`BufferedReader` and :class:`BufferedWriter`. The constructor creates a reader and writer for a seekable raw stream, given @@ -834,7 +834,7 @@ than raw I/O does. A buffered binary stream providing higher-level access to two non seekable :class:`RawIOBase` raw binary streams---one readable, the other writeable. - It inherits :class:`BufferedIOBase`. + It inherits from :class:`BufferedIOBase`. *reader* and *writer* are :class:`RawIOBase` objects that are readable and writeable respectively. If the *buffer_size* is omitted it defaults to @@ -857,7 +857,7 @@ Text I/O .. class:: TextIOBase Base class for text streams. This class provides a character and line based - interface to stream I/O. It inherits :class:`IOBase`. + interface to stream I/O. It inherits from :class:`IOBase`. :class:`TextIOBase` provides or overrides these data attributes and methods in addition to those from :class:`IOBase`: @@ -946,7 +946,7 @@ Text I/O line_buffering=False, write_through=False) A buffered text stream providing higher-level access to a - :class:`BufferedIOBase` buffered binary stream. It inherits + :class:`BufferedIOBase` buffered binary stream. It inherits from :class:`TextIOBase`. *encoding* gives the name of the encoding that the stream will be decoded or @@ -1073,7 +1073,7 @@ Text I/O .. class:: StringIO(initial_value='', newline='\n') - A text stream using an in-memory text buffer. It inherits + A text stream using an in-memory text buffer. It inherits from :class:`TextIOBase`. The text buffer is discarded when the :meth:`~IOBase.close` method is @@ -1124,7 +1124,7 @@ Text I/O .. class:: IncrementalNewlineDecoder A helper codec that decodes newlines for :term:`universal newlines` mode. - It inherits :class:`codecs.IncrementalDecoder`. + It inherits from :class:`codecs.IncrementalDecoder`. Performance diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 4b756d10b4c586c..39eb41ce1f16708 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -531,12 +531,12 @@ subclasses. However, the :meth:`!__init__` method in subclasses needs to call This method should be called from handlers when an exception is encountered during an :meth:`emit` call. If the module-level attribute - ``raiseExceptions`` is ``False``, exceptions get silently ignored. This is + :data:`raiseExceptions` is ``False``, exceptions get silently ignored. This is what is mostly wanted for a logging system - most users will not care about errors in the logging system, they are more interested in application errors. You could, however, replace this with a custom handler if you wish. The specified record is the one which was being processed when the exception - occurred. (The default value of ``raiseExceptions`` is ``True``, as that is + occurred. (The default value of :data:`raiseExceptions` is ``True``, as that is more useful during development). @@ -1494,6 +1494,18 @@ Module-Level Attributes .. versionadded:: 3.2 +.. attribute:: raiseExceptions + + Used to see if exceptions during handling should be propagated. + + Default: ``True``. + + If :data:`raiseExceptions` is ``False``, + exceptions get silently ignored. This is what is mostly wanted + for a logging system - most users will not care about errors in + the logging system, they are more interested in application errors. + + Integration with the warnings module ------------------------------------ diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index fa5b273093f583f..a613548c9e518e0 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -1136,8 +1136,8 @@ When a :class:`!MaildirMessage` instance is created based upon a leading "From " or trailing newline. For convenience, *time_* may be specified and will be formatted appropriately and appended to *from_*. If *time_* is specified, it should be a :class:`time.struct_time` instance, a - tuple suitable for passing to :meth:`time.strftime`, or ``True`` (to use - :meth:`time.gmtime`). + tuple suitable for passing to :func:`time.strftime`, or ``True`` (to use + :func:`time.gmtime`). .. method:: get_flags() @@ -1508,8 +1508,8 @@ When a :class:`!BabylMessage` instance is created based upon an leading "From " or trailing newline. For convenience, *time_* may be specified and will be formatted appropriately and appended to *from_*. If *time_* is specified, it should be a :class:`time.struct_time` instance, a - tuple suitable for passing to :meth:`time.strftime`, or ``True`` (to use - :meth:`time.gmtime`). + tuple suitable for passing to :func:`time.strftime`, or ``True`` (to use + :func:`time.gmtime`). .. method:: get_flags() diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 789a84b02d59d23..b104a6483b70e6f 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -56,7 +56,7 @@ will print to standard output :: The :class:`Process` class -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^ In :mod:`multiprocessing`, processes are spawned by creating a :class:`Process` object and then calling its :meth:`~Process.start` method. :class:`Process` @@ -102,7 +102,7 @@ necessary, see :ref:`multiprocessing-programming`. .. _multiprocessing-start-methods: Contexts and start methods -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^ Depending on the platform, :mod:`multiprocessing` supports three ways to start a process. These *start methods* are @@ -231,7 +231,7 @@ library user. Exchanging objects between processes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :mod:`multiprocessing` supports two types of communication channel between processes: @@ -283,7 +283,7 @@ processes: Synchronization between processes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :mod:`multiprocessing` contains equivalents of all the synchronization primitives from :mod:`threading`. For instance one can use a lock to ensure @@ -309,7 +309,7 @@ mixed up. Sharing state between processes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ As mentioned above, when doing concurrent programming it is usually best to avoid using shared state as far as possible. This is particularly true when @@ -399,7 +399,7 @@ However, if you really do need to use some shared data then Using a pool of workers -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^ The :class:`~multiprocessing.pool.Pool` class represents a pool of worker processes. It has methods which allows tasks to be offloaded to the worker @@ -490,7 +490,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the :class:`Process` and exceptions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: Process(group=None, target=None, name=None, args=(), kwargs={}, \ *, daemon=None) @@ -724,7 +724,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the Raised by methods with a timeout when the timeout expires. Pipes and Queues -~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^ When using multiple processes, one generally uses message passing for communication between processes and avoids having to use any synchronization @@ -981,7 +981,7 @@ For an example of the usage of queues for interprocess communication see Miscellaneous -~~~~~~~~~~~~~ +^^^^^^^^^^^^^ .. function:: active_children() @@ -1150,7 +1150,7 @@ Miscellaneous Connection Objects -~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^ .. currentmodule:: multiprocessing.connection @@ -1292,7 +1292,7 @@ For example: Synchronization primitives -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^ .. currentmodule:: multiprocessing @@ -1481,7 +1481,7 @@ object -- see :ref:`multiprocessing-managers`. Shared :mod:`ctypes` Objects -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ It is possible to create shared objects using shared memory which can be inherited by child processes. @@ -1543,7 +1543,7 @@ inherited by child processes. The :mod:`multiprocessing.sharedctypes` module ->>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +"""""""""""""""""""""""""""""""""""""""""""""" .. module:: multiprocessing.sharedctypes :synopsis: Allocate ctypes objects from shared memory. @@ -1709,7 +1709,7 @@ The results printed are :: .. _multiprocessing-managers: Managers -~~~~~~~~ +^^^^^^^^ Managers provide a way to create data which can be shared between different processes, including sharing over a network between processes running on @@ -1954,7 +1954,7 @@ their parent process exits. The manager classes are defined in the Customized managers ->>>>>>>>>>>>>>>>>>> +""""""""""""""""""" To create one's own manager, one creates a subclass of :class:`BaseManager` and uses the :meth:`~BaseManager.register` classmethod to register new types or @@ -1981,7 +1981,7 @@ callables with the manager class. For example:: Using a remote manager ->>>>>>>>>>>>>>>>>>>>>> +"""""""""""""""""""""" It is possible to run a manager server on one machine and have clients use it from other machines (assuming that the firewalls involved allow it). @@ -2044,7 +2044,7 @@ client to access it remotely:: .. _multiprocessing-proxy_objects: Proxy Objects -~~~~~~~~~~~~~ +^^^^^^^^^^^^^ A proxy is an object which *refers* to a shared object which lives (presumably) in a different process. The shared object is said to be the *referent* of the @@ -2196,7 +2196,7 @@ demonstrates a level of control over the synchronization. Cleanup ->>>>>>> +""""""" A proxy object uses a weakref callback so that when it gets garbage collected it deregisters itself from the manager which owns its referent. @@ -2206,7 +2206,7 @@ any proxies referring to it. Process Pools -~~~~~~~~~~~~~ +^^^^^^^^^^^^^ .. module:: multiprocessing.pool :synopsis: Create pools of processes. @@ -2442,7 +2442,7 @@ The following example demonstrates the use of a pool:: .. _multiprocessing-listeners-clients: Listeners and Clients -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^ .. module:: multiprocessing.connection :synopsis: API for dealing with sockets. @@ -2665,7 +2665,7 @@ wait for messages from multiple processes at once:: .. _multiprocessing-address-formats: Address Formats ->>>>>>>>>>>>>>> +""""""""""""""" * An ``'AF_INET'`` address is a tuple of the form ``(hostname, port)`` where *hostname* is a string and *port* is an integer. @@ -2685,7 +2685,7 @@ an ``'AF_PIPE'`` address rather than an ``'AF_UNIX'`` address. .. _multiprocessing-auth-keys: Authentication keys -~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^ When one uses :meth:`Connection.recv <Connection.recv>`, the data received is automatically @@ -2711,7 +2711,7 @@ Suitable authentication keys can also be generated by using :func:`os.urandom`. Logging -~~~~~~~ +^^^^^^^ Some support for logging is available. Note, however, that the :mod:`logging` package does not use process shared locks so it is possible (depending on the @@ -2759,7 +2759,7 @@ For a full table of logging levels, see the :mod:`logging` module. The :mod:`multiprocessing.dummy` module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: multiprocessing.dummy :synopsis: Dumb wrapper around threading. @@ -2818,7 +2818,7 @@ There are certain guidelines and idioms which should be adhered to when using All start methods -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^ The following applies to all start methods. @@ -2977,7 +2977,7 @@ Beware of replacing :data:`sys.stdin` with a "file like object" For more information, see :issue:`5155`, :issue:`5313` and :issue:`5331` The *spawn* and *forkserver* start methods -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ There are a few extra restriction which don't apply to the *fork* start method. diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index 10d7f061fb759b7..933fd07d62418a8 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -23,7 +23,7 @@ processes, a :class:`~multiprocessing.managers.BaseManager` subclass, :class:`~multiprocessing.managers.SharedMemoryManager`, is also provided in the :mod:`multiprocessing.managers` module. -In this module, shared memory refers to "System V style" shared memory blocks +In this module, shared memory refers to "POSIX style" shared memory blocks (though is not necessarily implemented explicitly as such) and does not refer to "distributed shared memory". This style of shared memory permits distinct processes to potentially read and write to a common (or shared) region of diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 3cab7a260df0084..34bc76b231de921 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -326,6 +326,28 @@ the :mod:`glob` module.) .. versionadded:: 3.12 +.. function:: isreserved(path) + + Return ``True`` if *path* is a reserved pathname on the current system. + + On Windows, reserved filenames include those that end with a space or dot; + those that contain colons (i.e. file streams such as "name:stream"), + wildcard characters (i.e. ``'*?"<>'``), pipe, or ASCII control characters; + as well as DOS device names such as "NUL", "CON", "CONIN$", "CONOUT$", + "AUX", "PRN", "COM1", and "LPT1". + + .. note:: + + This function approximates rules for reserved paths on most Windows + systems. These rules change over time in various Windows releases. + This function may be updated in future Python releases as changes to + the rules become broadly available. + + .. availability:: Windows. + + .. versionadded:: 3.13 + + .. function:: join(path, *paths) Join one or more path segments intelligently. The return value is the diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 0008ec6a40c76ff..cc9f3e75a80c517 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1122,6 +1122,20 @@ as internal buffering of data. .. versionchanged:: 3.12 Added support for pipes on Windows. + +.. function:: grantpt(fd, /) + + Grant access to the slave pseudo-terminal device associated with the + master pseudo-terminal device to which the file descriptor *fd* refers. + The file descriptor *fd* is not closed upon failure. + + Calls the C standard library function :c:func:`grantpt`. + + .. availability:: Unix, not Emscripten, not WASI. + + .. versionadded:: 3.13 + + .. function:: isatty(fd, /) Return ``True`` if the file descriptor *fd* is open and connected to a @@ -1429,6 +1443,23 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo .. versionadded:: 3.3 +.. function:: posix_openpt(oflag, /) + + Open and return a file descriptor for a master pseudo-terminal device. + + Calls the C standard library function :c:func:`posix_openpt`. The *oflag* + argument is used to set file status flags and file access modes as + specified in the manual page of :c:func:`posix_openpt` of your system. + + The returned file descriptor is :ref:`non-inheritable <fd_inheritance>`. + If the value :data:`O_CLOEXEC` is available on the system, it is added to + *oflag*. + + .. availability:: Unix, not Emscripten, not WASI. + + .. versionadded:: 3.13 + + .. function:: preadv(fd, buffers, offset, flags=0, /) Read from a file descriptor *fd* at a position of *offset* into mutable @@ -1486,6 +1517,21 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo .. versionadded:: 3.7 +.. function:: ptsname(fd, /) + + Return the name of the slave pseudo-terminal device associated with the + master pseudo-terminal device to which the file descriptor *fd* refers. + The file descriptor *fd* is not closed upon failure. + + Calls the reentrant C standard library function :c:func:`ptsname_r` if + it is available; otherwise, the C standard library function + :c:func:`ptsname`, which is not guaranteed to be thread-safe, is called. + + .. availability:: Unix, not Emscripten, not WASI. + + .. versionadded:: 3.13 + + .. function:: pwrite(fd, str, offset, /) Write the bytestring in *str* to file descriptor *fd* at position of @@ -1738,6 +1784,19 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo .. availability:: Unix. +.. function:: unlockpt(fd, /) + + Unlock the slave pseudo-terminal device associated with the master + pseudo-terminal device to which the file descriptor *fd* refers. + The file descriptor *fd* is not closed upon failure. + + Calls the C standard library function :c:func:`unlockpt`. + + .. availability:: Unix, not Emscripten, not WASI. + + .. versionadded:: 3.13 + + .. function:: write(fd, str, /) Write the bytestring in *str* to file descriptor *fd*. diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index fcbc0bf489b3445..f94b6fb38056847 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -535,14 +535,13 @@ Pure paths provide the following methods and properties: reserved under Windows, ``False`` otherwise. With :class:`PurePosixPath`, ``False`` is always returned. - >>> PureWindowsPath('nul').is_reserved() - True - >>> PurePosixPath('nul').is_reserved() - False - - File system calls on reserved paths can fail mysteriously or have - unintended effects. + .. versionchanged:: 3.13 + Windows path names that contain a colon, or end with a dot or a space, + are considered reserved. UNC paths may be reserved. + .. deprecated-removed:: 3.13 3.15 + This method is deprecated; use :func:`os.path.isreserved` to detect + reserved paths on Windows. .. method:: PurePath.joinpath(*pathsegments) @@ -559,55 +558,55 @@ Pure paths provide the following methods and properties: PureWindowsPath('c:/Program Files') -.. method:: PurePath.match(pattern, *, case_sensitive=None) +.. method:: PurePath.full_match(pattern, *, case_sensitive=None) Match this path against the provided glob-style pattern. Return ``True`` - if matching is successful, ``False`` otherwise. + if matching is successful, ``False`` otherwise. For example:: - If *pattern* is relative, the path can be either relative or absolute, - and matching is done from the right:: - - >>> PurePath('a/b.py').match('*.py') - True - >>> PurePath('/a/b/c.py').match('b/*.py') + >>> PurePath('a/b.py').full_match('a/*.py') True - >>> PurePath('/a/b/c.py').match('a/*.py') + >>> PurePath('a/b.py').full_match('*.py') False + >>> PurePath('/a/b/c.py').full_match('/a/**') + True + >>> PurePath('/a/b/c.py').full_match('**/*.py') + True - If *pattern* is absolute, the path must be absolute, and the whole path - must match:: + As with other methods, case-sensitivity follows platform defaults:: - >>> PurePath('/a.py').match('/*.py') - True - >>> PurePath('a/b.py').match('/*.py') + >>> PurePosixPath('b.py').full_match('*.PY') False + >>> PureWindowsPath('b.py').full_match('*.PY') + True - The *pattern* may be another path object; this speeds up matching the same - pattern against multiple files:: + Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. - >>> pattern = PurePath('*.py') - >>> PurePath('a/b.py').match(pattern) - True + .. versionadded:: 3.13 - .. versionchanged:: 3.12 - Accepts an object implementing the :class:`os.PathLike` interface. - As with other methods, case-sensitivity follows platform defaults:: +.. method:: PurePath.match(pattern, *, case_sensitive=None) - >>> PurePosixPath('b.py').match('*.PY') - False - >>> PureWindowsPath('b.py').match('*.PY') + Match this path against the provided non-recursive glob-style pattern. + Return ``True`` if matching is successful, ``False`` otherwise. + + This method is similar to :meth:`~PurePath.full_match`, but empty patterns + aren't allowed (:exc:`ValueError` is raised), the recursive wildcard + "``**``" isn't supported (it acts like non-recursive "``*``"), and if a + relative pattern is provided, then matching is done from the right:: + + >>> PurePath('a/b.py').match('*.py') True + >>> PurePath('/a/b/c.py').match('b/*.py') + True + >>> PurePath('/a/b/c.py').match('a/*.py') + False - Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. + .. versionchanged:: 3.12 + The *pattern* parameter accepts a :term:`path-like object`. .. versionchanged:: 3.12 The *case_sensitive* parameter was added. - .. versionchanged:: 3.13 - Support for the recursive wildcard "``**``" was added. In previous - versions, it acted like the non-recursive wildcard "``*``". - .. method:: PurePath.relative_to(other, walk_up=False) @@ -1039,9 +1038,8 @@ call fails (for example because the path doesn't exist). The *follow_symlinks* parameter was added. .. versionchanged:: 3.13 - Emits :exc:`FutureWarning` if the pattern ends with "``**``". In a - future Python release, patterns with this ending will match both files - and directories. Add a trailing slash to match only directories. + Return files and directories if *pattern* ends with "``**``". In + previous versions, only directories were returned. .. versionchanged:: 3.13 The *pattern* parameter accepts a :term:`path-like object`. diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index cfb251fca5c7cd9..acada092afb679b 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -272,13 +272,13 @@ The :mod:`pickle` module defines three exceptions: .. exception:: PickleError - Common base class for the other pickling exceptions. It inherits + Common base class for the other pickling exceptions. It inherits from :exc:`Exception`. .. exception:: PicklingError Error raised when an unpicklable object is encountered by :class:`Pickler`. - It inherits :exc:`PickleError`. + It inherits from :exc:`PickleError`. Refer to :ref:`pickle-picklable` to learn what kinds of objects can be pickled. @@ -286,7 +286,7 @@ The :mod:`pickle` module defines three exceptions: .. exception:: UnpicklingError Error raised when there is a problem unpickling an object, such as a data - corruption or a security violation. It inherits :exc:`PickleError`. + corruption or a security violation. It inherits from :exc:`PickleError`. Note that other exceptions may also be raised during unpickling, including (but not necessarily limited to) AttributeError, EOFError, ImportError, and @@ -356,7 +356,7 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, :func:`copyreg.pickle`. It is a mapping whose keys are classes and whose values are reduction functions. A reduction function takes a single argument of the associated class and should - conform to the same interface as a :meth:`__reduce__` + conform to the same interface as a :meth:`~object.__reduce__` method. By default, a pickler object will not have a @@ -376,7 +376,7 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, Special reducer that can be defined in :class:`Pickler` subclasses. This method has priority over any reducer in the :attr:`dispatch_table`. It - should conform to the same interface as a :meth:`__reduce__` method, and + should conform to the same interface as a :meth:`~object.__reduce__` method, and can optionally return ``NotImplemented`` to fallback on :attr:`dispatch_table`-registered reducers to pickle ``obj``. @@ -516,7 +516,7 @@ The following types can be pickled: * classes accessible from the top level of a module; -* instances of such classes whose the result of calling :meth:`__getstate__` +* instances of such classes whose the result of calling :meth:`~object.__getstate__` is picklable (see section :ref:`pickle-inst` for details). Attempts to pickle unpicklable objects will raise the :exc:`PicklingError` @@ -552,7 +552,7 @@ purpose, so you can fix bugs in a class or add methods to the class and still load objects that were created with an earlier version of the class. If you plan to have long-lived objects that will see many versions of a class, it may be worthwhile to put a version number in the objects so that suitable -conversions can be made by the class's :meth:`__setstate__` method. +conversions can be made by the class's :meth:`~object.__setstate__` method. .. _pickle-inst: @@ -567,7 +567,7 @@ customize, and control how class instances are pickled and unpickled. In most cases, no additional code is needed to make instances picklable. By default, pickle will retrieve the class and the attributes of an instance via -introspection. When a class instance is unpickled, its :meth:`__init__` method +introspection. When a class instance is unpickled, its :meth:`~object.__init__` method is usually *not* invoked. The default behaviour first creates an uninitialized instance and then restores the saved attributes. The following code shows an implementation of this behaviour:: @@ -658,30 +658,30 @@ methods: Refer to the section :ref:`pickle-state` for more information about how to use -the methods :meth:`__getstate__` and :meth:`__setstate__`. +the methods :meth:`~object.__getstate__` and :meth:`~object.__setstate__`. .. note:: - At unpickling time, some methods like :meth:`__getattr__`, - :meth:`__getattribute__`, or :meth:`__setattr__` may be called upon the + At unpickling time, some methods like :meth:`~object.__getattr__`, + :meth:`~object.__getattribute__`, or :meth:`~object.__setattr__` may be called upon the instance. In case those methods rely on some internal invariant being - true, the type should implement :meth:`__new__` to establish such an - invariant, as :meth:`__init__` is not called when unpickling an + true, the type should implement :meth:`~object.__new__` to establish such an + invariant, as :meth:`~object.__init__` is not called when unpickling an instance. .. index:: pair: copy; protocol As we shall see, pickle does not use directly the methods described above. In fact, these methods are part of the copy protocol which implements the -:meth:`__reduce__` special method. The copy protocol provides a unified +:meth:`~object.__reduce__` special method. The copy protocol provides a unified interface for retrieving the data necessary for pickling and copying objects. [#]_ -Although powerful, implementing :meth:`__reduce__` directly in your classes is +Although powerful, implementing :meth:`~object.__reduce__` directly in your classes is error prone. For this reason, class designers should use the high-level -interface (i.e., :meth:`__getnewargs_ex__`, :meth:`__getstate__` and -:meth:`__setstate__`) whenever possible. We will show, however, cases where -using :meth:`__reduce__` is the only option or leads to more efficient pickling +interface (i.e., :meth:`~object.__getnewargs_ex__`, :meth:`~object.__getstate__` and +:meth:`~object.__setstate__`) whenever possible. We will show, however, cases where +using :meth:`!__reduce__` is the only option or leads to more efficient pickling or both. .. method:: object.__reduce__() @@ -716,8 +716,9 @@ or both. These items will be appended to the object either using ``obj.append(item)`` or, in batch, using ``obj.extend(list_of_items)``. This is primarily used for list subclasses, but may be used by other - classes as long as they have :meth:`append` and :meth:`extend` methods with - the appropriate signature. (Whether :meth:`append` or :meth:`extend` is + classes as long as they have + :ref:`append and extend methods <typesseq-common>` with + the appropriate signature. (Whether :meth:`!append` or :meth:`!extend` is used depends on which pickle protocol version is used as well as the number of items to append, so both must be supported.) @@ -793,8 +794,8 @@ any other code which depends on pickling, then one can create a pickler with a private dispatch table. The global dispatch table managed by the :mod:`copyreg` module is -available as :data:`copyreg.dispatch_table`. Therefore, one may -choose to use a modified copy of :data:`copyreg.dispatch_table` as a +available as :data:`!copyreg.dispatch_table`. Therefore, one may +choose to use a modified copy of :data:`!copyreg.dispatch_table` as a private dispatch table. For example :: @@ -833,12 +834,12 @@ Handling Stateful Objects single: __setstate__() (copy protocol) Here's an example that shows how to modify pickling behavior for a class. -The :class:`TextReader` class opens a text file, and returns the line number and +The :class:`!TextReader` class below opens a text file, and returns the line number and line contents each time its :meth:`!readline` method is called. If a -:class:`TextReader` instance is pickled, all attributes *except* the file object +:class:`!TextReader` instance is pickled, all attributes *except* the file object member are saved. When the instance is unpickled, the file is reopened, and -reading resumes from the last location. The :meth:`__setstate__` and -:meth:`__getstate__` methods are used to implement this behavior. :: +reading resumes from the last location. The :meth:`!__setstate__` and +:meth:`!__getstate__` methods are used to implement this behavior. :: class TextReader: """Print and number lines in a text file.""" @@ -903,7 +904,7 @@ functions and classes. For those cases, it is possible to subclass from the :class:`Pickler` class and implement a :meth:`~Pickler.reducer_override` method. This method can return an -arbitrary reduction tuple (see :meth:`__reduce__`). It can alternatively return +arbitrary reduction tuple (see :meth:`~object.__reduce__`). It can alternatively return ``NotImplemented`` to fallback to the traditional behavior. If both the :attr:`~Pickler.dispatch_table` and @@ -971,7 +972,7 @@ provided by pickle protocol 5 and higher. Provider API ^^^^^^^^^^^^ -The large data objects to be pickled must implement a :meth:`__reduce_ex__` +The large data objects to be pickled must implement a :meth:`~object.__reduce_ex__` method specialized for protocol 5 and higher, which returns a :class:`PickleBuffer` instance (instead of e.g. a :class:`bytes` object) for any large data. diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index b2b787c5a8260cf..1421fc2e552f0e3 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -93,6 +93,14 @@ The :mod:`queue` module defines the following classes and exceptions: on a :class:`Queue` object which is full. +.. exception:: ShutDown + + Exception raised when :meth:`~Queue.put` or :meth:`~Queue.get` is called on + a :class:`Queue` object which has been shut down. + + .. versionadded:: 3.13 + + .. _queueobjects: Queue Objects @@ -135,6 +143,8 @@ provide the public methods described below. immediately available, else raise the :exc:`Full` exception (*timeout* is ignored in that case). + Raises :exc:`ShutDown` if the queue has been shut down. + .. method:: Queue.put_nowait(item) @@ -155,6 +165,9 @@ provide the public methods described below. an uninterruptible wait on an underlying lock. This means that no exceptions can occur, and in particular a SIGINT will not trigger a :exc:`KeyboardInterrupt`. + Raises :exc:`ShutDown` if the queue has been shut down and is empty, or if + the queue has been shut down immediately. + .. method:: Queue.get_nowait() @@ -177,6 +190,8 @@ fully processed by daemon consumer threads. Raises a :exc:`ValueError` if called more times than there were items placed in the queue. + Raises :exc:`ShutDown` if the queue has been shut down immediately. + .. method:: Queue.join() @@ -187,6 +202,8 @@ fully processed by daemon consumer threads. indicate that the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, :meth:`join` unblocks. + Raises :exc:`ShutDown` if the queue has been shut down immediately. + Example of how to wait for enqueued tasks to be completed:: @@ -214,6 +231,27 @@ Example of how to wait for enqueued tasks to be completed:: print('All work completed') +Terminating queues +^^^^^^^^^^^^^^^^^^ + +:class:`Queue` objects can be made to prevent further interaction by shutting +them down. + +.. method:: Queue.shutdown(immediate=False) + + Shut down the queue, making :meth:`~Queue.get` and :meth:`~Queue.put` raise + :exc:`ShutDown`. + + By default, :meth:`~Queue.get` on a shut down queue will only raise once the + queue is empty. Set *immediate* to true to make :meth:`~Queue.get` raise + immediately instead. + + All blocked callers of :meth:`~Queue.put` will be unblocked. If *immediate* + is true, also unblock callers of :meth:`~Queue.get` and :meth:`~Queue.join`. + + .. versionadded:: 3.13 + + SimpleQueue Objects ------------------- diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 1adafcaa02eab97..54c6d9f3b32b1aa 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -5,7 +5,7 @@ :platform: Unix :synopsis: GNU readline support for Python. -.. sectionauthor:: Skip Montanaro <skip@pobox.com> +.. sectionauthor:: Skip Montanaro <skip.montanaro@gmail.com> -------------- diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index d9ec2cbc47e6114..ff8c9a189ab3de2 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -360,21 +360,24 @@ Directory and files operations .. function:: move(src, dst, copy_function=copy2) - Recursively move a file or directory (*src*) to another location (*dst*) - and return the destination. + Recursively move a file or directory (*src*) to another location and return + the destination. - If the destination is an existing directory, then *src* is moved inside that - directory. If the destination already exists but is not a directory, it may - be overwritten depending on :func:`os.rename` semantics. + If *dst* is an existing directory or a symlink to a directory, then *src* + is moved inside that directory. The destination path in that directory must + not already exist. + + If *dst* already exists but is not a directory, it may be overwritten + depending on :func:`os.rename` semantics. If the destination is on the current filesystem, then :func:`os.rename` is - used. Otherwise, *src* is copied to *dst* using *copy_function* and then - removed. In case of symlinks, a new symlink pointing to the target of *src* - will be created in or as *dst* and *src* will be removed. + used. Otherwise, *src* is copied to the destination using *copy_function* + and then removed. In case of symlinks, a new symlink pointing to the target + of *src* will be created as the destination and *src* will be removed. - If *copy_function* is given, it must be a callable that takes two arguments - *src* and *dst*, and will be used to copy *src* to *dst* if - :func:`os.rename` cannot be used. If the source is a directory, + If *copy_function* is given, it must be a callable that takes two arguments, + *src* and the destination, and will be used to copy *src* to the destination + if :func:`os.rename` cannot be used. If the source is a directory, :func:`copytree` is called, passing it the *copy_function*. The default *copy_function* is :func:`copy2`. Using :func:`~shutil.copy` as the *copy_function* allows the move to succeed when it is not possible to also @@ -586,7 +589,9 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Create an archive file (such as zip or tar) and return its name. *base_name* is the name of the file to create, including the path, minus - any format-specific extension. *format* is the archive format: one of + any format-specific extension. + + *format* is the archive format: one of "zip" (if the :mod:`zlib` module is available), "tar", "gztar" (if the :mod:`zlib` module is available), "bztar" (if the :mod:`bz2` module is available), or "xztar" (if the :mod:`lzma` module is available). diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 5fd213fa613c8d4..864b1dadb785623 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -494,7 +494,7 @@ This is the server side:: def handle(self): # self.request is the TCP socket connected to the client self.data = self.request.recv(1024).strip() - print("{} wrote:".format(self.client_address[0])) + print("Received from {}:".format(self.client_address[0])) print(self.data) # just send back the same data, but upper-cased self.request.sendall(self.data.upper()) @@ -525,8 +525,9 @@ objects that simplify communication by providing the standard file interface):: The difference is that the ``readline()`` call in the second handler will call ``recv()`` multiple times until it encounters a newline character, while the -single ``recv()`` call in the first handler will just return what has been sent -from the client in one ``sendall()`` call. +single ``recv()`` call in the first handler will just return what has been +received so far from the client's ``sendall()`` call (typically all of it, but +this is not guaranteed by the TCP protocol). This is the client side:: diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index c3406b166c3d89d..87d5ef1e42ca3ac 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1137,12 +1137,19 @@ Connection objects .. _Loading an Extension: https://www.sqlite.org/loadext.html#loading_an_extension_ - .. method:: iterdump + .. method:: iterdump(*, filter=None) Return an :term:`iterator` to dump the database as SQL source code. Useful when saving an in-memory database for later restoration. Similar to the ``.dump`` command in the :program:`sqlite3` shell. + :param filter: + + An optional ``LIKE`` pattern for database objects to dump, e.g. ``prefix_%``. + If ``None`` (the default), all database objects will be included. + + :type filter: str | None + Example: .. testcode:: @@ -1158,6 +1165,8 @@ Connection objects :ref:`sqlite3-howto-encoding` + .. versionchanged:: 3.13 + Added the *filter* parameter. .. method:: backup(target, *, pages=-1, progress=None, name="main", sleep=0.250) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index e8709b516ae07a2..f9648fa6744bdc0 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -2574,12 +2574,8 @@ provided. :exc:`SSLWantReadError` if it needs more data than the incoming BIO has available. - - There is no module-level ``wrap_bio()`` call like there is for - :meth:`~SSLContext.wrap_socket`. An :class:`SSLObject` is always created - via an :class:`SSLContext`. - .. versionchanged:: 3.7 - :class:`SSLObject` instances must to created with + :class:`SSLObject` instances must be created with :meth:`~SSLContext.wrap_bio`. In earlier versions, it was possible to create instances directly. This was never documented or officially supported. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 9028ff5c134fa92..1a4c12590c1018d 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1528,7 +1528,7 @@ between them will be implicitly converted to a single string literal. That is, ``("spam " "eggs") == "spam eggs"``. See :ref:`strings` for more about the various forms of string literal, -including supported escape sequences, and the ``r`` ("raw") prefix that +including supported :ref:`escape sequences <escape-sequences>`, and the ``r`` ("raw") prefix that disables most escape sequence processing. Strings may also be created from other objects using the :class:`str` diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index c437ce770b37d0d..f63ca73b3ec067e 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -664,7 +664,8 @@ functions. If given, *startupinfo* will be a :class:`STARTUPINFO` object, which is passed to the underlying ``CreateProcess`` function. - *creationflags*, if given, can be one or more of the following flags: + + If given, *creationflags*, can be one or more of the following flags: * :data:`CREATE_NEW_CONSOLE` * :data:`CREATE_NEW_PROCESS_GROUP` diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 46159dcef940e7c..47568387f9a7ce0 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -97,7 +97,7 @@ Examining Symbol Tables .. class:: Function - A namespace for a function or method. This class inherits + A namespace for a function or method. This class inherits from :class:`SymbolTable`. .. method:: get_parameters() @@ -123,7 +123,7 @@ Examining Symbol Tables .. class:: Class - A namespace of a class. This class inherits :class:`SymbolTable`. + A namespace of a class. This class inherits from :class:`SymbolTable`. .. method:: get_methods() diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 762581b7eda7f1d..4980227c60b21e3 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -75,9 +75,6 @@ following IDs are pre-defined to make co-operation of tools easier:: sys.monitoring.PROFILER_ID = 2 sys.monitoring.OPTIMIZER_ID = 5 -There is no obligation to set an ID, nor is there anything preventing a tool -from using an ID even it is already in use. -However, tools are encouraged to use a unique ID and respect other tools. Events ------ diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index abf2c393a44928d..a97a369b77b88a8 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1293,7 +1293,10 @@ always available. The list of the original command line arguments passed to the Python executable. - See also :data:`sys.argv`. + The elements of :data:`sys.orig_argv` are the arguments to the Python interpreter, + while the elements of :data:`sys.argv` are the arguments to the user's program. + Arguments consumed by the interpreter itself will be present in :data:`sys.orig_argv` + and missing from :data:`sys.argv`. .. versionadded:: 3.10 diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 34a738a7f1c41f0..2134293a0bb0de1 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -673,6 +673,7 @@ be finalized; only the internally used file object will be closed. See the .. attribute:: TarFile.pax_headers + :type: dict A dictionary containing key-value pairs of pax global headers. @@ -838,26 +839,31 @@ A ``TarInfo`` object has the following public data attributes: attribute. .. attribute:: TarInfo.chksum + :type: int Header checksum. .. attribute:: TarInfo.devmajor + :type: int Device major number. .. attribute:: TarInfo.devminor + :type: int Device minor number. .. attribute:: TarInfo.offset + :type: int The tar header starts here. .. attribute:: TarInfo.offset_data + :type: int The file's data starts here. diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 9173db07fd00718..7d28f6253457263 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -143,7 +143,7 @@ guidelines to be followed: arg = (1, 2, 3) When using this pattern, remember that all classes that inherit from - :class:`unittest.TestCase` are run as tests. The :class:`Mixin` class in the example above + :class:`unittest.TestCase` are run as tests. The :class:`!TestFuncAcceptsSequencesMixin` class in the example above does not have any data and so can't be run by itself, thus it does not inherit from :class:`unittest.TestCase`. @@ -324,9 +324,9 @@ The :mod:`test.support` module defines the following constants: .. data:: Py_DEBUG - True if Python is built with the :c:macro:`Py_DEBUG` macro defined: if - Python is :ref:`built in debug mode <debug-build>` - (:option:`./configure --with-pydebug <--with-pydebug>`). + True if Python was built with the :c:macro:`Py_DEBUG` macro + defined, that is, if + Python was :ref:`built in debug mode <debug-build>`. .. versionadded:: 3.12 diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index b85b7f008d1594c..5fbf9379b8202c0 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -534,9 +534,10 @@ All methods are executed atomically. lock, subsequent attempts to acquire it block, until it is released; any thread may release it. - Note that ``Lock`` is actually a factory function which returns an instance - of the most efficient version of the concrete Lock class that is supported - by the platform. + .. versionchanged:: 3.13 + ``Lock`` is now a class. In earlier Pythons, ``Lock`` was a factory + function which returned an instance of the underlying private lock + type. .. method:: acquire(blocking=True, timeout=-1) diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 8ce67cf77253c30..c8c981024c1aeb1 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -398,6 +398,10 @@ Standard names are defined for the following types: data members which use standard conversion functions; it has the same purpose as the :class:`property` type, but for classes defined in extension modules. + In addition, when a class is defined with a :attr:`~object.__slots__` attribute, then for + each slot, an instance of :class:`!MemberDescriptorType` will be added as an attribute + on the class. This allows the slot to appear in the class's :attr:`~object.__dict__`. + .. impl-detail:: In other implementations of Python, this type may be identical to diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index f063e463753e0b8..b5a49d9c5923876 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -5,7 +5,7 @@ :synopsis: Load a robots.txt file and answer questions about fetchability of other URLs. -.. sectionauthor:: Skip Montanaro <skip@pobox.com> +.. sectionauthor:: Skip Montanaro <skip.montanaro@gmail.com> **Source code:** :source:`Lib/urllib/robotparser.py` diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index a9c469707e82273..500398636e11ae8 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -396,7 +396,7 @@ Available Functions ------------------- -.. function:: warn(message, category=None, stacklevel=1, source=None, \*, skip_file_prefixes=None) +.. function:: warn(message, category=None, stacklevel=1, source=None, *, skip_file_prefixes=None) Issue a warning, or maybe ignore it or raise an exception. The *category* argument, if given, must be a :ref:`warning category class <warning-categories>`; it diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index fe92400fb08dfdc..75a7915c15240d2 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -625,6 +625,8 @@ Functions target. Returns an :term:`iterator` providing ``(event, elem)`` pairs; it has a ``root`` attribute that references the root element of the resulting XML tree once *source* is fully read. + The iterator has the :meth:`!close` method that closes the internal + file object if *source* is a filename. Note that while :func:`iterparse` builds the tree incrementally, it issues blocking reads on *source* (or the file it names). As such, it's unsuitable @@ -647,6 +649,9 @@ Functions .. versionchanged:: 3.8 The ``comment`` and ``pi`` events were added. + .. versionchanged:: 3.13 + Added the :meth:`!close` method. + .. function:: parse(source, parser=None) @@ -664,7 +669,7 @@ Functions given. Returns an element instance, representing a processing instruction. Note that :class:`XMLParser` skips over processing instructions - in the input instead of creating comment objects for them. An + in the input instead of creating PI objects for them. An :class:`ElementTree` will only contain processing instruction nodes if they have been inserted into to the tree using one of the :class:`Element` methods. @@ -1302,8 +1307,8 @@ TreeBuilder Objects .. method:: pi(target, text) - Creates a comment with the given *target* name and *text*. If - ``insert_pis`` is true, this will also add it to the tree. + Creates a process instruction with the given *target* name and *text*. + If ``insert_pis`` is true, this will also add it to the tree. .. versionadded:: 3.8 diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 0a1c1d58558e94f..885ee825c122965 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1988,8 +1988,8 @@ access (use of, assignment to, or deletion of ``x.name``) for class instances. .. method:: object.__dir__(self) - Called when :func:`dir` is called on the object. A sequence must be - returned. :func:`dir` converts the returned sequence to a list and sorts it. + Called when :func:`dir` is called on the object. An iterable must be + returned. :func:`dir` converts the returned iterable to a list and sorts it. Customizing module attribute access @@ -2009,7 +2009,7 @@ not found on a module object through the normal lookup, i.e. the module ``__dict__`` before raising an :exc:`AttributeError`. If found, it is called with the attribute name and the result is returned. -The ``__dir__`` function should accept no arguments, and return a sequence of +The ``__dir__`` function should accept no arguments, and return an iterable of strings that represents the names accessible on module. If present, this function overrides the standard :func:`dir` search on a module. diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 87ebdc1ca1c9c6a..50e0f97a6534af2 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -1890,8 +1890,9 @@ the unpacking. .. index:: pair: trailing; comma -The trailing comma is required only to create a single tuple (a.k.a. a -*singleton*); it is optional in all other cases. A single expression without a +A trailing comma is required only to create a one-item tuple, +such as ``1,``; it is optional in all other cases. +A single expression without a trailing comma doesn't create a tuple, but rather yields the value of that expression. (To create an empty tuple, use an empty pair of parentheses: ``()``.) diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index a7beeea29b45567..f8c9724114da9ed 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -327,14 +327,15 @@ modules, and one that knows how to import modules from an :term:`import path` finders replaced :meth:`!find_module`, which is now deprecated. While it will continue to work without change, the import machinery will try it only if the finder does not implement - ``find_spec()``. + :meth:`~importlib.abc.MetaPathFinder.find_spec`. .. versionchanged:: 3.10 Use of :meth:`!find_module` by the import system now raises :exc:`ImportWarning`. .. versionchanged:: 3.12 - ``find_module()`` has been removed. Use :meth:`find_spec` instead. + :meth:`!find_module` has been removed. + Use :meth:`~importlib.abc.MetaPathFinder.find_spec` instead. Loading @@ -812,7 +813,7 @@ attributes on package objects are also used. These provide additional ways that the import machinery can be customized. :data:`sys.path` contains a list of strings providing search locations for -modules and packages. It is initialized from the :data:`PYTHONPATH` +modules and packages. It is initialized from the :envvar:`PYTHONPATH` environment variable and various other installation- and implementation-specific defaults. Entries in :data:`sys.path` can name directories on the file system, zip files, and potentially other "locations" diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 221a1f05c11e49d..9db02c5c3c73c90 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -4,17 +4,13 @@ Doc/c-api/descriptor.rst Doc/c-api/exceptions.rst -Doc/c-api/file.rst Doc/c-api/float.rst Doc/c-api/gcsupport.rst Doc/c-api/init.rst Doc/c-api/init_config.rst Doc/c-api/intro.rst -Doc/c-api/memoryview.rst Doc/c-api/module.rst -Doc/c-api/object.rst Doc/c-api/stable.rst -Doc/c-api/structures.rst Doc/c-api/sys.rst Doc/c-api/type.rst Doc/c-api/typeobj.rst @@ -22,26 +18,19 @@ Doc/extending/extending.rst Doc/glossary.rst Doc/howto/descriptor.rst Doc/howto/enum.rst -Doc/howto/logging.rst Doc/library/ast.rst Doc/library/asyncio-extending.rst Doc/library/asyncio-policy.rst Doc/library/asyncio-subprocess.rst -Doc/library/asyncio-task.rst Doc/library/bdb.rst Doc/library/collections.rst -Doc/library/concurrent.futures.rst -Doc/library/csv.rst -Doc/library/datetime.rst Doc/library/dbm.rst Doc/library/decimal.rst Doc/library/email.charset.rst Doc/library/email.compat32-message.rst Doc/library/email.errors.rst -Doc/library/email.mime.rst Doc/library/email.parser.rst Doc/library/email.policy.rst -Doc/library/enum.rst Doc/library/exceptions.rst Doc/library/faulthandler.rst Doc/library/fcntl.rst @@ -56,7 +45,6 @@ Doc/library/mmap.rst Doc/library/multiprocessing.rst Doc/library/optparse.rst Doc/library/os.rst -Doc/library/pickle.rst Doc/library/pickletools.rst Doc/library/platform.rst Doc/library/plistlib.rst @@ -93,12 +81,10 @@ Doc/library/xmlrpc.server.rst Doc/library/zlib.rst Doc/reference/compound_stmts.rst Doc/reference/datamodel.rst -Doc/reference/import.rst Doc/tutorial/datastructures.rst Doc/using/windows.rst Doc/whatsnew/2.0.rst Doc/whatsnew/2.1.rst -Doc/whatsnew/2.2.rst Doc/whatsnew/2.4.rst Doc/whatsnew/2.5.rst Doc/whatsnew/2.6.rst diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 87614d082a1d4e0..de2827461e2f241 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -48,10 +48,9 @@ objects: :noindex: Remove the item at the given position in the list, and return it. If no index - is specified, ``a.pop()`` removes and returns the last item in the list. (The - square brackets around the *i* in the method signature denote that the parameter - is optional, not that you should type square brackets at that position. You - will see this notation frequently in the Python Library Reference.) + is specified, ``a.pop()`` removes and returns the last item in the list. + It raises an :exc:`IndexError` if the list is empty or the index is + outside the list range. .. method:: list.clear() diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index df8b07c61185998..53c95ca1a05c9b4 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -623,7 +623,7 @@ Setting the environment variable ``TERM`` to ``dumb`` will disable color. If the environment variable ``FORCE_COLOR`` is set, then color will be enabled regardless of the value of TERM. This is useful on CI systems which -aren’t terminals but can none-the-less display ANSI escape sequences. +aren’t terminals but can still display ANSI escape sequences. If the environment variable ``NO_COLOR`` is set, Python will disable all color in the output. This takes precedence over ``FORCE_COLOR``. diff --git a/Doc/whatsnew/2.0.rst b/Doc/whatsnew/2.0.rst index f4a9d23699de53c..af8171487fbcfa2 100644 --- a/Doc/whatsnew/2.0.rst +++ b/Doc/whatsnew/2.0.rst @@ -1039,12 +1039,12 @@ is an implementation of the Secure Socket Layer, which encrypts the data being sent over a socket. When compiling Python, you can edit :file:`Modules/Setup` to include SSL support, which adds an additional function to the :mod:`socket` module: ``socket.ssl(socket, keyfile, certfile)``, which takes a socket -object and returns an SSL socket. The :mod:`httplib` and :mod:`urllib` modules +object and returns an SSL socket. The :mod:`httplib <http>` and :mod:`urllib` modules were also changed to support ``https://`` URLs, though no one has implemented FTP or SMTP over SSL. -The :mod:`httplib` module has been rewritten by Greg Stein to support HTTP/1.1. -Backward compatibility with the 1.5 version of :mod:`httplib` is provided, +The :mod:`httplib <http>` module has been rewritten by Greg Stein to support HTTP/1.1. +Backward compatibility with the 1.5 version of :mod:`!httplib` is provided, though using HTTP/1.1 features such as pipelining will require rewriting code to use a different set of interfaces. @@ -1108,7 +1108,7 @@ module. * :mod:`pyexpat`: An interface to the Expat XML parser. (Contributed by Paul Prescod.) -* :mod:`robotparser`: Parse a :file:`robots.txt` file, which is used for writing +* :mod:`robotparser <urllib.robotparser>`: Parse a :file:`robots.txt` file, which is used for writing web spiders that politely avoid certain areas of a web site. The parser accepts the contents of a :file:`robots.txt` file, builds a set of rules from it, and can then answer questions about the fetchability of a given URL. (Contributed @@ -1129,10 +1129,10 @@ module. :file:`Tools/idle/BrowserControl.py`, and adapted for the standard library by Fred.) -* :mod:`_winreg`: An interface to the Windows registry. :mod:`_winreg` is an +* :mod:`_winreg <winreg>`: An interface to the Windows registry. :mod:`!_winreg` is an adaptation of functions that have been part of PythonWin since 1995, but has now been added to the core distribution, and enhanced to support Unicode. - :mod:`_winreg` was written by Bill Tutt and Mark Hammond. + :mod:`!_winreg` was written by Bill Tutt and Mark Hammond. * :mod:`zipfile`: A module for reading and writing ZIP-format archives. These are archives produced by :program:`PKZIP` on DOS/Windows or :program:`zip` on diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index 6efc23a82de9232..e6c13f957b8d54a 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -53,9 +53,9 @@ A long time ago I wrote a web page listing flaws in Python's design. One of the most significant flaws was that it's impossible to subclass Python types implemented in C. In particular, it's not possible to subclass built-in types, so you can't just subclass, say, lists in order to add a single useful method to -them. The :mod:`UserList` module provides a class that supports all of the +them. The :mod:`!UserList` module provides a class that supports all of the methods of lists and that can be subclassed further, but there's lots of C code -that expects a regular Python list and won't accept a :class:`UserList` +that expects a regular Python list and won't accept a :class:`~collections.UserList` instance. Python 2.2 fixes this, and in the process adds some exciting new capabilities. @@ -69,7 +69,7 @@ A brief summary: * It's also possible to automatically call methods on accessing or setting an instance attribute by using a new mechanism called :dfn:`properties`. Many uses - of :meth:`__getattr__` can be rewritten to use properties instead, making the + of :meth:`~object.__getattr__` can be rewritten to use properties instead, making the resulting code simpler and faster. As a small side benefit, attributes can now have docstrings, too. @@ -120,7 +120,7 @@ added so if no built-in type is suitable, you can just subclass This means that :keyword:`class` statements that don't have any base classes are always classic classes in Python 2.2. (Actually you can also change this by -setting a module-level variable named :attr:`__metaclass__` --- see :pep:`253` +setting a module-level variable named :attr:`!__metaclass__` --- see :pep:`253` for the details --- but it's easier to just subclass :class:`object`.) The type objects for the built-in types are available as built-ins, named using @@ -134,8 +134,8 @@ type objects that behave as factories when called. :: 123 To make the set of types complete, new type objects such as :func:`dict` and -:func:`file` have been added. Here's a more interesting example, adding a -:meth:`lock` method to file objects:: +:func:`!file` have been added. Here's a more interesting example, adding a +:meth:`!lock` method to file objects:: class LockableFile(file): def lock (self, operation, length=0, start=0, whence=0): @@ -146,7 +146,7 @@ To make the set of types complete, new type objects such as :func:`dict` and The now-obsolete :mod:`!posixfile` module contained a class that emulated all of a file object's methods and also added a :meth:`!lock` method, but this class couldn't be passed to internal functions that expected a built-in file, -something which is possible with our new :class:`LockableFile`. +something which is possible with our new :class:`!LockableFile`. Descriptors @@ -154,11 +154,11 @@ Descriptors In previous versions of Python, there was no consistent way to discover what attributes and methods were supported by an object. There were some informal -conventions, such as defining :attr:`__members__` and :attr:`__methods__` +conventions, such as defining :attr:`!__members__` and :attr:`!__methods__` attributes that were lists of names, but often the author of an extension type or a class wouldn't bother to define them. You could fall back on inspecting the :attr:`~object.__dict__` of an object, but when class inheritance or an arbitrary -:meth:`__getattr__` hook were in use this could still be inaccurate. +:meth:`!__getattr__` hook were in use this could still be inaccurate. The one big idea underlying the new class model is that an API for describing the attributes of an object using :dfn:`descriptors` has been formalized. @@ -171,7 +171,7 @@ attributes of their own: * :attr:`~definition.__name__` is the attribute's name. -* :attr:`__doc__` is the attribute's docstring. +* :attr:`!__doc__` is the attribute's docstring. * ``__get__(object)`` is a method that retrieves the attribute value from *object*. @@ -186,7 +186,7 @@ are:: descriptor = obj.__class__.x descriptor.__get__(obj) -For methods, :meth:`descriptor.__get__` returns a temporary object that's +For methods, :meth:`!descriptor.__get__` returns a temporary object that's callable, and wraps up the instance and the method to be called on it. This is also why static methods and class methods are now possible; they have descriptors that wrap up just the method, or the method and the class. As a @@ -204,7 +204,7 @@ methods are defined like this:: ... g = classmethod(g) -The :func:`staticmethod` function takes the function :func:`f`, and returns it +The :func:`staticmethod` function takes the function :func:`!f`, and returns it wrapped up in a descriptor so it can be stored in the class object. You might expect there to be special syntax for creating such methods (``def static f``, ``defstatic f()``, or something like that) but no such syntax has been defined @@ -232,10 +232,10 @@ like this:: f = eiffelmethod(f, pre_f, post_f) -Note that a person using the new :func:`eiffelmethod` doesn't have to understand +Note that a person using the new :func:`!eiffelmethod` doesn't have to understand anything about descriptors. This is why I think the new features don't increase the basic complexity of the language. There will be a few wizards who need to -know about it in order to write :func:`eiffelmethod` or the ZODB or whatever, +know about it in order to write :func:`!eiffelmethod` or the ZODB or whatever, but most users will just write code on top of the resulting libraries and ignore the implementation details. @@ -263,10 +263,10 @@ from :pep:`253` by Guido van Rossum):: The lookup rule for classic classes is simple but not very smart; the base classes are searched depth-first, going from left to right. A reference to -:meth:`D.save` will search the classes :class:`D`, :class:`B`, and then -:class:`A`, where :meth:`save` would be found and returned. :meth:`C.save` -would never be found at all. This is bad, because if :class:`C`'s :meth:`save` -method is saving some internal state specific to :class:`C`, not calling it will +:meth:`!D.save` will search the classes :class:`!D`, :class:`!B`, and then +:class:`!A`, where :meth:`!save` would be found and returned. :meth:`!C.save` +would never be found at all. This is bad, because if :class:`!C`'s :meth:`!save` +method is saving some internal state specific to :class:`!C`, not calling it will result in that state never getting saved. New-style classes follow a different algorithm that's a bit more complicated to @@ -276,22 +276,22 @@ produces more useful results for really complicated inheritance graphs.) #. List all the base classes, following the classic lookup rule and include a class multiple times if it's visited repeatedly. In the above example, the list - of visited classes is [:class:`D`, :class:`B`, :class:`A`, :class:`C`, - :class:`A`]. + of visited classes is [:class:`!D`, :class:`!B`, :class:`!A`, :class:`!C`, + :class:`!A`]. #. Scan the list for duplicated classes. If any are found, remove all but one occurrence, leaving the *last* one in the list. In the above example, the list - becomes [:class:`D`, :class:`B`, :class:`C`, :class:`A`] after dropping + becomes [:class:`!D`, :class:`!B`, :class:`!C`, :class:`!A`] after dropping duplicates. -Following this rule, referring to :meth:`D.save` will return :meth:`C.save`, +Following this rule, referring to :meth:`!D.save` will return :meth:`!C.save`, which is the behaviour we're after. This lookup rule is the same as the one followed by Common Lisp. A new built-in function, :func:`super`, provides a way to get at a class's superclasses without having to reimplement Python's algorithm. The most commonly used form will be ``super(class, obj)``, which returns a bound superclass object (not the actual class object). This form will be used in methods to call a method in the superclass; for example, -:class:`D`'s :meth:`save` method would look like this:: +:class:`!D`'s :meth:`!save` method would look like this:: class D (B,C): def save (self): @@ -309,7 +309,7 @@ Attribute Access ---------------- A fair number of sophisticated Python classes define hooks for attribute access -using :meth:`__getattr__`; most commonly this is done for convenience, to make +using :meth:`~object.__getattr__`; most commonly this is done for convenience, to make code more readable by automatically mapping an attribute access such as ``obj.parent`` into a method call such as ``obj.get_parent``. Python 2.2 adds some new ways of controlling attribute access. @@ -321,22 +321,22 @@ instance's dictionary. New-style classes also support a new method, ``__getattribute__(attr_name)``. The difference between the two methods is -that :meth:`__getattribute__` is *always* called whenever any attribute is -accessed, while the old :meth:`__getattr__` is only called if ``foo`` isn't +that :meth:`~object.__getattribute__` is *always* called whenever any attribute is +accessed, while the old :meth:`~object.__getattr__` is only called if ``foo`` isn't found in the instance's dictionary. However, Python 2.2's support for :dfn:`properties` will often be a simpler way -to trap attribute references. Writing a :meth:`__getattr__` method is +to trap attribute references. Writing a :meth:`!__getattr__` method is complicated because to avoid recursion you can't use regular attribute accesses inside them, and instead have to mess around with the contents of -:attr:`~object.__dict__`. :meth:`__getattr__` methods also end up being called by Python -when it checks for other methods such as :meth:`__repr__` or :meth:`__coerce__`, +:attr:`~object.__dict__`. :meth:`~object.__getattr__` methods also end up being called by Python +when it checks for other methods such as :meth:`~object.__repr__` or :meth:`!__coerce__`, and so have to be written with this in mind. Finally, calling a function on every attribute access results in a sizable performance loss. :class:`property` is a new built-in type that packages up three functions that get, set, or delete an attribute, and a docstring. For example, if you want to -define a :attr:`size` attribute that's computed, but also settable, you could +define a :attr:`!size` attribute that's computed, but also settable, you could write:: class C(object): @@ -355,9 +355,9 @@ write:: "Storage size of this instance") That is certainly clearer and easier to write than a pair of -:meth:`__getattr__`/:meth:`__setattr__` methods that check for the :attr:`size` +:meth:`!__getattr__`/:meth:`!__setattr__` methods that check for the :attr:`!size` attribute and handle it specially while retrieving all other attributes from the -instance's :attr:`~object.__dict__`. Accesses to :attr:`size` are also the only ones +instance's :attr:`~object.__dict__`. Accesses to :attr:`!size` are also the only ones which have to perform the work of calling a function, so references to other attributes run at their usual speed. @@ -447,7 +447,7 @@ an iterator for the object *obj*, while ``iter(C, sentinel)`` returns an iterator that will invoke the callable object *C* until it returns *sentinel* to signal that the iterator is done. -Python classes can define an :meth:`__iter__` method, which should create and +Python classes can define an :meth:`!__iter__` method, which should create and return a new iterator for the object; if the object is its own iterator, this method can just return ``self``. In particular, iterators will usually be their own iterators. Extension types implemented in C can implement a :c:member:`~PyTypeObject.tp_iter` @@ -478,7 +478,7 @@ there are no more values to be returned, calling :meth:`next` should raise the In 2.2, Python's :keyword:`for` statement no longer expects a sequence; it expects something for which :func:`iter` will return an iterator. For backward compatibility and convenience, an iterator is automatically constructed for -sequences that don't implement :meth:`__iter__` or a :c:member:`~PyTypeObject.tp_iter` slot, so +sequences that don't implement :meth:`!__iter__` or a :c:member:`~PyTypeObject.tp_iter` slot, so ``for i in [1,2,3]`` will still work. Wherever the Python interpreter loops over a sequence, it's been changed to use the iterator protocol. This means you can do things like this:: @@ -510,8 +510,8 @@ Iterator support has been added to some of Python's basic types. Calling Oct 10 That's just the default behaviour. If you want to iterate over keys, values, or -key/value pairs, you can explicitly call the :meth:`iterkeys`, -:meth:`itervalues`, or :meth:`iteritems` methods to get an appropriate iterator. +key/value pairs, you can explicitly call the :meth:`!iterkeys`, +:meth:`!itervalues`, or :meth:`!iteritems` methods to get an appropriate iterator. In a minor related change, the :keyword:`in` operator now works on dictionaries, so ``key in dict`` is now equivalent to ``dict.has_key(key)``. @@ -580,7 +580,7 @@ allowed inside the :keyword:`!try` block of a :keyword:`try`...\ :keyword:`finally` statement; read :pep:`255` for a full explanation of the interaction between :keyword:`!yield` and exceptions.) -Here's a sample usage of the :func:`generate_ints` generator:: +Here's a sample usage of the :func:`!generate_ints` generator:: >>> gen = generate_ints(3) >>> gen @@ -641,7 +641,7 @@ like:: sentence := "Store it in the neighboring harbor" if (i := find("or", sentence)) > 5 then write(i) -In Icon the :func:`find` function returns the indexes at which the substring +In Icon the :func:`!find` function returns the indexes at which the substring "or" is found: 3, 23, 33. In the :keyword:`if` statement, ``i`` is first assigned a value of 3, but 3 is less than 5, so the comparison fails, and Icon retries it with the second value of 23. 23 is greater than 5, so the comparison @@ -671,7 +671,7 @@ PEP 237: Unifying Long Integers and Integers In recent versions, the distinction between regular integers, which are 32-bit values on most machines, and long integers, which can be of arbitrary size, was becoming an annoyance. For example, on platforms that support files larger than -``2**32`` bytes, the :meth:`tell` method of file objects has to return a long +``2**32`` bytes, the :meth:`!tell` method of file objects has to return a long integer. However, there were various bits of Python that expected plain integers and would raise an error if a long integer was provided instead. For example, in Python 1.5, only regular integers could be used as a slice index, and @@ -752,7 +752,7 @@ Here are the changes 2.2 introduces: 0.5. Without the ``__future__`` statement, ``/`` still means classic division. The default meaning of ``/`` will not change until Python 3.0. -* Classes can define methods called :meth:`__truediv__` and :meth:`__floordiv__` +* Classes can define methods called :meth:`~object.__truediv__` and :meth:`~object.__floordiv__` to overload the two division operators. At the C level, there are also slots in the :c:type:`PyNumberMethods` structure so extension types can define the two operators. @@ -785,17 +785,17 @@ support.) When built to use UCS-4 (a "wide Python"), the interpreter can natively handle Unicode characters from U+000000 to U+110000, so the range of legal values for -the :func:`unichr` function is expanded accordingly. Using an interpreter +the :func:`!unichr` function is expanded accordingly. Using an interpreter compiled to use UCS-2 (a "narrow Python"), values greater than 65535 will still -cause :func:`unichr` to raise a :exc:`ValueError` exception. This is all +cause :func:`!unichr` to raise a :exc:`ValueError` exception. This is all described in :pep:`261`, "Support for 'wide' Unicode characters"; consult it for further details. Another change is simpler to explain. Since their introduction, Unicode strings -have supported an :meth:`encode` method to convert the string to a selected +have supported an :meth:`!encode` method to convert the string to a selected encoding such as UTF-8 or Latin-1. A symmetric ``decode([*encoding*])`` method has been added to 8-bit strings (though not to Unicode strings) in 2.2. -:meth:`decode` assumes that the string is in the specified encoding and decodes +:meth:`!decode` assumes that the string is in the specified encoding and decodes it, returning whatever is returned by the codec. Using this new feature, codecs have been added for tasks not directly related to @@ -819,10 +819,10 @@ encoding, and compression with the :mod:`zlib` module:: >>> "sheesh".encode('rot-13') 'furrfu' -To convert a class instance to Unicode, a :meth:`__unicode__` method can be -defined by a class, analogous to :meth:`__str__`. +To convert a class instance to Unicode, a :meth:`!__unicode__` method can be +defined by a class, analogous to :meth:`!__str__`. -:meth:`encode`, :meth:`decode`, and :meth:`__unicode__` were implemented by +:meth:`!encode`, :meth:`!decode`, and :meth:`!__unicode__` were implemented by Marc-André Lemburg. The changes to support using UCS-4 internally were implemented by Fredrik Lundh and Martin von Löwis. @@ -859,7 +859,7 @@ doesn't work:: return g(value-1) + 1 ... -The function :func:`g` will always raise a :exc:`NameError` exception, because +The function :func:`!g` will always raise a :exc:`NameError` exception, because the binding of the name ``g`` isn't in either its local namespace or in the module-level namespace. This isn't much of a problem in practice (how often do you recursively define interior functions like this?), but this also made using @@ -915,7 +915,7 @@ To make the preceding explanation a bit clearer, here's an example:: Line 4 containing the ``exec`` statement is a syntax error, since ``exec`` would define a new local variable named ``x`` whose value should -be accessed by :func:`g`. +be accessed by :func:`!g`. This shouldn't be much of a limitation, since ``exec`` is rarely used in most Python code (and when it is used, it's often a sign of a poor design @@ -933,7 +933,7 @@ anyway). New and Improved Modules ======================== -* The :mod:`xmlrpclib` module was contributed to the standard library by Fredrik +* The :mod:`xmlrpclib <xmlrpc.client>` module was contributed to the standard library by Fredrik Lundh, providing support for writing XML-RPC clients. XML-RPC is a simple remote procedure call protocol built on top of HTTP and XML. For example, the following snippet retrieves a list of RSS channels from the O'Reilly Network, @@ -956,7 +956,7 @@ New and Improved Modules # 'description': 'A utility which converts HTML to XSL FO.', # 'title': 'html2fo 0.3 (Default)'}, ... ] - The :mod:`SimpleXMLRPCServer` module makes it easy to create straightforward + The :mod:`SimpleXMLRPCServer <xmlrpc.server>` module makes it easy to create straightforward XML-RPC servers. See http://xmlrpc.scripting.com/ for more information about XML-RPC. * The new :mod:`hmac` module implements the HMAC algorithm described by @@ -964,9 +964,9 @@ New and Improved Modules * Several functions that originally returned lengthy tuples now return pseudo-sequences that still behave like tuples but also have mnemonic attributes such - as memberst_mtime or :attr:`tm_year`. The enhanced functions include - :func:`stat`, :func:`fstat`, :func:`statvfs`, and :func:`fstatvfs` in the - :mod:`os` module, and :func:`localtime`, :func:`gmtime`, and :func:`strptime` in + as :attr:`!memberst_mtime` or :attr:`~time.struct_time.tm_year`. The enhanced functions include + :func:`~os.stat`, :func:`~os.fstat`, :func:`~os.statvfs`, and :func:`~os.fstatvfs` in the + :mod:`os` module, and :func:`~time.localtime`, :func:`~time.gmtime`, and :func:`~time.strptime` in the :mod:`time` module. For example, to obtain a file's size using the old tuples, you'd end up writing @@ -999,7 +999,7 @@ New and Improved Modules underlying the :mod:`re` module. For example, the :func:`re.sub` and :func:`re.split` functions have been rewritten in C. Another contributed patch speeds up certain Unicode character ranges by a factor of two, and a new - :meth:`finditer` method that returns an iterator over all the non-overlapping + :meth:`~re.finditer` method that returns an iterator over all the non-overlapping matches in a given string. (SRE is maintained by Fredrik Lundh. The BIGCHARSET patch was contributed by Martin von Löwis.) @@ -1012,33 +1012,33 @@ New and Improved Modules new extensions: the NAMESPACE extension defined in :rfc:`2342`, SORT, GETACL and SETACL. (Contributed by Anthony Baxter and Michel Pelletier.) -* The :mod:`rfc822` module's parsing of email addresses is now compliant with +* The :mod:`!rfc822` module's parsing of email addresses is now compliant with :rfc:`2822`, an update to :rfc:`822`. (The module's name is *not* going to be changed to ``rfc2822``.) A new package, :mod:`email`, has also been added for parsing and generating e-mail messages. (Contributed by Barry Warsaw, and arising out of his work on Mailman.) -* The :mod:`difflib` module now contains a new :class:`Differ` class for +* The :mod:`difflib` module now contains a new :class:`!Differ` class for producing human-readable lists of changes (a "delta") between two sequences of - lines of text. There are also two generator functions, :func:`ndiff` and - :func:`restore`, which respectively return a delta from two sequences, or one of + lines of text. There are also two generator functions, :func:`!ndiff` and + :func:`!restore`, which respectively return a delta from two sequences, or one of the original sequences from a delta. (Grunt work contributed by David Goodger, from ndiff.py code by Tim Peters who then did the generatorization.) -* New constants :const:`ascii_letters`, :const:`ascii_lowercase`, and - :const:`ascii_uppercase` were added to the :mod:`string` module. There were - several modules in the standard library that used :const:`string.letters` to +* New constants :const:`!ascii_letters`, :const:`!ascii_lowercase`, and + :const:`!ascii_uppercase` were added to the :mod:`string` module. There were + several modules in the standard library that used :const:`!string.letters` to mean the ranges A-Za-z, but that assumption is incorrect when locales are in - use, because :const:`string.letters` varies depending on the set of legal + use, because :const:`!string.letters` varies depending on the set of legal characters defined by the current locale. The buggy modules have all been fixed - to use :const:`ascii_letters` instead. (Reported by an unknown person; fixed by + to use :const:`!ascii_letters` instead. (Reported by an unknown person; fixed by Fred L. Drake, Jr.) * The :mod:`mimetypes` module now makes it easier to use alternative MIME-type - databases by the addition of a :class:`MimeTypes` class, which takes a list of + databases by the addition of a :class:`~mimetypes.MimeTypes` class, which takes a list of filenames to be parsed. (Contributed by Fred L. Drake, Jr.) -* A :class:`Timer` class was added to the :mod:`threading` module that allows +* A :class:`~threading.Timer` class was added to the :mod:`threading` module that allows scheduling an activity to happen at some future time. (Contributed by Itamar Shtull-Trauring.) @@ -1114,7 +1114,7 @@ code, none of the changes described here will affect you very much. * Two new wrapper functions, :c:func:`PyOS_snprintf` and :c:func:`PyOS_vsnprintf` were added to provide cross-platform implementations for the relatively new :c:func:`snprintf` and :c:func:`vsnprintf` C lib APIs. In contrast to the standard - :c:func:`sprintf` and :c:func:`vsprintf` functions, the Python versions check the + :c:func:`sprintf` and :c:func:`!vsprintf` functions, the Python versions check the bounds of the buffer used to protect against buffer overruns. (Contributed by M.-A. Lemburg.) @@ -1212,12 +1212,12 @@ Some of the more notable changes are: * The :file:`Tools/scripts/ftpmirror.py` script now parses a :file:`.netrc` file, if you have one. (Contributed by Mike Romberg.) -* Some features of the object returned by the :func:`xrange` function are now +* Some features of the object returned by the :func:`!xrange` function are now deprecated, and trigger warnings when they're accessed; they'll disappear in - Python 2.3. :class:`xrange` objects tried to pretend they were full sequence + Python 2.3. :class:`!xrange` objects tried to pretend they were full sequence types by supporting slicing, sequence multiplication, and the :keyword:`in` operator, but these features were rarely used and therefore buggy. The - :meth:`tolist` method and the :attr:`start`, :attr:`stop`, and :attr:`step` + :meth:`!tolist` method and the :attr:`!start`, :attr:`!stop`, and :attr:`!step` attributes are also being deprecated. At the C level, the fourth argument to the :c:func:`!PyRange_New` function, ``repeat``, has also been deprecated. diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index 15d4003622c5065..7e235d4370edaa9 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -1081,7 +1081,7 @@ complete list of changes, or look through the CVS logs for all the details. :func:`nsmallest` that use heaps to find the N largest or smallest values in a dataset without the expense of a full sort. (Contributed by Raymond Hettinger.) -* The :mod:`httplib` module now contains constants for HTTP status codes defined +* The :mod:`httplib <http>` module now contains constants for HTTP status codes defined in various HTTP-related RFC documents. Constants have names such as :const:`OK`, :const:`CREATED`, :const:`CONTINUE`, and :const:`MOVED_PERMANENTLY`; use pydoc to get a full list. (Contributed by @@ -1218,10 +1218,10 @@ complete list of changes, or look through the CVS logs for all the details. now include the string ``'%default'``, which will be replaced by the option's default value. (Contributed by Greg Ward.) -* The long-term plan is to deprecate the :mod:`rfc822` module in some future +* The long-term plan is to deprecate the :mod:`!rfc822` module in some future Python release in favor of the :mod:`email` package. To this end, the - :func:`email.Utils.formatdate` function has been changed to make it usable as a - replacement for :func:`rfc822.formatdate`. You may want to write new e-mail + :func:`email.Utils.formatdate <email.utils.formatdate>` function has been changed to make it usable as a + replacement for :func:`!rfc822.formatdate`. You may want to write new e-mail processing code with this in mind. (Change implemented by Anthony Baxter.) * A new ``urandom(n)`` function was added to the :mod:`os` module, returning @@ -1308,7 +1308,7 @@ complete list of changes, or look through the CVS logs for all the details. sockets, and regular expression pattern objects. (Contributed by Raymond Hettinger.) -* The :mod:`xmlrpclib` module now supports a multi-call extension for +* The :mod:`xmlrpclib <xmlrpc.client>` module now supports a multi-call extension for transmitting multiple XML-RPC calls in a single HTTP operation. (Contributed by Brian Quinlan.) @@ -1323,8 +1323,8 @@ complete list of changes, or look through the CVS logs for all the details. cookielib --------- -The :mod:`cookielib` library supports client-side handling for HTTP cookies, -mirroring the :mod:`Cookie` module's server-side cookie support. Cookies are +The :mod:`cookielib <http.cookiejar>` library supports client-side handling for HTTP cookies, +mirroring the :mod:`Cookie <http.cookies>` module's server-side cookie support. Cookies are stored in cookie jars; the library transparently stores cookies offered by the web server in the cookie jar, and fetches the cookie from the jar when connecting to the server. As in web browsers, policy objects control whether @@ -1335,7 +1335,7 @@ are provided: one that stores cookies in the Netscape format so applications can use the Mozilla or Lynx cookie files, and one that stores cookies in the same format as the Perl libwww library. -:mod:`urllib2` has been changed to interact with :mod:`cookielib`: +:mod:`urllib2 <urllib.request>` has been changed to interact with :mod:`cookielib <http.cookiejar>`: :class:`HTTPCookieProcessor` manages a cookie jar that is used when accessing URLs. diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index f45d70ea5a19a03..2ae26e7a106a0bf 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -1478,8 +1478,8 @@ complete list of changes, or look through the SVN logs for all the details. .. Patch 790710 -* The :mod:`pickle` and :mod:`cPickle` modules no longer accept a return value - of ``None`` from the :meth:`__reduce__` method; the method must return a tuple +* The :mod:`pickle` and :mod:`!cPickle` modules no longer accept a return value + of ``None`` from the :meth:`~object.__reduce__` method; the method must return a tuple of arguments instead. The ability to return ``None`` was deprecated in Python 2.4, so this completes the removal of the feature. @@ -1519,7 +1519,7 @@ complete list of changes, or look through the SVN logs for all the details. .. Patch #1472854 -* The :mod:`SimpleXMLRPCServer` and :mod:`DocXMLRPCServer` classes now have a +* The :mod:`SimpleXMLRPCServer <xmlrpc.server>` and :mod:`DocXMLRPCServer <xmlrpc.server>` classes now have a :attr:`rpc_paths` attribute that constrains XML-RPC operations to a limited set of URL paths; the default is to allow only ``'/'`` and ``'/RPC2'``. Setting :attr:`rpc_paths` to ``None`` or an empty tuple disables this path checking. @@ -1650,9 +1650,9 @@ complete list of changes, or look through the SVN logs for all the details. .. Patch #754022 -* The :mod:`xmlrpclib` module now supports returning :class:`~datetime.datetime` objects - for the XML-RPC date type. Supply ``use_datetime=True`` to the :func:`loads` - function or the :class:`Unmarshaller` class to enable this feature. (Contributed +* The :mod:`xmlrpclib <xmlrpc.client>` module now supports returning :class:`~datetime.datetime` objects + for the XML-RPC date type. Supply ``use_datetime=True`` to the :func:`~xmlrpc.client.loads` + function or the :class:`!Unmarshaller` class to enable this feature. (Contributed by Skip Montanaro.) .. Patch 1120353 @@ -2253,12 +2253,12 @@ code: appeared. In Python 2.5, the argument must be exactly one %char specifier with no surrounding text. -* Library: The :mod:`pickle` and :mod:`cPickle` modules no longer accept a - return value of ``None`` from the :meth:`__reduce__` method; the method must +* Library: The :mod:`pickle` and :mod:`!cPickle` modules no longer accept a + return value of ``None`` from the :meth:`~object.__reduce__` method; the method must return a tuple of arguments instead. The modules also no longer accept the deprecated *bin* keyword parameter. -* Library: The :mod:`SimpleXMLRPCServer` and :mod:`DocXMLRPCServer` classes now +* Library: The :mod:`SimpleXMLRPCServer <xmlrpc.server>` and :mod:`DocXMLRPCServer <xmlrpc.server>` classes now have a :attr:`rpc_paths` attribute that constrains XML-RPC operations to a limited set of URL paths; the default is to allow only ``'/'`` and ``'/RPC2'``. Setting :attr:`rpc_paths` to ``None`` or an empty tuple disables this path diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index c6bab93b7efdda1..7d3769a22286e28 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -1082,7 +1082,7 @@ the :mod:`io` module: (In Python 2.6, :class:`io.StringIO` is implemented in pure Python, so it's pretty slow. You should therefore stick with the - existing :mod:`StringIO` module or :mod:`cStringIO` for now. At some + existing :mod:`!StringIO` module or :mod:`!cStringIO` for now. At some point Python 3.0's :mod:`io` module will be rewritten into C for speed, and perhaps the C implementation will be backported to the 2.x releases.) @@ -1807,7 +1807,7 @@ changes, or look through the Subversion logs for all the details. Nubis; :issue:`1817`.) The :func:`parse_qs` and :func:`parse_qsl` functions have been - relocated from the :mod:`!cgi` module to the :mod:`urlparse` module. + relocated from the :mod:`!cgi` module to the :mod:`urlparse <urllib.parse>` module. The versions still available in the :mod:`!cgi` module will trigger :exc:`PendingDeprecationWarning` messages in 2.6 (:issue:`600362`). @@ -1895,8 +1895,8 @@ changes, or look through the Subversion logs for all the details. (Contributed by Raymond Hettinger.) -* The :mod:`Cookie` module's :class:`Morsel` objects now support an - :attr:`httponly` attribute. In some browsers. cookies with this attribute +* The :mod:`Cookie <http.cookies>` module's :class:`~http.cookies.Morsel` objects now support an + :attr:`~http.cookies.Morsel.httponly` attribute. In some browsers. cookies with this attribute set cannot be accessed or manipulated by JavaScript code. (Contributed by Arvin Schnell; :issue:`1638033`.) @@ -1987,8 +1987,8 @@ changes, or look through the Subversion logs for all the details. (Contributed by Raymond Hettinger.) * An optional ``timeout`` parameter, specifying a timeout measured in - seconds, was added to the :class:`httplib.HTTPConnection` and - :class:`HTTPSConnection` class constructors. (Added by Facundo + seconds, was added to the :class:`httplib.HTTPConnection <http.client.HTTPConnection>` and + :class:`HTTPSConnection <http.client.HTTPSConnection>` class constructors. (Added by Facundo Batista.) * Most of the :mod:`inspect` module's functions, such as @@ -2371,10 +2371,10 @@ changes, or look through the Subversion logs for all the details. ``socket(socket.AF_INET, ...)`` may be all that's required to make your code work with IPv6. -* The base classes in the :mod:`SocketServer` module now support - calling a :meth:`handle_timeout` method after a span of inactivity - specified by the server's :attr:`timeout` attribute. (Contributed - by Michael Pomraning.) The :meth:`serve_forever` method +* The base classes in the :mod:`SocketServer <socketserver>` module now support + calling a :meth:`~socketserver.BaseServer.handle_timeout` method after a span of inactivity + specified by the server's :attr:`~socketserver.BaseServer.timeout` attribute. (Contributed + by Michael Pomraning.) The :meth:`~socketserver.BaseServer.serve_forever` method now takes an optional poll interval measured in seconds, controlling how often the server will check for a shutdown request. (Contributed by Pedro Werneck and Jeffrey Yasskin; @@ -2478,9 +2478,9 @@ changes, or look through the Subversion logs for all the details. ``with tempfile.NamedTemporaryFile() as tmp: ...``. (Contributed by Alexander Belopolsky; :issue:`2021`.) -* The :mod:`test.test_support` module gained a number +* The :mod:`test.test_support <test.support>` module gained a number of context managers useful for writing tests. - :func:`EnvironmentVarGuard` is a + :func:`~test.support.os_helper.EnvironmentVarGuard` is a context manager that temporarily changes environment variables and automatically restores them to their old values. @@ -2577,9 +2577,9 @@ changes, or look through the Subversion logs for all the details. (:issue:`1513695`) * An optional ``timeout`` parameter was added to the - :func:`urllib.urlopen` function and the + :func:`urllib.urlopen <urllib.request.urlopen>` function and the :class:`urllib.ftpwrapper` class constructor, as well as the - :func:`urllib2.urlopen` function. The parameter specifies a timeout + :func:`urllib2.urlopen <urllib.request.urlopen>` function. The parameter specifies a timeout measured in seconds. For example:: >>> u = urllib2.urlopen("http://slow.example.com", @@ -2604,7 +2604,7 @@ changes, or look through the Subversion logs for all the details. intended for testing purposes that lets you temporarily modify the warning filters and then restore their original values (:issue:`3781`). -* The XML-RPC :class:`SimpleXMLRPCServer` and :class:`DocXMLRPCServer` +* The XML-RPC :class:`SimpleXMLRPCServer <xmlrpc.server>` and :class:`DocXMLRPCServer <xmlrpc.server>` classes can now be prevented from immediately opening and binding to their socket by passing ``False`` as the *bind_and_activate* constructor parameter. This can be used to modify the instance's @@ -2621,11 +2621,11 @@ changes, or look through the Subversion logs for all the details. information. (Contributed by Alan McIntyre as part of his project for Google's Summer of Code 2007.) -* The :mod:`xmlrpclib` module no longer automatically converts +* The :mod:`xmlrpclib <xmlrpc.client>` module no longer automatically converts :class:`datetime.date` and :class:`datetime.time` to the - :class:`xmlrpclib.DateTime` type; the conversion semantics were + :class:`xmlrpclib.DateTime <xmlrpc.client.DateTime>` type; the conversion semantics were not necessarily correct for all applications. Code using - :mod:`xmlrpclib` should convert :class:`date` and :class:`~datetime.time` + :mod:`!xmlrpclib` should convert :class:`date` and :class:`~datetime.time` instances. (:issue:`1330538`) The code can also handle dates before 1900 (contributed by Ralf Schmitt; :issue:`2014`) and 64-bit integers represented by using ``<i8>`` in XML-RPC responses @@ -3274,11 +3274,11 @@ that may require changes to your code: :exc:`StandardError` but now it is, through :exc:`IOError`. (Implemented by Gregory P. Smith; :issue:`1706815`.) -* The :mod:`xmlrpclib` module no longer automatically converts +* The :mod:`xmlrpclib <xmlrpc.client>` module no longer automatically converts :class:`datetime.date` and :class:`datetime.time` to the - :class:`xmlrpclib.DateTime` type; the conversion semantics were + :class:`xmlrpclib.DateTime <xmlrpc.client.DateTime>` type; the conversion semantics were not necessarily correct for all applications. Code using - :mod:`xmlrpclib` should convert :class:`date` and :class:`~datetime.time` + :mod:`!xmlrpclib` should convert :class:`date` and :class:`~datetime.time` instances. (:issue:`1330538`) * (3.0-warning mode) The :class:`Exception` class now warns diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 241d58720399aff..ada05aa22b46f66 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -915,7 +915,7 @@ used with the :option:`-W` switch, separated by commas. (Contributed by Brian Curtin; :issue:`7301`.) For example, the following setting will print warnings every time -they occur, but turn warnings from the :mod:`Cookie` module into an +they occur, but turn warnings from the :mod:`Cookie <http.cookies>` module into an error. (The exact syntax for setting an environment variable varies across operating systems and shells.) @@ -1012,12 +1012,12 @@ Several performance enhancements have been added: scan. This is sometimes faster by a factor of 10. (Added by Florent Xicluna; :issue:`7462` and :issue:`7622`.) -* The :mod:`pickle` and :mod:`cPickle` modules now automatically +* The :mod:`pickle` and :mod:`!cPickle` modules now automatically intern the strings used for attribute names, reducing memory usage of the objects resulting from unpickling. (Contributed by Jake McGuire; :issue:`5084`.) -* The :mod:`cPickle` module now special-cases dictionaries, +* The :mod:`!cPickle` module now special-cases dictionaries, nearly halving the time required to pickle them. (Contributed by Collin Winter; :issue:`5670`.) @@ -1163,7 +1163,7 @@ changes, or look through the Subversion logs for all the details. statement, has been deprecated, because the :keyword:`!with` statement now supports multiple context managers. -* The :mod:`cookielib` module now ignores cookies that have an invalid +* The :mod:`cookielib <http.cookiejar>` module now ignores cookies that have an invalid version field, one that doesn't contain an integer value. (Fixed by John J. Lee; :issue:`3924`.) @@ -1306,11 +1306,11 @@ changes, or look through the Subversion logs for all the details. ``('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')``. (Contributed by Carl Chenet; :issue:`7418`.) -* The default :class:`~httplib.HTTPResponse` class used by the :mod:`httplib` module now +* The default :class:`~http.client.HTTPResponse` class used by the :mod:`httplib <http>` module now supports buffering, resulting in much faster reading of HTTP responses. (Contributed by Kristján Valur Jónsson; :issue:`4879`.) - The :class:`~httplib.HTTPConnection` and :class:`~httplib.HTTPSConnection` classes + The :class:`~http.client.HTTPConnection` and :class:`~http.client.HTTPSConnection` classes now support a *source_address* parameter, a ``(host, port)`` 2-tuple giving the source address that will be used for the connection. (Contributed by Eldon Ziegler; :issue:`3972`.) @@ -1518,16 +1518,16 @@ changes, or look through the Subversion logs for all the details. the :class:`bytearray` and :class:`memoryview` objects. (Implemented by Antoine Pitrou; :issue:`8104`.) -* The :mod:`SocketServer` module's :class:`~SocketServer.TCPServer` class now +* The :mod:`SocketServer <socketserver>` module's :class:`~socketserver.TCPServer` class now supports socket timeouts and disabling the Nagle algorithm. - The :attr:`~SocketServer.TCPServer.disable_nagle_algorithm` class attribute + The :attr:`!disable_nagle_algorithm` class attribute defaults to ``False``; if overridden to be true, new request connections will have the TCP_NODELAY option set to prevent buffering many small sends into a single TCP packet. - The :attr:`~SocketServer.BaseServer.timeout` class attribute can hold + The :attr:`~socketserver.BaseServer.timeout` class attribute can hold a timeout in seconds that will be applied to the request socket; if - no request is received within that time, :meth:`~SocketServer.BaseServer.handle_timeout` - will be called and :meth:`~SocketServer.BaseServer.handle_request` will return. + no request is received within that time, :meth:`~socketserver.BaseServer.handle_timeout` + will be called and :meth:`~socketserver.BaseServer.handle_request` will return. (Contributed by Kristján Valur Jónsson; :issue:`6192` and :issue:`6267`.) * Updated module: the :mod:`sqlite3` module has been updated to @@ -1648,7 +1648,7 @@ changes, or look through the Subversion logs for all the details. and has been updated to version 5.2.0 (updated by Florent Xicluna; :issue:`8024`). -* The :mod:`urlparse` module's :func:`~urlparse.urlsplit` now handles +* The :mod:`urlparse <urllib.parse>` module's :func:`~urllib.parse.urlsplit` now handles unknown URL schemes in a fashion compliant with :rfc:`3986`: if the URL is of the form ``"<something>://..."``, the text before the ``://`` is treated as the scheme, even if it's a made-up scheme that @@ -1675,7 +1675,7 @@ changes, or look through the Subversion logs for all the details. (Python 2.7 actually produces slightly different output, since it returns a named tuple instead of a standard tuple.) - The :mod:`urlparse` module also supports IPv6 literal addresses as defined by + The :mod:`urlparse <urllib.parse>` module also supports IPv6 literal addresses as defined by :rfc:`2732` (contributed by Senthil Kumaran; :issue:`2987`). .. doctest:: @@ -1697,8 +1697,8 @@ changes, or look through the Subversion logs for all the details. or comment (which looks like ``<!-- comment -->``). (Patch by Neil Muller; :issue:`2746`.) -* The XML-RPC client and server, provided by the :mod:`xmlrpclib` and - :mod:`SimpleXMLRPCServer` modules, have improved performance by +* The XML-RPC client and server, provided by the :mod:`xmlrpclib <xmlrpc.client>` and + :mod:`SimpleXMLRPCServer <xmlrpc.server>` modules, have improved performance by supporting HTTP/1.1 keep-alive and by optionally using gzip encoding to compress the XML being exchanged. The gzip compression is controlled by the :attr:`encode_threshold` attribute of @@ -2130,7 +2130,7 @@ Changes to Python's build process and to the C API include: only the filename, function name, and first line number are required. This is useful for extension modules that are attempting to construct a more useful traceback stack. Previously such - extensions needed to call :c:func:`PyCode_New`, which had many + extensions needed to call :c:func:`!PyCode_New`, which had many more arguments. (Added by Jeffrey Yasskin.) * New function: :c:func:`PyErr_NewExceptionWithDoc` creates a new @@ -2334,11 +2334,11 @@ Port-Specific Changes: Windows and :data:`LIBRARIES_ASSEMBLY_NAME_PREFIX`. (Contributed by David Cournapeau; :issue:`4365`.) -* The :mod:`_winreg` module for accessing the registry now implements - the :func:`~_winreg.CreateKeyEx` and :func:`~_winreg.DeleteKeyEx` +* The :mod:`_winreg <winreg>` module for accessing the registry now implements + the :func:`~winreg.CreateKeyEx` and :func:`~winreg.DeleteKeyEx` functions, extended versions of previously supported functions that - take several extra arguments. The :func:`~_winreg.DisableReflectionKey`, - :func:`~_winreg.EnableReflectionKey`, and :func:`~_winreg.QueryReflectionKey` + take several extra arguments. The :func:`~winreg.DisableReflectionKey`, + :func:`~winreg.EnableReflectionKey`, and :func:`~winreg.QueryReflectionKey` were also tested and documented. (Implemented by Brian Curtin: :issue:`7347`.) @@ -2508,7 +2508,7 @@ In the standard library: which raises an exception if there's an error. (Changed by Lars Gustäbel; :issue:`7357`.) -* The :mod:`urlparse` module's :func:`~urlparse.urlsplit` now handles +* The :mod:`urlparse <urllib.parse>` module's :func:`~urllib.parse.urlsplit` now handles unknown URL schemes in a fashion compliant with :rfc:`3986`: if the URL is of the form ``"<something>://..."``, the text before the ``://`` is treated as the scheme, even if it's a made-up scheme that @@ -2711,8 +2711,8 @@ and :ref:`setuptools-index`. PEP 476: Enabling certificate verification by default for stdlib http clients ----------------------------------------------------------------------------- -:pep:`476` updated :mod:`httplib` and modules which use it, such as -:mod:`urllib2` and :mod:`xmlrpclib`, to now verify that the server +:pep:`476` updated :mod:`httplib <http>` and modules which use it, such as +:mod:`urllib2 <urllib.request>` and :mod:`xmlrpclib`, to now verify that the server presents a certificate which is signed by a Certificate Authority in the platform trust store and whose hostname matches the hostname being requested by default, significantly improving security for many applications. This diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index 1df5209f22c6a5b..888e6279754fc29 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -337,7 +337,7 @@ changed. (However, the standard library remains ASCII-only with the exception of contributor names in comments.) -* The :mod:`StringIO` and :mod:`cStringIO` modules are gone. Instead, +* The :mod:`!StringIO` and :mod:`!cStringIO` modules are gone. Instead, import the :mod:`io` module and use :class:`io.StringIO` or :class:`io.BytesIO` for text and data respectively. @@ -563,7 +563,7 @@ review: removal in Python 3.0 due to lack of use or because a better replacement exists. See :pep:`3108` for an exhaustive list. -* The :mod:`bsddb3` package was removed because its presence in the +* The :mod:`!bsddb3` package was removed because its presence in the core standard library has proved over time to be a particular burden for the core developers due to testing instability and Berkeley DB's release schedule. However, the package is alive and well, @@ -588,40 +588,40 @@ review: * A common pattern in Python 2.x is to have one version of a module implemented in pure Python, with an optional accelerated version implemented as a C extension; for example, :mod:`pickle` and - :mod:`cPickle`. This places the burden of importing the accelerated + :mod:`!cPickle`. This places the burden of importing the accelerated version and falling back on the pure Python version on each user of these modules. In Python 3.0, the accelerated versions are considered implementation details of the pure Python versions. Users should always import the standard version, which attempts to import the accelerated version and falls back to the pure Python - version. The :mod:`pickle` / :mod:`cPickle` pair received this + version. The :mod:`pickle` / :mod:`!cPickle` pair received this treatment. The :mod:`profile` module is on the list for 3.1. The - :mod:`StringIO` module has been turned into a class in the :mod:`io` + :mod:`!StringIO` module has been turned into a class in the :mod:`io` module. * Some related modules have been grouped into packages, and usually the submodule names have been simplified. The resulting new packages are: - * :mod:`dbm` (:mod:`anydbm`, :mod:`dbhash`, :mod:`dbm`, - :mod:`dumbdbm`, :mod:`gdbm`, :mod:`whichdb`). + * :mod:`dbm` (:mod:`!anydbm`, :mod:`!dbhash`, :mod:`!dbm`, + :mod:`!dumbdbm`, :mod:`!gdbm`, :mod:`!whichdb`). - * :mod:`html` (:mod:`HTMLParser`, :mod:`htmlentitydefs`). + * :mod:`html` (:mod:`!HTMLParser`, :mod:`!htmlentitydefs`). - * :mod:`http` (:mod:`httplib`, :mod:`BaseHTTPServer`, - :mod:`CGIHTTPServer`, :mod:`SimpleHTTPServer`, :mod:`Cookie`, - :mod:`cookielib`). + * :mod:`http` (:mod:`!httplib`, :mod:`!BaseHTTPServer`, + :mod:`!CGIHTTPServer`, :mod:`!SimpleHTTPServer`, :mod:`!Cookie`, + :mod:`!cookielib`). * :mod:`tkinter` (all :mod:`Tkinter`-related modules except :mod:`turtle`). The target audience of :mod:`turtle` doesn't really care about :mod:`tkinter`. Also note that as of Python 2.6, the functionality of :mod:`turtle` has been greatly enhanced. - * :mod:`urllib` (:mod:`urllib`, :mod:`urllib2`, :mod:`urlparse`, - :mod:`robotparse`). + * :mod:`urllib` (:mod:`!urllib`, :mod:`!urllib2`, :mod:`!urlparse`, + :mod:`!robotparse`). - * :mod:`xmlrpc` (:mod:`xmlrpclib`, :mod:`DocXMLRPCServer`, - :mod:`SimpleXMLRPCServer`). + * :mod:`xmlrpc` (:mod:`!xmlrpclib`, :mod:`!DocXMLRPCServer`, + :mod:`!SimpleXMLRPCServer`). Some other changes to standard library modules, not covered by :pep:`3108`: @@ -642,9 +642,9 @@ Some other changes to standard library modules, not covered by * Cleanup of the :mod:`operator` module: removed :func:`sequenceIncludes` and :func:`isCallable`. -* Cleanup of the :mod:`thread` module: :func:`acquire_lock` and - :func:`release_lock` are gone; use :func:`acquire` and - :func:`release` instead. +* Cleanup of the :mod:`!thread` module: :func:`!acquire_lock` and + :func:`!release_lock` are gone; use :meth:`~threading.Lock.acquire` and + :meth:`~threading.Lock.release` instead. * Cleanup of the :mod:`random` module: removed the :func:`jumpahead` API. diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index cb646a54df36070..4f4c1de8d8d5964 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -672,7 +672,7 @@ enum * Changed :meth:`Enum.__format__() <enum.Enum.__format__>` (the default for :func:`format`, :meth:`str.format` and :term:`f-string`\s) to always produce - the same result as :meth:`Enum.__str__()`: for enums inheriting from + the same result as :meth:`Enum.__str__() <enum.Enum.__str__>`: for enums inheriting from :class:`~enum.ReprEnum` it will be the member's value; for all other enums it will be the enum and member name (e.g. ``Color.RED``). @@ -1604,7 +1604,7 @@ raw, adaptive bytecode containing quickened data. New opcodes ----------- -* :opcode:`ASYNC_GEN_WRAP`, :opcode:`RETURN_GENERATOR` and :opcode:`SEND`, +* :opcode:`!ASYNC_GEN_WRAP`, :opcode:`RETURN_GENERATOR` and :opcode:`SEND`, used in generators and co-routines. * :opcode:`COPY_FREE_VARS`, @@ -1615,7 +1615,7 @@ New opcodes * :opcode:`MAKE_CELL`, to create :ref:`cell-objects`. -* :opcode:`CHECK_EG_MATCH` and :opcode:`PREP_RERAISE_STAR`, +* :opcode:`CHECK_EG_MATCH` and :opcode:`!PREP_RERAISE_STAR`, to handle the :ref:`new exception groups and except* <whatsnew311-pep654>` added in :pep:`654`. @@ -1630,38 +1630,38 @@ New opcodes Replaced opcodes ---------------- -+------------------------------------+-----------------------------------+-----------------------------------------+ -| Replaced Opcode(s) | New Opcode(s) | Notes | -+====================================+===================================+=========================================+ -| | :opcode:`!BINARY_*` | :opcode:`BINARY_OP` | Replaced all numeric binary/in-place | -| | :opcode:`!INPLACE_*` | | opcodes with a single opcode | -+------------------------------------+-----------------------------------+-----------------------------------------+ -| | :opcode:`!CALL_FUNCTION` | | :opcode:`CALL` | Decouples argument shifting for methods | -| | :opcode:`!CALL_FUNCTION_KW` | | :opcode:`KW_NAMES` | from handling of keyword arguments; | -| | :opcode:`!CALL_METHOD` | | :opcode:`PRECALL` | allows better specialization of calls | -| | | :opcode:`PUSH_NULL` | | -+------------------------------------+-----------------------------------+-----------------------------------------+ -| | :opcode:`!DUP_TOP` | | :opcode:`COPY` | Stack manipulation instructions | -| | :opcode:`!DUP_TOP_TWO` | | :opcode:`SWAP` | | -| | :opcode:`!ROT_TWO` | | | -| | :opcode:`!ROT_THREE` | | | -| | :opcode:`!ROT_FOUR` | | | -| | :opcode:`!ROT_N` | | | -+------------------------------------+-----------------------------------+-----------------------------------------+ -| | :opcode:`!JUMP_IF_NOT_EXC_MATCH` | | :opcode:`CHECK_EXC_MATCH` | Now performs check but doesn't jump | -+------------------------------------+-----------------------------------+-----------------------------------------+ -| | :opcode:`!JUMP_ABSOLUTE` | | :opcode:`JUMP_BACKWARD` | See [#bytecode-jump]_; | -| | :opcode:`!POP_JUMP_IF_FALSE` | | :opcode:`POP_JUMP_BACKWARD_IF_* | ``TRUE``, ``FALSE``, | -| | :opcode:`!POP_JUMP_IF_TRUE` | <POP_JUMP_BACKWARD_IF_TRUE>` | ``NONE`` and ``NOT_NONE`` variants | -| | | :opcode:`POP_JUMP_FORWARD_IF_* | for each direction | -| | <POP_JUMP_FORWARD_IF_TRUE>` | | -+------------------------------------+-----------------------------------+-----------------------------------------+ -| | :opcode:`!SETUP_WITH` | :opcode:`BEFORE_WITH` | :keyword:`with` block setup | -| | :opcode:`!SETUP_ASYNC_WITH` | | | -+------------------------------------+-----------------------------------+-----------------------------------------+ ++------------------------------------+------------------------------------+-----------------------------------------+ +| Replaced Opcode(s) | New Opcode(s) | Notes | ++====================================+====================================+=========================================+ +| | :opcode:`!BINARY_*` | :opcode:`BINARY_OP` | Replaced all numeric binary/in-place | +| | :opcode:`!INPLACE_*` | | opcodes with a single opcode | ++------------------------------------+------------------------------------+-----------------------------------------+ +| | :opcode:`!CALL_FUNCTION` | | :opcode:`CALL` | Decouples argument shifting for methods | +| | :opcode:`!CALL_FUNCTION_KW` | | :opcode:`!KW_NAMES` | from handling of keyword arguments; | +| | :opcode:`!CALL_METHOD` | | :opcode:`!PRECALL` | allows better specialization of calls | +| | | :opcode:`PUSH_NULL` | | ++------------------------------------+------------------------------------+-----------------------------------------+ +| | :opcode:`!DUP_TOP` | | :opcode:`COPY` | Stack manipulation instructions | +| | :opcode:`!DUP_TOP_TWO` | | :opcode:`SWAP` | | +| | :opcode:`!ROT_TWO` | | | +| | :opcode:`!ROT_THREE` | | | +| | :opcode:`!ROT_FOUR` | | | +| | :opcode:`!ROT_N` | | | ++------------------------------------+------------------------------------+-----------------------------------------+ +| | :opcode:`!JUMP_IF_NOT_EXC_MATCH` | | :opcode:`CHECK_EXC_MATCH` | Now performs check but doesn't jump | ++------------------------------------+------------------------------------+-----------------------------------------+ +| | :opcode:`!JUMP_ABSOLUTE` | | :opcode:`JUMP_BACKWARD` | See [#bytecode-jump]_; | +| | :opcode:`!POP_JUMP_IF_FALSE` | | :opcode:`!POP_JUMP_BACKWARD_IF_*`| ``TRUE``, ``FALSE``, | +| | :opcode:`!POP_JUMP_IF_TRUE` | | :opcode:`!POP_JUMP_FORWARD_IF_*` | ``NONE`` and ``NOT_NONE`` variants | +| | | for each direction | +| | | | ++------------------------------------+------------------------------------+-----------------------------------------+ +| | :opcode:`!SETUP_WITH` | :opcode:`BEFORE_WITH` | :keyword:`with` block setup | +| | :opcode:`!SETUP_ASYNC_WITH` | | | ++------------------------------------+------------------------------------+-----------------------------------------+ .. [#bytecode-jump] All jump opcodes are now relative, including the - existing :opcode:`JUMP_IF_TRUE_OR_POP` and :opcode:`JUMP_IF_FALSE_OR_POP`. + existing :opcode:`!JUMP_IF_TRUE_OR_POP` and :opcode:`!JUMP_IF_FALSE_OR_POP`. The argument is now an offset from the current instruction rather than an absolute location. @@ -1789,13 +1789,13 @@ Standard Library and will be removed in a future Python version, due to not supporting resources located within package subdirectories: - * :func:`importlib.resources.contents` - * :func:`importlib.resources.is_resource` - * :func:`importlib.resources.open_binary` - * :func:`importlib.resources.open_text` - * :func:`importlib.resources.read_binary` - * :func:`importlib.resources.read_text` - * :func:`importlib.resources.path` + * :func:`!importlib.resources.contents` + * :func:`!importlib.resources.is_resource` + * :func:`!importlib.resources.open_binary` + * :func:`!importlib.resources.open_text` + * :func:`!importlib.resources.read_binary` + * :func:`!importlib.resources.read_text` + * :func:`!importlib.resources.path` * The :func:`locale.getdefaultlocale` function is deprecated and will be removed in Python 3.15. Use :func:`locale.setlocale`, @@ -1803,7 +1803,7 @@ Standard Library :func:`locale.getlocale` functions instead. (Contributed by Victor Stinner in :gh:`90817`.) -* The :func:`locale.resetlocale` function is deprecated and will be +* The :func:`!locale.resetlocale` function is deprecated and will be removed in Python 3.13. Use ``locale.setlocale(locale.LC_ALL, "")`` instead. (Contributed by Victor Stinner in :gh:`90817`.) @@ -1967,7 +1967,7 @@ Removed C APIs are :ref:`listed separately <whatsnew311-c-api-removed>`. (Contributed by Victor Stinner in :issue:`45085`.) -* Removed the :mod:`distutils` ``bdist_msi`` command deprecated in Python 3.9. +* Removed the :mod:`!distutils` ``bdist_msi`` command deprecated in Python 3.9. Use ``bdist_wheel`` (wheel packages) instead. (Contributed by Hugo van Kemenade in :issue:`45124`.) @@ -2295,7 +2295,7 @@ Porting to Python 3.11 as its second parameter, instead of ``PyFrameObject*``. See :pep:`523` for more details of how to use this function pointer type. -* :c:func:`PyCode_New` and :c:func:`PyCode_NewWithPosOnlyArgs` now take +* :c:func:`!PyCode_New` and :c:func:`!PyCode_NewWithPosOnlyArgs` now take an additional ``exception_table`` argument. Using these functions should be avoided, if at all possible. To get a custom code object: create a code object using the compiler, @@ -2402,7 +2402,7 @@ Porting to Python 3.11 been included directly, consider including ``Python.h`` instead. (Contributed by Victor Stinner in :issue:`35134`.) -* The :c:func:`PyUnicode_CHECK_INTERNED` macro has been excluded from the +* The :c:func:`!PyUnicode_CHECK_INTERNED` macro has been excluded from the limited C API. It was never usable there, because it used internal structures which are not available in the limited C API. (Contributed by Victor Stinner in :issue:`46007`.) @@ -2465,7 +2465,7 @@ Porting to Python 3.11 Debuggers that accessed the :attr:`~frame.f_locals` directly *must* call :c:func:`PyFrame_GetLocals` instead. They no longer need to call - :c:func:`PyFrame_FastToLocalsWithError` or :c:func:`PyFrame_LocalsToFast`, + :c:func:`!PyFrame_FastToLocalsWithError` or :c:func:`!PyFrame_LocalsToFast`, in fact they should not call those functions. The necessary updating of the frame is now managed by the virtual machine. @@ -2604,8 +2604,8 @@ and will be removed in Python 3.12. * :c:func:`!PyUnicode_GET_DATA_SIZE` * :c:func:`!PyUnicode_GET_SIZE` * :c:func:`!PyUnicode_GetSize` -* :c:func:`PyUnicode_IS_COMPACT` -* :c:func:`PyUnicode_IS_READY` +* :c:func:`!PyUnicode_IS_COMPACT` +* :c:func:`!PyUnicode_IS_READY` * :c:func:`PyUnicode_READY` * :c:func:`!PyUnicode_WSTR_LENGTH` * :c:func:`!_PyUnicode_AsUnicode` @@ -2660,7 +2660,7 @@ Removed (Contributed by Victor Stinner in :issue:`45474`.) * Exclude :c:func:`PyWeakref_GET_OBJECT` from the limited C API. It never - worked since the :c:type:`PyWeakReference` structure is opaque in the + worked since the :c:type:`!PyWeakReference` structure is opaque in the limited C API. (Contributed by Victor Stinner in :issue:`35134`.) @@ -2701,4 +2701,30 @@ Removed (Contributed by Inada Naoki in :issue:`44029`.) +Notable changes in 3.11.4 +========================= + +tarfile +------- + +* The extraction methods in :mod:`tarfile`, and :func:`shutil.unpack_archive`, + have a new a *filter* argument that allows limiting tar features than may be + surprising or dangerous, such as creating files outside the destination + directory. + See :ref:`tarfile-extraction-filter` for details. + In Python 3.12, use without the *filter* argument will show a + :exc:`DeprecationWarning`. + In Python 3.14, the default will switch to ``'data'``. + (Contributed by Petr Viktorin in :pep:`706`.) + + +Notable changes in 3.11.5 +========================= + +OpenSSL +------- + +* Windows builds and macOS installers from python.org now use OpenSSL 3.0. + + .. _libb2: https://www.blake2.net/ diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 77b12f9284ba0d9..100312a5940b796 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -690,7 +690,7 @@ csv * Add :const:`csv.QUOTE_NOTNULL` and :const:`csv.QUOTE_STRINGS` flags to provide finer grained control of ``None`` and empty strings by - :class:`csv.writer` objects. + :class:`~csv.reader` and :class:`~csv.writer` objects. dis --- diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 40f0cd37fe93185..de79bd979aff80e 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -81,6 +81,13 @@ Important deprecations, removals or restrictions: * Python 3.13 and later have two years of full support, followed by three years of security fixes. +Interpreter improvements: + +* A basic :ref:`JIT compiler <whatsnew313-jit-compiler>` was added. + It is currently disabled by default (though we may turn it on later). + Performance improvements are modest -- we expect to be improving this + over the next few releases. + New Features ============ @@ -94,6 +101,17 @@ Improved Error Messages variables. See also :ref:`using-on-controlling-color`. (Contributed by Pablo Galindo Salgado in :gh:`112730`.) +* When an incorrect keyword argument is passed to a function, the error message + now potentially suggests the correct keyword argument. + (Contributed by Pablo Galindo Salgado and Shantanu Jain in :gh:`107944`.) + + >>> "better error messages!".split(max_split=1) + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + "better error messages!".split(max_split=1) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^ + TypeError: split() got an unexpected keyword argument 'max_split'. Did you mean 'maxsplit'? + Other Language Changes ====================== @@ -127,6 +145,11 @@ Other Language Changes is rejected when the global is used in the :keyword:`else` block. (Contributed by Irit Katriel in :gh:`111123`.) +* Many functions now emit a warning if a boolean value is passed as + a file descriptor argument. + This can help catch some errors earlier. + (Contributed by Serhiy Storchaka in :gh:`82626`.) + * Added a new environment variable :envvar:`PYTHON_FROZEN_MODULES`. It determines whether or not frozen modules are ignored by the import machinery, equivalent of the :option:`-X frozen_modules <-X>` command-line option. @@ -146,13 +169,14 @@ New Modules Improved Modules ================ -ast ---- +argparse +-------- -* :func:`ast.parse` now accepts an optional argument ``optimize`` - which is passed on to the :func:`compile` built-in. This makes it - possible to obtain an optimized ``AST``. - (Contributed by Irit Katriel in :gh:`108113`). +* Add parameter *deprecated* in methods + :meth:`~argparse.ArgumentParser.add_argument` and :meth:`!add_parser` + which allows to deprecate command-line options, positional arguments and + subcommands. + (Contributed by Serhiy Storchaka in :gh:`83648`). array ----- @@ -161,6 +185,14 @@ array It can be used instead of ``'u'`` type code, which is deprecated. (Contributed by Inada Naoki in :gh:`80480`.) +ast +--- + +* :func:`ast.parse` now accepts an optional argument ``optimize`` + which is passed on to the :func:`compile` built-in. This makes it + possible to obtain an optimized ``AST``. + (Contributed by Irit Katriel in :gh:`108113`.) + asyncio ------- @@ -180,6 +212,13 @@ copy any user classes which define the :meth:`!__replace__` method. (Contributed by Serhiy Storchaka in :gh:`108751`.) +dbm +--- + +* Add :meth:`dbm.gnu.gdbm.clear` and :meth:`dbm.ndbm.ndbm.clear` methods that remove all items + from the database. + (Contributed by Donghee Na in :gh:`107122`.) + dis --- @@ -189,13 +228,6 @@ dis the ``show_offsets`` parameter. (Contributed by Irit Katriel in :gh:`112137`.) -dbm ---- - -* Add :meth:`dbm.gnu.gdbm.clear` and :meth:`dbm.ndbm.ndbm.clear` methods that remove all items - from the database. - (Contributed by Donghee Na in :gh:`107122`.) - doctest ------- @@ -213,7 +245,7 @@ email parameter to these two functions: use ``strict=False`` to get the old behavior, accept malformed inputs. ``getattr(email.utils, 'supports_strict_parsing', False)`` can be use to - check if the *strict* paramater is available. + check if the *strict* parameter is available. (Contributed by Thomas Dwyer and Victor Stinner for :gh:`102988` to improve the CVE-2023-27043 fix.) @@ -223,7 +255,7 @@ fractions * Formatting for objects of type :class:`fractions.Fraction` now supports the standard format specification mini-language rules for fill, alignment, sign handling, minimum width and grouping. (Contributed by Mark Dickinson - in :gh:`111320`) + in :gh:`111320`.) glob ---- @@ -259,8 +291,8 @@ mmap ---- * The :class:`mmap.mmap` class now has an :meth:`~mmap.mmap.seekable` method - that can be used where it requires a file-like object with seekable and - the :meth:`~mmap.mmap.seek` method return the new absolute position. + that can be used when a seekable file-like object is required. + The :meth:`~mmap.mmap.seek` method now returns the new absolute position. (Contributed by Donghee Na and Sylvie Liberman in :gh:`111835`.) * :class:`mmap.mmap` now has a *trackfd* parameter on Unix; if it is ``False``, the file descriptor specified by *fileno* will not be duplicated. @@ -297,17 +329,17 @@ os the new environment variable :envvar:`PYTHON_CPU_COUNT` or the new command-line option :option:`-X cpu_count <-X>`. This option is useful for users who need to limit CPU resources of a container system without having to modify the container (application code). - (Contributed by Donghee Na in :gh:`109595`) + (Contributed by Donghee Na in :gh:`109595`.) * Add support of :func:`os.lchmod` and the *follow_symlinks* argument in :func:`os.chmod` on Windows. Note that the default value of *follow_symlinks* in :func:`!os.lchmod` is ``False`` on Windows. - (Contributed by Serhiy Storchaka in :gh:`59616`) + (Contributed by Serhiy Storchaka in :gh:`59616`.) * Add support of :func:`os.fchmod` and a file descriptor in :func:`os.chmod` on Windows. - (Contributed by Serhiy Storchaka in :gh:`113191`) + (Contributed by Serhiy Storchaka in :gh:`113191`.) * :func:`os.posix_spawn` now accepts ``env=None``, which makes the newly spawned process use the current process environment. @@ -321,6 +353,9 @@ os os.path ------- +* Add :func:`os.path.isreserved` to check if a path is reserved on the current + system. This function is only available on Windows. + (Contributed by Barney Gale in :gh:`88569`.) * On Windows, :func:`os.path.isabs` no longer considers paths starting with exactly one (back)slash to be absolute. (Contributed by Barney Gale and Jon Foster in :gh:`44626`.) @@ -333,10 +368,11 @@ pathlib (Contributed by Barney Gale in :gh:`89812`.) * Add :meth:`pathlib.Path.from_uri`, a new constructor to create a :class:`pathlib.Path` - object from a 'file' URI (``file:/``). + object from a 'file' URI (``file://``). (Contributed by Barney Gale in :gh:`107465`.) -* Add support for recursive wildcards in :meth:`pathlib.PurePath.match`. +* Add :meth:`pathlib.PurePath.full_match` for matching paths with + shell-style wildcards, including the recursive wildcard "``**``". (Contributed by Barney Gale in :gh:`73435`.) * Add *follow_symlinks* keyword-only argument to :meth:`pathlib.Path.glob`, @@ -346,6 +382,11 @@ pathlib (Contributed by Barney Gale in :gh:`77609` and :gh:`105793`, and Kamil Turek in :gh:`107962`). +* Return files and directories from :meth:`pathlib.Path.glob` and + :meth:`~pathlib.Path.rglob` when given a pattern that ends with "``**``". In + earlier versions, only directories were returned. + (Contributed by Barney Gale in :gh:`70303`). + pdb --- @@ -353,7 +394,7 @@ pdb the new ``exceptions [exc_number]`` command for Pdb. (Contributed by Matthias Bussonnier in :gh:`106676`.) -* Expressions/Statements whose prefix is a pdb command are now correctly +* Expressions/statements whose prefix is a pdb command are now correctly identified and executed. (Contributed by Tian Gao in :gh:`108464`.) @@ -362,6 +403,13 @@ pdb command line option or :envvar:`PYTHONSAFEPATH` environment variable). (Contributed by Tian Gao and Christian Walther in :gh:`111762`.) +queue +----- + +* Add :meth:`queue.Queue.shutdown` (along with :exc:`queue.ShutDown`) for queue + termination. + (Contributed by Laurie Opperman and Yves Duprat in :gh:`104750`.) + re -- * Rename :exc:`!re.error` to :exc:`re.PatternError` for improved clarity. @@ -374,6 +422,10 @@ sqlite3 object is not :meth:`closed <sqlite3.Connection.close>` explicitly. (Contributed by Erlend E. Aasland in :gh:`105539`.) +* Add *filter* keyword-only parameter to :meth:`sqlite3.Connection.iterdump` + for filtering database objects to dump. + (Contributed by Mariusz Felisiak in :gh:`91602`.) + subprocess ---------- @@ -405,6 +457,15 @@ tkinter :meth:`!tk_busy_current`, and :meth:`!tk_busy_status`. (Contributed by Miguel, klappnase and Serhiy Storchaka in :gh:`72684`.) +* The :mod:`tkinter` widget method :meth:`!wm_attributes` now accepts + the attribute name without the minus prefix to get window attributes, + e.g. ``w.wm_attributes('alpha')`` and allows to specify attributes and + values to set as keyword arguments, e.g. ``w.wm_attributes(alpha=0.5)``. + Add new optional keyword-only parameter *return_python_dict*: calling + ``w.wm_attributes(return_python_dict=True)`` returns the attributes as + a dict instead of a tuple. + (Contributed by Serhiy Storchaka in :gh:`43457`.) + * Add support of the "vsapi" element type in the :meth:`~tkinter.ttk.Style.element_create` method of :class:`tkinter.ttk.Style`. @@ -456,6 +517,14 @@ warnings warning may also be emitted when a decorated function or class is used at runtime. See :pep:`702`. (Contributed by Jelle Zijlstra in :gh:`104003`.) +xml.etree.ElementTree +--------------------- + +* Add the :meth:`!close` method for the iterator returned by + :func:`~xml.etree.ElementTree.iterparse` for explicit cleaning up. + (Contributed by Serhiy Storchaka in :gh:`69893`.) + + Optimizations ============= @@ -468,6 +537,46 @@ Optimizations FreeBSD and Solaris. See the ``subprocess`` section above for details. (Contributed by Jakub Kulik in :gh:`113117`.) +.. _whatsnew313-jit-compiler: + +Experimental JIT Compiler +========================= + +When CPython is configured using the ``--enable-experimental-jit`` option, +a just-in-time compiler is added which can speed up some Python programs. + +The internal architecture is roughly as follows. + +* We start with specialized *Tier 1 bytecode*. + See :ref:`What's new in 3.11 <whatsnew311-pep659>` for details. + +* When the Tier 1 bytecode gets hot enough, it gets translated + to a new, purely internal *Tier 2 IR*, a.k.a. micro-ops ("uops"). + +* The Tier 2 IR uses the same stack-based VM as Tier 1, but the + instruction format is better suited to translation to machine code. + +* We have several optimization passes for Tier 2 IR, which are applied + before it is interpreted or translated to machine code. + +* There is a Tier 2 interpreter, but it is mostly intended for debugging + the earlier stages of the optimization pipeline. If the JIT is not + enabled, the Tier 2 interpreter can be invoked by passing Python the + ``-X uops`` option or by setting the ``PYTHON_UOPS`` environment + variable to ``1``. + +* When the ``--enable-experimental-jit`` option is used, the optimized + Tier 2 IR is translated to machine code, which is then executed. + This does not require additional runtime options. + +* The machine code translation process uses an architecture called + *copy-and-patch*. It has no runtime dependencies, but there is a new + build-time dependency on LLVM. + +(JIT by Brandt Bucher, inspired by a paper by Haoran Xu and Fredrik Kjolstad. +Tier 2 IR by Mark Shannon and Guido van Rossum. +Tier 2 optimizer by Ken Jin.) + Deprecated ========== @@ -483,28 +592,69 @@ Deprecated Replace ``ctypes.ARRAY(item_type, size)`` with ``item_type * size``. (Contributed by Victor Stinner in :gh:`105733`.) +* :mod:`decimal`: Deprecate non-standard format specifier "N" for + :class:`decimal.Decimal`. + It was not documented and only supported in the C implementation. + (Contributed by Serhiy Storchaka in :gh:`89902`.) + +* :mod:`dis`: The ``dis.HAVE_ARGUMENT`` separator is deprecated. Check + membership in :data:`~dis.hasarg` instead. + (Contributed by Irit Katriel in :gh:`109319`.) + * :mod:`getopt` and :mod:`optparse` modules: They are now - :term:`soft deprecated`: the :mod:`argparse` should be used for new projects. + :term:`soft deprecated`: the :mod:`argparse` module should be used for new projects. Previously, the :mod:`optparse` module was already deprecated, its removal was not scheduled, and no warnings was emitted: so there is no change in practice. (Contributed by Victor Stinner in :gh:`106535`.) +* :mod:`gettext`: Emit deprecation warning for non-integer numbers in + :mod:`gettext` functions and methods that consider plural forms even if the + translation was not found. + (Contributed by Serhiy Storchaka in :gh:`88434`.) + * :mod:`http.server`: :class:`http.server.CGIHTTPRequestHandler` now emits a - :exc:`DeprecationWarning` as it will be removed in 3.15. Process based CGI - http servers have been out of favor for a very long time. This code was + :exc:`DeprecationWarning` as it will be removed in 3.15. Process-based CGI + HTTP servers have been out of favor for a very long time. This code was outdated, unmaintained, and rarely used. It has a high potential for both security and functionality bugs. This includes removal of the ``--cgi`` flag to the ``python -m http.server`` command line in 3.15. +* :mod:`pathlib`: + :meth:`pathlib.PurePath.is_reserved` is deprecated and scheduled for + removal in Python 3.15. Use :func:`os.path.isreserved` to detect reserved + paths on Windows. + +* :mod:`pydoc`: Deprecate undocumented :func:`!pydoc.ispackage` function. + (Contributed by Zackery Spytz in :gh:`64020`.) + +* :mod:`sqlite3`: Passing more than one positional argument to + :func:`sqlite3.connect` and the :class:`sqlite3.Connection` constructor is + deprecated. The remaining parameters will become keyword-only in Python 3.15. + + Deprecate passing name, number of arguments, and the callable as keyword + arguments for the following :class:`sqlite3.Connection` APIs: + + * :meth:`~sqlite3.Connection.create_function` + * :meth:`~sqlite3.Connection.create_aggregate` + + Deprecate passing the callback callable by keyword for the following + :class:`sqlite3.Connection` APIs: + + * :meth:`~sqlite3.Connection.set_authorizer` + * :meth:`~sqlite3.Connection.set_progress_handler` + * :meth:`~sqlite3.Connection.set_trace_callback` + + The affected parameters will become positional-only in Python 3.15. + + (Contributed by Erlend E. Aasland in :gh:`107948` and :gh:`108278`.) + * :mod:`sys`: :func:`sys._enablelegacywindowsfsencoding` function. - Replace it with :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable. + Replace it with the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable. (Contributed by Inada Naoki in :gh:`73427`.) -* :mod:`traceback`: - - * The field *exc_type* of :class:`traceback.TracebackException` is - deprecated. Use *exc_type_str* instead. +* :mod:`traceback`: The field *exc_type* of :class:`traceback.TracebackException` + is deprecated. Use *exc_type_str* instead. * :mod:`typing`: @@ -540,39 +690,6 @@ Deprecated They will be removed in Python 3.15. (Contributed by Victor Stinner in :gh:`105096`.) -* Passing more than one positional argument to :func:`sqlite3.connect` and the - :class:`sqlite3.Connection` constructor is deprecated. The remaining - parameters will become keyword-only in Python 3.15. - - Deprecate passing name, number of arguments, and the callable as keyword - arguments, for the following :class:`sqlite3.Connection` APIs: - - * :meth:`~sqlite3.Connection.create_function` - * :meth:`~sqlite3.Connection.create_aggregate` - - Deprecate passing the callback callable by keyword for the following - :class:`sqlite3.Connection` APIs: - - * :meth:`~sqlite3.Connection.set_authorizer` - * :meth:`~sqlite3.Connection.set_progress_handler` - * :meth:`~sqlite3.Connection.set_trace_callback` - - The affected parameters will become positional-only in Python 3.15. - - (Contributed by Erlend E. Aasland in :gh:`107948` and :gh:`108278`.) - -* The ``dis.HAVE_ARGUMENT`` separator is deprecated. Check membership - in :data:`~dis.hasarg` instead. - (Contributed by Irit Katriel in :gh:`109319`.) - -* Deprecate non-standard format specifier "N" for :class:`decimal.Decimal`. - It was not documented and only supported in the C implementation. - (Contributed by Serhiy Storchaka in :gh:`89902`.) - -* Emit deprecation warning for non-integer numbers in :mod:`gettext` functions - and methods that consider plural forms even if the translation was not found. - (Contributed by Serhiy Storchaka in :gh:`88434`.) - * Calling :meth:`frame.clear` on a suspended frame raises :exc:`RuntimeError` (as has always been the case for an executing frame). (Contributed by Irit Katriel in :gh:`79932`.) @@ -583,9 +700,6 @@ Deprecated coroutine. (Contributed by Irit Katriel in :gh:`81137`.) -* Deprecate undocumented :func:`!pydoc.ispackage` function. - (Contributed by Zackery Spytz in :gh:`64020`.) - Pending Removal in Python 3.14 ------------------------------ @@ -647,11 +761,11 @@ Pending Removal in Python 3.14 :func:`~multiprocessing.set_start_method` APIs to explicitly specify when your code *requires* ``'fork'``. See :ref:`multiprocessing-start-methods`. -* :mod:`pathlib`: :meth:`~pathlib.PurePath.is_relative_to`, +* :mod:`pathlib`: :meth:`~pathlib.PurePath.is_relative_to` and :meth:`~pathlib.PurePath.relative_to`: passing additional arguments is deprecated. -* :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` +* :mod:`pkgutil`: :func:`~pkgutil.find_loader` and :func:`~pkgutil.get_loader` now raise :exc:`DeprecationWarning`; use :func:`importlib.util.find_spec` instead. (Contributed by Nikita Sobolev in :gh:`97850`.) @@ -708,6 +822,18 @@ Pending Removal in Python 3.15 :func:`locale.getlocale()` instead. (Contributed by Hugo van Kemenade in :gh:`111187`.) +* :mod:`pathlib`: + :meth:`pathlib.PurePath.is_reserved` is deprecated and scheduled for + removal in Python 3.15. Use :func:`os.path.isreserved` to detect reserved + paths on Windows. + +* :mod:`threading`: + Passing any arguments to :func:`threading.RLock` is now deprecated. + C version allows any numbers of args and kwargs, + but they are just ignored. Python version does not allow any arguments. + All arguments will be removed from :func:`threading.RLock` in Python 3.15. + (Contributed by Nikita Sobolev in :gh:`102029`.) + * :class:`typing.NamedTuple`: * The undocumented keyword argument syntax for creating NamedTuple classes @@ -733,12 +859,6 @@ Pending Removal in Python 3.15 They will be removed in Python 3.15. (Contributed by Victor Stinner in :gh:`105096`.) -* Passing any arguments to :func:`threading.RLock` is now deprecated. - C version allows any numbers of args and kwargs, - but they are just ignored. Python version does not allow any arguments. - All arguments will be removed from :func:`threading.RLock` in Python 3.15. - (Contributed by Nikita Sobolev in :gh:`102029`.) - Pending Removal in Python 3.16 ------------------------------ @@ -785,6 +905,9 @@ although there is currently no date scheduled for their removal. :data:`calendar.FEBRUARY`. (Contributed by Prince Roshan in :gh:`103636`.) +* :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method + instead. + * :mod:`datetime`: * :meth:`~datetime.datetime.utcnow`: @@ -820,11 +943,13 @@ although there is currently no date scheduled for their removal. underscore. (Contributed by Serhiy Storchaka in :gh:`91760`.) +* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. + * :mod:`ssl` options and protocols: * :class:`ssl.SSLContext` without protocol argument is deprecated. * :class:`ssl.SSLContext`: :meth:`~ssl.SSLContext.set_npn_protocols` and - :meth:`!~ssl.SSLContext.selected_npn_protocol` are deprecated: use ALPN + :meth:`!selected_npn_protocol` are deprecated: use ALPN instead. * ``ssl.OP_NO_SSL*`` options * ``ssl.OP_NO_TLS*`` options @@ -837,13 +962,6 @@ although there is currently no date scheduled for their removal. * ``ssl.TLSVersion.TLSv1`` * ``ssl.TLSVersion.TLSv1_1`` -* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. - -* :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method - instead. - -* :class:`typing.Text` (:gh:`92332`). - * :func:`sysconfig.is_python_build` *check_home* parameter is deprecated and ignored. @@ -858,14 +976,10 @@ although there is currently no date scheduled for their removal. * :meth:`!threading.currentThread`: use :meth:`threading.current_thread`. * :meth:`!threading.activeCount`: use :meth:`threading.active_count`. -* :class:`unittest.IsolatedAsyncioTestCase`: it is deprecated to return a value - that is not None from a test case. - -* :mod:`urllib.request`: :class:`~urllib.request.URLopener` and - :class:`~urllib.request.FancyURLopener` style of invoking requests is - deprecated. Use newer :func:`~urllib.request.urlopen` functions and methods. +* :class:`typing.Text` (:gh:`92332`). -* :func:`!urllib.parse.to_bytes`. +* :class:`unittest.IsolatedAsyncioTestCase`: it is deprecated to return a value + that is not ``None`` from a test case. * :mod:`urllib.parse` deprecated functions: :func:`~urllib.parse.urlparse` instead @@ -879,6 +993,11 @@ although there is currently no date scheduled for their removal. * ``splittype()`` * ``splituser()`` * ``splitvalue()`` + * ``to_bytes()`` + +* :mod:`urllib.request`: :class:`~urllib.request.URLopener` and + :class:`~urllib.request.FancyURLopener` style of invoking requests is + deprecated. Use newer :func:`~urllib.request.urlopen` functions and methods. * :mod:`wsgiref`: ``SimpleHandler.stdout.write()`` should not do partial writes. @@ -1174,10 +1293,10 @@ Changes in the Python API * Functions :c:func:`PyDict_GetItem`, :c:func:`PyDict_GetItemString`, :c:func:`PyMapping_HasKey`, :c:func:`PyMapping_HasKeyString`, :c:func:`PyObject_HasAttr`, :c:func:`PyObject_HasAttrString`, and - :c:func:`PySys_GetObject`, which clear all errors occurred during calling - the function, report now them using :func:`sys.unraisablehook`. - You can consider to replace these functions with other functions as - recomended in the documentation. + :c:func:`PySys_GetObject`, which clear all errors which occurred when calling + them, now report them using :func:`sys.unraisablehook`. + You may replace them with other functions as + recommended in the documentation. (Contributed by Serhiy Storchaka in :gh:`106672`.) * An :exc:`OSError` is now raised by :func:`getpass.getuser` for any failure to @@ -1186,7 +1305,7 @@ Changes in the Python API * The :mod:`threading` module now expects the :mod:`!_thread` module to have an ``_is_main_interpreter`` attribute. It is a function with no - arguments that returns ``True`` if the current interpreter is the + arguments that return ``True`` if the current interpreter is the main interpreter. Any library or application that provides a custom ``_thread`` module @@ -1197,6 +1316,11 @@ Changes in the Python API * :class:`mailbox.Maildir` now ignores files with a leading dot. (Contributed by Zackery Spytz in :gh:`65559`.) +* :meth:`pathlib.Path.glob` and :meth:`~pathlib.Path.rglob` now return both + files and directories if a pattern that ends with "``**``" is given, rather + than directories only. Users may add a trailing slash to match only + directories. + Build Changes ============= @@ -1209,7 +1333,7 @@ Build Changes (Contributed by Erlend Aasland in :gh:`105875`.) * Python built with :file:`configure` :option:`--with-trace-refs` (tracing - references) is now ABI compatible with Python release build and + references) is now ABI compatible with the Python release build and :ref:`debug build <debug-build>`. (Contributed by Victor Stinner in :gh:`108634`.) @@ -1222,6 +1346,12 @@ Build Changes :ref:`limited C API <limited-c-api>`. (Contributed by Victor Stinner in :gh:`85283`.) +* ``wasm32-wasi`` is now a tier 2 platform. + (Contributed by Brett Cannon in :gh:`115192`.) + +* ``wasm32-emscripten`` is no longer a supported platform. + (Contributed by Brett Cannon in :gh:`115192`.) + C API Changes ============= @@ -1236,7 +1366,7 @@ New Features (Contributed by Inada Naoki in :gh:`104922`.) * The *keywords* parameter of :c:func:`PyArg_ParseTupleAndKeywords` and - :c:func:`PyArg_VaParseTupleAndKeywords` has now type :c:expr:`char * const *` + :c:func:`PyArg_VaParseTupleAndKeywords` now has type :c:expr:`char * const *` in C and :c:expr:`const char * const *` in C++, instead of :c:expr:`char **`. It makes these functions compatible with arguments of type :c:expr:`const char * const *`, :c:expr:`const char **` or @@ -1293,18 +1423,28 @@ New Features always steals a reference to the value. (Contributed by Serhiy Storchaka in :gh:`86493`.) -* Added :c:func:`PyDict_GetItemRef` and :c:func:`PyDict_GetItemStringRef` +* Add :c:func:`PyDict_GetItemRef` and :c:func:`PyDict_GetItemStringRef` functions: similar to :c:func:`PyDict_GetItemWithError` but returning a :term:`strong reference` instead of a :term:`borrowed reference`. Moreover, these functions return -1 on error and so checking ``PyErr_Occurred()`` is not needed. (Contributed by Victor Stinner in :gh:`106004`.) -* Added :c:func:`PyDict_ContainsString` function: same as +* Added :c:func:`PyDict_SetDefaultRef`, which is similar to + :c:func:`PyDict_SetDefault` but returns a :term:`strong reference` instead of + a :term:`borrowed reference`. This function returns ``-1`` on error, ``0`` on + insertion, and ``1`` if the key was already present in the dictionary. + (Contributed by Sam Gross in :gh:`112066`.) + +* Add :c:func:`PyDict_ContainsString` function: same as :c:func:`PyDict_Contains`, but *key* is specified as a :c:expr:`const char*` UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`. (Contributed by Victor Stinner in :gh:`108314`.) +* Added :c:func:`PyList_GetItemRef` function: similar to + :c:func:`PyList_GetItem` but returns a :term:`strong reference` instead of + a :term:`borrowed reference`. + * Add :c:func:`Py_IsFinalizing` function: check if the main Python interpreter is :term:`shutting down <interpreter shutdown>`. (Contributed by Victor Stinner in :gh:`108014`.) @@ -1358,7 +1498,7 @@ New Features (Contributed by Victor Stinner in :gh:`85283`.) * Add :c:func:`PyErr_FormatUnraisable` function: similar to - :c:func:`PyErr_WriteUnraisable`, but allow to customize the warning mesage. + :c:func:`PyErr_WriteUnraisable`, but allow customizing the warning message. (Contributed by Serhiy Storchaka in :gh:`108082`.) * Add :c:func:`PyList_Extend` and :c:func:`PyList_Clear` functions: similar to @@ -1368,7 +1508,7 @@ New Features * Add :c:func:`PyDict_Pop` and :c:func:`PyDict_PopString` functions: remove a key from a dictionary and optionally return the removed value. This is similar to :meth:`dict.pop`, but without the default value and not raising - :exc:`KeyError` if the key missing. + :exc:`KeyError` if the key is missing. (Contributed by Stefan Behnel and Victor Stinner in :gh:`111262`.) * Add :c:func:`Py_HashPointer` function to hash a pointer. @@ -1481,7 +1621,7 @@ Removed ------- * Removed chained :class:`classmethod` descriptors (introduced in - :issue:`19072`). This can no longer be used to wrap other descriptors + :gh:`63272`). This can no longer be used to wrap other descriptors such as :class:`property`. The core design of this feature was flawed and caused a number of downstream problems. To "pass-through" a :class:`classmethod`, consider using the :attr:`!__wrapped__` @@ -1495,14 +1635,14 @@ Removed add ``cc @vstinner`` to the issue to notify Victor Stinner. (Contributed by Victor Stinner in :gh:`106320`.) -* Remove functions deprecated in Python 3.9. +* Remove functions deprecated in Python 3.9: * ``PyEval_CallObject()``, ``PyEval_CallObjectWithKeywords()``: use :c:func:`PyObject_CallNoArgs` or :c:func:`PyObject_Call` instead. Warning: :c:func:`PyObject_Call` positional arguments must be a - :class:`tuple` and must not be *NULL*, keyword arguments must be a - :class:`dict` or *NULL*, whereas removed functions checked arguments type - and accepted *NULL* positional and keyword arguments. + :class:`tuple` and must not be ``NULL``, keyword arguments must be a + :class:`dict` or ``NULL``, whereas removed functions checked arguments type + and accepted ``NULL`` positional and keyword arguments. To replace ``PyEval_CallObjectWithKeywords(func, NULL, kwargs)`` with :c:func:`PyObject_Call`, pass an empty tuple as positional arguments using :c:func:`PyTuple_New(0) <PyTuple_New>`. diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index 1c7a9270af0aabc..5c2ec230441b426 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -2418,7 +2418,7 @@ Changes in the Python API (Contributed by Victor Stinner in :issue:`21205`.) * The deprecated "strict" mode and argument of :class:`~html.parser.HTMLParser`, - :meth:`HTMLParser.error`, and the :exc:`HTMLParserError` exception have been + :meth:`!HTMLParser.error`, and the :exc:`!HTMLParserError` exception have been removed. (Contributed by Ezio Melotti in :issue:`15114`.) The *convert_charrefs* argument of :class:`~html.parser.HTMLParser` is now ``True`` by default. (Contributed by Berker Peksag in :issue:`21047`.) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index d373fa163ff737d..b041e592d61ed11 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -754,8 +754,8 @@ datetime -------- Added new alternate constructors :meth:`datetime.date.fromisocalendar` and -:meth:`datetime.datetime.fromisocalendar`, which construct :class:`date` and -:class:`datetime` objects respectively from ISO year, week number, and weekday; +:meth:`datetime.datetime.fromisocalendar`, which construct :class:`~datetime.date` and +:class:`~datetime.datetime` objects respectively from ISO year, week number, and weekday; these are the inverse of each class's ``isocalendar`` method. (Contributed by Paul Ganssle in :issue:`36004`.) @@ -1623,8 +1623,8 @@ Build and C API Changes allocation or deallocation may need to be adjusted. (Contributed by Eddie Elizondo in :issue:`35810`.) -* The new function :c:func:`PyCode_NewWithPosOnlyArgs` allows to create - code objects like :c:func:`PyCode_New`, but with an extra *posonlyargcount* +* The new function :c:func:`!PyCode_NewWithPosOnlyArgs` allows to create + code objects like :c:func:`!PyCode_New`, but with an extra *posonlyargcount* parameter for indicating the number of positional-only arguments. (Contributed by Pablo Galindo in :issue:`37221`.) diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index 944965fb9e5351a..35b6a822a0dfffd 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -17,6 +17,9 @@ typedef struct { /* Dictionary version: globally unique, value change each time the dictionary is modified */ #ifdef Py_BUILD_CORE + /* Bits 0-7 are for dict watchers. + * Bits 8-11 are for the watched mutation counter (used by tier2 optimization) + * The remaining bits (12-63) are the actual version tag. */ uint64_t ma_version_tag; #else Py_DEPRECATED(3.12) uint64_t ma_version_tag; @@ -38,6 +41,16 @@ PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObject *, const char *); PyAPI_FUNC(PyObject *) PyDict_SetDefault( PyObject *mp, PyObject *key, PyObject *defaultobj); +// Inserts `key` with a value `default_value`, if `key` is not already present +// in the dictionary. If `result` is not NULL, then the value associated +// with `key` is returned in `*result` (either the existing value, or the now +// inserted `default_value`). +// Returns: +// -1 on error +// 0 if `key` was not present and `default_value` was inserted +// 1 if `key` was present and `default_value` was not inserted +PyAPI_FUNC(int) PyDict_SetDefaultRef(PyObject *mp, PyObject *key, PyObject *default_value, PyObject **result); + /* Get the number of items of a dictionary. */ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { PyDictObject *mp; diff --git a/Include/cpython/listobject.h b/Include/cpython/listobject.h index 8ade1b164681f9e..49f5e8d6d1a0d6c 100644 --- a/Include/cpython/listobject.h +++ b/Include/cpython/listobject.h @@ -29,7 +29,11 @@ typedef struct { static inline Py_ssize_t PyList_GET_SIZE(PyObject *op) { PyListObject *list = _PyList_CAST(op); +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&(_PyVarObject_CAST(list)->ob_size)); +#else return Py_SIZE(list); +#endif } #define PyList_GET_SIZE(op) PyList_GET_SIZE(_PyObject_CAST(op)) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index c93931634fee051..7512bb70c760fdd 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -229,6 +229,7 @@ struct _typeobject { /* bitset of which type-watchers care about this type */ unsigned char tp_watched; + uint16_t tp_versions_used; }; /* This struct is used by the specializer diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 96e829f8fbe97db..5a9ccaea3b22098 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -39,13 +39,18 @@ typedef struct { typedef struct _PyExecutorObject { PyObject_VAR_HEAD _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ + void *jit_code; + size_t jit_size; _PyUOpInstruction trace[1]; } _PyExecutorObject; typedef struct _PyOptimizerObject _PyOptimizerObject; /* Should return > 0 if a new executor is created. O if no executor is produced and < 0 if an error occurred. */ -typedef int (*optimize_func)(_PyOptimizerObject* self, PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutorObject **, int curr_stackentries); +typedef int (*optimize_func)( + _PyOptimizerObject* self, struct _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, _PyExecutorObject **exec_ptr, + int curr_stackentries); typedef struct _PyOptimizerObject { PyObject_HEAD @@ -92,6 +97,9 @@ PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewUOpOptimizer(void); /* Minimum of 16 additional executions before retry */ #define MINIMUM_TIER2_BACKOFF 4 +#define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3 +#define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6 + #ifdef __cplusplus } #endif diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 60b056bdcc2f1f2..9bc8758e72bd8f2 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -217,16 +217,19 @@ struct _ts { #ifdef Py_DEBUG // A debug build is likely built with low optimization level which implies // higher stack memory usage than a release build: use a lower limit. -# define Py_C_RECURSION_LIMIT 500 +# if defined(__wasi__) + // Based on wasmtime 16. +# define Py_C_RECURSION_LIMIT 150 +# else +# define Py_C_RECURSION_LIMIT 500 +# endif #elif defined(__wasi__) - // WASI has limited call stack. Python's recursion limit depends on code - // layout, optimization, and WASI runtime. Wasmtime can handle about 700 - // recursions, sometimes less. 500 is a more conservative limit. + // Based on wasmtime 16. # define Py_C_RECURSION_LIMIT 500 #elif defined(__s390x__) # define Py_C_RECURSION_LIMIT 800 #elif defined(_WIN32) -# define Py_C_RECURSION_LIMIT 4000 +# define Py_C_RECURSION_LIMIT 3000 #elif defined(_Py_ADDRESS_SANITIZER) # define Py_C_RECURSION_LIMIT 4000 #else diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index ba67eefef3e37ad..bf0cfe4cb695b45 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -122,11 +122,25 @@ typedef struct _optimization_stats { uint64_t optimized_trace_length_hist[_Py_UOP_HIST_SIZE]; } OptimizationStats; +typedef struct _rare_event_stats { + /* Setting an object's class, obj.__class__ = ... */ + uint64_t set_class; + /* Setting the bases of a class, cls.__bases__ = ... */ + uint64_t set_bases; + /* Setting the PEP 523 frame eval function, _PyInterpreterState_SetFrameEvalFunc() */ + uint64_t set_eval_frame_func; + /* Modifying the builtins, __builtins__.__dict__[var] = ... */ + uint64_t builtin_dict; + /* Modifying a function, e.g. func.__defaults__ = ..., etc. */ + uint64_t func_modification; +} RareEventStats; + typedef struct _stats { OpcodeStats opcode_stats[256]; CallStats call_stats; ObjectStats object_stats; OptimizationStats optimization_stats; + RareEventStats rare_event_stats; GCStats *gc_stats; } PyStats; diff --git a/Include/internal/pycore_brc.h b/Include/internal/pycore_brc.h new file mode 100644 index 000000000000000..3453d83b57ca97e --- /dev/null +++ b/Include/internal/pycore_brc.h @@ -0,0 +1,74 @@ +#ifndef Py_INTERNAL_BRC_H +#define Py_INTERNAL_BRC_H + +#include <stdint.h> +#include "pycore_llist.h" // struct llist_node +#include "pycore_lock.h" // PyMutex +#include "pycore_object_stack.h" // _PyObjectStack + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef Py_GIL_DISABLED + +// Prime number to avoid correlations with memory addresses. +#define _Py_BRC_NUM_BUCKETS 257 + +// Hash table bucket +struct _brc_bucket { + // Mutex protects both the bucket and thread state queues in this bucket. + PyMutex mutex; + + // Linked list of _PyThreadStateImpl objects hashed to this bucket. + struct llist_node root; +}; + +// Per-interpreter biased reference counting state +struct _brc_state { + // Hash table of thread states by thread-id. Thread states within a bucket + // are chained using a doubly-linked list. + struct _brc_bucket table[_Py_BRC_NUM_BUCKETS]; +}; + +// Per-thread biased reference counting state +struct _brc_thread_state { + // Linked-list of thread states per hash bucket + struct llist_node bucket_node; + + // Thread-id as determined by _PyThread_Id() + uintptr_t tid; + + // Objects with refcounts to be merged (protected by bucket mutex) + _PyObjectStack objects_to_merge; + + // Local stack of objects to be merged (not accessed by other threads) + _PyObjectStack local_objects_to_merge; +}; + +// Initialize/finalize the per-thread biased reference counting state +void _Py_brc_init_thread(PyThreadState *tstate); +void _Py_brc_remove_thread(PyThreadState *tstate); + +// Initialize per-interpreter state +void _Py_brc_init_state(PyInterpreterState *interp); + +void _Py_brc_after_fork(PyInterpreterState *interp); + +// Enqueues an object to be merged by it's owning thread (tid). This +// steals a reference to the object. +void _Py_brc_queue_object(PyObject *ob); + +// Merge the refcounts of queued objects for the current thread. +void _Py_brc_merge_refcounts(PyThreadState *tstate); + +#endif /* Py_GIL_DISABLED */ + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_BRC_H */ diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index a66af1389541dd7..b158fc9ff5ebc1b 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -206,6 +206,7 @@ void _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame) #define _PY_ASYNC_EXCEPTION_BIT 3 #define _PY_GC_SCHEDULED_BIT 4 #define _PY_EVAL_PLEASE_STOP_BIT 5 +#define _PY_EVAL_EXPLICIT_MERGE_BIT 6 /* Reserve a few bits for future use */ #define _PY_EVAL_EVENTS_BITS 8 diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 73df6c3568ffe0c..fdd5918228455d7 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -295,6 +295,7 @@ extern int _PyStaticCode_Init(PyCodeObject *co); _Py_stats->optimization_stats.name[bucket]++; \ } \ } while (0) +#define RARE_EVENT_STAT_INC(name) do { if (_Py_stats) _Py_stats->rare_event_stats.name++; } while (0) // Export for '_opcode' shared extension PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); @@ -313,6 +314,7 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define UOP_STAT_INC(opname, name) ((void)0) #define OPT_UNSUPPORTED_OPCODE(opname) ((void)0) #define OPT_HIST(length, name) ((void)0) +#define RARE_EVENT_STAT_INC(name) ((void)0) #endif // !Py_STATS // Utility functions for reading/writing 32/64-bit values in the inline caches. diff --git a/Include/internal/pycore_condvar.h b/Include/internal/pycore_condvar.h index 34c21aaad43197b..ee9533484e80488 100644 --- a/Include/internal/pycore_condvar.h +++ b/Include/internal/pycore_condvar.h @@ -35,14 +35,14 @@ #include <windows.h> // CRITICAL_SECTION /* options */ -/* non-emulated condition variables are provided for those that want - * to target Windows Vista. Modify this macro to enable them. +/* emulated condition variables are provided for those that want + * to target Windows XP or earlier. Modify this macro to enable them. */ #ifndef _PY_EMULATED_WIN_CV -#define _PY_EMULATED_WIN_CV 1 /* use emulated condition variables */ +#define _PY_EMULATED_WIN_CV 0 /* use non-emulated condition variables */ #endif -/* fall back to emulation if not targeting Vista */ +/* fall back to emulation if targeting earlier than Vista */ #if !defined NTDDI_VISTA || NTDDI_VERSION < NTDDI_VISTA #undef _PY_EMULATED_WIN_CV #define _PY_EMULATED_WIN_CV 1 @@ -77,7 +77,7 @@ typedef struct _PyCOND_T #else /* !_PY_EMULATED_WIN_CV */ -/* Use native Win7 primitives if build target is Win7 or higher */ +/* Use native Windows primitives if build target is Vista or higher */ /* SRWLOCK is faster and better than CriticalSection */ typedef SRWLOCK PyMUTEX_T; diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h index 3284efba2b6f4cb..ae5c47f195eb7f0 100644 --- a/Include/internal/pycore_context.h +++ b/Include/internal/pycore_context.h @@ -14,7 +14,6 @@ extern PyTypeObject _PyContextTokenMissing_Type; /* runtime lifecycle */ PyStatus _PyContext_Init(PyInterpreterState *); -void _PyContext_Fini(_PyFreeListState *); /* other API */ diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index bf2bbfffc38bd0f..38ed8cd69804ba9 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -104,12 +104,37 @@ extern "C" { # define Py_END_CRITICAL_SECTION2() \ _PyCriticalSection2_End(&_cs2); \ } + +// Asserts that the mutex is locked. The mutex must be held by the +// top-most critical section otherwise there's the possibility +// that the mutex would be swalled out in some code paths. +#define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) \ + _PyCriticalSection_AssertHeld(mutex) + +// Asserts that the mutex for the given object is locked. The mutex must +// be held by the top-most critical section otherwise there's the +// possibility that the mutex would be swalled out in some code paths. +#ifdef Py_DEBUG + +#define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) \ + if (Py_REFCNT(op) != 1) { \ + _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(&_PyObject_CAST(op)->ob_mutex); \ + } + +#else /* Py_DEBUG */ + +#define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) + +#endif /* Py_DEBUG */ + #else /* !Py_GIL_DISABLED */ // The critical section APIs are no-ops with the GIL. # define Py_BEGIN_CRITICAL_SECTION(op) # define Py_END_CRITICAL_SECTION() # define Py_BEGIN_CRITICAL_SECTION2(a, b) # define Py_END_CRITICAL_SECTION2() +# define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) +# define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) #endif /* !Py_GIL_DISABLED */ typedef struct { @@ -236,6 +261,27 @@ _PyCriticalSection2_End(_PyCriticalSection2 *c) PyAPI_FUNC(void) _PyCriticalSection_SuspendAll(PyThreadState *tstate); +#ifdef Py_GIL_DISABLED + +static inline void +_PyCriticalSection_AssertHeld(PyMutex *mutex) { +#ifdef Py_DEBUG + PyThreadState *tstate = _PyThreadState_GET(); + uintptr_t prev = tstate->critical_section; + if (prev & _Py_CRITICAL_SECTION_TWO_MUTEXES) { + _PyCriticalSection2 *cs = (_PyCriticalSection2 *)(prev & ~_Py_CRITICAL_SECTION_MASK); + assert(cs != NULL && (cs->base.mutex == mutex || cs->mutex2 == mutex)); + } + else { + _PyCriticalSection *cs = (_PyCriticalSection *)(tstate->critical_section & ~_Py_CRITICAL_SECTION_MASK); + assert(cs != NULL && cs->mutex == mutex); + } + +#endif +} + +#endif + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index d96870e9197bbf0..233da058f464d17 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -9,6 +9,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_freelist.h" // _PyFreeListState #include "pycore_identifier.h" // _Py_Identifier #include "pycore_object.h" // PyDictOrValues @@ -69,7 +70,7 @@ extern PyObject* _PyDictView_Intersect(PyObject* self, PyObject *other); /* runtime lifecycle */ -extern void _PyDict_Fini(PyInterpreterState *interp); +extern void _PyDict_Fini(PyInterpreterState *state); /* other API */ @@ -206,11 +207,17 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { #define DK_IS_UNICODE(dk) ((dk)->dk_kind != DICT_KEYS_GENERAL) -#define DICT_VERSION_INCREMENT (1 << DICT_MAX_WATCHERS) -#define DICT_VERSION_MASK (DICT_VERSION_INCREMENT - 1) +#define DICT_VERSION_INCREMENT (1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) +#define DICT_WATCHER_MASK ((1 << DICT_MAX_WATCHERS) - 1) +#ifdef Py_GIL_DISABLED +#define DICT_NEXT_VERSION(INTERP) \ + (_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version, DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT) + +#else #define DICT_NEXT_VERSION(INTERP) \ ((INTERP)->dict_state.global_version += DICT_VERSION_INCREMENT) +#endif void _PyDict_SendEvent(int watcher_bits, @@ -227,7 +234,7 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, PyObject *value) { assert(Py_REFCNT((PyObject*)mp) > 0); - int watcher_bits = mp->ma_version_tag & DICT_VERSION_MASK; + int watcher_bits = mp->ma_version_tag & DICT_WATCHER_MASK; if (watcher_bits) { _PyDict_SendEvent(watcher_bits, event, mp, key, value); return DICT_NEXT_VERSION(interp) | watcher_bits; diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index ece0f10ca251707..1a44755c7a01a3a 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -8,17 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif - -#ifndef WITH_FREELISTS -// without freelists -# define PyDict_MAXFREELIST 0 -#endif - -#ifndef PyDict_MAXFREELIST -# define PyDict_MAXFREELIST 80 -#endif - #define DICT_MAX_WATCHERS 8 +#define DICT_WATCHED_MUTATION_BITS 4 struct _Py_dict_state { /*Global counter used to set ma_version_tag field of dictionary. @@ -26,15 +17,6 @@ struct _Py_dict_state { * time that a dictionary is modified. */ uint64_t global_version; uint32_t next_keys_version; - -#if PyDict_MAXFREELIST > 0 - /* Dictionary reuse scheme to save calls to malloc and free */ - PyDictObject *free_list[PyDict_MAXFREELIST]; - PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST]; - int numfree; - int keys_numfree; -#endif - PyDict_WatchCallback watchers[DICT_MAX_WATCHERS]; }; diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index 038578e1f9680a6..3767df5506d43fb 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -15,7 +15,6 @@ extern "C" { extern void _PyFloat_InitState(PyInterpreterState *); extern PyStatus _PyFloat_InitTypes(PyInterpreterState *); -extern void _PyFloat_Fini(_PyFreeListState *); extern void _PyFloat_FiniType(PyInterpreterState *); diff --git a/Include/internal/pycore_freelist.h b/Include/internal/pycore_freelist.h index 4ab93ee2bf6c329..1bc551914794f0c 100644 --- a/Include/internal/pycore_freelist.h +++ b/Include/internal/pycore_freelist.h @@ -17,16 +17,20 @@ extern "C" { # define PyTuple_NFREELISTS PyTuple_MAXSAVESIZE # define PyTuple_MAXFREELIST 2000 # define PyList_MAXFREELIST 80 +# define PyDict_MAXFREELIST 80 # define PyFloat_MAXFREELIST 100 # define PyContext_MAXFREELIST 255 # define _PyAsyncGen_MAXFREELIST 80 +# define _PyObjectStackChunk_MAXFREELIST 4 #else # define PyTuple_NFREELISTS 0 # define PyTuple_MAXFREELIST 0 # define PyList_MAXFREELIST 0 +# define PyDict_MAXFREELIST 0 # define PyFloat_MAXFREELIST 0 # define PyContext_MAXFREELIST 0 # define _PyAsyncGen_MAXFREELIST 0 +# define _PyObjectStackChunk_MAXFREELIST 0 #endif struct _Py_list_state { @@ -63,6 +67,16 @@ struct _Py_float_state { #endif }; +struct _Py_dict_freelist { +#ifdef WITH_FREELISTS + /* Dictionary reuse scheme to save calls to malloc and free */ + PyDictObject *free_list[PyDict_MAXFREELIST]; + PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST]; + int numfree; + int keys_numfree; +#endif +}; + struct _Py_slice_state { #ifdef WITH_FREELISTS /* Using a cache is very effective since typically only a single slice is @@ -93,15 +107,34 @@ struct _Py_async_gen_state { #endif }; +struct _PyObjectStackChunk; + +struct _Py_object_stack_state { + struct _PyObjectStackChunk *free_list; + Py_ssize_t numfree; +}; + typedef struct _Py_freelist_state { - struct _Py_float_state float_state; - struct _Py_tuple_state tuple_state; - struct _Py_list_state list_state; - struct _Py_slice_state slice_state; - struct _Py_context_state context_state; - struct _Py_async_gen_state async_gen_state; + struct _Py_float_state floats; + struct _Py_tuple_state tuples; + struct _Py_list_state lists; + struct _Py_dict_freelist dicts; + struct _Py_slice_state slices; + struct _Py_context_state contexts; + struct _Py_async_gen_state async_gens; + struct _Py_object_stack_state object_stacks; } _PyFreeListState; +extern void _PyObject_ClearFreeLists(_PyFreeListState *state, int is_finalization); +extern void _PyTuple_ClearFreeList(_PyFreeListState *state, int is_finalization); +extern void _PyFloat_ClearFreeList(_PyFreeListState *state, int is_finalization); +extern void _PyList_ClearFreeList(_PyFreeListState *state, int is_finalization); +extern void _PySlice_ClearFreeList(_PyFreeListState *state, int is_finalization); +extern void _PyDict_ClearFreeList(_PyFreeListState *state, int is_finalization); +extern void _PyAsyncGen_ClearFreeLists(_PyFreeListState *state, int is_finalization); +extern void _PyContext_ClearFreeList(_PyFreeListState *state, int is_finalization); +extern void _PyObjectStackChunk_ClearFreeList(_PyFreeListState *state, int is_finalization); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index d53de97709a782f..582a16bf5218cef 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -37,10 +37,23 @@ static inline PyObject* _Py_FROM_GC(PyGC_Head *gc) { } +/* Bit flags for ob_gc_bits (in Py_GIL_DISABLED builds) */ +#ifdef Py_GIL_DISABLED +# define _PyGC_BITS_TRACKED (1) +# define _PyGC_BITS_FINALIZED (2) +# define _PyGC_BITS_UNREACHABLE (4) +# define _PyGC_BITS_FROZEN (8) +# define _PyGC_BITS_SHARED (16) +#endif + /* True if the object is currently tracked by the GC. */ static inline int _PyObject_GC_IS_TRACKED(PyObject *op) { +#ifdef Py_GIL_DISABLED + return (op->ob_gc_bits & _PyGC_BITS_TRACKED) != 0; +#else PyGC_Head *gc = _Py_AS_GC(op); return (gc->_gc_next != 0); +#endif } #define _PyObject_GC_IS_TRACKED(op) _PyObject_GC_IS_TRACKED(_Py_CAST(PyObject*, op)) @@ -56,6 +69,22 @@ static inline int _PyObject_GC_MAY_BE_TRACKED(PyObject *obj) { return 1; } +#ifdef Py_GIL_DISABLED + +/* True if an object is shared between multiple threads and + * needs special purpose when freeing to do the possibility + * of in-flight lock-free reads occuring */ +static inline int _PyObject_GC_IS_SHARED(PyObject *op) { + return (op->ob_gc_bits & _PyGC_BITS_SHARED) != 0; +} +#define _PyObject_GC_IS_SHARED(op) _PyObject_GC_IS_SHARED(_Py_CAST(PyObject*, op)) + +static inline void _PyObject_GC_SET_SHARED(PyObject *op) { + op->ob_gc_bits |= _PyGC_BITS_SHARED; +} +#define _PyObject_GC_SET_SHARED(op) _PyObject_GC_SET_SHARED(_Py_CAST(PyObject*, op)) + +#endif /* Bit flags for _gc_prev */ /* Bit 0 is set when tp_finalize is called */ @@ -107,24 +136,29 @@ static inline void _PyGCHead_SET_PREV(PyGC_Head *gc, PyGC_Head *prev) { gc->_gc_prev = ((gc->_gc_prev & ~_PyGC_PREV_MASK) | uprev); } -static inline int _PyGCHead_FINALIZED(PyGC_Head *gc) { - return ((gc->_gc_prev & _PyGC_PREV_MASK_FINALIZED) != 0); -} -static inline void _PyGCHead_SET_FINALIZED(PyGC_Head *gc) { - gc->_gc_prev |= _PyGC_PREV_MASK_FINALIZED; -} - static inline int _PyGC_FINALIZED(PyObject *op) { +#ifdef Py_GIL_DISABLED + return (op->ob_gc_bits & _PyGC_BITS_FINALIZED) != 0; +#else PyGC_Head *gc = _Py_AS_GC(op); - return _PyGCHead_FINALIZED(gc); + return ((gc->_gc_prev & _PyGC_PREV_MASK_FINALIZED) != 0); +#endif } static inline void _PyGC_SET_FINALIZED(PyObject *op) { +#ifdef Py_GIL_DISABLED + op->ob_gc_bits |= _PyGC_BITS_FINALIZED; +#else PyGC_Head *gc = _Py_AS_GC(op); - _PyGCHead_SET_FINALIZED(gc); + gc->_gc_prev |= _PyGC_PREV_MASK_FINALIZED; +#endif } static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) { +#ifdef Py_GIL_DISABLED + op->ob_gc_bits &= ~_PyGC_BITS_FINALIZED; +#else PyGC_Head *gc = _Py_AS_GC(op); gc->_gc_prev &= ~_PyGC_PREV_MASK_FINALIZED; +#endif } @@ -245,14 +279,6 @@ extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs); // Functions to clear types free lists extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); -extern void _Py_ClearFreeLists(_PyFreeListState *state, int is_finalization); -extern void _PyTuple_ClearFreeList(_PyFreeListState *state, int is_finalization); -extern void _PyFloat_ClearFreeList(_PyFreeListState *state, int is_finalization); -extern void _PyList_ClearFreeList(_PyFreeListState *state, int is_finalization); -extern void _PySlice_ClearCache(_PyFreeListState *state); -extern void _PyDict_ClearFreeList(PyInterpreterState *interp); -extern void _PyAsyncGen_ClearFreeLists(_PyFreeListState *state, int is_finalization); -extern void _PyContext_ClearFreeList(_PyFreeListState *state, int is_finalization); extern void _Py_ScheduleGC(PyInterpreterState *interp); extern void _Py_RunGC(PyThreadState *tstate); diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index 5ad63658051e86d..b2aa017598409f7 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -26,10 +26,6 @@ extern PyTypeObject _PyCoroWrapper_Type; extern PyTypeObject _PyAsyncGenWrappedValue_Type; extern PyTypeObject _PyAsyncGenAThrow_Type; -/* runtime lifecycle */ - -extern void _PyAsyncGen_Fini(_PyFreeListState *); - #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 0a24b127192c9b8..932738c3049882b 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -787,9 +787,11 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_child)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_parent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aggregate_class)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(allow_code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argdefs)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(args)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arguments)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argv)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(as_integer_ratio)); @@ -874,6 +876,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(d)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(data)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(database)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(day)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(default)); @@ -913,6 +916,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(errors)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(event)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eventmask)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_type)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(excepthook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exception)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(existing_file_name)); @@ -935,12 +940,14 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fileno)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(filepath)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fillvalue)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(filter)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(filters)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(final)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(find_class)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fix_imports)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(flags)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(flush)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fold)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(follow_symlinks)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(format)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(from_param)); @@ -971,6 +978,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(headers)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hi)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hook)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hour)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(id)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ident)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(identity_hint)); @@ -1045,6 +1053,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(max_length)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxdigits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxevents)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxlen)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxmem)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxsplit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxvalue)); @@ -1054,11 +1063,14 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(metaclass)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(metadata)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(method)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(microsecond)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(minute)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mod)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(module_globals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(modules)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(month)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mro)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(msg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mycmp)); @@ -1163,9 +1175,11 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(salt)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sched_priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(scheduler)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(second)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seek)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seekable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(selectors)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(self)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(send)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sep)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sequence)); @@ -1228,6 +1242,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timetuple)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(top)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(trace_callback)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(traceback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(trailers)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(translate)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(true)); @@ -1237,6 +1252,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(type)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(type_params)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tz)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tzinfo)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tzname)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(uid)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(unlink)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index efb659c5806e6e7..da62b4f0a951ff8 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -276,9 +276,11 @@ struct _Py_global_strings { STRUCT_FOR_ID(after_in_child) STRUCT_FOR_ID(after_in_parent) STRUCT_FOR_ID(aggregate_class) + STRUCT_FOR_ID(alias) STRUCT_FOR_ID(allow_code) STRUCT_FOR_ID(append) STRUCT_FOR_ID(argdefs) + STRUCT_FOR_ID(args) STRUCT_FOR_ID(arguments) STRUCT_FOR_ID(argv) STRUCT_FOR_ID(as_integer_ratio) @@ -363,6 +365,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(d) STRUCT_FOR_ID(data) STRUCT_FOR_ID(database) + STRUCT_FOR_ID(day) STRUCT_FOR_ID(decode) STRUCT_FOR_ID(decoder) STRUCT_FOR_ID(default) @@ -402,6 +405,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(errors) STRUCT_FOR_ID(event) STRUCT_FOR_ID(eventmask) + STRUCT_FOR_ID(exc_type) + STRUCT_FOR_ID(exc_value) STRUCT_FOR_ID(excepthook) STRUCT_FOR_ID(exception) STRUCT_FOR_ID(existing_file_name) @@ -424,12 +429,14 @@ struct _Py_global_strings { STRUCT_FOR_ID(fileno) STRUCT_FOR_ID(filepath) STRUCT_FOR_ID(fillvalue) + STRUCT_FOR_ID(filter) STRUCT_FOR_ID(filters) STRUCT_FOR_ID(final) STRUCT_FOR_ID(find_class) STRUCT_FOR_ID(fix_imports) STRUCT_FOR_ID(flags) STRUCT_FOR_ID(flush) + STRUCT_FOR_ID(fold) STRUCT_FOR_ID(follow_symlinks) STRUCT_FOR_ID(format) STRUCT_FOR_ID(from_param) @@ -460,6 +467,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(headers) STRUCT_FOR_ID(hi) STRUCT_FOR_ID(hook) + STRUCT_FOR_ID(hour) STRUCT_FOR_ID(id) STRUCT_FOR_ID(ident) STRUCT_FOR_ID(identity_hint) @@ -534,6 +542,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(max_length) STRUCT_FOR_ID(maxdigits) STRUCT_FOR_ID(maxevents) + STRUCT_FOR_ID(maxlen) STRUCT_FOR_ID(maxmem) STRUCT_FOR_ID(maxsplit) STRUCT_FOR_ID(maxvalue) @@ -543,11 +552,14 @@ struct _Py_global_strings { STRUCT_FOR_ID(metaclass) STRUCT_FOR_ID(metadata) STRUCT_FOR_ID(method) + STRUCT_FOR_ID(microsecond) + STRUCT_FOR_ID(minute) STRUCT_FOR_ID(mod) STRUCT_FOR_ID(mode) STRUCT_FOR_ID(module) STRUCT_FOR_ID(module_globals) STRUCT_FOR_ID(modules) + STRUCT_FOR_ID(month) STRUCT_FOR_ID(mro) STRUCT_FOR_ID(msg) STRUCT_FOR_ID(mycmp) @@ -652,9 +664,11 @@ struct _Py_global_strings { STRUCT_FOR_ID(salt) STRUCT_FOR_ID(sched_priority) STRUCT_FOR_ID(scheduler) + STRUCT_FOR_ID(second) STRUCT_FOR_ID(seek) STRUCT_FOR_ID(seekable) STRUCT_FOR_ID(selectors) + STRUCT_FOR_ID(self) STRUCT_FOR_ID(send) STRUCT_FOR_ID(sep) STRUCT_FOR_ID(sequence) @@ -717,6 +731,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(timetuple) STRUCT_FOR_ID(top) STRUCT_FOR_ID(trace_callback) + STRUCT_FOR_ID(traceback) STRUCT_FOR_ID(trailers) STRUCT_FOR_ID(translate) STRUCT_FOR_ID(true) @@ -726,6 +741,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(type) STRUCT_FOR_ID(type_params) STRUCT_FOR_ID(tz) + STRUCT_FOR_ID(tzinfo) STRUCT_FOR_ID(tzname) STRUCT_FOR_ID(uid) STRUCT_FOR_ID(unlink) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index f953b8426e180a0..31d88071e19d0cf 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -20,6 +20,7 @@ extern "C" { #include "pycore_dtoa.h" // struct _dtoa_state #include "pycore_exceptions.h" // struct _Py_exc_state #include "pycore_floatobject.h" // struct _Py_float_state +#include "pycore_freelist.h" // struct _Py_freelist_state #include "pycore_function.h" // FUNC_MAX_WATCHERS #include "pycore_gc.h" // struct _gc_runtime_state #include "pycore_genobject.h" // struct _Py_async_gen_state @@ -60,6 +61,20 @@ struct _stoptheworld_state { /* cross-interpreter data registry */ +/* Tracks some rare events per-interpreter, used by the optimizer to turn on/off + specific optimizations. */ +typedef struct _rare_events { + /* Setting an object's class, obj.__class__ = ... */ + uint8_t set_class; + /* Setting the bases of a class, cls.__bases__ = ... */ + uint8_t set_bases; + /* Setting the PEP 523 frame eval function, _PyInterpreterState_SetFrameEvalFunc() */ + uint8_t set_eval_frame_func; + /* Modifying the builtins, __builtins__.__dict__[var] = ... */ + uint8_t builtin_dict; + /* Modifying a function, e.g. func.__defaults__ = ..., etc. */ + uint8_t func_modification; +} _rare_events; /* interpreter state */ @@ -186,9 +201,20 @@ struct _is { #if defined(Py_GIL_DISABLED) struct _mimalloc_interp_state mimalloc; + struct _brc_state brc; // biased reference counting state #endif - struct _obmalloc_state obmalloc; + // Per-interpreter state for the obmalloc allocator. For the main + // interpreter and for all interpreters that don't have their + // own obmalloc state, this points to the static structure in + // obmalloc.c obmalloc_state_main. For other interpreters, it is + // heap allocated by _PyMem_init_obmalloc() and freed when the + // interpreter structure is freed. In the case of a heap allocated + // obmalloc state, it is not safe to hold on to or use memory after + // the interpreter is freed. The obmalloc state corresponding to + // that allocated memory is gone. See free_obmalloc_arenas() for + // more comments. + struct _obmalloc_state *obmalloc; PyObject *audit_hooks; PyType_WatchCallback type_watchers[TYPE_MAX_WATCHERS]; @@ -205,7 +231,6 @@ struct _is { struct _dtoa_state dtoa; struct _py_func_state func_state; - struct _Py_tuple_state tuple; struct _Py_dict_state dict_state; struct _Py_exc_state exc_state; @@ -217,6 +242,8 @@ struct _is { uint16_t optimizer_resume_threshold; uint16_t optimizer_backedge_threshold; uint32_t next_func_version; + _rare_events rare_events; + PyDict_WatchCallback builtins_dict_watcher; _Py_GlobalMonitors monitors; bool sys_profile_initialized; @@ -347,6 +374,19 @@ PyAPI_FUNC(PyStatus) _PyInterpreterState_New( PyInterpreterState **pinterp); +#define RARE_EVENT_INTERP_INC(interp, name) \ + do { \ + /* saturating add */ \ + if (interp->rare_events.name < UINT8_MAX) interp->rare_events.name++; \ + RARE_EVENT_STAT_INC(name); \ + } while (0); \ + +#define RARE_EVENT_INC(name) \ + do { \ + PyInterpreterState *interp = PyInterpreterState_Get(); \ + RARE_EVENT_INTERP_INC(interp, name); \ + } while (0); \ + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h new file mode 100644 index 000000000000000..0b71eb6f758ac62 --- /dev/null +++ b/Include/internal/pycore_jit.h @@ -0,0 +1,25 @@ +#ifndef Py_INTERNAL_JIT_H +#define Py_INTERNAL_JIT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef _Py_JIT + +typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate); + +int _PyJIT_Compile(_PyExecutorObject *executor, _PyUOpInstruction *trace, size_t length); +void _PyJIT_Free(_PyExecutorObject *executor); + +#endif // _Py_JIT + +#ifdef __cplusplus +} +#endif + +#endif // !Py_INTERNAL_JIT_H diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 6c29d882335512e..50dc13c4da4487c 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -13,23 +13,18 @@ extern "C" { extern PyObject* _PyList_Extend(PyListObject *, PyObject *); extern void _PyList_DebugMallocStats(FILE *out); - -/* runtime lifecycle */ - -extern void _PyList_Fini(_PyFreeListState *); - - #define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item) extern int _PyList_AppendTakeRefListResize(PyListObject *self, PyObject *newitem); +// In free-threaded build: self should be locked by the caller, if it should be thread-safe. static inline int _PyList_AppendTakeRef(PyListObject *self, PyObject *newitem) { assert(self != NULL && newitem != NULL); assert(PyList_Check(self)); - Py_ssize_t len = PyList_GET_SIZE(self); + Py_ssize_t len = Py_SIZE(self); Py_ssize_t allocated = self->allocated; assert((size_t)len + 1 < PY_SSIZE_T_MAX); if (allocated > len) { diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index f413b8451e5ab49..34a83ea228e8b10 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -178,7 +178,7 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) } _Py_DECREF_STAT_INC(); #ifdef Py_REF_DEBUG - _Py_DEC_REFTOTAL(_PyInterpreterState_GET()); + _Py_DEC_REFTOTAL(PyInterpreterState_Get()); #endif if (--op->ob_refcnt != 0) { assert(op->ob_refcnt > 0); @@ -199,7 +199,7 @@ _Py_DECREF_NO_DEALLOC(PyObject *op) } _Py_DECREF_STAT_INC(); #ifdef Py_REF_DEBUG - _Py_DEC_REFTOTAL(_PyInterpreterState_GET()); + _Py_DEC_REFTOTAL(PyInterpreterState_Get()); #endif op->ob_refcnt--; #ifdef Py_DEBUG @@ -315,7 +315,9 @@ static inline void _PyObject_GC_TRACK( _PyObject_ASSERT_FROM(op, !_PyObject_GC_IS_TRACKED(op), "object already tracked by the garbage collector", filename, lineno, __func__); - +#ifdef Py_GIL_DISABLED + op->ob_gc_bits |= _PyGC_BITS_TRACKED; +#else PyGC_Head *gc = _Py_AS_GC(op); _PyObject_ASSERT_FROM(op, (gc->_gc_prev & _PyGC_PREV_MASK_COLLECTING) == 0, @@ -329,6 +331,7 @@ static inline void _PyObject_GC_TRACK( _PyGCHead_SET_PREV(gc, last); _PyGCHead_SET_NEXT(gc, generation0); generation0->_gc_prev = (uintptr_t)gc; +#endif } /* Tell the GC to stop tracking this object. @@ -352,6 +355,9 @@ static inline void _PyObject_GC_UNTRACK( "object not tracked by the garbage collector", filename, lineno, __func__); +#ifdef Py_GIL_DISABLED + op->ob_gc_bits &= ~_PyGC_BITS_TRACKED; +#else PyGC_Head *gc = _Py_AS_GC(op); PyGC_Head *prev = _PyGCHead_PREV(gc); PyGC_Head *next = _PyGCHead_NEXT(gc); @@ -359,6 +365,7 @@ static inline void _PyObject_GC_UNTRACK( _PyGCHead_SET_PREV(next, prev); gc->_gc_next = 0; gc->_gc_prev &= _PyGC_PREV_MASK_FINALIZED; +#endif } // Macros to accept any type for the parameter, and to automatically pass @@ -376,6 +383,142 @@ static inline void _PyObject_GC_UNTRACK( _PyObject_GC_UNTRACK(__FILE__, __LINE__, _PyObject_CAST(op)) #endif +#ifdef Py_GIL_DISABLED + +/* Tries to increment an object's reference count + * + * This is a specialized version of _Py_TryIncref that only succeeds if the + * object is immortal or local to this thread. It does not handle the case + * where the reference count modification requires an atomic operation. This + * allows call sites to specialize for the immortal/local case. + */ +static inline int +_Py_TryIncrefFast(PyObject *op) { + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + local += 1; + if (local == 0) { + // immortal + return 1; + } + if (_Py_IsOwnedByCurrentThread(op)) { + _Py_INCREF_STAT_INC(); + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyInterpreterState_GET()); +#endif + return 1; + } + return 0; +} + +static inline int +_Py_TryIncRefShared(PyObject *op) +{ + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); + for (;;) { + // If the shared refcount is zero and the object is either merged + // or may not have weak references, then we cannot incref it. + if (shared == 0 || shared == _Py_REF_MERGED) { + return 0; + } + + if (_Py_atomic_compare_exchange_ssize( + &op->ob_ref_shared, + &shared, + shared + (1 << _Py_REF_SHARED_SHIFT))) { +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyInterpreterState_GET()); +#endif + _Py_INCREF_STAT_INC(); + return 1; + } + } +} + +/* Tries to incref the object op and ensures that *src still points to it. */ +static inline int +_Py_TryIncref(PyObject **src, PyObject *op) +{ + if (_Py_TryIncrefFast(op)) { + return 1; + } + if (!_Py_TryIncRefShared(op)) { + return 0; + } + if (op != _Py_atomic_load_ptr(src)) { + Py_DECREF(op); + return 0; + } + return 1; +} + +/* Loads and increfs an object from ptr, which may contain a NULL value. + Safe with concurrent (atomic) updates to ptr. + NOTE: The writer must set maybe-weakref on the stored object! */ +static inline PyObject * +_Py_XGetRef(PyObject **ptr) +{ + for (;;) { + PyObject *value = _Py_atomic_load_ptr(ptr); + if (value == NULL) { + return value; + } + if (_Py_TryIncref(ptr, value)) { + return value; + } + } +} + +/* Attempts to loads and increfs an object from ptr. Returns NULL + on failure, which may be due to a NULL value or a concurrent update. */ +static inline PyObject * +_Py_TryXGetRef(PyObject **ptr) +{ + PyObject *value = _Py_atomic_load_ptr(ptr); + if (value == NULL) { + return value; + } + if (_Py_TryIncref(ptr, value)) { + return value; + } + return NULL; +} + +/* Like Py_NewRef but also optimistically sets _Py_REF_MAYBE_WEAKREF + on objects owned by a different thread. */ +static inline PyObject * +_Py_NewRefWithLock(PyObject *op) +{ + if (_Py_TryIncrefFast(op)) { + return op; + } + _Py_INCREF_STAT_INC(); + for (;;) { + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); + Py_ssize_t new_shared = shared + (1 << _Py_REF_SHARED_SHIFT); + if ((shared & _Py_REF_SHARED_FLAG_MASK) == 0) { + new_shared |= _Py_REF_MAYBE_WEAKREF; + } + if (_Py_atomic_compare_exchange_ssize( + &op->ob_ref_shared, + &shared, + new_shared)) { + return op; + } + } +} + +static inline PyObject * +_Py_XNewRefWithLock(PyObject *obj) +{ + if (obj == NULL) { + return NULL; + } + return _Py_NewRefWithLock(obj); +} + +#endif + #ifdef Py_REF_DEBUG extern void _PyInterpreterState_FinalizeRefTotal(PyInterpreterState *); extern void _Py_FinalizeRefTotal(_PyRuntimeState *); @@ -450,8 +593,12 @@ _PyObject_IS_GC(PyObject *obj) static inline size_t _PyType_PreHeaderSize(PyTypeObject *tp) { - return _PyType_IS_GC(tp) * sizeof(PyGC_Head) + - _PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER) * 2 * sizeof(PyObject *); + return ( +#ifndef Py_GIL_DISABLED + _PyType_IS_GC(tp) * sizeof(PyGC_Head) + +#endif + _PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER) * 2 * sizeof(PyObject *) + ); } void _PyObject_GC_Link(PyObject *op); @@ -481,6 +628,14 @@ extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, PyObject *name); +#ifdef Py_GIL_DISABLED +# define MANAGED_DICT_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-1) +# define MANAGED_WEAKREF_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-2) +#else +# define MANAGED_DICT_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-3) +# define MANAGED_WEAKREF_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-4) +#endif + typedef union { PyObject *dict; /* Use a char* to generate a warning if directly assigning a PyDictValues */ @@ -491,7 +646,7 @@ static inline PyDictOrValues * _PyObject_DictOrValuesPointer(PyObject *obj) { assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - return ((PyDictOrValues *)obj)-3; + return (PyDictOrValues *)((char *)obj + MANAGED_DICT_OFFSET); } static inline int @@ -520,8 +675,6 @@ _PyDictOrValues_SetValues(PyDictOrValues *ptr, PyDictValues *values) ptr->values = ((char *)values) - 1; } -#define MANAGED_WEAKREF_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-4) - extern PyObject ** _PyObject_ComputedDictPointer(PyObject *); extern void _PyObject_FreeInstanceAttributes(PyObject *obj); extern int _PyObject_IsInstanceDictEmpty(PyObject *); diff --git a/Include/internal/pycore_object_stack.h b/Include/internal/pycore_object_stack.h new file mode 100644 index 000000000000000..fc130b1e9920b41 --- /dev/null +++ b/Include/internal/pycore_object_stack.h @@ -0,0 +1,87 @@ +#ifndef Py_INTERNAL_OBJECT_STACK_H +#define Py_INTERNAL_OBJECT_STACK_H + +#include "pycore_freelist.h" // _PyFreeListState + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// _PyObjectStack is a stack of Python objects implemented as a linked list of +// fixed size buffers. + +// Chosen so that _PyObjectStackChunk is a power-of-two size. +#define _Py_OBJECT_STACK_CHUNK_SIZE 254 + +typedef struct _PyObjectStackChunk { + struct _PyObjectStackChunk *prev; + Py_ssize_t n; + PyObject *objs[_Py_OBJECT_STACK_CHUNK_SIZE]; +} _PyObjectStackChunk; + +typedef struct _PyObjectStack { + _PyObjectStackChunk *head; +} _PyObjectStack; + + +extern _PyObjectStackChunk * +_PyObjectStackChunk_New(void); + +extern void +_PyObjectStackChunk_Free(_PyObjectStackChunk *); + +// Push an item onto the stack. Return -1 on allocation failure, 0 on success. +static inline int +_PyObjectStack_Push(_PyObjectStack *stack, PyObject *obj) +{ + _PyObjectStackChunk *buf = stack->head; + if (buf == NULL || buf->n == _Py_OBJECT_STACK_CHUNK_SIZE) { + buf = _PyObjectStackChunk_New(); + if (buf == NULL) { + return -1; + } + buf->prev = stack->head; + buf->n = 0; + stack->head = buf; + } + + assert(buf->n >= 0 && buf->n < _Py_OBJECT_STACK_CHUNK_SIZE); + buf->objs[buf->n] = obj; + buf->n++; + return 0; +} + +// Pop the top item from the stack. Return NULL if the stack is empty. +static inline PyObject * +_PyObjectStack_Pop(_PyObjectStack *stack) +{ + _PyObjectStackChunk *buf = stack->head; + if (buf == NULL) { + return NULL; + } + assert(buf->n > 0 && buf->n <= _Py_OBJECT_STACK_CHUNK_SIZE); + buf->n--; + PyObject *obj = buf->objs[buf->n]; + if (buf->n == 0) { + stack->head = buf->prev; + _PyObjectStackChunk_Free(buf); + } + return obj; +} + +// Merge src into dst, leaving src empty +extern void +_PyObjectStack_Merge(_PyObjectStack *dst, _PyObjectStack *src); + +// Remove all items from the stack +extern void +_PyObjectStack_Clear(_PyObjectStack *stack); + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_OBJECT_STACK_H diff --git a/Include/internal/pycore_obmalloc.h b/Include/internal/pycore_obmalloc.h index 17572dba65487d3..9140d8f08f0af1e 100644 --- a/Include/internal/pycore_obmalloc.h +++ b/Include/internal/pycore_obmalloc.h @@ -686,6 +686,8 @@ extern Py_ssize_t _Py_GetGlobalAllocatedBlocks(void); _Py_GetGlobalAllocatedBlocks() extern Py_ssize_t _PyInterpreterState_GetAllocatedBlocks(PyInterpreterState *); extern void _PyInterpreterState_FinalizeAllocatedBlocks(PyInterpreterState *); +extern int _PyMem_init_obmalloc(PyInterpreterState *interp); +extern bool _PyMem_obmalloc_state_on_heap(PyInterpreterState *interp); #ifdef WITH_PYMALLOC diff --git a/Include/internal/pycore_obmalloc_init.h b/Include/internal/pycore_obmalloc_init.h index 8ee72ff2d4126f9..e6811b7aeca73c1 100644 --- a/Include/internal/pycore_obmalloc_init.h +++ b/Include/internal/pycore_obmalloc_init.h @@ -59,13 +59,6 @@ extern "C" { .dump_debug_stats = -1, \ } -#define _obmalloc_state_INIT(obmalloc) \ - { \ - .pools = { \ - .used = _obmalloc_pools_INIT(obmalloc.pools), \ - }, \ - } - #ifdef __cplusplus } diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index fbb448f663369a4..75d7f44025328ee 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -176,7 +176,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case END_ASYNC_FOR: return 2; case END_FOR: - return 2; + return 1; case END_SEND: return 2; case ENTER_EXECUTOR: @@ -647,7 +647,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case INSTRUMENTED_CALL_KW: return 0; case INSTRUMENTED_END_FOR: - return 0; + return 1; case INSTRUMENTED_END_SEND: return 1; case INSTRUMENTED_FOR_ITER: @@ -1232,7 +1232,7 @@ _PyOpcode_macro_expansion[256] = { [DELETE_SUBSCR] = { .nuops = 1, .uops = { { _DELETE_SUBSCR, 0, 0 } } }, [DICT_MERGE] = { .nuops = 1, .uops = { { _DICT_MERGE, 0, 0 } } }, [DICT_UPDATE] = { .nuops = 1, .uops = { { _DICT_UPDATE, 0, 0 } } }, - [END_FOR] = { .nuops = 2, .uops = { { _POP_TOP, 0, 0 }, { _POP_TOP, 0, 0 } } }, + [END_FOR] = { .nuops = 1, .uops = { { _POP_TOP, 0, 0 } } }, [END_SEND] = { .nuops = 1, .uops = { { _END_SEND, 0, 0 } } }, [EXIT_INIT_CHECK] = { .nuops = 1, .uops = { { _EXIT_INIT_CHECK, 0, 0 } } }, [FORMAT_SIMPLE] = { .nuops = 1, .uops = { { _FORMAT_SIMPLE, 0, 0 } } }, diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 31f30c673f207a1..e21412fc815540d 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -8,8 +8,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -int _Py_uop_analyze_and_optimize(PyCodeObject *code, - _PyUOpInstruction *trace, int trace_len, int curr_stackentries); +int _Py_uop_analyze_and_optimize(_PyInterpreterFrame *frame, + _PyUOpInstruction *trace, int trace_len, int curr_stackentries, + _PyBloomFilter *dependencies); extern PyTypeObject _PyCounterExecutor_Type; extern PyTypeObject _PyCounterOptimizer_Type; diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h index 9c9a09f60f34418..265299d7574838b 100644 --- a/Include/internal/pycore_pythread.h +++ b/Include/internal/pycore_pythread.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "dynamic_annotations.h" // _Py_ANNOTATE_PURE_HAPPENS_BEFORE_MUTEX +#include "pycore_llist.h" // struct llist_node // Get _POSIX_THREADS and _POSIX_SEMAPHORES macros if available #if (defined(HAVE_UNISTD_H) && !defined(_POSIX_THREADS) \ @@ -75,14 +76,22 @@ struct _pythread_runtime_state { struct py_stub_tls_entry tls_entries[PTHREAD_KEYS_MAX]; } stubs; #endif + + // Linked list of ThreadHandleObjects + struct llist_node handles; }; +#define _pythread_RUNTIME_INIT(pythread) \ + { \ + .handles = LLIST_INIT(pythread.handles), \ + } #ifdef HAVE_FORK /* Private function to reinitialize a lock at fork in the child process. Reset the lock to the unlocked state. Return 0 on success, return -1 on error. */ extern int _PyThread_at_fork_reinit(PyThread_type_lock *lock); +extern void _PyThread_AfterFork(struct _pythread_runtime_state *state); #endif /* HAVE_FORK */ @@ -143,12 +152,6 @@ PyAPI_FUNC(int) PyThread_join_thread(PyThread_handle_t); */ PyAPI_FUNC(int) PyThread_detach_thread(PyThread_handle_t); -/* - * Obtain the new thread ident and handle in a forked child process. - */ -PyAPI_FUNC(void) PyThread_update_thread_after_fork(PyThread_ident_t* ident, - PyThread_handle_t* handle); - #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 02ab22b967b38ff..7c705d1224f915b 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -268,7 +268,7 @@ typedef struct pyruntimestate { a pointer type. */ - /* PyInterpreterState.interpreters.main */ + /* _PyRuntimeState.interpreters.main */ PyInterpreterState _main_interpreter; #if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index b4806ab09fd145f..571a7d612c94e25 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -16,6 +16,7 @@ extern "C" { #include "pycore_parser.h" // _parser_runtime_state_INIT #include "pycore_pyhash.h" // pyhash_state_INIT #include "pycore_pymem_init.h" // _pymem_allocators_standard_INIT +#include "pycore_pythread.h" // _pythread_RUNTIME_INIT #include "pycore_runtime_init_generated.h" // _Py_bytes_characters_INIT #include "pycore_signal.h" // _signals_RUNTIME_INIT #include "pycore_tracemalloc.h" // _tracemalloc_runtime_state_INIT @@ -90,6 +91,7 @@ extern PyTypeObject _PyExc_MemoryError; }, \ .obmalloc = _obmalloc_global_state_INIT, \ .pyhash_state = pyhash_state_INIT, \ + .threads = _pythread_RUNTIME_INIT(runtime.threads), \ .signals = _signals_RUNTIME_INIT, \ .interpreters = { \ /* This prevents interpreters from getting created \ @@ -155,7 +157,6 @@ extern PyTypeObject _PyExc_MemoryError; { \ .id_refcount = -1, \ .imports = IMPORTS_INIT, \ - .obmalloc = _obmalloc_state_INIT(INTERP.obmalloc), \ .ceval = { \ .recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \ }, \ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index e3ebd80745e6103..68fbbcb4378e17f 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -785,9 +785,11 @@ extern "C" { INIT_ID(after_in_child), \ INIT_ID(after_in_parent), \ INIT_ID(aggregate_class), \ + INIT_ID(alias), \ INIT_ID(allow_code), \ INIT_ID(append), \ INIT_ID(argdefs), \ + INIT_ID(args), \ INIT_ID(arguments), \ INIT_ID(argv), \ INIT_ID(as_integer_ratio), \ @@ -872,6 +874,7 @@ extern "C" { INIT_ID(d), \ INIT_ID(data), \ INIT_ID(database), \ + INIT_ID(day), \ INIT_ID(decode), \ INIT_ID(decoder), \ INIT_ID(default), \ @@ -911,6 +914,8 @@ extern "C" { INIT_ID(errors), \ INIT_ID(event), \ INIT_ID(eventmask), \ + INIT_ID(exc_type), \ + INIT_ID(exc_value), \ INIT_ID(excepthook), \ INIT_ID(exception), \ INIT_ID(existing_file_name), \ @@ -933,12 +938,14 @@ extern "C" { INIT_ID(fileno), \ INIT_ID(filepath), \ INIT_ID(fillvalue), \ + INIT_ID(filter), \ INIT_ID(filters), \ INIT_ID(final), \ INIT_ID(find_class), \ INIT_ID(fix_imports), \ INIT_ID(flags), \ INIT_ID(flush), \ + INIT_ID(fold), \ INIT_ID(follow_symlinks), \ INIT_ID(format), \ INIT_ID(from_param), \ @@ -969,6 +976,7 @@ extern "C" { INIT_ID(headers), \ INIT_ID(hi), \ INIT_ID(hook), \ + INIT_ID(hour), \ INIT_ID(id), \ INIT_ID(ident), \ INIT_ID(identity_hint), \ @@ -1043,6 +1051,7 @@ extern "C" { INIT_ID(max_length), \ INIT_ID(maxdigits), \ INIT_ID(maxevents), \ + INIT_ID(maxlen), \ INIT_ID(maxmem), \ INIT_ID(maxsplit), \ INIT_ID(maxvalue), \ @@ -1052,11 +1061,14 @@ extern "C" { INIT_ID(metaclass), \ INIT_ID(metadata), \ INIT_ID(method), \ + INIT_ID(microsecond), \ + INIT_ID(minute), \ INIT_ID(mod), \ INIT_ID(mode), \ INIT_ID(module), \ INIT_ID(module_globals), \ INIT_ID(modules), \ + INIT_ID(month), \ INIT_ID(mro), \ INIT_ID(msg), \ INIT_ID(mycmp), \ @@ -1161,9 +1173,11 @@ extern "C" { INIT_ID(salt), \ INIT_ID(sched_priority), \ INIT_ID(scheduler), \ + INIT_ID(second), \ INIT_ID(seek), \ INIT_ID(seekable), \ INIT_ID(selectors), \ + INIT_ID(self), \ INIT_ID(send), \ INIT_ID(sep), \ INIT_ID(sequence), \ @@ -1226,6 +1240,7 @@ extern "C" { INIT_ID(timetuple), \ INIT_ID(top), \ INIT_ID(trace_callback), \ + INIT_ID(traceback), \ INIT_ID(trailers), \ INIT_ID(translate), \ INIT_ID(true), \ @@ -1235,6 +1250,7 @@ extern "C" { INIT_ID(type), \ INIT_ID(type_params), \ INIT_ID(tz), \ + INIT_ID(tzinfo), \ INIT_ID(tzname), \ INIT_ID(uid), \ INIT_ID(unlink), \ diff --git a/Include/internal/pycore_sliceobject.h b/Include/internal/pycore_sliceobject.h index 0c72d3ee6225c51..89086f67683a2f5 100644 --- a/Include/internal/pycore_sliceobject.h +++ b/Include/internal/pycore_sliceobject.h @@ -11,8 +11,6 @@ extern "C" { /* runtime lifecycle */ -extern void _PySlice_Fini(_PyFreeListState *); - extern PyObject * _PyBuildSlice_ConsumeRefs(PyObject *start, PyObject *stop); diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 1d782ca2c96e055..b44393b56446735 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -109,18 +109,18 @@ extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); /* Flags for def-use information */ -#define DEF_GLOBAL 1 /* global stmt */ -#define DEF_LOCAL 2 /* assignment in code block */ -#define DEF_PARAM 2<<1 /* formal parameter */ -#define DEF_NONLOCAL 2<<2 /* nonlocal stmt */ -#define USE 2<<3 /* name is used */ -#define DEF_FREE 2<<4 /* name used but not defined in nested block */ -#define DEF_FREE_CLASS 2<<5 /* free variable from class's method */ -#define DEF_IMPORT 2<<6 /* assignment occurred via import */ -#define DEF_ANNOT 2<<7 /* this name is annotated */ -#define DEF_COMP_ITER 2<<8 /* this name is a comprehension iteration variable */ -#define DEF_TYPE_PARAM 2<<9 /* this name is a type parameter */ -#define DEF_COMP_CELL 2<<10 /* this name is a cell in an inlined comprehension */ +#define DEF_GLOBAL 1 /* global stmt */ +#define DEF_LOCAL 2 /* assignment in code block */ +#define DEF_PARAM (2<<1) /* formal parameter */ +#define DEF_NONLOCAL (2<<2) /* nonlocal stmt */ +#define USE (2<<3) /* name is used */ +#define DEF_FREE (2<<4) /* name used but not defined in nested block */ +#define DEF_FREE_CLASS (2<<5) /* free variable from class's method */ +#define DEF_IMPORT (2<<6) /* assignment occurred via import */ +#define DEF_ANNOT (2<<7) /* this name is annotated */ +#define DEF_COMP_ITER (2<<8) /* this name is a comprehension iteration variable */ +#define DEF_TYPE_PARAM (2<<9) /* this name is a type parameter */ +#define DEF_COMP_CELL (2<<10) /* this name is a cell in an inlined comprehension */ #define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT) diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index 472fa08154e8f92..77a1dc59163d212 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -10,6 +10,7 @@ extern "C" { #include "pycore_freelist.h" // struct _Py_freelist_state #include "pycore_mimalloc.h" // struct _mimalloc_thread_state +#include "pycore_brc.h" // struct _brc_thread_state // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The @@ -22,6 +23,7 @@ typedef struct _PyThreadStateImpl { #ifdef Py_GIL_DISABLED struct _mimalloc_thread_state mimalloc; struct _Py_freelist_state freelist_state; + struct _brc_thread_state brc; #endif } _PyThreadStateImpl; diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index b348339a505b0f7..4605f355ccbc38f 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -14,7 +14,6 @@ extern void _PyTuple_DebugMallocStats(FILE *out); /* runtime lifecycle */ extern PyStatus _PyTuple_InitGlobalObjects(PyInterpreterState *); -extern void _PyTuple_Fini(_PyFreeListState *); /* other API */ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 9fa6c896c1a3285..c8458b4e36ccc93 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -669,6 +669,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(aggregate_class); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(alias); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(allow_code); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -678,6 +681,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(argdefs); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(args); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(arguments); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -930,6 +936,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(database); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(day); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(decode); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1047,6 +1056,12 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(eventmask); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(exc_type); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(exc_value); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(excepthook); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1113,6 +1128,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(fillvalue); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(filter); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(filters); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1131,6 +1149,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(flush); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(fold); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(follow_symlinks); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1221,6 +1242,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(hook); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(hour); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(id); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1443,6 +1467,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(maxevents); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(maxlen); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(maxmem); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1470,6 +1497,12 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(method); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(microsecond); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(minute); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(mod); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1485,6 +1518,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(modules); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(month); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(mro); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1797,6 +1833,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(scheduler); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(second); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(seek); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1806,6 +1845,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(selectors); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(self); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(send); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1992,6 +2034,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(trace_callback); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(traceback); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(trailers); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -2019,6 +2064,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(tz); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(tzinfo); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(tzname); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 8ee90d79a13c2f8..b2476e1c6e5c4b5 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -230,9 +230,14 @@ extern "C" { #define _JUMP_TO_TOP 377 #define _SAVE_RETURN_OFFSET 378 #define _CHECK_VALIDITY 379 -#define _LOAD_CONST_INLINE_BORROW 380 -#define _INTERNAL_INCREMENT_OPT_COUNTER 381 -#define MAX_UOP_ID 381 +#define _LOAD_CONST_INLINE 380 +#define _LOAD_CONST_INLINE_BORROW 381 +#define _LOAD_CONST_INLINE_WITH_NULL 382 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 383 +#define _CHECK_GLOBALS 384 +#define _CHECK_BUILTINS 385 +#define _INTERNAL_INCREMENT_OPT_COUNTER 386 +#define MAX_UOP_ID 386 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 9bfb4f4f3a4dea3..2b5b37e6b8d6a43 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -202,7 +202,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, [_EXIT_TRACE] = HAS_DEOPT_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, + [_LOAD_CONST_INLINE] = 0, [_LOAD_CONST_INLINE_BORROW] = 0, + [_LOAD_CONST_INLINE_WITH_NULL] = 0, + [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = 0, + [_CHECK_GLOBALS] = HAS_DEOPT_FLAG, + [_CHECK_BUILTINS] = HAS_DEOPT_FLAG, [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, }; @@ -249,10 +254,12 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", [_CHECK_ATTR_MODULE] = "_CHECK_ATTR_MODULE", [_CHECK_ATTR_WITH_HINT] = "_CHECK_ATTR_WITH_HINT", + [_CHECK_BUILTINS] = "_CHECK_BUILTINS", [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", [_CHECK_EG_MATCH] = "_CHECK_EG_MATCH", [_CHECK_EXC_MATCH] = "_CHECK_EXC_MATCH", [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", + [_CHECK_GLOBALS] = "_CHECK_GLOBALS", [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", [_CHECK_PEP_523] = "_CHECK_PEP_523", [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", @@ -329,7 +336,10 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS", [_LOAD_CONST] = "_LOAD_CONST", + [_LOAD_CONST_INLINE] = "_LOAD_CONST_INLINE", [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW", + [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = "_LOAD_CONST_INLINE_BORROW_WITH_NULL", + [_LOAD_CONST_INLINE_WITH_NULL] = "_LOAD_CONST_INLINE_WITH_NULL", [_LOAD_DEREF] = "_LOAD_DEREF", [_LOAD_FAST] = "_LOAD_FAST", [_LOAD_FAST_AND_CLEAR] = "_LOAD_FAST_AND_CLEAR", diff --git a/Include/listobject.h b/Include/listobject.h index 6b7041ba0b05d59..4e4084b43483a2d 100644 --- a/Include/listobject.h +++ b/Include/listobject.h @@ -29,6 +29,7 @@ PyAPI_FUNC(PyObject *) PyList_New(Py_ssize_t size); PyAPI_FUNC(Py_ssize_t) PyList_Size(PyObject *); PyAPI_FUNC(PyObject *) PyList_GetItem(PyObject *, Py_ssize_t); +PyAPI_FUNC(PyObject *) PyList_GetItemRef(PyObject *, Py_ssize_t); PyAPI_FUNC(int) PyList_SetItem(PyObject *, Py_ssize_t, PyObject *); PyAPI_FUNC(int) PyList_Insert(PyObject *, Py_ssize_t, PyObject *); PyAPI_FUNC(int) PyList_Append(PyObject *, PyObject *); diff --git a/Include/object.h b/Include/object.h index 48f1ddf7510887d..05187fe5dc4f20d 100644 --- a/Include/object.h +++ b/Include/object.h @@ -212,7 +212,10 @@ struct _object { struct _PyMutex { uint8_t v; }; struct _object { - uintptr_t ob_tid; // thread id (or zero) + // ob_tid stores the thread id (or zero). It is also used by the GC and the + // trashcan mechanism as a linked list pointer and by the GC to store the + // computed "gc_refs" refcount. + uintptr_t ob_tid; uint16_t _padding; struct _PyMutex ob_mutex; // per-object lock uint8_t ob_gc_bits; // gc-related state @@ -426,7 +429,11 @@ static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { assert(ob->ob_base.ob_type != &PyLong_Type); assert(ob->ob_base.ob_type != &PyBool_Type); +#ifdef Py_GIL_DISABLED + _Py_atomic_store_ssize_relaxed(&ob->ob_size, size); +#else ob->ob_size = size; +#endif } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SET_SIZE(ob, size) Py_SET_SIZE(_PyVarObject_CAST(ob), (size)) diff --git a/Include/pyerrors.h b/Include/pyerrors.h index 5d0028c116e2d86..68d7985dac8876b 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -108,6 +108,7 @@ PyAPI_DATA(PyObject *) PyExc_NotImplementedError; PyAPI_DATA(PyObject *) PyExc_SyntaxError; PyAPI_DATA(PyObject *) PyExc_IndentationError; PyAPI_DATA(PyObject *) PyExc_TabError; +PyAPI_DATA(PyObject *) PyExc_IncompleteInputError; PyAPI_DATA(PyObject *) PyExc_ReferenceError; PyAPI_DATA(PyObject *) PyExc_SystemError; PyAPI_DATA(PyObject *) PyExc_SystemExit; diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index bca2acf1fc88cf9..54c12d3b2f3f16a 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -1809,7 +1809,7 @@ def fromtimestamp(cls, timestamp, tz=None): def utcfromtimestamp(cls, t): """Construct a naive UTC datetime from a POSIX timestamp.""" import warnings - warnings.warn("datetime.utcfromtimestamp() is deprecated and scheduled " + warnings.warn("datetime.datetime.utcfromtimestamp() is deprecated and scheduled " "for removal in a future version. Use timezone-aware " "objects to represent datetimes in UTC: " "datetime.datetime.fromtimestamp(t, datetime.UTC).", @@ -1827,8 +1827,8 @@ def now(cls, tz=None): def utcnow(cls): "Construct a UTC datetime from time.time()." import warnings - warnings.warn("datetime.utcnow() is deprecated and scheduled for " - "removal in a future version. Instead, Use timezone-aware " + warnings.warn("datetime.datetime.utcnow() is deprecated and scheduled for " + "removal in a future version. Use timezone-aware " "objects to represent datetimes in UTC: " "datetime.datetime.now(datetime.UTC).", DeprecationWarning, @@ -2347,6 +2347,9 @@ def __new__(cls, offset, name=_Omitted): "timedelta(hours=24).") return cls._create(offset, name) + def __init_subclass__(cls): + raise TypeError("type 'datetime.timezone' is not an acceptable base type") + @classmethod def _create(cls, offset, name=None): self = tzinfo.__new__(cls) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index df2c29bfa9caeed..8a0d0dc4b1a0b85 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -1495,6 +1495,11 @@ def __init__(self, file, mode='r', closefd=True, opener=None): if isinstance(file, float): raise TypeError('integer argument expected, got float') if isinstance(file, int): + if isinstance(file, bool): + import warnings + warnings.warn("bool is used as a file descriptor", + RuntimeWarning, stacklevel=2) + file = int(file) fd = file if fd < 0: raise ValueError('negative file descriptor') diff --git a/Lib/argparse.py b/Lib/argparse.py index a32884db80d1ea1..04ee3b19aca755e 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -564,22 +564,18 @@ def _format_action_invocation(self, action): return metavar else: - parts = [] # if the Optional doesn't take a value, format is: # -s, --long if action.nargs == 0: - parts.extend(action.option_strings) + return ', '.join(action.option_strings) # if the Optional takes a value, format is: - # -s ARGS, --long ARGS + # -s, --long ARGS else: default = self._get_default_metavar_for_optional(action) args_string = self._format_args(action, default) - for option_string in action.option_strings: - parts.append('%s %s' % (option_string, args_string)) - - return ', '.join(parts) + return ', '.join(action.option_strings) + ' ' + args_string def _metavar_formatter(self, action, default_metavar): if action.metavar is not None: @@ -847,7 +843,8 @@ def __init__(self, choices=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): self.option_strings = option_strings self.dest = dest self.nargs = nargs @@ -858,6 +855,7 @@ def __init__(self, self.required = required self.help = help self.metavar = metavar + self.deprecated = deprecated def _get_kwargs(self): names = [ @@ -871,6 +869,7 @@ def _get_kwargs(self): 'required', 'help', 'metavar', + 'deprecated', ] return [(name, getattr(self, name)) for name in names] @@ -893,7 +892,8 @@ def __init__(self, choices=_deprecated_default, required=False, help=None, - metavar=_deprecated_default): + metavar=_deprecated_default, + deprecated=False): _option_strings = [] for option_string in option_strings: @@ -931,7 +931,8 @@ def __init__(self, choices=choices, required=required, help=help, - metavar=metavar) + metavar=metavar, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): @@ -954,7 +955,8 @@ def __init__(self, choices=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): if nargs == 0: raise ValueError('nargs for store actions must be != 0; if you ' 'have nothing to store, actions such as store ' @@ -971,7 +973,8 @@ def __init__(self, choices=choices, required=required, help=help, - metavar=metavar) + metavar=metavar, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, values) @@ -986,7 +989,8 @@ def __init__(self, default=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): super(_StoreConstAction, self).__init__( option_strings=option_strings, dest=dest, @@ -994,7 +998,8 @@ def __init__(self, const=const, default=default, required=required, - help=help) + help=help, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, self.const) @@ -1007,14 +1012,16 @@ def __init__(self, dest, default=False, required=False, - help=None): + help=None, + deprecated=False): super(_StoreTrueAction, self).__init__( option_strings=option_strings, dest=dest, const=True, - default=default, + deprecated=deprecated, required=required, - help=help) + help=help, + default=default) class _StoreFalseAction(_StoreConstAction): @@ -1024,14 +1031,16 @@ def __init__(self, dest, default=True, required=False, - help=None): + help=None, + deprecated=False): super(_StoreFalseAction, self).__init__( option_strings=option_strings, dest=dest, const=False, default=default, required=required, - help=help) + help=help, + deprecated=deprecated) class _AppendAction(Action): @@ -1046,7 +1055,8 @@ def __init__(self, choices=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): if nargs == 0: raise ValueError('nargs for append actions must be != 0; if arg ' 'strings are not supplying the value to append, ' @@ -1063,7 +1073,8 @@ def __init__(self, choices=choices, required=required, help=help, - metavar=metavar) + metavar=metavar, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): items = getattr(namespace, self.dest, None) @@ -1081,7 +1092,8 @@ def __init__(self, default=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): super(_AppendConstAction, self).__init__( option_strings=option_strings, dest=dest, @@ -1090,7 +1102,8 @@ def __init__(self, default=default, required=required, help=help, - metavar=metavar) + metavar=metavar, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): items = getattr(namespace, self.dest, None) @@ -1106,14 +1119,16 @@ def __init__(self, dest, default=None, required=False, - help=None): + help=None, + deprecated=False): super(_CountAction, self).__init__( option_strings=option_strings, dest=dest, nargs=0, default=default, required=required, - help=help) + help=help, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): count = getattr(namespace, self.dest, None) @@ -1128,13 +1143,15 @@ def __init__(self, option_strings, dest=SUPPRESS, default=SUPPRESS, - help=None): + help=None, + deprecated=False): super(_HelpAction, self).__init__( option_strings=option_strings, dest=dest, default=default, nargs=0, - help=help) + help=help, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): parser.print_help() @@ -1148,7 +1165,8 @@ def __init__(self, version=None, dest=SUPPRESS, default=SUPPRESS, - help="show program's version number and exit"): + help="show program's version number and exit", + deprecated=False): super(_VersionAction, self).__init__( option_strings=option_strings, dest=dest, @@ -1192,6 +1210,7 @@ def __init__(self, self._parser_class = parser_class self._name_parser_map = {} self._choices_actions = [] + self._deprecated = set() super(_SubParsersAction, self).__init__( option_strings=option_strings, @@ -1202,7 +1221,7 @@ def __init__(self, help=help, metavar=metavar) - def add_parser(self, name, **kwargs): + def add_parser(self, name, *, deprecated=False, **kwargs): # set prog from the existing prefix if kwargs.get('prog') is None: kwargs['prog'] = '%s %s' % (self._prog_prefix, name) @@ -1230,6 +1249,10 @@ def add_parser(self, name, **kwargs): for alias in aliases: self._name_parser_map[alias] = parser + if deprecated: + self._deprecated.add(name) + self._deprecated.update(aliases) + return parser def _get_subactions(self): @@ -1245,13 +1268,17 @@ def __call__(self, parser, namespace, values, option_string=None): # select the parser try: - parser = self._name_parser_map[parser_name] + subparser = self._name_parser_map[parser_name] except KeyError: args = {'parser_name': parser_name, 'choices': ', '.join(self._name_parser_map)} msg = _('unknown parser %(parser_name)r (choices: %(choices)s)') % args raise ArgumentError(self, msg) + if parser_name in self._deprecated: + parser._warning(_("command '%(parser_name)s' is deprecated") % + {'parser_name': parser_name}) + # parse all the remaining options into the namespace # store any unrecognized options on the object, so that the top # level parser can decide what to do with them @@ -1259,7 +1286,7 @@ def __call__(self, parser, namespace, values, option_string=None): # In case this subparser defines new defaults, we parse them # in a new namespace object and then update the original # namespace for the relevant parts. - subnamespace, arg_strings = parser.parse_known_args(arg_strings, None) + subnamespace, arg_strings = subparser.parse_known_args(arg_strings, None) for key, value in vars(subnamespace).items(): setattr(namespace, key, value) @@ -1979,6 +2006,7 @@ def _parse_known_args(self, arg_strings, namespace): # converts arg strings to the appropriate and then takes the action seen_actions = set() seen_non_default_actions = set() + warned = set() def take_action(action, argument_strings, option_string=None): seen_actions.add(action) @@ -2074,6 +2102,10 @@ def consume_optional(start_index): # the Optional's string args stopped assert action_tuples for action, args, option_string in action_tuples: + if action.deprecated and option_string not in warned: + self._warning(_("option '%(option)s' is deprecated") % + {'option': option_string}) + warned.add(option_string) take_action(action, args, option_string) return stop @@ -2093,6 +2125,10 @@ def consume_positionals(start_index): for action, arg_count in zip(positionals, arg_counts): args = arg_strings[start_index: start_index + arg_count] start_index += arg_count + if args and action.deprecated and action.dest not in warned: + self._warning(_("argument '%(argument_name)s' is deprecated") % + {'argument_name': action.dest}) + warned.add(action.dest) take_action(action, args) # slice off the Positionals that we just parsed and return the @@ -2489,7 +2525,7 @@ def parse_known_intermixed_args(self, args=None, namespace=None): # ======================== def _get_values(self, action, arg_strings): # for everything but PARSER, REMAINDER args, strip out first '--' - if action.nargs not in [PARSER, REMAINDER]: + if not action.option_strings and action.nargs not in [PARSER, REMAINDER]: try: arg_strings.remove('--') except ValueError: @@ -2654,3 +2690,7 @@ def error(self, message): self.print_usage(_sys.stderr) args = {'prog': self.prog, 'message': message} self.exit(2, _('%(prog)s: error: %(message)s\n') % args) + + def _warning(self, message): + args = {'prog': self.prog, 'message': message} + self._print_message(_('%(prog)s: warning: %(message)s\n') % args, _sys.stderr) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index c60d7688ef8c772..aadc4f478f8b560 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -1340,9 +1340,9 @@ async def create_datagram_endpoint(self, protocol_factory, allow_broadcast=None, sock=None): """Create datagram connection.""" if sock is not None: - if sock.type != socket.SOCK_DGRAM: + if sock.type == socket.SOCK_STREAM: raise ValueError( - f'A UDP Socket was expected, got {sock!r}') + f'A datagram socket was expected, got {sock!r}') if (local_addr or remote_addr or family or proto or flags or reuse_port or allow_broadcast): diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 04158e667a895fc..aaee8ff07029233 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -24,25 +24,23 @@ class Lock(_ContextManagerMixin, mixins._LoopBoundMixin): """Primitive lock objects. A primitive lock is a synchronization primitive that is not owned - by a particular coroutine when locked. A primitive lock is in one + by a particular task when locked. A primitive lock is in one of two states, 'locked' or 'unlocked'. It is created in the unlocked state. It has two basic methods, acquire() and release(). When the state is unlocked, acquire() changes the state to locked and returns immediately. When the state is locked, acquire() blocks until a call to release() in - another coroutine changes it to unlocked, then the acquire() call + another task changes it to unlocked, then the acquire() call resets it to locked and returns. The release() method should only be called in the locked state; it changes the state to unlocked and returns immediately. If an attempt is made to release an unlocked lock, a RuntimeError will be raised. - When more than one coroutine is blocked in acquire() waiting for - the state to turn to unlocked, only one coroutine proceeds when a - release() call resets the state to unlocked; first coroutine which - is blocked in acquire() is being processed. - - acquire() is a coroutine and should be called with 'await'. + When more than one task is blocked in acquire() waiting for + the state to turn to unlocked, only one task proceeds when a + release() call resets the state to unlocked; successive release() + calls will unblock tasks in FIFO order. Locks also support the asynchronous context management protocol. 'async with lock' statement should be used. @@ -130,7 +128,7 @@ def release(self): """Release a lock. When the lock is locked, reset it to unlocked, and return. - If any other coroutines are blocked waiting for the lock to become + If any other tasks are blocked waiting for the lock to become unlocked, allow exactly one of them to proceed. When invoked on an unlocked lock, a RuntimeError is raised. @@ -182,8 +180,8 @@ def is_set(self): return self._value def set(self): - """Set the internal flag to true. All coroutines waiting for it to - become true are awakened. Coroutine that call wait() once the flag is + """Set the internal flag to true. All tasks waiting for it to + become true are awakened. Tasks that call wait() once the flag is true will not block at all. """ if not self._value: @@ -194,7 +192,7 @@ def set(self): fut.set_result(True) def clear(self): - """Reset the internal flag to false. Subsequently, coroutines calling + """Reset the internal flag to false. Subsequently, tasks calling wait() will block until set() is called to set the internal flag to true again.""" self._value = False @@ -203,7 +201,7 @@ async def wait(self): """Block until the internal flag is true. If the internal flag is true on entry, return True - immediately. Otherwise, block until another coroutine calls + immediately. Otherwise, block until another task calls set() to set the flag to true, then return True. """ if self._value: @@ -222,8 +220,8 @@ class Condition(_ContextManagerMixin, mixins._LoopBoundMixin): """Asynchronous equivalent to threading.Condition. This class implements condition variable objects. A condition variable - allows one or more coroutines to wait until they are notified by another - coroutine. + allows one or more tasks to wait until they are notified by another + task. A new Lock object is created and used as the underlying lock. """ @@ -250,50 +248,64 @@ def __repr__(self): async def wait(self): """Wait until notified. - If the calling coroutine has not acquired the lock when this + If the calling task has not acquired the lock when this method is called, a RuntimeError is raised. This method releases the underlying lock, and then blocks until it is awakened by a notify() or notify_all() call for - the same condition variable in another coroutine. Once + the same condition variable in another task. Once awakened, it re-acquires the lock and returns True. + + This method may return spuriously, + which is why the caller should always + re-check the state and be prepared to wait() again. """ if not self.locked(): raise RuntimeError('cannot wait on un-acquired lock') + fut = self._get_loop().create_future() self.release() try: - fut = self._get_loop().create_future() - self._waiters.append(fut) try: - await fut - return True - finally: - self._waiters.remove(fut) - - finally: - # Must re-acquire lock even if wait is cancelled. - # We only catch CancelledError here, since we don't want any - # other (fatal) errors with the future to cause us to spin. - err = None - while True: - try: - await self.acquire() - break - except exceptions.CancelledError as e: - err = e - - if err: + self._waiters.append(fut) try: - raise err # Re-raise most recent exception instance. + await fut + return True finally: - err = None # Break reference cycles. + self._waiters.remove(fut) + + finally: + # Must re-acquire lock even if wait is cancelled. + # We only catch CancelledError here, since we don't want any + # other (fatal) errors with the future to cause us to spin. + err = None + while True: + try: + await self.acquire() + break + except exceptions.CancelledError as e: + err = e + + if err is not None: + try: + raise err # Re-raise most recent exception instance. + finally: + err = None # Break reference cycles. + except BaseException: + # Any error raised out of here _may_ have occurred after this Task + # believed to have been successfully notified. + # Make sure to notify another Task instead. This may result + # in a "spurious wakeup", which is allowed as part of the + # Condition Variable protocol. + self._notify(1) + raise async def wait_for(self, predicate): """Wait until a predicate becomes true. - The predicate should be a callable which result will be - interpreted as a boolean value. The final predicate value is + The predicate should be a callable whose result will be + interpreted as a boolean value. The method will repeatedly + wait() until it evaluates to true. The final predicate value is the return value. """ result = predicate() @@ -303,20 +315,22 @@ async def wait_for(self, predicate): return result def notify(self, n=1): - """By default, wake up one coroutine waiting on this condition, if any. - If the calling coroutine has not acquired the lock when this method + """By default, wake up one task waiting on this condition, if any. + If the calling task has not acquired the lock when this method is called, a RuntimeError is raised. - This method wakes up at most n of the coroutines waiting for the - condition variable; it is a no-op if no coroutines are waiting. + This method wakes up n of the tasks waiting for the condition + variable; if fewer than n are waiting, they are all awoken. - Note: an awakened coroutine does not actually return from its + Note: an awakened task does not actually return from its wait() call until it can reacquire the lock. Since notify() does not release the lock, its caller should. """ if not self.locked(): raise RuntimeError('cannot notify on un-acquired lock') + self._notify(n) + def _notify(self, n): idx = 0 for fut in self._waiters: if idx >= n: @@ -374,7 +388,7 @@ async def acquire(self): If the internal counter is larger than zero on entry, decrement it by one and return True immediately. If it is - zero on entry, block, waiting until some other coroutine has + zero on entry, block, waiting until some other task has called release() to make it larger than 0, and then return True. """ @@ -414,8 +428,8 @@ async def acquire(self): def release(self): """Release a semaphore, incrementing the internal counter by one. - When it was zero on entry and another coroutine is waiting for it to - become larger than zero again, wake up that coroutine. + When it was zero on entry and another task is waiting for it to + become larger than zero again, wake up that task. """ self._value += 1 self._wake_up_next() diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index dcd5e0aa345029a..10fbdd76e93f79c 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -235,6 +235,10 @@ async def _accept_connection2( await waiter except BaseException: transport.close() + # gh-109534: When an exception is raised by the SSLProtocol object the + # exception set in this future can keep the protocol object alive and + # cause a reference cycle. + waiter = None raise # It's now up to the protocol to handle the connection. diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 599e91ba0003d10..fa99d4533aa0a6a 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -579,6 +579,7 @@ def _on_handshake_complete(self, handshake_exc): peercert = sslobj.getpeercert() except Exception as exc: + handshake_exc = None self._set_state(SSLProtocolState.UNWRAPPED) if isinstance(exc, ssl.CertificateError): msg = 'SSL handshake failed on verifying the certificate' diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index e1c56d140bef7de..f322b1f6653f6a2 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -132,7 +132,7 @@ async def __aexit__(self, et, exc, tb): # Propagate CancelledError if there is one, except if there # are other errors -- those have priority. - if propagate_cancellation_error and not self._errors: + if propagate_cancellation_error is not None and not self._errors: raise propagate_cancellation_error if et is not None and not issubclass(et, exceptions.CancelledError): diff --git a/Lib/calendar.py b/Lib/calendar.py index 03469d8ac96bcd9..833ce331b14a0cc 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -734,10 +734,15 @@ def main(args=None): choices=("text", "html"), help="output type (text or html)" ) + parser.add_argument( + "-f", "--first-weekday", + type=int, default=0, + help="weekday (0 is Monday, 6 is Sunday) to start each week (default 0)" + ) parser.add_argument( "year", nargs='?', type=int, - help="year number (1-9999)" + help="year number" ) parser.add_argument( "month", @@ -761,6 +766,7 @@ def main(args=None): cal = LocaleHTMLCalendar(locale=locale) else: cal = HTMLCalendar() + cal.setfirstweekday(options.first_weekday) encoding = options.encoding if encoding is None: encoding = sys.getdefaultencoding() @@ -775,6 +781,7 @@ def main(args=None): cal = LocaleTextCalendar(locale=locale) else: cal = TextCalendar() + cal.setfirstweekday(options.first_weekday) optdict = dict(w=options.width, l=options.lines) if options.month is None: optdict["c"] = options.spacing diff --git a/Lib/codeop.py b/Lib/codeop.py index 91146be2c438e2c..6ad60e7f85098d8 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -65,9 +65,10 @@ def _maybe_compile(compiler, source, filename, symbol): try: compiler(source + "\n", filename, symbol) return None + except IncompleteInputError as e: + return None except SyntaxError as e: - if "incomplete input" in str(e): - return None + pass # fallthrough return compiler(source, filename, symbol, incomplete_input=False) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index ffaffdb8b3d0aad..ca843e11eeb83dc 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -190,16 +190,6 @@ def _on_queue_feeder_error(self, e, obj): super()._on_queue_feeder_error(e, obj) -def _get_chunks(*iterables, chunksize): - """ Iterates over zip()ed iterables in chunks. """ - it = zip(*iterables) - while True: - chunk = tuple(itertools.islice(it, chunksize)) - if not chunk: - return - yield chunk - - def _process_chunk(fn, chunk): """ Processes a chunk of an iterable passed to map. @@ -847,7 +837,7 @@ def map(self, fn, *iterables, timeout=None, chunksize=1): raise ValueError("chunksize must be >= 1.") results = super().map(partial(_process_chunk, fn), - _get_chunks(*iterables, chunksize=chunksize), + itertools.batched(zip(*iterables), chunksize), timeout=timeout) return _chain_from_iterable_of_lists(results) diff --git a/Lib/csv.py b/Lib/csv.py index a079279b8b8cbce..75e35b23236795f 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -113,8 +113,8 @@ def _validate(self): try: _Dialect(self) except TypeError as e: - # We do this for compatibility with py2.3 - raise Error(str(e)) + # Re-raise to get a traceback showing more user code. + raise Error(str(e)) from None class excel(Dialect): """Describe the usual properties of Excel-generated CSV files.""" diff --git a/Lib/dbm/dumb.py b/Lib/dbm/dumb.py index 754624ccc8f5008..def120ffc3778b6 100644 --- a/Lib/dbm/dumb.py +++ b/Lib/dbm/dumb.py @@ -98,7 +98,8 @@ def _update(self, flag): except OSError: if flag not in ('c', 'n'): raise - self._modified = True + with self._io.open(self._dirfile, 'w', encoding="Latin-1") as f: + self._chmod(self._dirfile) else: with f: for line in f: @@ -134,6 +135,7 @@ def _commit(self): # position; UTF-8, though, does care sometimes. entry = "%r, %r\n" % (key.decode('Latin-1'), pos_and_siz_pair) f.write(entry) + self._modified = False sync = _commit diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py index febe411355d6be3..0f1bf8e4253ec4b 100644 --- a/Lib/email/_parseaddr.py +++ b/Lib/email/_parseaddr.py @@ -13,7 +13,7 @@ 'quote', ] -import time, calendar +import time SPACE = ' ' EMPTYSTRING = '' @@ -194,6 +194,9 @@ def mktime_tz(data): # No zone info, so localtime is better assumption than GMT return time.mktime(data[:8] + (-1,)) else: + # Delay the import, since mktime_tz is rarely used + import calendar + t = calendar.timegm(data) return t - data[9] diff --git a/Lib/email/policy.py b/Lib/email/policy.py index 611deb50bb5290e..8816c84ed175a70 100644 --- a/Lib/email/policy.py +++ b/Lib/email/policy.py @@ -210,8 +210,15 @@ def _fold(self, name, value, refold_binary=False): self.refold_source == 'long' and (lines and len(lines[0])+len(name)+2 > maxlen or any(len(x) > maxlen for x in lines[1:]))) - if refold or refold_binary and _has_surrogates(value): + + if not refold: + if not self.utf8: + refold = not value.isascii() + elif refold_binary: + refold = _has_surrogates(value) + if refold: return self.header_factory(name, ''.join(lines)).fold(policy=self) + return name + ': ' + self.linesep.join(lines) + self.linesep diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index a09bf3201e1fb7d..e8dd253bb555209 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -1,78 +1,64 @@ -import collections import os -import os.path import subprocess import sys import sysconfig import tempfile +from contextlib import nullcontext from importlib import resources +from pathlib import Path +from shutil import copy2 __all__ = ["version", "bootstrap"] -_PACKAGE_NAMES = ('pip',) -_PIP_VERSION = "23.3.2" -_PROJECTS = [ - ("pip", _PIP_VERSION, "py3"), -] - -# Packages bundled in ensurepip._bundled have wheel_name set. -# Packages from WHEEL_PKG_DIR have wheel_path set. -_Package = collections.namedtuple('Package', - ('version', 'wheel_name', 'wheel_path')) +_PIP_VERSION = "24.0" # Directory of system wheel packages. Some Linux distribution packaging # policies recommend against bundling dependencies. For example, Fedora # installs wheel packages in the /usr/share/python-wheels/ directory and don't # install the ensurepip._bundled package. -_WHEEL_PKG_DIR = sysconfig.get_config_var('WHEEL_PKG_DIR') +if (_pkg_dir := sysconfig.get_config_var('WHEEL_PKG_DIR')) is not None: + _WHEEL_PKG_DIR = Path(_pkg_dir).resolve() +else: + _WHEEL_PKG_DIR = None + +def _find_wheel_pkg_dir_pip(): + if _WHEEL_PKG_DIR is None: + # NOTE: The compile-time `WHEEL_PKG_DIR` is unset so there is no place + # NOTE: for looking up the wheels. + return None -def _find_packages(path): - packages = {} + dist_matching_wheels = _WHEEL_PKG_DIR.glob('pip-*.whl') try: - filenames = os.listdir(path) - except OSError: - # Ignore: path doesn't exist or permission error - filenames = () - # Make the code deterministic if a directory contains multiple wheel files - # of the same package, but don't attempt to implement correct version - # comparison since this case should not happen. - filenames = sorted(filenames) - for filename in filenames: - # filename is like 'pip-21.2.4-py3-none-any.whl' - if not filename.endswith(".whl"): - continue - for name in _PACKAGE_NAMES: - prefix = name + '-' - if filename.startswith(prefix): - break - else: - continue - - # Extract '21.2.4' from 'pip-21.2.4-py3-none-any.whl' - version = filename.removeprefix(prefix).partition('-')[0] - wheel_path = os.path.join(path, filename) - packages[name] = _Package(version, None, wheel_path) - return packages - - -def _get_packages(): - global _PACKAGES, _WHEEL_PKG_DIR - if _PACKAGES is not None: - return _PACKAGES - - packages = {} - for name, version, py_tag in _PROJECTS: - wheel_name = f"{name}-{version}-{py_tag}-none-any.whl" - packages[name] = _Package(version, wheel_name, None) - if _WHEEL_PKG_DIR: - dir_packages = _find_packages(_WHEEL_PKG_DIR) - # only used the wheel package directory if all packages are found there - if all(name in dir_packages for name in _PACKAGE_NAMES): - packages = dir_packages - _PACKAGES = packages - return packages -_PACKAGES = None + last_matching_dist_wheel = sorted(dist_matching_wheels)[-1] + except IndexError: + # NOTE: `WHEEL_PKG_DIR` does not contain any wheel files for `pip`. + return None + + return nullcontext(last_matching_dist_wheel) + + +def _get_pip_whl_path_ctx(): + # Prefer pip from the wheel package directory, if present. + if (alternative_pip_wheel_path := _find_wheel_pkg_dir_pip()) is not None: + return alternative_pip_wheel_path + + return resources.as_file( + resources.files('ensurepip') + / '_bundled' + / f'pip-{_PIP_VERSION}-py3-none-any.whl' + ) + + +def _get_pip_version(): + with _get_pip_whl_path_ctx() as bundled_wheel_path: + wheel_name = bundled_wheel_path.name + return ( + # Extract '21.2.4' from 'pip-21.2.4-py3-none-any.whl' + wheel_name. + removeprefix('pip-'). + partition('-')[0] + ) def _run_pip(args, additional_paths=None): @@ -105,7 +91,7 @@ def version(): """ Returns a string specifying the bundled version of pip. """ - return _get_packages()['pip'].version + return _get_pip_version() def _disable_pip_configuration_settings(): @@ -167,24 +153,10 @@ def _bootstrap(*, root=None, upgrade=False, user=False, with tempfile.TemporaryDirectory() as tmpdir: # Put our bundled wheels into a temporary directory and construct the # additional paths that need added to sys.path - additional_paths = [] - for name, package in _get_packages().items(): - if package.wheel_name: - # Use bundled wheel package - wheel_name = package.wheel_name - wheel_path = resources.files("ensurepip") / "_bundled" / wheel_name - whl = wheel_path.read_bytes() - else: - # Use the wheel package directory - with open(package.wheel_path, "rb") as fp: - whl = fp.read() - wheel_name = os.path.basename(package.wheel_path) - - filename = os.path.join(tmpdir, wheel_name) - with open(filename, "wb") as fp: - fp.write(whl) - - additional_paths.append(filename) + tmpdir_path = Path(tmpdir) + with _get_pip_whl_path_ctx() as bundled_wheel_path: + tmp_wheel_path = tmpdir_path / bundled_wheel_path.name + copy2(bundled_wheel_path, tmp_wheel_path) # Construct the arguments to be passed to the pip command args = ["install", "--no-cache-dir", "--no-index", "--find-links", tmpdir] @@ -197,7 +169,8 @@ def _bootstrap(*, root=None, upgrade=False, user=False, if verbosity: args += ["-" + "v" * verbosity] - return _run_pip([*args, *_PACKAGE_NAMES], additional_paths) + return _run_pip([*args, "pip"], [os.fsdecode(tmp_wheel_path)]) + def _uninstall_helper(*, verbosity=0): """Helper to support a clean default uninstall process on Windows @@ -227,7 +200,7 @@ def _uninstall_helper(*, verbosity=0): if verbosity: args += ["-" + "v" * verbosity] - return _run_pip([*args, *reversed(_PACKAGE_NAMES)]) + return _run_pip([*args, "pip"]) def _main(argv=None): diff --git a/Lib/ensurepip/_bundled/pip-23.3.2-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-24.0-py3-none-any.whl similarity index 83% rename from Lib/ensurepip/_bundled/pip-23.3.2-py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-24.0-py3-none-any.whl index ae78b8a6ce07379..2e6aa9d2cb99232 100644 Binary files a/Lib/ensurepip/_bundled/pip-23.3.2-py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-24.0-py3-none-any.whl differ diff --git a/Lib/enum.py b/Lib/enum.py index a8a50a583803758..98a8966f5eb1599 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -409,10 +409,11 @@ def __setitem__(self, key, value): if isinstance(value, auto): single = True value = (value, ) - if type(value) is tuple and any(isinstance(v, auto) for v in value): + if isinstance(value, tuple) and any(isinstance(v, auto) for v in value): # insist on an actual tuple, no subclasses, in keeping with only supporting # top-level auto() usage (not contained in any other data structure) auto_valued = [] + t = type(value) for v in value: if isinstance(v, auto): non_auto_store = False @@ -427,7 +428,12 @@ def __setitem__(self, key, value): if single: value = auto_valued[0] else: - value = tuple(auto_valued) + try: + # accepts iterable as multiple arguments? + value = t(auto_valued) + except TypeError: + # then pass them in singlely + value = t(*auto_valued) self._member_names[key] = None if non_auto_store: self._last_values.append(value) diff --git a/Lib/fileinput.py b/Lib/fileinput.py index 1b25f28f3d34326..3dba3d2fbfa967b 100644 --- a/Lib/fileinput.py +++ b/Lib/fileinput.py @@ -53,7 +53,7 @@ sequence must be accessed in strictly sequential order; sequence access and readline() cannot be mixed. -Optional in-place filtering: if the keyword argument inplace=1 is +Optional in-place filtering: if the keyword argument inplace=True is passed to input() or to the FileInput constructor, the file is moved to a backup file and standard output is directed to the input file. This makes it possible to write a filter that rewrites its input file diff --git a/Lib/fractions.py b/Lib/fractions.py index 389ab386b6a8a4d..f8c6c9c438c7379 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -579,7 +579,8 @@ def __format__(self, format_spec, /): f"for object of type {type(self).__name__!r}" ) - def _operator_fallbacks(monomorphic_operator, fallback_operator): + def _operator_fallbacks(monomorphic_operator, fallback_operator, + handle_complex=True): """Generates forward and reverse operators given a purely-rational operator and a function from the operator module. @@ -666,7 +667,7 @@ def forward(a, b): return monomorphic_operator(a, Fraction(b)) elif isinstance(b, float): return fallback_operator(float(a), b) - elif isinstance(b, complex): + elif handle_complex and isinstance(b, complex): return fallback_operator(complex(a), b) else: return NotImplemented @@ -679,7 +680,7 @@ def reverse(b, a): return monomorphic_operator(Fraction(a), b) elif isinstance(a, numbers.Real): return fallback_operator(float(a), float(b)) - elif isinstance(a, numbers.Complex): + elif handle_complex and isinstance(a, numbers.Complex): return fallback_operator(complex(a), complex(b)) else: return NotImplemented @@ -830,7 +831,7 @@ def _floordiv(a, b): """a // b""" return (a.numerator * b.denominator) // (a.denominator * b.numerator) - __floordiv__, __rfloordiv__ = _operator_fallbacks(_floordiv, operator.floordiv) + __floordiv__, __rfloordiv__ = _operator_fallbacks(_floordiv, operator.floordiv, False) def _divmod(a, b): """(a // b, a % b)""" @@ -838,14 +839,14 @@ def _divmod(a, b): div, n_mod = divmod(a.numerator * db, da * b.numerator) return div, Fraction(n_mod, da * db) - __divmod__, __rdivmod__ = _operator_fallbacks(_divmod, divmod) + __divmod__, __rdivmod__ = _operator_fallbacks(_divmod, divmod, False) def _mod(a, b): """a % b""" da, db = a.denominator, b.denominator return Fraction((a.numerator * db) % (b.numerator * da), da * db) - __mod__, __rmod__ = _operator_fallbacks(_mod, operator.mod) + __mod__, __rmod__ = _operator_fallbacks(_mod, operator.mod, False) def __pow__(a, b): """a ** b diff --git a/Lib/gzip.py b/Lib/gzip.py index 177f9080dc5af8b..fda93e0261e0285 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -349,7 +349,7 @@ def closed(self): def close(self): fileobj = self.fileobj - if fileobj is None: + if fileobj is None or self._buffer.closed: return try: if self.mode == WRITE: diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 6ad383f460c7ee2..8ee8eba64367a50 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -166,8 +166,9 @@ def __init__(self, flist=None, filename=None, key=None, root=None): text.bind("<3>",self.right_menu_event) text.bind('<MouseWheel>', wheel_event) - text.bind('<Button-4>', wheel_event) - text.bind('<Button-5>', wheel_event) + if text._windowingsystem == 'x11': + text.bind('<Button-4>', wheel_event) + text.bind('<Button-5>', wheel_event) text.bind('<Configure>', self.handle_winconfig) text.bind("<<cut>>", self.cut) text.bind("<<copy>>", self.copy) diff --git a/Lib/idlelib/idle_test/test_sidebar.py b/Lib/idlelib/idle_test/test_sidebar.py index fb52b3a0179553d..605e7a892570d7d 100644 --- a/Lib/idlelib/idle_test/test_sidebar.py +++ b/Lib/idlelib/idle_test/test_sidebar.py @@ -690,16 +690,22 @@ def test_mousewheel(self): last_lineno = get_end_linenumber(text) self.assertIsNotNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) - # Scroll up using the <MouseWheel> event. - # The meaning of delta is platform-dependent. - delta = -1 if sys.platform == 'darwin' else 120 - sidebar.canvas.event_generate('<MouseWheel>', x=0, y=0, delta=delta) + # Delta for <MouseWheel>, whose meaning is platform-dependent. + delta = 1 if sidebar.canvas._windowingsystem == 'aqua' else 120 + + # Scroll up. + if sidebar.canvas._windowingsystem == 'x11': + sidebar.canvas.event_generate('<Button-4>', x=0, y=0) + else: + sidebar.canvas.event_generate('<MouseWheel>', x=0, y=0, delta=delta) yield - if sys.platform != 'darwin': # .update_idletasks() does not work. - self.assertIsNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) + self.assertIsNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) - # Scroll back down using the <Button-5> event. - sidebar.canvas.event_generate('<Button-5>', x=0, y=0) + # Scroll back down. + if sidebar.canvas._windowingsystem == 'x11': + sidebar.canvas.event_generate('<Button-5>', x=0, y=0) + else: + sidebar.canvas.event_generate('<MouseWheel>', x=0, y=0, delta=-delta) yield self.assertIsNotNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) diff --git a/Lib/idlelib/tree.py b/Lib/idlelib/tree.py index 9c2eb47b24aec94..0726d7e23660f64 100644 --- a/Lib/idlelib/tree.py +++ b/Lib/idlelib/tree.py @@ -285,8 +285,9 @@ def drawtext(self): self.label.bind("<1>", self.select_or_edit) self.label.bind("<Double-1>", self.flip) self.label.bind("<MouseWheel>", lambda e: wheel_event(e, self.canvas)) - self.label.bind("<Button-4>", lambda e: wheel_event(e, self.canvas)) - self.label.bind("<Button-5>", lambda e: wheel_event(e, self.canvas)) + if self.label._windowingsystem == 'x11': + self.label.bind("<Button-4>", lambda e: wheel_event(e, self.canvas)) + self.label.bind("<Button-5>", lambda e: wheel_event(e, self.canvas)) self.text_id = id def select_or_edit(self, event=None): @@ -460,8 +461,9 @@ def __init__(self, master, **opts): self.canvas.bind("<Key-Up>", self.unit_up) self.canvas.bind("<Key-Down>", self.unit_down) self.canvas.bind("<MouseWheel>", wheel_event) - self.canvas.bind("<Button-4>", wheel_event) - self.canvas.bind("<Button-5>", wheel_event) + if self.canvas._windowingsystem == 'x11': + self.canvas.bind("<Button-4>", wheel_event) + self.canvas.bind("<Button-5>", wheel_event) #if isinstance(master, Toplevel) or isinstance(master, Tk): self.canvas.bind("<Alt-Key-2>", self.zoom_height) self.canvas.focus_set() diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index d942045f3de666f..6d6292f95592534 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -53,7 +53,7 @@ def _new_module(name): # For a list that can have a weakref to it. class _List(list): - pass + __slots__ = ("__weakref__",) # Copied from weakref.py with some simplifications and modifications unique to diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index a4d2b7e01844092..2a9aef03179f6fb 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -464,6 +464,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.13a1 3565 (Oparg of YIELD_VALUE indicates whether it is in a yield-from) # Python 3.13a1 3566 (Emit JUMP_NO_INTERRUPT instead of JUMP for non-loop no-lineno cases) # Python 3.13a1 3567 (Reimplement line number propagation by the compiler) +# Python 3.13a1 3568 (Change semantics of END_FOR) # Python 3.14 will start with 3600 @@ -480,7 +481,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3567).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3568).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py index 7b142e786e829ef..c612fbefee2e802 100644 --- a/Lib/importlib/metadata/__init__.py +++ b/Lib/importlib/metadata/__init__.py @@ -1,7 +1,6 @@ import os import re import abc -import csv import sys import json import email @@ -478,6 +477,10 @@ def make_file(name, hash=None, size_str=None): @pass_none def make_files(lines): + # Delay csv import, since Distribution.files is not as widely used + # as other parts of importlib.metadata + import csv + return starmap(make_file, csv.reader(lines)) @pass_none diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index eb7e020d1edfc04..684b58d5548f91f 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -1493,7 +1493,7 @@ def debug(self, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.debug("Houston, we have a %s", "thorny problem", exc_info=1) + logger.debug("Houston, we have a %s", "thorny problem", exc_info=True) """ if self.isEnabledFor(DEBUG): self._log(DEBUG, msg, args, **kwargs) @@ -1505,7 +1505,7 @@ def info(self, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.info("Houston, we have a %s", "notable problem", exc_info=1) + logger.info("Houston, we have a %s", "notable problem", exc_info=True) """ if self.isEnabledFor(INFO): self._log(INFO, msg, args, **kwargs) @@ -1517,7 +1517,7 @@ def warning(self, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.warning("Houston, we have a %s", "bit of a problem", exc_info=1) + logger.warning("Houston, we have a %s", "bit of a problem", exc_info=True) """ if self.isEnabledFor(WARNING): self._log(WARNING, msg, args, **kwargs) @@ -1529,7 +1529,7 @@ def error(self, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.error("Houston, we have a %s", "major problem", exc_info=1) + logger.error("Houston, we have a %s", "major problem", exc_info=True) """ if self.isEnabledFor(ERROR): self._log(ERROR, msg, args, **kwargs) @@ -1547,7 +1547,7 @@ def critical(self, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.critical("Houston, we have a %s", "major disaster", exc_info=1) + logger.critical("Houston, we have a %s", "major disaster", exc_info=True) """ if self.isEnabledFor(CRITICAL): self._log(CRITICAL, msg, args, **kwargs) @@ -1565,7 +1565,7 @@ def log(self, level, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.log(level, "We have a %s", "mysterious problem", exc_info=1) + logger.log(level, "We have a %s", "mysterious problem", exc_info=True) """ if not isinstance(level, int): if raiseExceptions: diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 9840b7b0aeba884..e7f1322e4ba3d9d 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -1586,6 +1586,7 @@ def stop(self): Note that if you don't call this before your application exits, there may be some records still left on the queue, which won't be processed. """ - self.enqueue_sentinel() - self._thread.join() - self._thread = None + if self._thread: # see gh-114706 - allow calling this more than once + self.enqueue_sentinel() + self._thread.join() + self._thread = None diff --git a/Lib/mailbox.py b/Lib/mailbox.py index 81ea210cf815a48..746811bd559412d 100644 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -830,10 +830,11 @@ def get_message(self, key): """Return a Message representation or raise a KeyError.""" start, stop = self._lookup(key) self._file.seek(start) - from_line = self._file.readline().replace(linesep, b'') + from_line = self._file.readline().replace(linesep, b'').decode('ascii') string = self._file.read(stop - self._file.tell()) msg = self._message_factory(string.replace(linesep, b'\n')) - msg.set_from(from_line[5:].decode('ascii')) + msg.set_unixfrom(from_line) + msg.set_from(from_line[5:]) return msg def get_string(self, key, from_=False): diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 37228de4828de59..51b99701c9d727b 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -528,6 +528,7 @@ def _default_mime_types(): '.tiff' : 'image/tiff', '.tif' : 'image/tiff', '.ico' : 'image/vnd.microsoft.icon', + '.webp' : 'image/webp', '.ras' : 'image/x-cmu-raster', '.pnm' : 'image/x-portable-anymap', '.pbm' : 'image/x-portable-bitmap', @@ -587,7 +588,6 @@ def _default_mime_types(): '.pict': 'image/pict', '.pct' : 'image/pict', '.pic' : 'image/pict', - '.webp': 'image/webp', '.xul' : 'text/xul', } diff --git a/Lib/ntpath.py b/Lib/ntpath.py index aa0e018eb668c2e..e7cbfe17ecb3c84 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -26,8 +26,8 @@ __all__ = ["normcase","isabs","join","splitdrive","splitroot","split","splitext", "basename","dirname","commonprefix","getsize","getmtime", "getatime","getctime", "islink","exists","lexists","isdir","isfile", - "ismount", "expanduser","expandvars","normpath","abspath", - "curdir","pardir","sep","pathsep","defpath","altsep", + "ismount","isreserved","expanduser","expandvars","normpath", + "abspath","curdir","pardir","sep","pathsep","defpath","altsep", "extsep","devnull","realpath","supports_unicode_filenames","relpath", "samefile", "sameopenfile", "samestat", "commonpath", "isjunction"] @@ -330,6 +330,42 @@ def ismount(path): return False +_reserved_chars = frozenset( + {chr(i) for i in range(32)} | + {'"', '*', ':', '<', '>', '?', '|', '/', '\\'} +) + +_reserved_names = frozenset( + {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} | + {f'COM{c}' for c in '123456789\xb9\xb2\xb3'} | + {f'LPT{c}' for c in '123456789\xb9\xb2\xb3'} +) + +def isreserved(path): + """Return true if the pathname is reserved by the system.""" + # Refer to "Naming Files, Paths, and Namespaces": + # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file + path = os.fsdecode(splitroot(path)[2]).replace(altsep, sep) + return any(_isreservedname(name) for name in reversed(path.split(sep))) + +def _isreservedname(name): + """Return true if the filename is reserved by the system.""" + # Trailing dots and spaces are reserved. + if name.endswith(('.', ' ')) and name not in ('.', '..'): + return True + # Wildcards, separators, colon, and pipe (*?"<>/\:|) are reserved. + # ASCII control characters (0-31) are reserved. + # Colon is reserved for file streams (e.g. "name:stream[:type]"). + if _reserved_chars.intersection(name): + return True + # DOS device names are reserved (e.g. "nul" or "nul .txt"). The rules + # are complex and vary across Windows versions. On the side of + # caution, return True for names that may not be reserved. + if name.partition('.')[0].rstrip(' ').upper() in _reserved_names: + return True + return False + + # Expand paths beginning with '~' or '~user'. # '~' means $HOME; '~user' means that user's home directory. # If the path doesn't begin with '~', or if the user or $HOME is unknown, diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index b043aed12b3849e..65ce836765c42bb 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -33,15 +33,6 @@ ] -# Reference for Windows paths can be found at -# https://learn.microsoft.com/en-gb/windows/win32/fileio/naming-a-file . -_WIN_RESERVED_NAMES = frozenset( - {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} | - {f'COM{c}' for c in '123456789\xb9\xb2\xb3'} | - {f'LPT{c}' for c in '123456789\xb9\xb2\xb3'} -) - - class _PathParents(Sequence): """This object provides sequence-like access to the logical ancestors of a path. Don't try to construct it yourself.""" @@ -433,18 +424,13 @@ def is_absolute(self): def is_reserved(self): """Return True if the path contains one of the special names reserved by the system, if any.""" - if self.pathmod is not ntpath or not self.name: - return False - - # NOTE: the rules for reserved names seem somewhat complicated - # (e.g. r"..\NUL" is reserved but not r"foo\NUL" if "foo" does not - # exist). We err on the side of caution and return True for paths - # which are not considered reserved by Windows. - if self.drive.startswith('\\\\'): - # UNC paths are never reserved. - return False - name = self.name.partition('.')[0].partition(':')[0].rstrip(' ') - return name.upper() in _WIN_RESERVED_NAMES + msg = ("pathlib.PurePath.is_reserved() is deprecated and scheduled " + "for removal in Python 3.15. Use os.path.isreserved() to " + "detect reserved paths on Windows.") + warnings.warn(msg, DeprecationWarning, stacklevel=2) + if self.pathmod is ntpath: + return self.pathmod.isreserved(self) + return False def as_uri(self): """Return the path as a URI.""" @@ -479,17 +465,16 @@ def _pattern_stack(self): elif pattern[-1] in (self.pathmod.sep, self.pathmod.altsep): # GH-65238: pathlib doesn't preserve trailing slash. Add it back. parts.append('') - elif parts[-1] == '**': - # GH-70303: '**' only matches directories. Add trailing slash. - warnings.warn( - "Pattern ending '**' will match files and directories in a " - "future Python release. Add a trailing slash to match only " - "directories and remove this warning.", - FutureWarning, 4) - parts.append('') parts.reverse() return parts + @property + def _pattern_str(self): + """The path expressed as a string, for use in pattern-matching.""" + # The string representation of an empty path is a single dot ('.'). Empty + # paths shouldn't match wildcards, so we change it to the empty string. + path_str = str(self) + return '' if path_str == '.' else path_str # Subclassing os.PathLike makes isinstance() checks slower, # which in turn makes Path construction slower. Register instead! @@ -529,9 +514,8 @@ class Path(_abc.PathBase, PurePath): as_uri = PurePath.as_uri @classmethod - def _unsupported(cls, method_name): - msg = f"{cls.__name__}.{method_name}() is unsupported on this system" - raise UnsupportedOperation(msg) + def _unsupported_msg(cls, attribute): + return f"{cls.__name__}.{attribute} is unsupported on this system" def __init__(self, *args, **kwargs): if kwargs: diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 553e1a399061d3d..e4b1201a3703c32 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -47,8 +47,8 @@ def _is_case_sensitive(pathmod): re = glob = None -@functools.lru_cache(maxsize=256) -def _compile_pattern(pat, sep, case_sensitive): +@functools.lru_cache(maxsize=512) +def _compile_pattern(pat, sep, case_sensitive, recursive=True): """Compile given glob pattern to a re.Pattern object (observing case sensitivity).""" global re, glob @@ -56,10 +56,7 @@ def _compile_pattern(pat, sep, case_sensitive): import re, glob flags = re.NOFLAG if case_sensitive else re.IGNORECASE - regex = glob.translate(pat, recursive=True, include_hidden=True, seps=sep) - # The string representation of an empty path is a single dot ('.'). Empty - # paths shouldn't match wildcards, so we consume it with an atomic group. - regex = r'(\.\Z)?+' + regex + regex = glob.translate(pat, recursive=recursive, include_hidden=True, seps=sep) return re.compile(regex, flags=flags).match @@ -98,7 +95,7 @@ def _select_recursive(parent_paths, dir_only, follow_symlinks): if follow_symlinks is None: follow_symlinks = False for parent_path in parent_paths: - paths = [parent_path] + paths = [parent_path._make_child_relpath('')] while paths: path = paths.pop() yield path @@ -152,39 +149,39 @@ class PathModuleBase: """ @classmethod - def _unsupported(cls, attr): - raise UnsupportedOperation(f"{cls.__name__}.{attr} is unsupported") + def _unsupported_msg(cls, attribute): + return f"{cls.__name__}.{attribute} is unsupported" @property def sep(self): """The character used to separate path components.""" - self._unsupported('sep') + raise UnsupportedOperation(self._unsupported_msg('sep')) def join(self, path, *paths): """Join path segments.""" - self._unsupported('join()') + raise UnsupportedOperation(self._unsupported_msg('join()')) def split(self, path): """Split the path into a pair (head, tail), where *head* is everything before the final path separator, and *tail* is everything after. Either part may be empty. """ - self._unsupported('split()') + raise UnsupportedOperation(self._unsupported_msg('split()')) def splitdrive(self, path): """Split the path into a 2-item tuple (drive, tail), where *drive* is a device name or mount point, and *tail* is everything after the drive. Either part may be empty.""" - self._unsupported('splitdrive()') + raise UnsupportedOperation(self._unsupported_msg('splitdrive()')) def normcase(self, path): """Normalize the case of the path.""" - self._unsupported('normcase()') + raise UnsupportedOperation(self._unsupported_msg('normcase()')) def isabs(self, path): """Returns whether the path is absolute, i.e. unaffected by the current directory or drive.""" - self._unsupported('isabs()') + raise UnsupportedOperation(self._unsupported_msg('isabs()')) class PurePathBase: @@ -210,6 +207,9 @@ class PurePathBase: def __init__(self, path, *paths): self._raw_path = self.pathmod.join(path, *paths) if paths else path + if not isinstance(self._raw_path, str): + raise TypeError( + f"path should be a str, not {type(self._raw_path).__name__!r}") self._resolving = False def with_segments(self, *pathsegments): @@ -302,10 +302,13 @@ def with_suffix(self, suffix): has no suffix, add given suffix. If the given suffix is an empty string, remove the suffix from the path. """ + stem = self.stem if not suffix: - return self.with_name(self.stem) + return self.with_name(stem) + elif not stem: + raise ValueError(f"{self!r} has an empty name") elif suffix.startswith('.') and len(suffix) > 1: - return self.with_name(self.stem + suffix) + return self.with_name(stem + suffix) else: raise ValueError(f"Invalid suffix {suffix!r}") @@ -321,8 +324,6 @@ def relative_to(self, other, *, walk_up=False): other = self.with_segments(other) anchor0, parts0 = self._stack anchor1, parts1 = other._stack - if isinstance(anchor0, str) != isinstance(anchor1, str): - raise TypeError(f"{self._raw_path!r} and {other._raw_path!r} have different types") if anchor0 != anchor1: raise ValueError(f"{self._raw_path!r} and {other._raw_path!r} have different anchors") while parts0 and parts1 and parts0[-1] == parts1[-1]: @@ -346,8 +347,6 @@ def is_relative_to(self, other): other = self.with_segments(other) anchor0, parts0 = self._stack anchor1, parts1 = other._stack - if isinstance(anchor0, str) != isinstance(anchor1, str): - raise TypeError(f"{self._raw_path!r} and {other._raw_path!r} have different types") if anchor0 != anchor1: return False while parts0 and parts1 and parts0[-1] == parts1[-1]: @@ -441,23 +440,48 @@ def _pattern_stack(self): raise NotImplementedError("Non-relative patterns are unsupported") return parts + @property + def _pattern_str(self): + """The path expressed as a string, for use in pattern-matching.""" + return str(self) + def match(self, path_pattern, *, case_sensitive=None): """ - Return True if this path matches the given pattern. + Return True if this path matches the given pattern. If the pattern is + relative, matching is done from the right; otherwise, the entire path + is matched. The recursive wildcard '**' is *not* supported by this + method. """ if not isinstance(path_pattern, PurePathBase): path_pattern = self.with_segments(path_pattern) if case_sensitive is None: case_sensitive = _is_case_sensitive(self.pathmod) sep = path_pattern.pathmod.sep - if path_pattern.anchor: - pattern_str = str(path_pattern) - elif path_pattern.parts: - pattern_str = str('**' / path_pattern) - else: + path_parts = self.parts[::-1] + pattern_parts = path_pattern.parts[::-1] + if not pattern_parts: raise ValueError("empty pattern") - match = _compile_pattern(pattern_str, sep, case_sensitive) - return match(str(self)) is not None + if len(path_parts) < len(pattern_parts): + return False + if len(path_parts) > len(pattern_parts) and path_pattern.anchor: + return False + for path_part, pattern_part in zip(path_parts, pattern_parts): + match = _compile_pattern(pattern_part, sep, case_sensitive, recursive=False) + if match(path_part) is None: + return False + return True + + def full_match(self, pattern, *, case_sensitive=None): + """ + Return True if this path matches the given glob-style pattern. The + pattern is matched against the entire path. + """ + if not isinstance(pattern, PurePathBase): + pattern = self.with_segments(pattern) + if case_sensitive is None: + case_sensitive = _is_case_sensitive(self.pathmod) + match = _compile_pattern(pattern._pattern_str, pattern.pathmod.sep, case_sensitive) + return match(self._pattern_str) is not None @@ -480,16 +504,15 @@ class PathBase(PurePathBase): _max_symlinks = 40 @classmethod - def _unsupported(cls, method_name): - msg = f"{cls.__name__}.{method_name}() is unsupported" - raise UnsupportedOperation(msg) + def _unsupported_msg(cls, attribute): + return f"{cls.__name__}.{attribute} is unsupported" def stat(self, *, follow_symlinks=True): """ Return the result of the stat() system call on this path, like os.stat() does. """ - self._unsupported("stat") + raise UnsupportedOperation(self._unsupported_msg('stat()')) def lstat(self): """ @@ -678,7 +701,7 @@ def open(self, mode='r', buffering=-1, encoding=None, Open the file pointed by this path and return a file object, as the built-in open() function does. """ - self._unsupported("open") + raise UnsupportedOperation(self._unsupported_msg('open()')) def read_bytes(self): """ @@ -719,7 +742,7 @@ def iterdir(self): The children are yielded in arbitrary order, and the special entries '.' and '..' are not included. """ - self._unsupported("iterdir") + raise UnsupportedOperation(self._unsupported_msg('iterdir()')) def _scandir(self): # Emulate os.scandir(), which returns an object that can be used as a @@ -749,7 +772,7 @@ def glob(self, pattern, *, case_sensitive=None, follow_symlinks=None): filter_paths = False deduplicate_paths = False sep = self.pathmod.sep - paths = iter([self.joinpath('')] if self.is_dir() else []) + paths = iter([self] if self.is_dir() else []) while stack: part = stack.pop() if part in specials: @@ -781,8 +804,8 @@ def glob(self, pattern, *, case_sensitive=None, follow_symlinks=None): if filter_paths: # Filter out paths that don't match pattern. prefix_len = len(str(self._make_child_relpath('_'))) - 1 - match = _compile_pattern(str(pattern), sep, case_sensitive) - paths = (path for path in paths if match(str(path), prefix_len)) + match = _compile_pattern(pattern._pattern_str, sep, case_sensitive) + paths = (path for path in paths if match(path._pattern_str, prefix_len)) return paths def rglob(self, pattern, *, case_sensitive=None, follow_symlinks=None): @@ -846,7 +869,7 @@ def absolute(self): Use resolve() to resolve symlinks and remove '..' segments. """ - self._unsupported("absolute") + raise UnsupportedOperation(self._unsupported_msg('absolute()')) @classmethod def cwd(cls): @@ -861,7 +884,7 @@ def expanduser(self): """ Return a new path with expanded ~ and ~user constructs (as returned by os.path.expanduser) """ - self._unsupported("expanduser") + raise UnsupportedOperation(self._unsupported_msg('expanduser()')) @classmethod def home(cls): @@ -873,7 +896,7 @@ def readlink(self): """ Return the path to which the symbolic link points. """ - self._unsupported("readlink") + raise UnsupportedOperation(self._unsupported_msg('readlink()')) readlink._supported = False def resolve(self, strict=False): @@ -948,7 +971,7 @@ def symlink_to(self, target, target_is_directory=False): Make this path a symlink pointing to the target path. Note the order of arguments (link, target) is the reverse of os.symlink. """ - self._unsupported("symlink_to") + raise UnsupportedOperation(self._unsupported_msg('symlink_to()')) def hardlink_to(self, target): """ @@ -956,19 +979,19 @@ def hardlink_to(self, target): Note the order of arguments (self, target) is the reverse of os.link's. """ - self._unsupported("hardlink_to") + raise UnsupportedOperation(self._unsupported_msg('hardlink_to()')) def touch(self, mode=0o666, exist_ok=True): """ Create this file with the given access mode, if it doesn't exist. """ - self._unsupported("touch") + raise UnsupportedOperation(self._unsupported_msg('touch()')) def mkdir(self, mode=0o777, parents=False, exist_ok=False): """ Create a new directory at this given path. """ - self._unsupported("mkdir") + raise UnsupportedOperation(self._unsupported_msg('mkdir()')) def rename(self, target): """ @@ -980,7 +1003,7 @@ def rename(self, target): Returns the new Path instance pointing to the target path. """ - self._unsupported("rename") + raise UnsupportedOperation(self._unsupported_msg('rename()')) def replace(self, target): """ @@ -992,13 +1015,13 @@ def replace(self, target): Returns the new Path instance pointing to the target path. """ - self._unsupported("replace") + raise UnsupportedOperation(self._unsupported_msg('replace()')) def chmod(self, mode, *, follow_symlinks=True): """ Change the permissions of the path, like os.chmod(). """ - self._unsupported("chmod") + raise UnsupportedOperation(self._unsupported_msg('chmod()')) def lchmod(self, mode): """ @@ -1012,31 +1035,31 @@ def unlink(self, missing_ok=False): Remove this file or link. If the path is a directory, use rmdir() instead. """ - self._unsupported("unlink") + raise UnsupportedOperation(self._unsupported_msg('unlink()')) def rmdir(self): """ Remove this directory. The directory must be empty. """ - self._unsupported("rmdir") + raise UnsupportedOperation(self._unsupported_msg('rmdir()')) def owner(self, *, follow_symlinks=True): """ Return the login name of the file owner. """ - self._unsupported("owner") + raise UnsupportedOperation(self._unsupported_msg('owner()')) def group(self, *, follow_symlinks=True): """ Return the group name of the file gid. """ - self._unsupported("group") + raise UnsupportedOperation(self._unsupported_msg('group()')) @classmethod def from_uri(cls, uri): """Return a new path from the given 'file' URI.""" - cls._unsupported("from_uri") + raise UnsupportedOperation(cls._unsupported_msg('from_uri()')) def as_uri(self): """Return the path as a URI.""" - self._unsupported("as_uri") + raise UnsupportedOperation(self._unsupported_msg('as_uri()')) diff --git a/Lib/pdb.py b/Lib/pdb.py index 68f810620f88263..0754e8b628cf570 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -97,17 +97,47 @@ class Restart(Exception): __all__ = ["run", "pm", "Pdb", "runeval", "runctx", "runcall", "set_trace", "post_mortem", "help"] + +def find_first_executable_line(code): + """ Try to find the first executable line of the code object. + + Equivalently, find the line number of the instruction that's + after RESUME + + Return code.co_firstlineno if no executable line is found. + """ + prev = None + for instr in dis.get_instructions(code): + if prev is not None and prev.opname == 'RESUME': + if instr.positions.lineno is not None: + return instr.positions.lineno + return code.co_firstlineno + prev = instr + return code.co_firstlineno + def find_function(funcname, filename): cre = re.compile(r'def\s+%s\s*[(]' % re.escape(funcname)) try: fp = tokenize.open(filename) except OSError: return None + funcdef = "" + funcstart = None # consumer of this info expects the first line to be 1 with fp: for lineno, line in enumerate(fp, start=1): if cre.match(line): - return funcname, filename, lineno + funcstart, funcdef = lineno, line + elif funcdef: + funcdef += line + + if funcdef: + try: + funccode = compile(funcdef, filename, 'exec').co_consts[0] + except SyntaxError: + continue + lineno_offset = find_first_executable_line(funccode) + return funcname, filename, funcstart + lineno_offset - 1 return None def lasti2lineno(code, lasti): @@ -233,6 +263,8 @@ class Pdb(bdb.Bdb, cmd.Cmd): # but in case there are recursions, we stop at 999. MAX_CHAINED_EXCEPTION_DEPTH = 999 + _file_mtime_table = {} + def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, nosigint=False, readrc=True): bdb.Bdb.__init__(self, skip=skip) @@ -437,6 +469,20 @@ def _cmdloop(self): except KeyboardInterrupt: self.message('--KeyboardInterrupt--') + def _validate_file_mtime(self): + """Check if the source file of the current frame has been modified since + the last time we saw it. If so, give a warning.""" + try: + filename = self.curframe.f_code.co_filename + mtime = os.path.getmtime(filename) + except Exception: + return + if (filename in self._file_mtime_table and + mtime != self._file_mtime_table[filename]): + self.message(f"*** WARNING: file '{filename}' was edited, " + "running stale code until the program is rerun") + self._file_mtime_table[filename] = mtime + # Called before loop, handles display expressions # Set up convenience variable containers def preloop(self): @@ -681,6 +727,7 @@ def onecmd(self, line): a breakpoint command list definition. """ if not self.commands_defining: + self._validate_file_mtime() return cmd.Cmd.onecmd(self, line) else: return self.handle_command_def(line) @@ -958,7 +1005,7 @@ def do_break(self, arg, temporary = 0): #use co_name to identify the bkpt (function names #could be aliased, but co_name is invariant) funcname = code.co_name - lineno = self._find_first_executable_line(code) + lineno = find_first_executable_line(code) filename = code.co_filename except: # last thing to try @@ -1061,23 +1108,6 @@ def checkline(self, filename, lineno): return 0 return lineno - def _find_first_executable_line(self, code): - """ Try to find the first executable line of the code object. - - Equivalently, find the line number of the instruction that's - after RESUME - - Return code.co_firstlineno if no executable line is found. - """ - prev = None - for instr in dis.get_instructions(code): - if prev is not None and prev.opname == 'RESUME': - if instr.positions.lineno is not None: - return instr.positions.lineno - return code.co_firstlineno - prev = instr - return code.co_firstlineno - def do_enable(self, arg): """enable bpnumber [bpnumber ...] @@ -2021,6 +2051,10 @@ def _run(self, target: Union[_ModuleTarget, _ScriptTarget]): __main__.__dict__.clear() __main__.__dict__.update(target.namespace) + # Clear the mtime table for program reruns, assume all the files + # are up to date. + self._file_mtime_table.clear() + self.run(target.code) def _format_exc(self, exc: BaseException): diff --git a/Lib/pickletools.py b/Lib/pickletools.py index 95a77aeb2afe2af..51ee4a7a2632ac0 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -1253,7 +1253,7 @@ def __init__(self, name, code, arg, stack_before=[], stack_after=[pyint], proto=2, - doc="""Long integer using found-byte length. + doc="""Long integer using four-byte length. A more efficient encoding of a Python long; the long4 encoding says it all."""), diff --git a/Lib/platform.py b/Lib/platform.py index 75aa55510858fd4..b56472235ee9e42 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -752,6 +752,8 @@ def architecture(executable=sys.executable, bits='', linkage=''): # Linkage if 'ELF' in fileout: linkage = 'ELF' + elif 'Mach-O' in fileout: + linkage = "Mach-O" elif 'PE' in fileout: # E.g. Windows uses this format if 'Windows' in fileout: diff --git a/Lib/queue.py b/Lib/queue.py index 55f50088460f9e5..467ff4fcecb1340 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -25,6 +25,10 @@ class Full(Exception): pass +class ShutDown(Exception): + '''Raised when put/get with shut-down queue.''' + + class Queue: '''Create a queue object with a given maximum size. @@ -54,6 +58,9 @@ def __init__(self, maxsize=0): self.all_tasks_done = threading.Condition(self.mutex) self.unfinished_tasks = 0 + # Queue shutdown state + self.is_shutdown = False + def task_done(self): '''Indicate that a formerly enqueued task is complete. @@ -67,6 +74,8 @@ def task_done(self): Raises a ValueError if called more times than there were items placed in the queue. + + Raises ShutDown if the queue has been shut down immediately. ''' with self.all_tasks_done: unfinished = self.unfinished_tasks - 1 @@ -84,6 +93,8 @@ def join(self): to indicate the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, join() unblocks. + + Raises ShutDown if the queue has been shut down immediately. ''' with self.all_tasks_done: while self.unfinished_tasks: @@ -129,8 +140,12 @@ def put(self, item, block=True, timeout=None): Otherwise ('block' is false), put an item on the queue if a free slot is immediately available, else raise the Full exception ('timeout' is ignored in that case). + + Raises ShutDown if the queue has been shut down. ''' with self.not_full: + if self.is_shutdown: + raise ShutDown if self.maxsize > 0: if not block: if self._qsize() >= self.maxsize: @@ -138,6 +153,8 @@ def put(self, item, block=True, timeout=None): elif timeout is None: while self._qsize() >= self.maxsize: self.not_full.wait() + if self.is_shutdown: + raise ShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: @@ -147,6 +164,8 @@ def put(self, item, block=True, timeout=None): if remaining <= 0.0: raise Full self.not_full.wait(remaining) + if self.is_shutdown: + raise ShutDown self._put(item) self.unfinished_tasks += 1 self.not_empty.notify() @@ -161,14 +180,21 @@ def get(self, block=True, timeout=None): Otherwise ('block' is false), return an item if one is immediately available, else raise the Empty exception ('timeout' is ignored in that case). + + Raises ShutDown if the queue has been shut down and is empty, + or if the queue has been shut down immediately. ''' with self.not_empty: + if self.is_shutdown and not self._qsize(): + raise ShutDown if not block: if not self._qsize(): raise Empty elif timeout is None: while not self._qsize(): self.not_empty.wait() + if self.is_shutdown and not self._qsize(): + raise ShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: @@ -178,6 +204,8 @@ def get(self, block=True, timeout=None): if remaining <= 0.0: raise Empty self.not_empty.wait(remaining) + if self.is_shutdown and not self._qsize(): + raise ShutDown item = self._get() self.not_full.notify() return item @@ -198,6 +226,28 @@ def get_nowait(self): ''' return self.get(block=False) + def shutdown(self, immediate=False): + '''Shut-down the queue, making queue gets and puts raise. + + By default, gets will only raise once the queue is empty. Set + 'immediate' to True to make gets raise immediately instead. + + All blocked callers of put() will be unblocked, and also get() + and join() if 'immediate'. The ShutDown exception is raised. + ''' + with self.mutex: + self.is_shutdown = True + if immediate: + n_items = self._qsize() + while self._qsize(): + self._get() + if self.unfinished_tasks > 0: + self.unfinished_tasks -= 1 + self.not_empty.notify_all() + # release all blocked threads in `join()` + self.all_tasks_done.notify_all() + self.not_full.notify_all() + # Override these methods to implement other queue organizations # (e.g. stack or priority queue). # These will only be called with appropriate locks held diff --git a/Lib/shutil.py b/Lib/shutil.py index acc9419be4dfca6..c19ea0607208afa 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -861,12 +861,12 @@ def move(src, dst, copy_function=copy2): similar to the Unix "mv" command. Return the file or directory's destination. - If the destination is a directory or a symlink to a directory, the source - is moved inside the directory. The destination path must not already - exist. + If dst is an existing directory or a symlink to a directory, then src is + moved inside that directory. The destination path in that directory must + not already exist. - If the destination already exists but is not a directory, it may be - overwritten depending on os.rename() semantics. + If dst already exists but is not a directory, it may be overwritten + depending on os.rename() semantics. If the destination is on our current filesystem, then rename() is used. Otherwise, src is copied to the destination and then removed. Symlinks are diff --git a/Lib/sqlite3/dump.py b/Lib/sqlite3/dump.py index 719dfc8947697d4..9dcce7dc76ced48 100644 --- a/Lib/sqlite3/dump.py +++ b/Lib/sqlite3/dump.py @@ -15,7 +15,7 @@ def _quote_value(value): return "'{0}'".format(value.replace("'", "''")) -def _iterdump(connection): +def _iterdump(connection, *, filter=None): """ Returns an iterator to the dump of the database in an SQL text format. @@ -32,15 +32,23 @@ def _iterdump(connection): yield('PRAGMA foreign_keys=OFF;') yield('BEGIN TRANSACTION;') + if filter: + # Return database objects which match the filter pattern. + filter_name_clause = 'AND "name" LIKE ?' + params = [filter] + else: + filter_name_clause = "" + params = [] # sqlite_master table contains the SQL CREATE statements for the database. - q = """ + q = f""" SELECT "name", "type", "sql" FROM "sqlite_master" WHERE "sql" NOT NULL AND "type" == 'table' + {filter_name_clause} ORDER BY "name" """ - schema_res = cu.execute(q) + schema_res = cu.execute(q, params) sqlite_sequence = [] for table_name, type, sql in schema_res.fetchall(): if table_name == 'sqlite_sequence': @@ -82,13 +90,14 @@ def _iterdump(connection): yield("{0};".format(row[0])) # Now when the type is 'index', 'trigger', or 'view' - q = """ + q = f""" SELECT "name", "type", "sql" FROM "sqlite_master" WHERE "sql" NOT NULL AND "type" IN ('index', 'trigger', 'view') + {filter_name_clause} """ - schema_res = cu.execute(q) + schema_res = cu.execute(q, params) for name, type, sql in schema_res.fetchall(): yield('{0};'.format(sql)) diff --git a/Lib/ssl.py b/Lib/ssl.py index 74a9d2d8fd4fb01..03d0121891ff4cb 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -994,71 +994,67 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, if context.check_hostname and not server_hostname: raise ValueError("check_hostname requires server_hostname") + sock_timeout = sock.gettimeout() kwargs = dict( family=sock.family, type=sock.type, proto=sock.proto, fileno=sock.fileno() ) self = cls.__new__(cls, **kwargs) super(SSLSocket, self).__init__(**kwargs) - sock_timeout = sock.gettimeout() sock.detach() - - self._context = context - self._session = session - self._closed = False - self._sslobj = None - self.server_side = server_side - self.server_hostname = context._encode_hostname(server_hostname) - self.do_handshake_on_connect = do_handshake_on_connect - self.suppress_ragged_eofs = suppress_ragged_eofs - - # See if we are connected + # Now SSLSocket is responsible for closing the file descriptor. try: - self.getpeername() - except OSError as e: - if e.errno != errno.ENOTCONN: - raise - connected = False - blocking = self.getblocking() - self.setblocking(False) + self._context = context + self._session = session + self._closed = False + self._sslobj = None + self.server_side = server_side + self.server_hostname = context._encode_hostname(server_hostname) + self.do_handshake_on_connect = do_handshake_on_connect + self.suppress_ragged_eofs = suppress_ragged_eofs + + # See if we are connected try: - # We are not connected so this is not supposed to block, but - # testing revealed otherwise on macOS and Windows so we do - # the non-blocking dance regardless. Our raise when any data - # is found means consuming the data is harmless. - notconn_pre_handshake_data = self.recv(1) + self.getpeername() except OSError as e: - # EINVAL occurs for recv(1) on non-connected on unix sockets. - if e.errno not in (errno.ENOTCONN, errno.EINVAL): + if e.errno != errno.ENOTCONN: raise - notconn_pre_handshake_data = b'' - self.setblocking(blocking) - if notconn_pre_handshake_data: - # This prevents pending data sent to the socket before it was - # closed from escaping to the caller who could otherwise - # presume it came through a successful TLS connection. - reason = "Closed before TLS handshake with data in recv buffer." - notconn_pre_handshake_data_error = SSLError(e.errno, reason) - # Add the SSLError attributes that _ssl.c always adds. - notconn_pre_handshake_data_error.reason = reason - notconn_pre_handshake_data_error.library = None - try: - self.close() - except OSError: - pass + connected = False + blocking = self.getblocking() + self.setblocking(False) try: - raise notconn_pre_handshake_data_error - finally: - # Explicitly break the reference cycle. - notconn_pre_handshake_data_error = None - else: - connected = True + # We are not connected so this is not supposed to block, but + # testing revealed otherwise on macOS and Windows so we do + # the non-blocking dance regardless. Our raise when any data + # is found means consuming the data is harmless. + notconn_pre_handshake_data = self.recv(1) + except OSError as e: + # EINVAL occurs for recv(1) on non-connected on unix sockets. + if e.errno not in (errno.ENOTCONN, errno.EINVAL): + raise + notconn_pre_handshake_data = b'' + self.setblocking(blocking) + if notconn_pre_handshake_data: + # This prevents pending data sent to the socket before it was + # closed from escaping to the caller who could otherwise + # presume it came through a successful TLS connection. + reason = "Closed before TLS handshake with data in recv buffer." + notconn_pre_handshake_data_error = SSLError(e.errno, reason) + # Add the SSLError attributes that _ssl.c always adds. + notconn_pre_handshake_data_error.reason = reason + notconn_pre_handshake_data_error.library = None + try: + raise notconn_pre_handshake_data_error + finally: + # Explicitly break the reference cycle. + notconn_pre_handshake_data_error = None + else: + connected = True - self.settimeout(sock_timeout) # Must come after setblocking() calls. - self._connected = connected - if connected: - # create the SSL object - try: + self.settimeout(sock_timeout) # Must come after setblocking() calls. + self._connected = connected + if connected: + # create the SSL object self._sslobj = self._context._wrap_socket( self, server_side, self.server_hostname, owner=self, session=self._session, @@ -1069,9 +1065,12 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, # non-blocking raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets") self.do_handshake() - except (OSError, ValueError): + except: + try: self.close() - raise + except OSError: + pass + raise return self @property diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 20e0394507f5db1..9775040cbe372cd 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -2456,7 +2456,8 @@ def makedir(self, tarinfo, targetpath): # later in _extract_member(). os.mkdir(targetpath, 0o700) except FileExistsError: - pass + if not os.path.isdir(targetpath): + raise def makefile(self, tarinfo, targetpath): """Make a file called targetpath. diff --git a/Lib/test/.ruff.toml b/Lib/test/.ruff.toml index d6c1d8745036ece..1c9bac507209b12 100644 --- a/Lib/test/.ruff.toml +++ b/Lib/test/.ruff.toml @@ -1,7 +1,4 @@ fix = true -select = [ - "F811", # Redefinition of unused variable (useful for finding test methods with the same name) -] extend-exclude = [ # Excluded (run with the other AC files in its own separate ruff job in pre-commit) "test_clinic.py", @@ -21,3 +18,8 @@ extend-exclude = [ "test_pkg.py", "test_yield_from.py", ] + +[lint] +select = [ + "F811", # Redefinition of unused variable (useful for finding test methods with the same name) +] diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 6a050fa541db1ed..c0d3ca50f17d69d 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -2705,6 +2705,7 @@ def test_terminate(self): p = self.Pool(3) args = [sleep_time for i in range(10_000)] result = p.map_async(time.sleep, args, chunksize=1) + time.sleep(0.2) # give some tasks a chance to start p.terminate() p.join() diff --git a/Lib/test/archiver_tests.py b/Lib/test/archiver_tests.py new file mode 100644 index 000000000000000..1a4bbb9e5706c53 --- /dev/null +++ b/Lib/test/archiver_tests.py @@ -0,0 +1,155 @@ +"""Tests common to tarfile and zipfile.""" + +import os +import sys + +from test.support import os_helper + +class OverwriteTests: + + def setUp(self): + os.makedirs(self.testdir) + self.addCleanup(os_helper.rmtree, self.testdir) + + def create_file(self, path, content=b''): + with open(path, 'wb') as f: + f.write(content) + + def open(self, path): + raise NotImplementedError + + def extractall(self, ar): + raise NotImplementedError + + + def test_overwrite_file_as_file(self): + target = os.path.join(self.testdir, 'test') + self.create_file(target, b'content') + with self.open(self.ar_with_file) as ar: + self.extractall(ar) + self.assertTrue(os.path.isfile(target)) + with open(target, 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + def test_overwrite_dir_as_dir(self): + target = os.path.join(self.testdir, 'test') + os.mkdir(target) + with self.open(self.ar_with_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + + def test_overwrite_dir_as_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + os.mkdir(target) + with self.open(self.ar_with_implicit_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + self.assertTrue(os.path.isfile(os.path.join(target, 'file'))) + with open(os.path.join(target, 'file'), 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + def test_overwrite_dir_as_file(self): + target = os.path.join(self.testdir, 'test') + os.mkdir(target) + with self.open(self.ar_with_file) as ar: + with self.assertRaises(PermissionError if sys.platform == 'win32' + else IsADirectoryError): + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + + def test_overwrite_file_as_dir(self): + target = os.path.join(self.testdir, 'test') + self.create_file(target, b'content') + with self.open(self.ar_with_dir) as ar: + with self.assertRaises(FileExistsError): + self.extractall(ar) + self.assertTrue(os.path.isfile(target)) + with open(target, 'rb') as f: + self.assertEqual(f.read(), b'content') + + def test_overwrite_file_as_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + self.create_file(target, b'content') + with self.open(self.ar_with_implicit_dir) as ar: + with self.assertRaises(FileNotFoundError if sys.platform == 'win32' + else NotADirectoryError): + self.extractall(ar) + self.assertTrue(os.path.isfile(target)) + with open(target, 'rb') as f: + self.assertEqual(f.read(), b'content') + + @os_helper.skip_unless_symlink + def test_overwrite_file_symlink_as_file(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + self.create_file(target2, b'content') + os.symlink('test2', target) + with self.open(self.ar_with_file) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isfile(target2)) + with open(target2, 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + @os_helper.skip_unless_symlink + def test_overwrite_broken_file_symlink_as_file(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.symlink('test2', target) + with self.open(self.ar_with_file) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isfile(target2)) + with open(target2, 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + @os_helper.skip_unless_symlink + def test_overwrite_dir_symlink_as_dir(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.mkdir(target2) + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isdir(target2)) + + @os_helper.skip_unless_symlink + def test_overwrite_dir_symlink_as_implicit_dir(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.mkdir(target2) + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_implicit_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isdir(target2)) + self.assertTrue(os.path.isfile(os.path.join(target2, 'file'))) + with open(os.path.join(target2, 'file'), 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + @os_helper.skip_unless_symlink + def test_overwrite_broken_dir_symlink_as_dir(self): + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_dir) as ar: + with self.assertRaises(FileExistsError): + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertFalse(os.path.exists(target2)) + + @os_helper.skip_unless_symlink + def test_overwrite_broken_dir_symlink_as_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_implicit_dir) as ar: + with self.assertRaises(FileExistsError): + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertFalse(os.path.exists(target2)) diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index ce4a11b119c9007..de7d0da560a1c79 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -487,7 +487,13 @@ def hook(event, args): print(event, args[0]) sys.addaudithook(hook) - _wmi.exec_query("SELECT * FROM Win32_OperatingSystem") + try: + _wmi.exec_query("SELECT * FROM Win32_OperatingSystem") + except WindowsError as e: + # gh-112278: WMI may be slow response when first called, but we still + # get the audit event, so just ignore the timeout + if e.winerror != 258: + raise def test_syslog(): import syslog diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index b15aeb898d35a12..168f6f73f6186fa 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -4625,7 +4625,7 @@ Test_cls_no_params_impl(TestObj *self, PyTypeObject *cls); static PyObject * Test_cls_no_params(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "cls_no_params() takes no arguments"); return NULL; } @@ -4634,7 +4634,7 @@ Test_cls_no_params(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_s static PyObject * Test_cls_no_params_impl(TestObj *self, PyTypeObject *cls) -/*[clinic end generated code: output=cc8845f22cff3dcb input=e7e2e4e344e96a11]*/ +/*[clinic end generated code: output=4d68b4652c144af3 input=e7e2e4e344e96a11]*/ /*[clinic input] diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 8bda17358db87f9..53ad5e57ada0178 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -301,6 +301,10 @@ def test_inheritance(self): self.assertIsInstance(timezone.utc, tzinfo) self.assertIsInstance(self.EST, tzinfo) + def test_cannot_subclass(self): + with self.assertRaises(TypeError): + class MyTimezone(timezone): pass + def test_utcoffset(self): dummy = self.DT for h in [0, 1.5, 12]: diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt index 1eca123be0fecbf..217ee15d4c8af54 100644 --- a/Lib/test/exception_hierarchy.txt +++ b/Lib/test/exception_hierarchy.txt @@ -44,6 +44,7 @@ BaseException ├── StopAsyncIteration ├── StopIteration ├── SyntaxError + │ └── IncompleteInputError │ └── IndentationError │ └── TabError ├── SystemError diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index f2e6af078a5f291..5b091fb2fd32dc6 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -43,7 +43,7 @@ "requires_limited_api", "requires_specialization", # sys "MS_WINDOWS", "is_jython", "is_android", "is_emscripten", "is_wasi", - "check_impl_detail", "unix_shell", "setswitchinterval", + "is_apple_mobile", "check_impl_detail", "unix_shell", "setswitchinterval", # os "get_pagesize", # network @@ -522,7 +522,7 @@ def requires_debug_ranges(reason='requires co_positions / debug_ranges'): is_android = hasattr(sys, 'getandroidapilevel') -if sys.platform not in ('win32', 'vxworks'): +if sys.platform not in {"win32", "vxworks", "ios", "tvos", "watchos"}: unix_shell = '/system/bin/sh' if is_android else '/bin/sh' else: unix_shell = None @@ -532,19 +532,35 @@ def requires_debug_ranges(reason='requires co_positions / debug_ranges'): is_emscripten = sys.platform == "emscripten" is_wasi = sys.platform == "wasi" -has_fork_support = hasattr(os, "fork") and not is_emscripten and not is_wasi +# Apple mobile platforms (iOS/tvOS/watchOS) are POSIX-like but do not +# have subprocess or fork support. +is_apple_mobile = sys.platform in {"ios", "tvos", "watchos"} +is_apple = is_apple_mobile or sys.platform == "darwin" + +has_fork_support = hasattr(os, "fork") and not ( + is_emscripten + or is_wasi + or is_apple_mobile +) def requires_fork(): return unittest.skipUnless(has_fork_support, "requires working os.fork()") -has_subprocess_support = not is_emscripten and not is_wasi +has_subprocess_support = not ( + is_emscripten + or is_wasi + or is_apple_mobile +) def requires_subprocess(): """Used for subprocess, os.spawn calls, fd inheritance""" return unittest.skipUnless(has_subprocess_support, "requires subprocess support") # Emscripten's socket emulation and WASI sockets have limitations. -has_socket_support = not is_emscripten and not is_wasi +has_socket_support = not ( + is_emscripten + or is_wasi +) def requires_working_socket(*, module=False): """Skip tests or modules that require working sockets diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index 20f38fd36a8876e..22787e32b5f3abe 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -22,8 +22,8 @@ # TESTFN_UNICODE is a non-ascii filename TESTFN_UNICODE = TESTFN_ASCII + "-\xe0\xf2\u0258\u0141\u011f" -if sys.platform == 'darwin': - # In Mac OS X's VFS API file names are, by definition, canonically +if support.is_apple: + # On Apple's VFS API file names are, by definition, canonically # decomposed Unicode, encoded using UTF-8. See QA1173: # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html import unicodedata @@ -48,8 +48,8 @@ 'encoding (%s). Unicode filename tests may not be effective' % (TESTFN_UNENCODABLE, sys.getfilesystemencoding())) TESTFN_UNENCODABLE = None -# macOS and Emscripten deny unencodable filenames (invalid utf-8) -elif sys.platform not in {'darwin', 'emscripten', 'wasi'}: +# Apple and Emscripten deny unencodable filenames (invalid utf-8) +elif not support.is_apple and sys.platform not in {"emscripten", "wasi"}: try: # ascii and utf-8 cannot encode the byte 0xff b'\xff'.decode(sys.getfilesystemencoding()) diff --git a/Lib/test/test_abstract_numbers.py b/Lib/test/test_abstract_numbers.py index 2e06f0d16fdd05b..72232b670cdb89e 100644 --- a/Lib/test/test_abstract_numbers.py +++ b/Lib/test/test_abstract_numbers.py @@ -1,14 +1,34 @@ """Unit tests for numbers.py.""" +import abc import math import operator import unittest -from numbers import Complex, Real, Rational, Integral +from numbers import Complex, Real, Rational, Integral, Number + + +def concretize(cls): + def not_implemented(*args, **kwargs): + raise NotImplementedError() + + for name in dir(cls): + try: + value = getattr(cls, name) + if value.__isabstractmethod__: + setattr(cls, name, not_implemented) + except AttributeError: + pass + abc.update_abstractmethods(cls) + return cls + class TestNumbers(unittest.TestCase): def test_int(self): self.assertTrue(issubclass(int, Integral)) + self.assertTrue(issubclass(int, Rational)) + self.assertTrue(issubclass(int, Real)) self.assertTrue(issubclass(int, Complex)) + self.assertTrue(issubclass(int, Number)) self.assertEqual(7, int(7).real) self.assertEqual(0, int(7).imag) @@ -18,8 +38,11 @@ def test_int(self): self.assertEqual(1, int(7).denominator) def test_float(self): + self.assertFalse(issubclass(float, Integral)) self.assertFalse(issubclass(float, Rational)) self.assertTrue(issubclass(float, Real)) + self.assertTrue(issubclass(float, Complex)) + self.assertTrue(issubclass(float, Number)) self.assertEqual(7.3, float(7.3).real) self.assertEqual(0, float(7.3).imag) @@ -27,8 +50,11 @@ def test_float(self): self.assertEqual(-7.3, float(-7.3).conjugate()) def test_complex(self): + self.assertFalse(issubclass(complex, Integral)) + self.assertFalse(issubclass(complex, Rational)) self.assertFalse(issubclass(complex, Real)) self.assertTrue(issubclass(complex, Complex)) + self.assertTrue(issubclass(complex, Number)) c1, c2 = complex(3, 2), complex(4,1) # XXX: This is not ideal, but see the comment in math_trunc(). @@ -40,5 +66,135 @@ def test_complex(self): self.assertRaises(TypeError, int, c1) +class TestNumbersDefaultMethods(unittest.TestCase): + def test_complex(self): + @concretize + class MyComplex(Complex): + def __init__(self, real, imag): + self.r = real + self.i = imag + + @property + def real(self): + return self.r + + @property + def imag(self): + return self.i + + def __add__(self, other): + if isinstance(other, Complex): + return MyComplex(self.imag + other.imag, + self.real + other.real) + raise NotImplementedError + + def __neg__(self): + return MyComplex(-self.real, -self.imag) + + def __eq__(self, other): + if isinstance(other, Complex): + return self.imag == other.imag and self.real == other.real + if isinstance(other, Number): + return self.imag == 0 and self.real == other.real + + # test __bool__ + self.assertTrue(bool(MyComplex(1, 1))) + self.assertTrue(bool(MyComplex(0, 1))) + self.assertTrue(bool(MyComplex(1, 0))) + self.assertFalse(bool(MyComplex(0, 0))) + + # test __sub__ + self.assertEqual(MyComplex(2, 3) - complex(1, 2), MyComplex(1, 1)) + + # test __rsub__ + self.assertEqual(complex(2, 3) - MyComplex(1, 2), MyComplex(1, 1)) + + def test_real(self): + @concretize + class MyReal(Real): + def __init__(self, n): + self.n = n + + def __pos__(self): + return self.n + + def __float__(self): + return float(self.n) + + def __floordiv__(self, other): + return self.n // other + + def __rfloordiv__(self, other): + return other // self.n + + def __mod__(self, other): + return self.n % other + + def __rmod__(self, other): + return other % self.n + + # test __divmod__ + self.assertEqual(divmod(MyReal(3), 2), (1, 1)) + + # test __rdivmod__ + self.assertEqual(divmod(3, MyReal(2)), (1, 1)) + + # test __complex__ + self.assertEqual(complex(MyReal(1)), 1+0j) + + # test real + self.assertEqual(MyReal(3).real, 3) + + # test imag + self.assertEqual(MyReal(3).imag, 0) + + # test conjugate + self.assertEqual(MyReal(123).conjugate(), 123) + + + def test_rational(self): + @concretize + class MyRational(Rational): + def __init__(self, numerator, denominator): + self.n = numerator + self.d = denominator + + @property + def numerator(self): + return self.n + + @property + def denominator(self): + return self.d + + # test__float__ + self.assertEqual(float(MyRational(5, 2)), 2.5) + + + def test_integral(self): + @concretize + class MyIntegral(Integral): + def __init__(self, n): + self.n = n + + def __pos__(self): + return self.n + + def __int__(self): + return self.n + + # test __index__ + self.assertEqual(operator.index(MyIntegral(123)), 123) + + # test __float__ + self.assertEqual(float(MyIntegral(123)), 123.0) + + # test numerator + self.assertEqual(MyIntegral(123).numerator, 123) + + # test denominator + self.assertEqual(MyIntegral(123).denominator, 1) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 7c1f5d36999a3d5..86d6e81a71642b7 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -3922,7 +3922,7 @@ class TestHelpUsageWithParentheses(HelpTestCase): options: -h, --help show this help message and exit - -p {1 (option A), 2 (option B)}, --optional {1 (option A), 2 (option B)} + -p, --optional {1 (option A), 2 (option B)} ''' version = '' @@ -4405,8 +4405,8 @@ class TestHelpAlternatePrefixChars(HelpTestCase): help = usage + '''\ options: - ^^foo foo help - ;b BAR, ;;bar BAR bar help + ^^foo foo help + ;b, ;;bar BAR bar help ''' version = '' @@ -5099,7 +5099,8 @@ def test_optional(self): string = ( "Action(option_strings=['--foo', '-a', '-b'], dest='b', " "nargs='+', const=None, default=42, type='int', " - "choices=[1, 2, 3], required=False, help='HELP', metavar='METAVAR')") + "choices=[1, 2, 3], required=False, help='HELP', " + "metavar='METAVAR', deprecated=False)") self.assertStringEqual(option, string) def test_argument(self): @@ -5116,7 +5117,8 @@ def test_argument(self): string = ( "Action(option_strings=[], dest='x', nargs='?', " "const=None, default=2.5, type=%r, choices=[0.5, 1.5, 2.5], " - "required=True, help='H HH H', metavar='MV MV MV')" % float) + "required=True, help='H HH H', metavar='MV MV MV', " + "deprecated=False)" % float) self.assertStringEqual(argument, string) def test_namespace(self): @@ -5308,6 +5310,139 @@ def spam(string_to_convert): args = parser.parse_args('--foo spam!'.split()) self.assertEqual(NS(foo='foo_converted'), args) + +# ============================================== +# Check that deprecated arguments output warning +# ============================================== + +class TestDeprecatedArguments(TestCase): + + def test_deprecated_option(self): + parser = argparse.ArgumentParser() + parser.add_argument('-f', '--foo', deprecated=True) + + with captured_stderr() as stderr: + parser.parse_args(['--foo', 'spam']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['-f', 'spam']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '-f' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['--foo', 'spam', '-f', 'ham']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertRegex(stderr, "warning: option '-f' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 2) + + with captured_stderr() as stderr: + parser.parse_args(['--foo', 'spam', '--foo', 'ham']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + def test_deprecated_boolean_option(self): + parser = argparse.ArgumentParser() + parser.add_argument('-f', '--foo', action=argparse.BooleanOptionalAction, deprecated=True) + + with captured_stderr() as stderr: + parser.parse_args(['--foo']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['-f']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '-f' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['--no-foo']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--no-foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['--foo', '--no-foo']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertRegex(stderr, "warning: option '--no-foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 2) + + def test_deprecated_arguments(self): + parser = argparse.ArgumentParser() + parser.add_argument('foo', nargs='?', deprecated=True) + parser.add_argument('bar', nargs='?', deprecated=True) + + with captured_stderr() as stderr: + parser.parse_args([]) + stderr = stderr.getvalue() + self.assertEqual(stderr.count('is deprecated'), 0) + + with captured_stderr() as stderr: + parser.parse_args(['spam']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: argument 'foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['spam', 'ham']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: argument 'foo' is deprecated") + self.assertRegex(stderr, "warning: argument 'bar' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 2) + + def test_deprecated_varargument(self): + parser = argparse.ArgumentParser() + parser.add_argument('foo', nargs='*', deprecated=True) + + with captured_stderr() as stderr: + parser.parse_args([]) + stderr = stderr.getvalue() + self.assertEqual(stderr.count('is deprecated'), 0) + + with captured_stderr() as stderr: + parser.parse_args(['spam']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: argument 'foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['spam', 'ham']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: argument 'foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + def test_deprecated_subparser(self): + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + subparsers.add_parser('foo', aliases=['baz'], deprecated=True) + subparsers.add_parser('bar') + + with captured_stderr() as stderr: + parser.parse_args(['bar']) + stderr = stderr.getvalue() + self.assertEqual(stderr.count('is deprecated'), 0) + + with captured_stderr() as stderr: + parser.parse_args(['foo']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: command 'foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['baz']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: command 'baz' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + # ================================================================== # Check semantics regarding the default argument and type conversion # ================================================================== @@ -5405,6 +5540,22 @@ def test_zero_or_more_optional(self): args = parser.parse_args([]) self.assertEqual(NS(x=[]), args) + def test_double_dash(self): + parser = argparse.ArgumentParser() + parser.add_argument('-f', '--foo', nargs='*') + parser.add_argument('bar', nargs='*') + + args = parser.parse_args(['--foo=--']) + self.assertEqual(NS(foo=['--'], bar=[]), args) + args = parser.parse_args(['--foo', '--']) + self.assertEqual(NS(foo=[], bar=[]), args) + args = parser.parse_args(['-f--']) + self.assertEqual(NS(foo=['--'], bar=[]), args) + args = parser.parse_args(['-f', '--']) + self.assertEqual(NS(foo=[], bar=[]), args) + args = parser.parse_args(['--foo', 'a', 'b', '--', 'c', 'd']) + self.assertEqual(NS(foo=['a', 'b'], bar=['c', 'd']), args) + # =========================== # parse_intermixed_args tests diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index c2080977e9d5877..82071edb2525706 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -1232,7 +1232,7 @@ def test_create_datagram_endpoint_wrong_sock(self): with sock: coro = self.loop.create_datagram_endpoint(MyProto, sock=sock) with self.assertRaisesRegex(ValueError, - 'A UDP Socket was expected'): + 'A datagram socket was expected'): self.loop.run_until_complete(coro) def test_create_connection_no_host_port_sock(self): diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index b25c0975736e20e..c92c88bd5b2429c 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1815,6 +1815,7 @@ def check_killed(self, returncode): else: self.assertEqual(-signal.SIGKILL, returncode) + @support.requires_subprocess() def test_subprocess_exec(self): prog = os.path.join(os.path.dirname(__file__), 'echo.py') @@ -1836,6 +1837,7 @@ def test_subprocess_exec(self): self.check_killed(proto.returncode) self.assertEqual(b'Python The Winner', proto.data[1]) + @support.requires_subprocess() def test_subprocess_interactive(self): prog = os.path.join(os.path.dirname(__file__), 'echo.py') @@ -1863,6 +1865,7 @@ def test_subprocess_interactive(self): self.loop.run_until_complete(proto.completed) self.check_killed(proto.returncode) + @support.requires_subprocess() def test_subprocess_shell(self): connect = self.loop.subprocess_shell( functools.partial(MySubprocessProtocol, self.loop), @@ -1879,6 +1882,7 @@ def test_subprocess_shell(self): self.assertEqual(proto.data[2], b'') transp.close() + @support.requires_subprocess() def test_subprocess_exitcode(self): connect = self.loop.subprocess_shell( functools.partial(MySubprocessProtocol, self.loop), @@ -1890,6 +1894,7 @@ def test_subprocess_exitcode(self): self.assertEqual(7, proto.returncode) transp.close() + @support.requires_subprocess() def test_subprocess_close_after_finish(self): connect = self.loop.subprocess_shell( functools.partial(MySubprocessProtocol, self.loop), @@ -1904,6 +1909,7 @@ def test_subprocess_close_after_finish(self): self.assertEqual(7, proto.returncode) self.assertIsNone(transp.close()) + @support.requires_subprocess() def test_subprocess_kill(self): prog = os.path.join(os.path.dirname(__file__), 'echo.py') @@ -1920,6 +1926,7 @@ def test_subprocess_kill(self): self.check_killed(proto.returncode) transp.close() + @support.requires_subprocess() def test_subprocess_terminate(self): prog = os.path.join(os.path.dirname(__file__), 'echo.py') @@ -1937,6 +1944,7 @@ def test_subprocess_terminate(self): transp.close() @unittest.skipIf(sys.platform == 'win32', "Don't have SIGHUP") + @support.requires_subprocess() def test_subprocess_send_signal(self): # bpo-31034: Make sure that we get the default signal handler (killing # the process). The parent process may have decided to ignore SIGHUP, @@ -1961,6 +1969,7 @@ def test_subprocess_send_signal(self): finally: signal.signal(signal.SIGHUP, old_handler) + @support.requires_subprocess() def test_subprocess_stderr(self): prog = os.path.join(os.path.dirname(__file__), 'echo2.py') @@ -1982,6 +1991,7 @@ def test_subprocess_stderr(self): self.assertTrue(proto.data[2].startswith(b'ERR:test'), proto.data[2]) self.assertEqual(0, proto.returncode) + @support.requires_subprocess() def test_subprocess_stderr_redirect_to_stdout(self): prog = os.path.join(os.path.dirname(__file__), 'echo2.py') @@ -2007,6 +2017,7 @@ def test_subprocess_stderr_redirect_to_stdout(self): transp.close() self.assertEqual(0, proto.returncode) + @support.requires_subprocess() def test_subprocess_close_client_stream(self): prog = os.path.join(os.path.dirname(__file__), 'echo3.py') @@ -2041,6 +2052,7 @@ def test_subprocess_close_client_stream(self): self.loop.run_until_complete(proto.completed) self.check_killed(proto.returncode) + @support.requires_subprocess() def test_subprocess_wait_no_same_group(self): # start the new process in a new session connect = self.loop.subprocess_shell( @@ -2053,6 +2065,7 @@ def test_subprocess_wait_no_same_group(self): self.assertEqual(7, proto.returncode) transp.close() + @support.requires_subprocess() def test_subprocess_exec_invalid_args(self): async def connect(**kwds): await self.loop.subprocess_exec( @@ -2066,6 +2079,7 @@ async def connect(**kwds): with self.assertRaises(ValueError): self.loop.run_until_complete(connect(shell=True)) + @support.requires_subprocess() def test_subprocess_shell_invalid_args(self): async def connect(cmd=None, **kwds): diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 9029efd2355b46f..a0884bffe6b0de1 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -816,6 +816,98 @@ async def func(): # originally raised. self.assertIs(err.exception, raised) + async def test_cancelled_wakeup(self): + # Test that a task cancelled at the "same" time as it is woken + # up as part of a Condition.notify() does not result in a lost wakeup. + # This test simulates a cancel while the target task is awaiting initial + # wakeup on the wakeup queue. + condition = asyncio.Condition() + state = 0 + async def consumer(): + nonlocal state + async with condition: + while True: + await condition.wait_for(lambda: state != 0) + if state < 0: + return + state -= 1 + + # create two consumers + c = [asyncio.create_task(consumer()) for _ in range(2)] + # wait for them to settle + await asyncio.sleep(0) + async with condition: + # produce one item and wake up one + state += 1 + condition.notify(1) + + # Cancel it while it is awaiting to be run. + # This cancellation could come from the outside + c[0].cancel() + + # now wait for the item to be consumed + # if it doesn't means that our "notify" didn"t take hold. + # because it raced with a cancel() + try: + async with asyncio.timeout(0.01): + await condition.wait_for(lambda: state == 0) + except TimeoutError: + pass + self.assertEqual(state, 0) + + # clean up + state = -1 + condition.notify_all() + await c[1] + + async def test_cancelled_wakeup_relock(self): + # Test that a task cancelled at the "same" time as it is woken + # up as part of a Condition.notify() does not result in a lost wakeup. + # This test simulates a cancel while the target task is acquiring the lock + # again. + condition = asyncio.Condition() + state = 0 + async def consumer(): + nonlocal state + async with condition: + while True: + await condition.wait_for(lambda: state != 0) + if state < 0: + return + state -= 1 + + # create two consumers + c = [asyncio.create_task(consumer()) for _ in range(2)] + # wait for them to settle + await asyncio.sleep(0) + async with condition: + # produce one item and wake up one + state += 1 + condition.notify(1) + + # now we sleep for a bit. This allows the target task to wake up and + # settle on re-aquiring the lock + await asyncio.sleep(0) + + # Cancel it while awaiting the lock + # This cancel could come the outside. + c[0].cancel() + + # now wait for the item to be consumed + # if it doesn't means that our "notify" didn"t take hold. + # because it raced with a cancel() + try: + async with asyncio.timeout(0.01): + await condition.wait_for(lambda: state == 0) + except TimeoutError: + pass + self.assertEqual(state, 0) + + # clean up + state = -1 + condition.notify_all() + await c[1] + class SemaphoreTests(unittest.IsolatedAsyncioTestCase): def test_initial_value_zero(self): diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 3c8cc5f3649180d..210990593adfa9b 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -10,7 +10,6 @@ import unittest from unittest import mock import warnings -from test.support import socket_helper try: import ssl except ImportError: @@ -18,6 +17,7 @@ import asyncio from test.test_asyncio import utils as test_utils +from test.support import requires_subprocess, socket_helper def tearDownModule(): @@ -770,6 +770,7 @@ async def client(addr): self.assertEqual(msg2, b"hello world 2!\n") @unittest.skipIf(sys.platform == 'win32', "Don't have pipes") + @requires_subprocess() def test_read_all_from_pipe_reader(self): # See asyncio issue 168. This test is derived from the example # subprocess_attach_read_pipe.py, but we configure the diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 859d2932c33fede..f50a9ebc031ba8a 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -47,6 +47,7 @@ def _start(self, *args, **kwargs): self._proc.pid = -1 +@support.requires_subprocess() class SubprocessTransportTests(test_utils.TestCase): def setUp(self): super().setUp() @@ -110,6 +111,7 @@ def test_subprocess_repr(self): transport.close() +@support.requires_subprocess() class SubprocessMixin: def test_stdin_stdout(self): @@ -207,7 +209,7 @@ def test_kill(self): def test_kill_issue43884(self): if sys.platform == 'win32': - blocking_shell_command = f'{sys.executable} -c "import time; time.sleep(2)"' + blocking_shell_command = f'"{sys.executable}" -c "import time; time.sleep(2)"' else: blocking_shell_command = 'sleep 1; sleep 1' creationflags = 0 @@ -745,7 +747,10 @@ async def check_stdout_output(self, coro, output): def test_create_subprocess_env_shell(self) -> None: async def main() -> None: - cmd = f'''{sys.executable} -c "import os, sys; sys.stdout.write(os.getenv('FOO'))"''' + executable = sys.executable + if sys.platform == "win32": + executable = f'"{executable}"' + cmd = f'''{executable} -c "import os, sys; sys.stdout.write(os.getenv('FOO'))"''' env = os.environ.copy() env["FOO"] = "bar" proc = await asyncio.create_subprocess_shell( diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index d2c8cba6acfa31c..59ef9f5f58cabc3 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1874,7 +1874,7 @@ async def runner(): wsock.close() -@unittest.skipUnless(hasattr(os, 'fork'), 'requires os.fork()') +@support.requires_fork() class TestFork(unittest.IsolatedAsyncioTestCase): async def test_fork_not_share_event_loop(self): diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 72a06d6af450e32..5b1b95b9c82064c 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4585,6 +4585,33 @@ def test_c_buffer(self): buf.__release_buffer__(mv) self.assertEqual(buf.references, 0) + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_buffer_invalid_flags(self): + buf = _testcapi.testBuf() + self.assertRaises(SystemError, buf.__buffer__, PyBUF_READ) + self.assertRaises(SystemError, buf.__buffer__, PyBUF_WRITE) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_fill_buffer_invalid_flags(self): + # PyBuffer_FillInfo + source = b"abc" + self.assertRaises(SystemError, _testcapi.buffer_fill_info, + source, 0, PyBUF_READ) + self.assertRaises(SystemError, _testcapi.buffer_fill_info, + source, 0, PyBUF_WRITE) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_fill_buffer_readonly_and_writable(self): + source = b"abc" + with _testcapi.buffer_fill_info(source, 1, PyBUF_SIMPLE) as m: + self.assertEqual(bytes(m), b"abc") + self.assertTrue(m.readonly) + with _testcapi.buffer_fill_info(source, 0, PyBUF_WRITABLE) as m: + self.assertEqual(bytes(m), b"abc") + self.assertFalse(m.readonly) + self.assertRaises(BufferError, _testcapi.buffer_fill_info, + source, 1, PyBUF_WRITABLE) + def test_inheritance(self): class A(bytearray): def __buffer__(self, flags): diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index fcddd147bac63ef..9a0bf524e3943fd 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -308,14 +308,13 @@ class C3(C2): pass self.assertTrue(callable(c3)) def test_chr(self): + self.assertEqual(chr(0), '\0') self.assertEqual(chr(32), ' ') self.assertEqual(chr(65), 'A') self.assertEqual(chr(97), 'a') self.assertEqual(chr(0xff), '\xff') - self.assertRaises(ValueError, chr, 1<<24) - self.assertEqual(chr(sys.maxunicode), - str('\\U0010ffff'.encode("ascii"), 'unicode-escape')) self.assertRaises(TypeError, chr) + self.assertRaises(TypeError, chr, 65.0) self.assertEqual(chr(0x0000FFFF), "\U0000FFFF") self.assertEqual(chr(0x00010000), "\U00010000") self.assertEqual(chr(0x00010001), "\U00010001") @@ -327,7 +326,11 @@ def test_chr(self): self.assertEqual(chr(0x0010FFFF), "\U0010FFFF") self.assertRaises(ValueError, chr, -1) self.assertRaises(ValueError, chr, 0x00110000) - self.assertRaises((OverflowError, ValueError), chr, 2**32) + self.assertRaises(ValueError, chr, 1<<24) + self.assertRaises(ValueError, chr, 2**32-1) + self.assertRaises(ValueError, chr, -2**32) + self.assertRaises(ValueError, chr, 2**1000) + self.assertRaises(ValueError, chr, -2**1000) def test_cmp(self): self.assertTrue(not hasattr(builtins, "cmp")) @@ -611,6 +614,14 @@ def __dir__(self): self.assertIsInstance(res, list) self.assertTrue(res == ["a", "b", "c"]) + # dir(obj__dir__iterable) + class Foo(object): + def __dir__(self): + return {"b", "c", "a"} + res = dir(Foo()) + self.assertIsInstance(res, list) + self.assertEqual(sorted(res), ["a", "b", "c"]) + # dir(obj__dir__not_sequence) class Foo(object): def __dir__(self): diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 3c8fc35e3c116d0..2a6a5d287b04ee9 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -155,7 +155,7 @@ def test_varargs16_kw(self): min, 0, default=1, key=2, foo=3) def test_varargs17_kw(self): - msg = r"'foo' is an invalid keyword argument for print\(\)$" + msg = r"print\(\) got an unexpected keyword argument 'foo'$" self.assertRaisesRegex(TypeError, msg, print, 0, sep=1, end=2, file=3, flush=4, foo=5) @@ -928,7 +928,7 @@ def check_suggestion_includes(self, message): self.assertIn(f"Did you mean '{message}'?", str(cm.exception)) @contextlib.contextmanager - def check_suggestion_not_pressent(self): + def check_suggestion_not_present(self): with self.assertRaises(TypeError) as cm: yield self.assertNotIn("Did you mean", str(cm.exception)) @@ -946,7 +946,7 @@ def foo(blech=None, /, aaa=None, *args, late1=None): for keyword, suggestion in cases: with self.subTest(keyword): - ctx = self.check_suggestion_includes(suggestion) if suggestion else self.check_suggestion_not_pressent() + ctx = self.check_suggestion_includes(suggestion) if suggestion else self.check_suggestion_not_present() with ctx: foo(**{keyword:None}) @@ -987,6 +987,32 @@ def case_change_over_substitution(BLuch=None, Luch = None, fluch = None): with self.check_suggestion_includes(suggestion): func(bluch=None) + def test_unexpected_keyword_suggestion_via_getargs(self): + with self.check_suggestion_includes("maxsplit"): + "foo".split(maxsplt=1) + + self.assertRaisesRegex( + TypeError, r"split\(\) got an unexpected keyword argument 'blech'$", + "foo".split, blech=1 + ) + with self.check_suggestion_not_present(): + "foo".split(blech=1) + with self.check_suggestion_not_present(): + "foo".split(more_noise=1, maxsplt=1) + + # Also test the vgetargskeywords path + with self.check_suggestion_includes("name"): + ImportError(namez="oops") + + self.assertRaisesRegex( + TypeError, r"ImportError\(\) got an unexpected keyword argument 'blech'$", + ImportError, blech=1 + ) + with self.check_suggestion_not_present(): + ImportError(blech=1) + with self.check_suggestion_not_present(): + ImportError(blech=1, namez="oops") + @cpython_only class TestRecursion(unittest.TestCase): diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py index 57a7238588eae09..cca6145bc90c047 100644 --- a/Lib/test/test_capi/test_dict.py +++ b/Lib/test/test_capi/test_dict.py @@ -339,6 +339,28 @@ def test_dict_setdefault(self): # CRASHES setdefault({}, 'a', NULL) # CRASHES setdefault(NULL, 'a', 5) + def test_dict_setdefaultref(self): + setdefault = _testcapi.dict_setdefaultref + dct = {} + self.assertEqual(setdefault(dct, 'a', 5), 5) + self.assertEqual(dct, {'a': 5}) + self.assertEqual(setdefault(dct, 'a', 8), 5) + self.assertEqual(dct, {'a': 5}) + + dct2 = DictSubclass() + self.assertEqual(setdefault(dct2, 'a', 5), 5) + self.assertEqual(dct2, {'a': 5}) + self.assertEqual(setdefault(dct2, 'a', 8), 5) + self.assertEqual(dct2, {'a': 5}) + + self.assertRaises(TypeError, setdefault, {}, [], 5) # unhashable + self.assertRaises(SystemError, setdefault, UserDict(), 'a', 5) + self.assertRaises(SystemError, setdefault, [1], 0, 5) + self.assertRaises(SystemError, setdefault, 42, 'a', 5) + # CRASHES setdefault({}, NULL, 5) + # CRASHES setdefault({}, 'a', NULL) + # CRASHES setdefault(NULL, 'a', 5) + def test_mapping_keys_valuesitems(self): class BadMapping(dict): def keys(self): diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index 9b6aef27625ad0e..12039803ba543eb 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -667,7 +667,7 @@ def test_invalid_keyword(self): try: getargs_keywords((1,2),3,arg5=10,arg666=666) except TypeError as err: - self.assertEqual(str(err), "'arg666' is an invalid keyword argument for this function") + self.assertEqual(str(err), "this function got an unexpected keyword argument 'arg666'") else: self.fail('TypeError should have been raised') @@ -675,7 +675,7 @@ def test_surrogate_keyword(self): try: getargs_keywords((1,2), 3, (4,(5,6)), (7,8,9), **{'\uDC80': 10}) except TypeError as err: - self.assertEqual(str(err), "'\udc80' is an invalid keyword argument for this function") + self.assertEqual(str(err), "this function got an unexpected keyword argument '\udc80'") else: self.fail('TypeError should have been raised') @@ -742,12 +742,12 @@ def test_too_many_args(self): def test_invalid_keyword(self): # extraneous keyword arg with self.assertRaisesRegex(TypeError, - "'monster' is an invalid keyword argument for this function"): + "this function got an unexpected keyword argument 'monster'"): getargs_keyword_only(1, 2, monster=666) def test_surrogate_keyword(self): with self.assertRaisesRegex(TypeError, - "'\udc80' is an invalid keyword argument for this function"): + "this function got an unexpected keyword argument '\udc80'"): getargs_keyword_only(1, 2, **{'\uDC80': 10}) def test_weird_str_subclass(self): @@ -761,7 +761,7 @@ def __hash__(self): "invalid keyword argument for this function"): getargs_keyword_only(1, 2, **{BadStr("keyword_only"): 3}) with self.assertRaisesRegex(TypeError, - "invalid keyword argument for this function"): + "this function got an unexpected keyword argument"): getargs_keyword_only(1, 2, **{BadStr("monster"): 666}) def test_weird_str_subclass2(self): @@ -774,7 +774,7 @@ def __hash__(self): "invalid keyword argument for this function"): getargs_keyword_only(1, 2, **{BadStr("keyword_only"): 3}) with self.assertRaisesRegex(TypeError, - "invalid keyword argument for this function"): + "this function got an unexpected keyword argument"): getargs_keyword_only(1, 2, **{BadStr("monster"): 666}) @@ -807,7 +807,7 @@ def test_required_args(self): def test_empty_keyword(self): with self.assertRaisesRegex(TypeError, - "'' is an invalid keyword argument for this function"): + "this function got an unexpected keyword argument ''"): self.getargs(1, 2, **{'': 666}) @@ -1204,7 +1204,7 @@ def test_basic(self): "function missing required argument 'a'"): parse((), {}, 'O', ['a']) with self.assertRaisesRegex(TypeError, - "'b' is an invalid keyword argument"): + "this function got an unexpected keyword argument 'b'"): parse((), {'b': 1}, '|O', ['a']) with self.assertRaisesRegex(TypeError, fr"argument for function given by name \('a'\) " @@ -1278,10 +1278,10 @@ def test_nonascii_keywords(self): fr"and position \(1\)"): parse((1,), {name: 2}, 'O|O', [name, 'b']) with self.assertRaisesRegex(TypeError, - f"'{name}' is an invalid keyword argument"): + f"this function got an unexpected keyword argument '{name}'"): parse((), {name: 1}, '|O', ['b']) with self.assertRaisesRegex(TypeError, - "'b' is an invalid keyword argument"): + "this function got an unexpected keyword argument 'b'"): parse((), {'b': 1}, '|O', [name]) invalid = name.encode() + (name.encode()[:-1] or b'\x80') @@ -1301,17 +1301,17 @@ def test_nonascii_keywords(self): for name2 in ('b', 'ë', 'ĉ', 'Ɐ', '𐀁'): with self.subTest(name2=name2): with self.assertRaisesRegex(TypeError, - f"'{name2}' is an invalid keyword argument"): + f"this function got an unexpected keyword argument '{name2}'"): parse((), {name2: 1}, '|O', [name]) name2 = name.encode().decode('latin1') if name2 != name: with self.assertRaisesRegex(TypeError, - f"'{name2}' is an invalid keyword argument"): + f"this function got an unexpected keyword argument '{name2}'"): parse((), {name2: 1}, '|O', [name]) name3 = name + '3' with self.assertRaisesRegex(TypeError, - f"'{name2}' is an invalid keyword argument"): + f"this function got an unexpected keyword argument '{name2}'"): parse((), {name2: 1, name3: 2}, '|OO', [name, name3]) def test_nested_tuple(self): diff --git a/Lib/test/test_capi/test_list.py b/Lib/test/test_capi/test_list.py index eb03d51d3def378..dceb4fce3c077bf 100644 --- a/Lib/test/test_capi/test_list.py +++ b/Lib/test/test_capi/test_list.py @@ -82,10 +82,8 @@ def test_list_get_size(self): # CRASHES size(UserList()) # CRASHES size(NULL) - - def test_list_getitem(self): - # Test PyList_GetItem() - getitem = _testcapi.list_getitem + def check_list_get_item(self, getitem, exctype): + # Common test cases for PyList_GetItem() and PyList_GetItemRef() lst = [1, 2, 3] self.assertEqual(getitem(lst, 0), 1) self.assertEqual(getitem(lst, 2), 3) @@ -93,12 +91,19 @@ def test_list_getitem(self): self.assertRaises(IndexError, getitem, lst, -1) self.assertRaises(IndexError, getitem, lst, PY_SSIZE_T_MIN) self.assertRaises(IndexError, getitem, lst, PY_SSIZE_T_MAX) - self.assertRaises(SystemError, getitem, 42, 1) - self.assertRaises(SystemError, getitem, (1, 2, 3), 1) - self.assertRaises(SystemError, getitem, {1: 2}, 1) - + self.assertRaises(exctype, getitem, 42, 1) + self.assertRaises(exctype, getitem, (1, 2, 3), 1) + self.assertRaises(exctype, getitem, {1: 2}, 1) # CRASHES getitem(NULL, 1) + def test_list_getitem(self): + # Test PyList_GetItem() + self.check_list_get_item(_testcapi.list_getitem, SystemError) + + def test_list_get_item_ref(self): + # Test PyList_GetItemRef() + self.check_list_get_item(_testcapi.list_get_item_ref, TypeError) + def test_list_get_item(self): # Test PyList_GET_ITEM() get_item = _testcapi.list_get_item @@ -112,7 +117,6 @@ def test_list_get_item(self): # CRASHES get_item(21, 2) # CRASHES get_item(NULL, 1) - def test_list_setitem(self): # Test PyList_SetItem() setitem = _testcapi.list_setitem diff --git a/Lib/test/test_capi/test_structmembers.py b/Lib/test/test_capi/test_structmembers.py index 2cf46b203478dc2..a294c3b13a5c30c 100644 --- a/Lib/test/test_capi/test_structmembers.py +++ b/Lib/test/test_capi/test_structmembers.py @@ -14,6 +14,13 @@ PY_SSIZE_T_MAX, PY_SSIZE_T_MIN, ) + +class Index: + def __init__(self, value): + self.value = value + def __index__(self): + return self.value + # There are two classes: one using <structmember.h> and another using # `Py_`-prefixed API. They should behave the same in Python @@ -38,75 +45,115 @@ class ReadWriteTests: def setUp(self): self.ts = _make_test_object(self.cls) + def _test_write(self, name, value, expected=None): + if expected is None: + expected = value + ts = self.ts + setattr(ts, name, value) + self.assertEqual(getattr(ts, name), expected) + + def _test_warn(self, name, value, expected=None): + ts = self.ts + self.assertWarns(RuntimeWarning, setattr, ts, name, value) + if expected is not None: + self.assertEqual(getattr(ts, name), expected) + + def _test_overflow(self, name, value): + ts = self.ts + self.assertRaises(OverflowError, setattr, ts, name, value) + + def _test_int_range(self, name, minval, maxval, *, hardlimit=None, + indexlimit=None): + if hardlimit is None: + hardlimit = (minval, maxval) + ts = self.ts + self._test_write(name, minval) + self._test_write(name, maxval) + hardminval, hardmaxval = hardlimit + self._test_overflow(name, hardminval-1) + self._test_overflow(name, hardmaxval+1) + self._test_overflow(name, 2**1000) + self._test_overflow(name, -2**1000) + if hardminval < minval: + self._test_warn(name, hardminval) + self._test_warn(name, minval-1, maxval) + if maxval < hardmaxval: + self._test_warn(name, maxval+1, minval) + self._test_warn(name, hardmaxval) + + if indexlimit is None: + indexlimit = hardlimit + if not indexlimit: + self.assertRaises(TypeError, setattr, ts, name, Index(minval)) + self.assertRaises(TypeError, setattr, ts, name, Index(maxval)) + else: + hardminindexval, hardmaxindexval = indexlimit + self._test_write(name, Index(minval), minval) + if minval < hardminindexval: + self._test_write(name, Index(hardminindexval), hardminindexval) + if maxval < hardmaxindexval: + self._test_write(name, Index(maxval), maxval) + else: + self._test_write(name, Index(hardmaxindexval), hardmaxindexval) + self._test_overflow(name, Index(hardminindexval-1)) + if name in ('T_UINT', 'T_ULONG'): + self.assertRaises(TypeError, setattr, self.ts, name, + Index(hardmaxindexval+1)) + self.assertRaises(TypeError, setattr, self.ts, name, + Index(2**1000)) + else: + self._test_overflow(name, Index(hardmaxindexval+1)) + self._test_overflow(name, Index(2**1000)) + self._test_overflow(name, Index(-2**1000)) + if hardminindexval < minval and name != 'T_ULONGLONG': + self._test_warn(name, Index(hardminindexval)) + self._test_warn(name, Index(minval-1)) + if maxval < hardmaxindexval: + self._test_warn(name, Index(maxval+1)) + self._test_warn(name, Index(hardmaxindexval)) + def test_bool(self): ts = self.ts ts.T_BOOL = True - self.assertEqual(ts.T_BOOL, True) + self.assertIs(ts.T_BOOL, True) ts.T_BOOL = False - self.assertEqual(ts.T_BOOL, False) + self.assertIs(ts.T_BOOL, False) self.assertRaises(TypeError, setattr, ts, 'T_BOOL', 1) + self.assertRaises(TypeError, setattr, ts, 'T_BOOL', 0) + self.assertRaises(TypeError, setattr, ts, 'T_BOOL', None) def test_byte(self): - ts = self.ts - ts.T_BYTE = CHAR_MAX - self.assertEqual(ts.T_BYTE, CHAR_MAX) - ts.T_BYTE = CHAR_MIN - self.assertEqual(ts.T_BYTE, CHAR_MIN) - ts.T_UBYTE = UCHAR_MAX - self.assertEqual(ts.T_UBYTE, UCHAR_MAX) + self._test_int_range('T_BYTE', CHAR_MIN, CHAR_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) + self._test_int_range('T_UBYTE', 0, UCHAR_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) def test_short(self): - ts = self.ts - ts.T_SHORT = SHRT_MAX - self.assertEqual(ts.T_SHORT, SHRT_MAX) - ts.T_SHORT = SHRT_MIN - self.assertEqual(ts.T_SHORT, SHRT_MIN) - ts.T_USHORT = USHRT_MAX - self.assertEqual(ts.T_USHORT, USHRT_MAX) + self._test_int_range('T_SHORT', SHRT_MIN, SHRT_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) + self._test_int_range('T_USHORT', 0, USHRT_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) def test_int(self): - ts = self.ts - ts.T_INT = INT_MAX - self.assertEqual(ts.T_INT, INT_MAX) - ts.T_INT = INT_MIN - self.assertEqual(ts.T_INT, INT_MIN) - ts.T_UINT = UINT_MAX - self.assertEqual(ts.T_UINT, UINT_MAX) + self._test_int_range('T_INT', INT_MIN, INT_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) + self._test_int_range('T_UINT', 0, UINT_MAX, + hardlimit=(LONG_MIN, ULONG_MAX), + indexlimit=(LONG_MIN, LONG_MAX)) def test_long(self): - ts = self.ts - ts.T_LONG = LONG_MAX - self.assertEqual(ts.T_LONG, LONG_MAX) - ts.T_LONG = LONG_MIN - self.assertEqual(ts.T_LONG, LONG_MIN) - ts.T_ULONG = ULONG_MAX - self.assertEqual(ts.T_ULONG, ULONG_MAX) + self._test_int_range('T_LONG', LONG_MIN, LONG_MAX) + self._test_int_range('T_ULONG', 0, ULONG_MAX, + hardlimit=(LONG_MIN, ULONG_MAX), + indexlimit=(LONG_MIN, LONG_MAX)) def test_py_ssize_t(self): - ts = self.ts - ts.T_PYSSIZET = PY_SSIZE_T_MAX - self.assertEqual(ts.T_PYSSIZET, PY_SSIZE_T_MAX) - ts.T_PYSSIZET = PY_SSIZE_T_MIN - self.assertEqual(ts.T_PYSSIZET, PY_SSIZE_T_MIN) + self._test_int_range('T_PYSSIZET', PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, indexlimit=False) def test_longlong(self): - ts = self.ts - if not hasattr(ts, "T_LONGLONG"): - self.skipTest("long long not present") - - ts.T_LONGLONG = LLONG_MAX - self.assertEqual(ts.T_LONGLONG, LLONG_MAX) - ts.T_LONGLONG = LLONG_MIN - self.assertEqual(ts.T_LONGLONG, LLONG_MIN) - - ts.T_ULONGLONG = ULLONG_MAX - self.assertEqual(ts.T_ULONGLONG, ULLONG_MAX) - - ## make sure these will accept a plain int as well as a long - ts.T_LONGLONG = 3 - self.assertEqual(ts.T_LONGLONG, 3) - ts.T_ULONGLONG = 4 - self.assertEqual(ts.T_ULONGLONG, 4) + self._test_int_range('T_LONGLONG', LLONG_MIN, LLONG_MAX) + self._test_int_range('T_ULONGLONG', 0, ULLONG_MAX, + indexlimit=(LONG_MIN, LONG_MAX)) def test_bad_assignments(self): ts = self.ts @@ -116,10 +163,9 @@ def test_bad_assignments(self): 'T_SHORT', 'T_USHORT', 'T_INT', 'T_UINT', 'T_LONG', 'T_ULONG', + 'T_LONGLONG', 'T_ULONGLONG', 'T_PYSSIZET' ] - if hasattr(ts, 'T_LONGLONG'): - integer_attributes.extend(['T_LONGLONG', 'T_ULONGLONG']) # issue8014: this produced 'bad argument to internal function' # internal error @@ -139,46 +185,6 @@ class ReadWriteTests_OldAPI(ReadWriteTests, unittest.TestCase): class ReadWriteTests_NewAPI(ReadWriteTests, unittest.TestCase): cls = _test_structmembersType_NewAPI -class TestWarnings: - def setUp(self): - self.ts = _make_test_object(self.cls) - - def test_byte_max(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_BYTE = CHAR_MAX+1 - - def test_byte_min(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_BYTE = CHAR_MIN-1 - - def test_ubyte_max(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_UBYTE = UCHAR_MAX+1 - - def test_short_max(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_SHORT = SHRT_MAX+1 - - def test_short_min(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_SHORT = SHRT_MIN-1 - - def test_ushort_max(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_USHORT = USHRT_MAX+1 - -class TestWarnings_OldAPI(TestWarnings, unittest.TestCase): - cls = _test_structmembersType_OldAPI - -class TestWarnings_NewAPI(TestWarnings, unittest.TestCase): - cls = _test_structmembersType_NewAPI - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 5981712c80c3a96..ae062b1bda26b7c 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -151,8 +151,8 @@ def test_watch_out_of_range_watcher_id(self): def test_watch_unassigned_watcher_id(self): d = {} - with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 1"): - self.watch(1, d) + with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 3"): + self.watch(3, d) def test_unwatch_non_dict(self): with self.watcher() as wid: @@ -168,8 +168,8 @@ def test_unwatch_out_of_range_watcher_id(self): def test_unwatch_unassigned_watcher_id(self): d = {} - with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 1"): - self.unwatch(1, d) + with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 3"): + self.unwatch(3, d) def test_clear_out_of_range_watcher_id(self): with self.assertRaisesRegex(ValueError, r"Invalid dict watcher ID -1"): @@ -178,8 +178,8 @@ def test_clear_out_of_range_watcher_id(self): self.clear_watcher(8) # DICT_MAX_WATCHERS = 8 def test_clear_unassigned_watcher_id(self): - with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 1"): - self.clear_watcher(1) + with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 3"): + self.clear_watcher(3) class TestTypeWatchers(unittest.TestCase): diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 1531aad4f1f779d..d59271435e9eb05 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -771,6 +771,22 @@ def add_one_level(): with self.assertRaises(RecursionError): add_one_level() + def testMetaclassCallOptimization(self): + calls = 0 + + class TypeMetaclass(type): + def __call__(cls, *args, **kwargs): + nonlocal calls + calls += 1 + return type.__call__(cls, *args, **kwargs) + + class Type(metaclass=TypeMetaclass): + def __init__(self, obj): + self._obj = obj + + for i in range(100): + Type(i) + self.assertEqual(calls, 100) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 7323bdd801f4be0..e987ce546054979 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -3288,6 +3288,26 @@ def test_cloned_func_with_converter_exception_message(self): func = getattr(ac_tester, name) self.assertEqual(func(), name) + def test_meth_method_no_params(self): + obj = ac_tester.TestClass() + meth = obj.meth_method_no_params + check = partial(self.assertRaisesRegex, TypeError, "no arguments") + check(meth, 1) + check(meth, a=1) + + def test_meth_method_no_params_capi(self): + from _testcapi import pyobject_vectorcall + obj = ac_tester.TestClass() + meth = obj.meth_method_no_params + pyobject_vectorcall(meth, None, None) + pyobject_vectorcall(meth, (), None) + pyobject_vectorcall(meth, (), ()) + pyobject_vectorcall(meth, None, ()) + + check = partial(self.assertRaisesRegex, TypeError, "no arguments") + check(pyobject_vectorcall, meth, (1,), None) + check(pyobject_vectorcall, meth, (1,), ("a",)) + def test_depr_star_new(self): cls = ac_tester.DeprStarNew cls() diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 48754d5a63da3b1..3a5a8abf81e43d4 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -14,8 +14,7 @@ import textwrap from test import support -from test.support import import_helper -from test.support import os_helper +from test.support import import_helper, is_apple, os_helper from test.support.script_helper import ( make_pkg, make_script, make_zip_pkg, make_zip_script, assert_python_ok, assert_python_failure, spawn_python, kill_python) @@ -557,12 +556,17 @@ def test_pep_409_verbiage(self): self.assertTrue(text[3].startswith('NameError')) def test_non_ascii(self): - # Mac OS X denies the creation of a file with an invalid UTF-8 name. + # Apple platforms deny the creation of a file with an invalid UTF-8 name. # Windows allows creating a name with an arbitrary bytes name, but # Python cannot a undecodable bytes argument to a subprocess. - # WASI does not permit invalid UTF-8 names. - if (os_helper.TESTFN_UNDECODABLE - and sys.platform not in ('win32', 'darwin', 'emscripten', 'wasi')): + # Emscripten/WASI does not permit invalid UTF-8 names. + if ( + os_helper.TESTFN_UNDECODABLE + and sys.platform not in { + "win32", "emscripten", "wasi" + } + and not is_apple + ): name = os.fsdecode(os_helper.TESTFN_UNDECODABLE) elif os_helper.TESTFN_NONASCII: name = os_helper.TESTFN_NONASCII diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index d8fb826edeb6816..46bebfc7af675b1 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -865,6 +865,7 @@ def __init__(self, f, test): self.test = test def run(self): del self.f + gc_collect() self.test.assertEqual(LAST_FREED, 500) SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(500)) diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 747c0f9683c19c7..259778a5cade98f 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -160,6 +160,7 @@ def setUp(self): self.console = code.InteractiveConsole(local_exit=True) self.mock_sys() + @unittest.skipIf(sys.flags.no_site, "exit() isn't defined unless there's a site module") def test_exit(self): # default exit message self.infunc.side_effect = ["exit()"] diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 3b1ceceaa6305f7..ebb479f2de7c633 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1104,6 +1104,17 @@ async def test(aseq): code_lines = self.get_code_lines(test.__code__) self.assertEqual(expected_lines, code_lines) + def check_line_numbers(self, code, opnames=None): + # Check that all instructions whose op matches opnames + # have a line number. opnames can be a single name, or + # a sequence of names. If it is None, match all ops. + + if isinstance(opnames, str): + opnames = (opnames, ) + for inst in dis.Bytecode(code): + if opnames and inst.opname in opnames: + self.assertIsNotNone(inst.positions.lineno) + def test_line_number_synthetic_jump_multiple_predecessors(self): def f(): for x in it: @@ -1113,25 +1124,52 @@ def f(): except OSError: pass - # Ensure that all JUMP_BACKWARDs have line number - code = f.__code__ - for inst in dis.Bytecode(code): - if inst.opname == 'JUMP_BACKWARD': - self.assertIsNotNone(inst.positions.lineno) + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') - def test_lineno_of_backward_jump(self): + def test_line_number_synthetic_jump_multiple_predecessors_nested(self): + def f(): + for x in it: + try: + X = 3 + except OSError: + try: + if C3: + X = 4 + except OSError: + pass + return 42 + + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') + + def test_line_number_synthetic_jump_multiple_predecessors_more_nested(self): + def f(): + for x in it: + try: + X = 3 + except OSError: + try: + if C3: + if C4: + X = 4 + except OSError: + try: + if C3: + if C4: + X = 5 + except OSError: + pass + return 42 + + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') + + def test_lineno_of_backward_jump_conditional_in_loop(self): # Issue gh-107901 def f(): for i in x: if y: pass - linenos = list(inst.positions.lineno - for inst in dis.get_instructions(f.__code__) - if inst.opname == 'JUMP_BACKWARD') - - self.assertTrue(len(linenos) > 0) - self.assertTrue(all(l is not None for l in linenos)) + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') def test_big_dict_literal(self): # The compiler has a flushing point in "compiler_dict" that calls compiles diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index b5d1e2f9e4752cd..dbeadd9ca47c63d 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -49,6 +49,7 @@ def test_for_loop(self): ('JUMP', loop_lbl), exit_lbl, ('END_FOR', None), + ('POP_TOP', None), ('LOAD_CONST', 0), ('RETURN_VALUE', None), ] diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 1e7d4344740943d..6a79fe69ec37cf4 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -1,8 +1,10 @@ import threading import time +import unittest import weakref from concurrent import futures from test import support +from test.support import Py_GIL_DISABLED def mul(x, y): @@ -83,10 +85,21 @@ def test_no_stale_references(self): my_object_collected = threading.Event() my_object_callback = weakref.ref( my_object, lambda obj: my_object_collected.set()) - # Deliberately discarding the future. - self.executor.submit(my_object.my_method) + fut = self.executor.submit(my_object.my_method) del my_object + if Py_GIL_DISABLED: + # Due to biased reference counting, my_object might only be + # deallocated while the thread that created it runs -- if the + # thread is paused waiting on an event, it may not merge the + # refcount of the queued object. For that reason, we wait for the + # task to finish (so that it's no longer referenced) and force a + # GC to ensure that it is collected. + fut.result() # Wait for the task to finish. + support.gc_collect() + else: + del fut # Deliberately discard the future. + collected = my_object_collected.wait(timeout=support.SHORT_TIMEOUT) self.assertTrue(collected, "Stale reference not collected within timeout.") diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py index 3e61b0c9387c6fa..7fc59a05f3deac0 100644 --- a/Lib/test/test_concurrent_futures/test_process_pool.py +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -98,6 +98,7 @@ def test_ressources_gced_in_workers(self): # explicitly destroy the object to ensure that EventfulGCObj.__del__() # is called while manager is still running. + support.gc_collect() obj = None support.gc_collect() diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 69fef5945ae66f5..21a4cb586ff6658 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -392,10 +392,26 @@ def test_read_quoting(self): # will this fail where locale uses comma for decimals? self._read_test([',3,"5",7.3, 9'], [['', 3, '5', 7.3, 9]], quoting=csv.QUOTE_NONNUMERIC) + self._read_test([',3,"5",7.3, 9'], [[None, '3', '5', '7.3', ' 9']], + quoting=csv.QUOTE_NOTNULL) + self._read_test([',3,"5",7.3, 9'], [[None, 3, '5', 7.3, 9]], + quoting=csv.QUOTE_STRINGS) + + self._read_test([',,"",'], [['', '', '', '']]) + self._read_test([',,"",'], [['', '', '', '']], + quoting=csv.QUOTE_NONNUMERIC) + self._read_test([',,"",'], [[None, None, '', None]], + quoting=csv.QUOTE_NOTNULL) + self._read_test([',,"",'], [[None, None, '', None]], + quoting=csv.QUOTE_STRINGS) + self._read_test(['"a\nb", 7'], [['a\nb', ' 7']]) self.assertRaises(ValueError, self._read_test, ['abc,3'], [[]], quoting=csv.QUOTE_NONNUMERIC) + self.assertRaises(ValueError, self._read_test, + ['abc,3'], [[]], + quoting=csv.QUOTE_STRINGS) self._read_test(['1,@,3,@,5'], [['1', ',3,', '5']], quotechar='@') self._read_test(['1,\0,3,\0,5'], [['1', ',3,', '5']], quotechar='\0') @@ -403,6 +419,15 @@ def test_read_skipinitialspace(self): self._read_test(['no space, space, spaces,\ttab'], [['no space', 'space', 'spaces', '\ttab']], skipinitialspace=True) + self._read_test([' , , '], + [['', '', '']], + skipinitialspace=True) + self._read_test([' , , '], + [[None, None, None]], + skipinitialspace=True, quoting=csv.QUOTE_NOTNULL) + self._read_test([' , , '], + [[None, None, None]], + skipinitialspace=True, quoting=csv.QUOTE_STRINGS) def test_read_bigfield(self): # This exercises the buffer realloc functionality and field size diff --git a/Lib/test/test_ctypes/test_as_parameter.py b/Lib/test/test_ctypes/test_as_parameter.py index a1a8745e737fa25..ca75e748256083b 100644 --- a/Lib/test/test_ctypes/test_as_parameter.py +++ b/Lib/test/test_ctypes/test_as_parameter.py @@ -221,5 +221,16 @@ class AsParamPropertyWrapperTestCase(BasicWrapTestCase): wrap = AsParamPropertyWrapper +class AsParamNestedWrapperTestCase(BasicWrapTestCase): + """Test that _as_parameter_ is evaluated recursively. + + The _as_parameter_ attribute can be another object which + defines its own _as_parameter_ attribute. + """ + + def wrap(self, param): + return AsParamWrapper(AsParamWrapper(AsParamWrapper(param))) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_dbm_dumb.py b/Lib/test/test_dbm_dumb.py index a481175b3bfdbde..672f9092207cf62 100644 --- a/Lib/test/test_dbm_dumb.py +++ b/Lib/test/test_dbm_dumb.py @@ -246,9 +246,27 @@ def test_missing_data(self): _delete_files() with self.assertRaises(FileNotFoundError): dumbdbm.open(_fname, value) + self.assertFalse(os.path.exists(_fname + '.dat')) self.assertFalse(os.path.exists(_fname + '.dir')) self.assertFalse(os.path.exists(_fname + '.bak')) + for value in ('c', 'n'): + _delete_files() + with dumbdbm.open(_fname, value) as f: + self.assertTrue(os.path.exists(_fname + '.dat')) + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertFalse(os.path.exists(_fname + '.bak')) + + for value in ('c', 'n'): + _delete_files() + with dumbdbm.open(_fname, value) as f: + f['key'] = 'value' + self.assertTrue(os.path.exists(_fname + '.dat')) + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertTrue(os.path.exists(_fname + '.bak')) + def test_missing_index(self): with dumbdbm.open(_fname, 'n') as f: pass @@ -259,6 +277,60 @@ def test_missing_index(self): self.assertFalse(os.path.exists(_fname + '.dir')) self.assertFalse(os.path.exists(_fname + '.bak')) + for value in ('c', 'n'): + with dumbdbm.open(_fname, value) as f: + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertFalse(os.path.exists(_fname + '.bak')) + os.unlink(_fname + '.dir') + + for value in ('c', 'n'): + with dumbdbm.open(_fname, value) as f: + f['key'] = 'value' + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertTrue(os.path.exists(_fname + '.bak')) + os.unlink(_fname + '.dir') + os.unlink(_fname + '.bak') + + def test_sync_empty_unmodified(self): + with dumbdbm.open(_fname, 'n') as f: + pass + os.unlink(_fname + '.dir') + for value in ('c', 'n'): + with dumbdbm.open(_fname, value) as f: + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + f.sync() + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + os.unlink(_fname + '.dir') + f.sync() + self.assertFalse(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertFalse(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + + def test_sync_nonempty_unmodified(self): + with dumbdbm.open(_fname, 'n') as f: + pass + os.unlink(_fname + '.dir') + for value in ('c', 'n'): + with dumbdbm.open(_fname, value) as f: + f['key'] = 'value' + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + f.sync() + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertTrue(os.path.exists(_fname + '.bak')) + os.unlink(_fname + '.dir') + os.unlink(_fname + '.bak') + f.sync() + self.assertFalse(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertFalse(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + def test_invalid_flag(self): for flag in ('x', 'rf', None): with self.assertRaisesRegex(ValueError, diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 7a5fe62b4673720..1423bc61c7f6906 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -41,6 +41,7 @@ darwin_malloc_err_warning, is_emscripten) from test.support.import_helper import import_fresh_module from test.support import threading_helper +from test.support import warnings_helper import random import inspect import threading @@ -1237,7 +1238,14 @@ def test_deprecated_N_format(self): else: self.assertRaises(ValueError, format, h, 'N') self.assertRaises(ValueError, format, h, '010.3N') - + with warnings_helper.check_no_warnings(self): + self.assertEqual(format(h, 'N>10.3'), 'NN6.63E-34') + self.assertEqual(format(h, 'N>10.3n'), 'NN6.63e-34') + self.assertEqual(format(h, 'N>10.3e'), 'N6.626e-34') + self.assertEqual(format(h, 'N>10.3f'), 'NNNNN0.000') + self.assertRaises(ValueError, format, h, '>Nf') + self.assertRaises(ValueError, format, h, '10Nf') + self.assertRaises(ValueError, format, h, 'Nx') @run_with_locale('LC_ALL', 'ps_AF') def test_wide_char_separator_decimal_point(self): diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index fd0af9b30a0a71a..beeab6cb7f254c1 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -4734,6 +4734,20 @@ class X(object): with self.assertRaises(AttributeError): del X.__abstractmethods__ + def test_gh55664(self): + # gh-55664: issue a warning when the + # __dict__ of a class contains non-string keys + with self.assertWarnsRegex(RuntimeWarning, 'MyClass'): + MyClass = type('MyClass', (), {1: 2}) + + class meta(type): + def __new__(mcls, name, bases, ns): + ns[1] = 2 + return super().__new__(mcls, name, bases, ns) + + with self.assertWarnsRegex(RuntimeWarning, 'MyClass'): + MyClass = meta('MyClass', (), {}) + def test_proxy_call(self): class FakeStr: __class__ = str @@ -5151,7 +5165,8 @@ class Base2(object): mykey = 'from Base2' mykey2 = 'from Base2' - X = type('X', (Base,), {MyKey(): 5}) + with self.assertWarnsRegex(RuntimeWarning, 'X'): + X = type('X', (Base,), {MyKey(): 5}) # mykey is read from Base self.assertEqual(X.mykey, 'from Base') # mykey2 is read from Base2 because MyKey.__eq__ has set __bases__ diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 3ae81b2f5d62b06..a5917da346dded5 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -158,6 +158,7 @@ def bug708901(): %3d JUMP_BACKWARD 5 (to L1) %3d L2: END_FOR + POP_TOP RETURN_CONST 0 (None) """ % (bug708901.__code__.co_firstlineno, bug708901.__code__.co_firstlineno + 1, @@ -791,6 +792,7 @@ def foo(x): POP_TOP JUMP_BACKWARD 12 (to L2) L3: END_FOR + POP_TOP RETURN_CONST 0 (None) -- L4: CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) @@ -843,6 +845,7 @@ def loop_test(): JUMP_BACKWARD 16 (to L1) %3d L2: END_FOR + POP_TOP RETURN_CONST 0 (None) """ % (loop_test.__code__.co_firstlineno, loop_test.__code__.co_firstlineno + 1, @@ -1648,122 +1651,123 @@ def _prepare_test_cases(): ] expected_opinfo_jumpy = [ - Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=1, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=1, argval=10, argrepr='10', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='GET_ITER', opcode=19, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='FOR_ITER', opcode=72, arg=30, argval=88, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, positions=None), - Instruction(opname='STORE_FAST', opcode=110, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=True, line_number=5, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval=4, argrepr='4', offset=54, start_offset=54, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=68, argrepr='to L2', offset=60, start_offset=60, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=22, argval=24, argrepr='to L1', offset=64, start_offset=64, starts_line=True, line_number=6, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=68, start_offset=68, starts_line=True, line_number=7, label=2, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=3, argval=6, argrepr='6', offset=70, start_offset=70, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=148, argval='>', argrepr='bool(>)', offset=72, start_offset=72, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=2, argval=84, argrepr='to L3', offset=76, start_offset=76, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=30, argval=24, argrepr='to L1', offset=80, start_offset=80, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=True, line_number=8, label=3, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=79, arg=12, argval=112, argrepr='to L5', offset=86, start_offset=86, starts_line=False, line_number=8, label=None, positions=None), - Instruction(opname='END_FOR', opcode=11, arg=None, argval=None, argrepr='', offset=88, start_offset=88, starts_line=True, line_number=3, label=4, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=90, start_offset=90, starts_line=True, line_number=10, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=100, start_offset=100, starts_line=False, line_number=10, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=102, start_offset=102, starts_line=False, line_number=10, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=110, start_offset=110, starts_line=False, line_number=10, label=None, positions=None), - Instruction(opname='LOAD_FAST_CHECK', opcode=87, arg=0, argval='i', argrepr='i', offset=112, start_offset=112, starts_line=True, line_number=11, label=5, positions=None), - Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=114, start_offset=114, starts_line=False, line_number=11, label=None, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=40, argval=206, argrepr='to L9', offset=122, start_offset=122, starts_line=False, line_number=11, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=126, start_offset=126, starts_line=True, line_number=12, label=6, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=136, start_offset=136, starts_line=False, line_number=12, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=138, start_offset=138, starts_line=False, line_number=12, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=146, start_offset=146, starts_line=False, line_number=12, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=148, start_offset=148, starts_line=True, line_number=13, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=5, argval=1, argrepr='1', offset=150, start_offset=150, starts_line=False, line_number=13, label=None, positions=None), - Instruction(opname='BINARY_OP', opcode=45, arg=23, argval=23, argrepr='-=', offset=152, start_offset=152, starts_line=False, line_number=13, label=None, positions=None), - Instruction(opname='STORE_FAST', opcode=110, arg=0, argval='i', argrepr='i', offset=156, start_offset=156, starts_line=False, line_number=13, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=158, start_offset=158, starts_line=True, line_number=14, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=3, argval=6, argrepr='6', offset=160, start_offset=160, starts_line=False, line_number=14, label=None, positions=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=148, argval='>', argrepr='bool(>)', offset=162, start_offset=162, starts_line=False, line_number=14, label=None, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=174, argrepr='to L7', offset=166, start_offset=166, starts_line=False, line_number=14, label=None, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=31, argval=112, argrepr='to L5', offset=170, start_offset=170, starts_line=True, line_number=15, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=174, start_offset=174, starts_line=True, line_number=16, label=7, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval=4, argrepr='4', offset=176, start_offset=176, starts_line=False, line_number=16, label=None, positions=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=18, argval='<', argrepr='bool(<)', offset=178, start_offset=178, starts_line=False, line_number=16, label=None, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=1, argval=188, argrepr='to L8', offset=182, start_offset=182, starts_line=False, line_number=16, label=None, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=79, arg=20, argval=228, argrepr='to L10', offset=186, start_offset=186, starts_line=True, line_number=17, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=188, start_offset=188, starts_line=True, line_number=11, label=8, positions=None), - Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=190, start_offset=190, starts_line=False, line_number=11, label=None, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=206, argrepr='to L9', offset=198, start_offset=198, starts_line=False, line_number=11, label=None, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=40, argval=126, argrepr='to L6', offset=202, start_offset=202, starts_line=False, line_number=11, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=206, start_offset=206, starts_line=True, line_number=19, label=9, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=216, start_offset=216, starts_line=False, line_number=19, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=218, start_offset=218, starts_line=False, line_number=19, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=226, start_offset=226, starts_line=False, line_number=19, label=None, positions=None), - Instruction(opname='NOP', opcode=30, arg=None, argval=None, argrepr='', offset=228, start_offset=228, starts_line=True, line_number=20, label=10, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=5, argval=1, argrepr='1', offset=230, start_offset=230, starts_line=True, line_number=21, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=7, argval=0, argrepr='0', offset=232, start_offset=232, starts_line=False, line_number=21, label=None, positions=None), - Instruction(opname='BINARY_OP', opcode=45, arg=11, argval=11, argrepr='/', offset=234, start_offset=234, starts_line=False, line_number=21, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=238, start_offset=238, starts_line=False, line_number=21, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=240, start_offset=240, starts_line=True, line_number=25, label=None, positions=None), - Instruction(opname='BEFORE_WITH', opcode=2, arg=None, argval=None, argrepr='', offset=242, start_offset=242, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='STORE_FAST', opcode=110, arg=1, argval='dodgy', argrepr='dodgy', offset=244, start_offset=244, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=246, start_offset=246, starts_line=True, line_number=26, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=256, start_offset=256, starts_line=False, line_number=26, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=258, start_offset=258, starts_line=False, line_number=26, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=266, start_offset=266, starts_line=False, line_number=26, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=268, start_offset=268, starts_line=True, line_number=25, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=270, start_offset=270, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=272, start_offset=272, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=2, argval=2, argrepr='', offset=274, start_offset=274, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=282, start_offset=282, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=284, start_offset=284, starts_line=True, line_number=28, label=11, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=294, start_offset=294, starts_line=False, line_number=28, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=296, start_offset=296, starts_line=False, line_number=28, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=304, start_offset=304, starts_line=False, line_number=28, label=None, positions=None), - Instruction(opname='RETURN_CONST', opcode=103, arg=0, argval=None, argrepr='None', offset=306, start_offset=306, starts_line=False, line_number=28, label=None, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=308, start_offset=308, starts_line=True, line_number=25, label=None, positions=None), - Instruction(opname='WITH_EXCEPT_START', opcode=44, arg=None, argval=None, argrepr='', offset=310, start_offset=310, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=1, argval=326, argrepr='to L12', offset=320, start_offset=320, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='RERAISE', opcode=102, arg=2, argval=2, argrepr='', offset=324, start_offset=324, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=326, start_offset=326, starts_line=False, line_number=25, label=12, positions=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=328, start_offset=328, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=332, start_offset=332, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=78, arg=26, argval=284, argrepr='to L11', offset=334, start_offset=334, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=336, start_offset=336, starts_line=True, line_number=None, label=None, positions=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=338, start_offset=338, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=340, start_offset=340, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=342, start_offset=342, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=344, start_offset=344, starts_line=True, line_number=22, label=None, positions=None), - Instruction(opname='CHECK_EXC_MATCH', opcode=7, arg=None, argval=None, argrepr='', offset=354, start_offset=354, starts_line=False, line_number=22, label=None, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=14, argval=388, argrepr='to L13', offset=356, start_offset=356, starts_line=False, line_number=22, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=360, start_offset=360, starts_line=False, line_number=22, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=362, start_offset=362, starts_line=True, line_number=23, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=372, start_offset=372, starts_line=False, line_number=23, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=374, start_offset=374, starts_line=False, line_number=23, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=382, start_offset=382, starts_line=False, line_number=23, label=None, positions=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=384, start_offset=384, starts_line=False, line_number=23, label=None, positions=None), - Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=78, arg=52, argval=284, argrepr='to L11', offset=386, start_offset=386, starts_line=False, line_number=23, label=None, positions=None), - Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=388, start_offset=388, starts_line=True, line_number=22, label=13, positions=None), - Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=390, start_offset=390, starts_line=True, line_number=None, label=None, positions=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=392, start_offset=392, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=394, start_offset=394, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=396, start_offset=396, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=398, start_offset=398, starts_line=True, line_number=28, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=408, start_offset=408, starts_line=False, line_number=28, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=410, start_offset=410, starts_line=False, line_number=28, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=418, start_offset=418, starts_line=False, line_number=28, label=None, positions=None), - Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=420, start_offset=420, starts_line=False, line_number=28, label=None, positions=None), - Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=422, start_offset=422, starts_line=True, line_number=None, label=None, positions=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=424, start_offset=424, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=426, start_offset=426, starts_line=False, line_number=None, label=None, positions=None), + Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=1, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=83, arg=1, argval=10, argrepr='10', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='GET_ITER', opcode=19, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='FOR_ITER', opcode=72, arg=30, argval=88, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=110, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=True, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval=4, argrepr='4', offset=54, start_offset=54, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=58, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=68, argrepr='to L2', offset=60, start_offset=60, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=77, arg=22, argval=24, argrepr='to L1', offset=64, start_offset=64, starts_line=True, line_number=6, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=68, start_offset=68, starts_line=True, line_number=7, label=2, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=3, argval=6, argrepr='6', offset=70, start_offset=70, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=58, arg=148, argval='>', argrepr='bool(>)', offset=72, start_offset=72, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=2, argval=84, argrepr='to L3', offset=76, start_offset=76, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=77, arg=30, argval=24, argrepr='to L1', offset=80, start_offset=80, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=True, line_number=8, label=3, positions=None, cache_info=None), + Instruction(opname='JUMP_FORWARD', opcode=79, arg=13, argval=114, argrepr='to L5', offset=86, start_offset=86, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='END_FOR', opcode=11, arg=None, argval=None, argrepr='', offset=88, start_offset=88, starts_line=True, line_number=3, label=4, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=90, start_offset=90, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=92, start_offset=92, starts_line=True, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=83, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=102, start_offset=102, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=104, start_offset=104, starts_line=False, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=112, start_offset=112, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST_CHECK', opcode=87, arg=0, argval='i', argrepr='i', offset=114, start_offset=114, starts_line=True, line_number=11, label=5, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=116, start_offset=116, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=40, argval=208, argrepr='to L9', offset=124, start_offset=124, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=128, start_offset=128, starts_line=True, line_number=12, label=6, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=138, start_offset=138, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=140, start_offset=140, starts_line=False, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=148, start_offset=148, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=150, start_offset=150, starts_line=True, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=5, argval=1, argrepr='1', offset=152, start_offset=152, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='BINARY_OP', opcode=45, arg=23, argval=23, argrepr='-=', offset=154, start_offset=154, starts_line=False, line_number=13, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=110, arg=0, argval='i', argrepr='i', offset=158, start_offset=158, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=160, start_offset=160, starts_line=True, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=3, argval=6, argrepr='6', offset=162, start_offset=162, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=58, arg=148, argval='>', argrepr='bool(>)', offset=164, start_offset=164, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=176, argrepr='to L7', offset=168, start_offset=168, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=77, arg=31, argval=114, argrepr='to L5', offset=172, start_offset=172, starts_line=True, line_number=15, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=176, start_offset=176, starts_line=True, line_number=16, label=7, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval=4, argrepr='4', offset=178, start_offset=178, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=58, arg=18, argval='<', argrepr='bool(<)', offset=180, start_offset=180, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=1, argval=190, argrepr='to L8', offset=184, start_offset=184, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_FORWARD', opcode=79, arg=20, argval=230, argrepr='to L10', offset=188, start_offset=188, starts_line=True, line_number=17, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=190, start_offset=190, starts_line=True, line_number=11, label=8, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=192, start_offset=192, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=208, argrepr='to L9', offset=200, start_offset=200, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=77, arg=40, argval=128, argrepr='to L6', offset=204, start_offset=204, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=208, start_offset=208, starts_line=True, line_number=19, label=9, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=83, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=218, start_offset=218, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=220, start_offset=220, starts_line=False, line_number=19, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=228, start_offset=228, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), + Instruction(opname='NOP', opcode=30, arg=None, argval=None, argrepr='', offset=230, start_offset=230, starts_line=True, line_number=20, label=10, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=5, argval=1, argrepr='1', offset=232, start_offset=232, starts_line=True, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=7, argval=0, argrepr='0', offset=234, start_offset=234, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='BINARY_OP', opcode=45, arg=11, argval=11, argrepr='/', offset=236, start_offset=236, starts_line=False, line_number=21, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=240, start_offset=240, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=242, start_offset=242, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='BEFORE_WITH', opcode=2, arg=None, argval=None, argrepr='', offset=244, start_offset=244, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='STORE_FAST', opcode=110, arg=1, argval='dodgy', argrepr='dodgy', offset=246, start_offset=246, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=248, start_offset=248, starts_line=True, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=83, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=258, start_offset=258, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=260, start_offset=260, starts_line=False, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=268, start_offset=268, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=270, start_offset=270, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=272, start_offset=272, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=274, start_offset=274, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=53, arg=2, argval=2, argrepr='', offset=276, start_offset=276, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=284, start_offset=284, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=286, start_offset=286, starts_line=True, line_number=28, label=11, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=83, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=296, start_offset=296, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=298, start_offset=298, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=306, start_offset=306, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_CONST', opcode=103, arg=0, argval=None, argrepr='None', offset=308, start_offset=308, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=310, start_offset=310, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='WITH_EXCEPT_START', opcode=44, arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=314, start_offset=314, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=1, argval=328, argrepr='to L12', offset=322, start_offset=322, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='RERAISE', opcode=102, arg=2, argval=2, argrepr='', offset=326, start_offset=326, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=328, start_offset=328, starts_line=False, line_number=25, label=12, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=332, start_offset=332, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=334, start_offset=334, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=78, arg=26, argval=286, argrepr='to L11', offset=336, start_offset=336, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=338, start_offset=338, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=342, start_offset=342, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=346, start_offset=346, starts_line=True, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='CHECK_EXC_MATCH', opcode=7, arg=None, argval=None, argrepr='', offset=356, start_offset=356, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=14, argval=390, argrepr='to L13', offset=358, start_offset=358, starts_line=False, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=364, start_offset=364, starts_line=True, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=83, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=374, start_offset=374, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=376, start_offset=376, starts_line=False, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=384, start_offset=384, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=386, start_offset=386, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=78, arg=52, argval=286, argrepr='to L11', offset=388, start_offset=388, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=390, start_offset=390, starts_line=True, line_number=22, label=13, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=392, start_offset=392, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=394, start_offset=394, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=396, start_offset=396, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=398, start_offset=398, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=400, start_offset=400, starts_line=True, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=83, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=410, start_offset=410, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=412, start_offset=412, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=420, start_offset=420, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=422, start_offset=422, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=424, start_offset=424, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=426, start_offset=426, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=428, start_offset=428, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), ] # One last piece of inspect fodder to check the default line number handling diff --git a/Lib/test/test_dynamic.py b/Lib/test/test_dynamic.py index 0aa3be6a1bde6ae..3928bbab4423c22 100644 --- a/Lib/test/test_dynamic.py +++ b/Lib/test/test_dynamic.py @@ -4,7 +4,7 @@ import sys import unittest -from test.support import swap_item, swap_attr +from test.support import is_wasi, Py_DEBUG, swap_item, swap_attr class RebindBuiltinsTests(unittest.TestCase): @@ -134,6 +134,7 @@ def test_eval_gives_lambda_custom_globals(self): self.assertEqual(foo(), 7) + @unittest.skipIf(is_wasi and Py_DEBUG, "stack depth too shallow in pydebug WASI") def test_load_global_specialization_failure_keeps_oparg(self): # https://github.com/python/cpython/issues/91625 class MyGlobals(dict): diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py index e87c275549406d1..c6b9c80efe1b54a 100644 --- a/Lib/test/test_email/test_policy.py +++ b/Lib/test/test_email/test_policy.py @@ -135,6 +135,23 @@ def test_policy_addition(self): for attr, value in expected.items(): self.assertEqual(getattr(added, attr), value) + def test_fold_utf8(self): + expected_ascii = 'Subject: =?utf-8?q?=C3=A1?=\n' + expected_utf8 = 'Subject: á\n' + + msg = email.message.EmailMessage() + s = 'á' + msg['Subject'] = s + + p_ascii = email.policy.default.clone() + p_utf8 = email.policy.default.clone(utf8=True) + + self.assertEqual(p_ascii.fold('Subject', msg['Subject']), expected_ascii) + self.assertEqual(p_utf8.fold('Subject', msg['Subject']), expected_utf8) + + self.assertEqual(p_ascii.fold('Subject', s), expected_ascii) + self.assertEqual(p_utf8.fold('Subject', s), expected_utf8) + def test_fold_zero_max_line_length(self): expected = 'Subject: =?utf-8?q?=C3=A1?=\n' diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index 69ab2a4feaa9389..a4b36a90d8815ea 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -6,6 +6,8 @@ import test.support import unittest import unittest.mock +from importlib.resources.abc import Traversable +from pathlib import Path import ensurepip import ensurepip._uninstall @@ -20,41 +22,35 @@ def test_version(self): # Test version() with tempfile.TemporaryDirectory() as tmpdir: self.touch(tmpdir, "pip-1.2.3b1-py2.py3-none-any.whl") - with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), - unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', tmpdir)): + with unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', Path(tmpdir)): self.assertEqual(ensurepip.version(), '1.2.3b1') - def test_get_packages_no_dir(self): - # Test _get_packages() without a wheel package directory - with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), - unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', None)): - packages = ensurepip._get_packages() - - # when bundled wheel packages are used, we get _PIP_VERSION + def test_version_no_dir(self): + # Test version() without a wheel package directory + with unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', None): + # when the bundled pip wheel is used, we get _PIP_VERSION self.assertEqual(ensurepip._PIP_VERSION, ensurepip.version()) - # use bundled wheel packages - self.assertIsNotNone(packages['pip'].wheel_name) + def test_selected_wheel_path_no_dir(self): + pip_filename = f'pip-{ensurepip._PIP_VERSION}-py3-none-any.whl' + with unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', None): + with ensurepip._get_pip_whl_path_ctx() as bundled_wheel_path: + self.assertEqual(pip_filename, bundled_wheel_path.name) - def test_get_packages_with_dir(self): - # Test _get_packages() with a wheel package directory + def test_selected_wheel_path_with_dir(self): + # Test _get_pip_whl_path_ctx() with a wheel package directory pip_filename = "pip-20.2.2-py2.py3-none-any.whl" with tempfile.TemporaryDirectory() as tmpdir: self.touch(tmpdir, pip_filename) - # not used, make sure that it's ignored + # not used, make sure that they're ignored + self.touch(tmpdir, "pip-1.2.3-py2.py3-none-any.whl") self.touch(tmpdir, "wheel-0.34.2-py2.py3-none-any.whl") + self.touch(tmpdir, "pip-script.py") - with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), - unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', tmpdir)): - packages = ensurepip._get_packages() - - self.assertEqual(packages['pip'].version, '20.2.2') - self.assertEqual(packages['pip'].wheel_path, - os.path.join(tmpdir, pip_filename)) - - # wheel package is ignored - self.assertEqual(sorted(packages), ['pip']) + with unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', Path(tmpdir)): + with ensurepip._get_pip_whl_path_ctx() as bundled_wheel_path: + self.assertEqual(pip_filename, bundled_wheel_path.name) class EnsurepipMixin: @@ -69,7 +65,7 @@ def setUp(self): real_devnull = os.devnull os_patch = unittest.mock.patch("ensurepip.os") patched_os = os_patch.start() - # But expose os.listdir() used by _find_packages() + # But expose os.listdir() used by _find_wheel_pkg_dir_pip() patched_os.listdir = os.listdir self.addCleanup(os_patch.stop) patched_os.devnull = real_devnull diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index d045739efa46b8b..39c1ae0ad5a0787 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -2344,6 +2344,40 @@ class SomeTuple(tuple, Enum): globals()['SomeTuple'] = SomeTuple test_pickle_dump_load(self.assertIs, SomeTuple.first) + def test_tuple_subclass_with_auto_1(self): + from collections import namedtuple + T = namedtuple('T', 'index desc') + class SomeEnum(T, Enum): + __qualname__ = 'SomeEnum' # needed for pickle protocol 4 + first = auto(), 'for the money' + second = auto(), 'for the show' + third = auto(), 'for the music' + self.assertIs(type(SomeEnum.first), SomeEnum) + self.assertEqual(SomeEnum.third.value, (3, 'for the music')) + self.assertIsInstance(SomeEnum.third.value, T) + self.assertEqual(SomeEnum.first.index, 1) + self.assertEqual(SomeEnum.second.desc, 'for the show') + globals()['SomeEnum'] = SomeEnum + globals()['T'] = T + test_pickle_dump_load(self.assertIs, SomeEnum.first) + + def test_tuple_subclass_with_auto_2(self): + from collections import namedtuple + T = namedtuple('T', 'index desc') + class SomeEnum(Enum): + __qualname__ = 'SomeEnum' # needed for pickle protocol 4 + first = T(auto(), 'for the money') + second = T(auto(), 'for the show') + third = T(auto(), 'for the music') + self.assertIs(type(SomeEnum.first), SomeEnum) + self.assertEqual(SomeEnum.third.value, (3, 'for the music')) + self.assertIsInstance(SomeEnum.third.value, T) + self.assertEqual(SomeEnum.first.value.index, 1) + self.assertEqual(SomeEnum.second.value.desc, 'for the show') + globals()['SomeEnum'] = SomeEnum + globals()['T'] = T + test_pickle_dump_load(self.assertIs, SomeEnum.first) + def test_duplicate_values_give_unique_enum_items(self): class AutoNumber(Enum): first = () diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index c57488e44aecc64..c7e76414ff07154 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1917,7 +1917,7 @@ def test_attributes(self): self.assertEqual(exc.name, 'somename') self.assertEqual(exc.path, 'somepath') - msg = "'invalid' is an invalid keyword argument for ImportError" + msg = r"ImportError\(\) got an unexpected keyword argument 'invalid'" with self.assertRaisesRegex(TypeError, msg): ImportError('test', invalid='keyword') diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 203dd6fe57dcd99..6d734d052454d34 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -6,7 +6,9 @@ import struct import sys import unittest -from test.support import verbose, cpython_only, get_pagesize +from test.support import ( + cpython_only, get_pagesize, is_apple, requires_subprocess, verbose +) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink @@ -56,8 +58,10 @@ def get_lockdata(): else: start_len = "qq" - if (sys.platform.startswith(('netbsd', 'freebsd', 'openbsd')) - or sys.platform == 'darwin'): + if ( + sys.platform.startswith(('netbsd', 'freebsd', 'openbsd')) + or is_apple + ): if struct.calcsize('l') == 8: off_t = 'l' pid_t = 'i' @@ -157,6 +161,7 @@ def test_flock(self): self.assertRaises(TypeError, fcntl.flock, 'spam', fcntl.LOCK_SH) @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError") + @requires_subprocess() def test_lockf_exclusive(self): self.f = open(TESTFN, 'wb+') cmd = fcntl.LOCK_EX | fcntl.LOCK_NB @@ -169,6 +174,7 @@ def test_lockf_exclusive(self): self.assertEqual(p.exitcode, 0) @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError") + @requires_subprocess() def test_lockf_share(self): self.f = open(TESTFN, 'wb+') cmd = fcntl.LOCK_SH | fcntl.LOCK_NB diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index 786d9186634305b..b3ad41d2588c4cb 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -151,7 +151,7 @@ def test_buffer_sizes(self): print('6. Inplace') savestdout = sys.stdout try: - fi = FileInput(files=(t1, t2, t3, t4), inplace=1, encoding="utf-8") + fi = FileInput(files=(t1, t2, t3, t4), inplace=True, encoding="utf-8") for line in fi: line = line[:-1].upper() print(line) @@ -256,7 +256,7 @@ def test_detached_stdin_binary_mode(self): def test_file_opening_hook(self): try: # cannot use openhook and inplace mode - fi = FileInput(inplace=1, openhook=lambda f, m: None) + fi = FileInput(inplace=True, openhook=lambda f, m: None) self.fail("FileInput should raise if both inplace " "and openhook arguments are given") except ValueError: diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index 06d9b454add34cd..06d5a8abf320835 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -484,6 +484,14 @@ def testInvalidFd(self): import msvcrt self.assertRaises(OSError, msvcrt.get_osfhandle, make_bad_fd()) + def testBooleanFd(self): + for fd in False, True: + with self.assertWarnsRegex(RuntimeWarning, + 'bool is used as a file descriptor') as cm: + f = self.FileIO(fd, closefd=False) + f.close() + self.assertEqual(cm.filename, __file__) + def testBadModeArgument(self): # verify that we get a sensible error message for bad mode argument bad_mode = "qwerty" diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index af3cb214ab0ac11..b45bd098a366845 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -1314,6 +1314,33 @@ def test_float_format_testfile(self): self.assertEqual(float(format(f, fmt2)), float(rhs)) self.assertEqual(float(format(-f, fmt2)), float('-' + rhs)) + def test_complex_handling(self): + # See issue gh-102840 for more details. + + a = F(1, 2) + b = 1j + message = "unsupported operand type(s) for %s: '%s' and '%s'" + # test forward + self.assertRaisesMessage(TypeError, + message % ("%", "Fraction", "complex"), + operator.mod, a, b) + self.assertRaisesMessage(TypeError, + message % ("//", "Fraction", "complex"), + operator.floordiv, a, b) + self.assertRaisesMessage(TypeError, + message % ("divmod()", "Fraction", "complex"), + divmod, a, b) + # test reverse + self.assertRaisesMessage(TypeError, + message % ("%", "complex", "Fraction"), + operator.mod, b, a) + self.assertRaisesMessage(TypeError, + message % ("//", "complex", "Fraction"), + operator.floordiv, b, a) + self.assertRaisesMessage(TypeError, + message % ("divmod()", "complex", "Fraction"), + divmod, b, a) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 7f17666a8d9697b..baed03d92b9e561 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -55,6 +55,27 @@ class C: # The reference was released by .clear() self.assertIs(None, wr()) + def test_clear_locals_after_f_locals_access(self): + # see gh-113939 + class C: + pass + + wr = None + def inner(): + nonlocal wr + c = C() + wr = weakref.ref(c) + 1/0 + + try: + inner() + except ZeroDivisionError as exc: + support.gc_collect() + self.assertIsNotNone(wr()) + exc.__traceback__.tb_next.tb_frame.clear() + support.gc_collect() + self.assertIsNone(wr()) + def test_clear_does_not_clear_specials(self): class C: pass diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index 2f191ea7a44c161..81115e9db888cf2 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -18,6 +18,7 @@ from unittest import TestCase, skipUnless from test import support +from test.support import requires_subprocess from test.support import threading_helper from test.support import socket_helper from test.support import warnings_helper @@ -900,6 +901,7 @@ def retr(): @skipUnless(ssl, "SSL not available") +@requires_subprocess() class TestTLS_FTPClassMixin(TestFTPClass): """Repeat TestFTPClass tests starting the TLS layer for both control and data connections first. @@ -916,6 +918,7 @@ def setUp(self, encoding=DEFAULT_ENCODING): @skipUnless(ssl, "SSL not available") +@requires_subprocess() class TestTLS_FTPClass(TestCase): """Specific TLS_FTP class tests.""" diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 1d71dd9e262a6ae..b01f344cb14a1a7 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1,7 +1,7 @@ import unittest import unittest.mock from test.support import (verbose, refcount_test, - cpython_only, requires_subprocess) + cpython_only, requires_subprocess, Py_GIL_DISABLED) from test.support.import_helper import import_module from test.support.os_helper import temp_dir, TESTFN, unlink from test.support.script_helper import assert_python_ok, make_script @@ -815,6 +815,15 @@ def test_freeze(self): self.assertEqual(gc.get_freeze_count(), 0) def test_get_objects(self): + gc.collect() + l = [] + l.append(l) + self.assertTrue( + any(l is element for element in gc.get_objects()) + ) + + @unittest.skipIf(Py_GIL_DISABLED, 'need generational GC') + def test_get_objects_generations(self): gc.collect() l = [] l.append(l) @@ -1225,7 +1234,7 @@ def test_refcount_errors(self): p.stderr.close() # Verify that stderr has a useful error message: self.assertRegex(stderr, - br'gc\.c:[0-9]+: gc_decref: Assertion "gc_get_refs\(g\) > 0" failed.') + br'gc.*\.c:[0-9]+: .*: Assertion "gc_get_refs\(.+\) .*" failed.') self.assertRegex(stderr, br'refcount is too small') # "address : 0x7fb5062efc18" diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index 4f311c2d498e9f4..f407ee3caf154c2 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -7,9 +7,9 @@ import sys import unittest import warnings -from test.support import is_emscripten -from test.support import os_helper -from test.support import warnings_helper +from test.support import ( + is_apple, is_emscripten, os_helper, warnings_helper +) from test.support.script_helper import assert_python_ok from test.support.os_helper import FakePath @@ -165,6 +165,12 @@ def test_exists_fd(self): os.close(w) self.assertFalse(self.pathmodule.exists(r)) + def test_exists_bool(self): + for fd in False, True: + with self.assertWarnsRegex(RuntimeWarning, + 'bool is used as a file descriptor'): + self.pathmodule.exists(fd) + def test_isdir(self): filename = os_helper.TESTFN bfilename = os.fsencode(filename) @@ -483,12 +489,16 @@ def test_abspath_issue3426(self): self.assertIsInstance(abspath(path), str) def test_nonascii_abspath(self): - if (os_helper.TESTFN_UNDECODABLE - # macOS and Emscripten deny the creation of a directory with an - # invalid UTF-8 name. Windows allows creating a directory with an - # arbitrary bytes name, but fails to enter this directory - # (when the bytes name is used). - and sys.platform not in ('win32', 'darwin', 'emscripten', 'wasi')): + if ( + os_helper.TESTFN_UNDECODABLE + # Apple platforms and Emscripten/WASI deny the creation of a + # directory with an invalid UTF-8 name. Windows allows creating a + # directory with an arbitrary bytes name, but fails to enter this + # directory (when the bytes name is used). + and sys.platform not in { + "win32", "emscripten", "wasi" + } and not is_apple + ): name = os_helper.TESTFN_UNDECODABLE elif os_helper.TESTFN_NONASCII: name = os_helper.TESTFN_NONASCII diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 9fa6ecf9c08e279..d762ec6102ab8aa 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -30,8 +30,9 @@ import unittest from test import support -from test.support import os_helper -from test.support import threading_helper +from test.support import ( + is_apple, os_helper, requires_subprocess, threading_helper +) support.requires_working_socket(module=True) @@ -410,8 +411,8 @@ def close_conn(): reader.close() return body - @unittest.skipIf(sys.platform == 'darwin', - 'undecodable name cannot always be decoded on macOS') + @unittest.skipIf(is_apple, + 'undecodable name cannot always be decoded on Apple platforms') @unittest.skipIf(sys.platform == 'win32', 'undecodable name cannot be decoded on win32') @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, @@ -422,11 +423,11 @@ def test_undecodable_filename(self): with open(os.path.join(self.tempdir, filename), 'wb') as f: f.write(os_helper.TESTFN_UNDECODABLE) response = self.request(self.base_url + '/') - if sys.platform == 'darwin': - # On Mac OS the HFS+ filesystem replaces bytes that aren't valid - # UTF-8 into a percent-encoded value. + if is_apple: + # On Apple platforms the HFS+ filesystem replaces bytes that + # aren't valid UTF-8 into a percent-encoded value. for name in os.listdir(self.tempdir): - if name != 'test': # Ignore a filename created in setUp(). + if name != 'test': # Ignore a filename created in setUp(). filename = name break body = self.check_status_and_reason(response, HTTPStatus.OK) @@ -697,6 +698,7 @@ def test_html_escape_filename(self): @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, "This test can't be run reliably as root (issue #13308).") +@requires_subprocess() class CGIHTTPServerTestCase(BaseTestCase): class request_handler(NoLogRequestHandler, CGIHTTPRequestHandler): _test_case_self = None # populated by each setUp() method call. diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 936edea3cad70c6..cc387afa3919099 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -39,11 +39,9 @@ from test import support from test.support.script_helper import ( assert_python_ok, assert_python_failure, run_python_until_end) -from test.support import import_helper -from test.support import os_helper -from test.support import threading_helper -from test.support import warnings_helper -from test.support import skip_if_sanitizer +from test.support import ( + import_helper, is_apple, os_helper, skip_if_sanitizer, threading_helper, warnings_helper +) from test.support.os_helper import FakePath import codecs @@ -606,10 +604,10 @@ def test_raw_bytes_io(self): self.read_ops(f, True) def test_large_file_ops(self): - # On Windows and Mac OSX this test consumes large resources; It takes - # a long time to build the >2 GiB file and takes >2 GiB of disk space - # therefore the resource must be enabled to run this test. - if sys.platform[:3] == 'win' or sys.platform == 'darwin': + # On Windows and Apple platforms this test consumes large resources; It + # takes a long time to build the >2 GiB file and takes >2 GiB of disk + # space therefore the resource must be enabled to run this test. + if sys.platform[:3] == 'win' or is_apple: support.requires( 'largefile', 'test requires %s bytes and a long time to run' % self.LARGE) @@ -2499,6 +2497,28 @@ def test_interleaved_read_write(self): f.flush() self.assertEqual(raw.getvalue(), b'a2c') + def test_read1_after_write(self): + with self.BytesIO(b'abcdef') as raw: + with self.tp(raw, 3) as f: + f.write(b"1") + self.assertEqual(f.read1(1), b'b') + f.flush() + self.assertEqual(raw.getvalue(), b'1bcdef') + with self.BytesIO(b'abcdef') as raw: + with self.tp(raw, 3) as f: + f.write(b"1") + self.assertEqual(f.read1(), b'bcd') + f.flush() + self.assertEqual(raw.getvalue(), b'1bcdef') + with self.BytesIO(b'abcdef') as raw: + with self.tp(raw, 3) as f: + f.write(b"1") + # XXX: read(100) returns different numbers of bytes + # in Python and C implementations. + self.assertEqual(f.read1(100)[:3], b'bcd') + f.flush() + self.assertEqual(raw.getvalue(), b'1bcdef') + def test_interleaved_readline_write(self): with self.BytesIO(b'ab\ncdef\ng\n') as raw: with self.tp(raw) as f: @@ -3652,10 +3672,8 @@ def _check_create_at_shutdown(self, **kwargs): codecs.lookup('utf-8') class C: - def __init__(self): - self.buf = io.BytesIO() def __del__(self): - io.TextIOWrapper(self.buf, **{kwargs}) + io.TextIOWrapper(io.BytesIO(), **{kwargs}) print("ok") c = C() """.format(iomod=iomod, kwargs=kwargs) diff --git a/Lib/test/test_launcher.py b/Lib/test/test_launcher.py index 3da6173cfd3f134..2528a51240fbf76 100644 --- a/Lib/test/test_launcher.py +++ b/Lib/test/test_launcher.py @@ -90,6 +90,12 @@ "test-command=TEST_EXE.exe", ]) + +def quote(s): + s = str(s) + return f'"{s}"' if " " in s else s + + def create_registry_data(root, data): def _create_registry_data(root, key, value): if isinstance(value, dict): @@ -542,10 +548,10 @@ def test_virtualenv_with_env(self): data1 = self.run_py([], env={**env, "PY_PYTHON": "PythonTestSuite/3"}) data2 = self.run_py(["-V:PythonTestSuite/3"], env={**env, "PY_PYTHON": "PythonTestSuite/3"}) # Compare stdout, because stderr goes via ascii - self.assertEqual(data1["stdout"].strip(), str(venv_exe)) + self.assertEqual(data1["stdout"].strip(), quote(venv_exe)) self.assertEqual(data1["SearchInfo.lowPriorityTag"], "True") # Ensure passing the argument doesn't trigger the same behaviour - self.assertNotEqual(data2["stdout"].strip(), str(venv_exe)) + self.assertNotEqual(data2["stdout"].strip(), quote(venv_exe)) self.assertNotEqual(data2["SearchInfo.lowPriorityTag"], "True") def test_py_shebang(self): @@ -554,7 +560,7 @@ def test_py_shebang(self): data = self.run_py([script, "-postarg"]) self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) self.assertEqual("3.100", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y.exe -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"X.Y.exe -prearg {quote(script)} -postarg", data["stdout"].strip()) def test_python_shebang(self): with self.py_ini(TEST_PY_DEFAULTS): @@ -562,7 +568,7 @@ def test_python_shebang(self): data = self.run_py([script, "-postarg"]) self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) self.assertEqual("3.100", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y.exe -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"X.Y.exe -prearg {quote(script)} -postarg", data["stdout"].strip()) def test_py2_shebang(self): with self.py_ini(TEST_PY_DEFAULTS): @@ -570,7 +576,8 @@ def test_py2_shebang(self): data = self.run_py([script, "-postarg"]) self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) self.assertEqual("3.100-32", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y-32.exe -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"X.Y-32.exe -prearg {quote(script)} -postarg", + data["stdout"].strip()) def test_py3_shebang(self): with self.py_ini(TEST_PY_DEFAULTS): @@ -578,7 +585,8 @@ def test_py3_shebang(self): data = self.run_py([script, "-postarg"]) self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) self.assertEqual("3.100-arm64", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y-arm64.exe -X fake_arg_for_test -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"X.Y-arm64.exe -X fake_arg_for_test -prearg {quote(script)} -postarg", + data["stdout"].strip()) def test_py_shebang_nl(self): with self.py_ini(TEST_PY_DEFAULTS): @@ -586,7 +594,8 @@ def test_py_shebang_nl(self): data = self.run_py([script, "-postarg"]) self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) self.assertEqual("3.100", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y.exe -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"X.Y.exe -prearg {quote(script)} -postarg", + data["stdout"].strip()) def test_py2_shebang_nl(self): with self.py_ini(TEST_PY_DEFAULTS): @@ -594,7 +603,8 @@ def test_py2_shebang_nl(self): data = self.run_py([script, "-postarg"]) self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) self.assertEqual("3.100-32", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y-32.exe -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"X.Y-32.exe -prearg {quote(script)} -postarg", + data["stdout"].strip()) def test_py3_shebang_nl(self): with self.py_ini(TEST_PY_DEFAULTS): @@ -602,7 +612,8 @@ def test_py3_shebang_nl(self): data = self.run_py([script, "-postarg"]) self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) self.assertEqual("3.100-arm64", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y-arm64.exe -X fake_arg_for_test -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"X.Y-arm64.exe -X fake_arg_for_test -prearg {quote(script)} -postarg", + data["stdout"].strip()) def test_py_shebang_short_argv0(self): with self.py_ini(TEST_PY_DEFAULTS): @@ -630,7 +641,8 @@ def test_search_path(self): [script, "-postarg"], env={"PATH": f"{exe.parent};{os.getenv('PATH')}"}, ) - self.assertEqual(f"{exe} -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"{quote(exe)} -prearg {quote(script)} -postarg", + data["stdout"].strip()) def test_search_path_exe(self): # Leave the .exe on the name to ensure we don't add it a second time @@ -643,7 +655,8 @@ def test_search_path_exe(self): [script, "-postarg"], env={"PATH": f"{exe.parent};{os.getenv('PATH')}"}, ) - self.assertEqual(f"{exe} -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"{quote(exe)} -prearg {quote(script)} -postarg", + data["stdout"].strip()) def test_recursive_search_path(self): stem = self.get_py_exe().stem @@ -654,7 +667,7 @@ def test_recursive_search_path(self): env={"PATH": f"{self.get_py_exe().parent};{os.getenv('PATH')}"}, ) # The recursive search is ignored and we get normal "py" behavior - self.assertEqual(f"X.Y.exe {script}", data["stdout"].strip()) + self.assertEqual(f"X.Y.exe {quote(script)}", data["stdout"].strip()) def test_install(self): data = self.run_py(["-V:3.10"], env={"PYLAUNCHER_ALWAYS_INSTALL": "1"}, expect_returncode=111) @@ -674,7 +687,7 @@ def test_literal_shebang_absolute(self): with self.script("#! C:/some_random_app -witharg") as script: data = self.run_py([script]) self.assertEqual( - f"C:\\some_random_app -witharg {script}", + f"C:\\some_random_app -witharg {quote(script)}", data["stdout"].strip(), ) @@ -682,7 +695,7 @@ def test_literal_shebang_relative(self): with self.script("#! ..\\some_random_app -witharg") as script: data = self.run_py([script]) self.assertEqual( - f"{script.parent.parent}\\some_random_app -witharg {script}", + f"{quote(script.parent.parent / 'some_random_app')} -witharg {quote(script)}", data["stdout"].strip(), ) @@ -690,14 +703,14 @@ def test_literal_shebang_quoted(self): with self.script('#! "some random app" -witharg') as script: data = self.run_py([script]) self.assertEqual( - f'"{script.parent}\\some random app" -witharg {script}', + f"{quote(script.parent / 'some random app')} -witharg {quote(script)}", data["stdout"].strip(), ) with self.script('#! some" random "app -witharg') as script: data = self.run_py([script]) self.assertEqual( - f'"{script.parent}\\some random app" -witharg {script}', + f"{quote(script.parent / 'some random app')} -witharg {quote(script)}", data["stdout"].strip(), ) @@ -705,7 +718,7 @@ def test_literal_shebang_quoted_escape(self): with self.script('#! some\\" random "app -witharg') as script: data = self.run_py([script]) self.assertEqual( - f'"{script.parent}\\some\\ random app" -witharg {script}', + f"{quote(script.parent / 'some/ random app')} -witharg {quote(script)}", data["stdout"].strip(), ) @@ -714,7 +727,7 @@ def test_literal_shebang_command(self): with self.script('#! test-command arg1') as script: data = self.run_py([script]) self.assertEqual( - f"TEST_EXE.exe arg1 {script}", + f"TEST_EXE.exe arg1 {quote(script)}", data["stdout"].strip(), ) @@ -723,7 +736,7 @@ def test_literal_shebang_invalid_template(self): data = self.run_py([script]) expect = script.parent / "/usr/bin/not-python" self.assertEqual( - f"{expect} arg1 {script}", + f"{quote(expect)} arg1 {quote(script)}", data["stdout"].strip(), ) @@ -746,8 +759,8 @@ def test_shebang_command_in_venv(self): with self.script(f'#! /usr/bin/env {stem} arg1') as script: data = self.run_py([script], env=env) - self.assertEqual(data["stdout"].strip(), f"{venv_exe} arg1 {script}") + self.assertEqual(data["stdout"].strip(), f"{quote(venv_exe)} arg1 {quote(script)}") with self.script(f'#! /usr/bin/env {exe.stem} arg1') as script: data = self.run_py([script], env=env) - self.assertEqual(data["stdout"].strip(), f"{exe} arg1 {script}") + self.assertEqual(data["stdout"].strip(), f"{quote(exe)} arg1 {quote(script)}") diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index f95a78aff0c7118..2868dd01545b95f 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -156,6 +156,18 @@ def method(self): self.assertEqual(C.y, [4, 4, 4, 4, 4]) self.assertIs(C().method(), C) + def test_references_super(self): + code = """ + res = [super for x in [1]] + """ + self._check_in_scopes(code, outputs={"res": [super]}) + + def test_references___class__(self): + code = """ + res = [__class__ for x in [1]] + """ + self._check_in_scopes(code, raises=NameError) + def test_inner_cell_shadows_outer(self): code = """ items = [(lambda: i) for i in range(5)] diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 908e242b85f5e7a..888523227c2ac43 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -4089,6 +4089,7 @@ def test_queue_listener(self): self.que_logger.critical(self.next_message()) finally: listener.stop() + listener.stop() # gh-114706 - ensure no crash if called again self.assertTrue(handler.matches(levelno=logging.WARNING, message='1')) self.assertTrue(handler.matches(levelno=logging.ERROR, message='2')) self.assertTrue(handler.matches(levelno=logging.CRITICAL, message='3')) diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index d84faad0eb34069..c52c014185bec7a 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -1127,12 +1127,14 @@ def test_add_from_string(self): # Add a string starting with 'From ' to the mailbox key = self._box.add('From foo@bar blah\nFrom: foo\n\n0\n') self.assertEqual(self._box[key].get_from(), 'foo@bar blah') + self.assertEqual(self._box[key].get_unixfrom(), 'From foo@bar blah') self.assertEqual(self._box[key].get_payload(), '0\n') def test_add_from_bytes(self): # Add a byte string starting with 'From ' to the mailbox key = self._box.add(b'From foo@bar blah\nFrom: foo\n\n0\n') self.assertEqual(self._box[key].get_from(), 'foo@bar blah') + self.assertEqual(self._box[key].get_unixfrom(), 'From foo@bar blah') self.assertEqual(self._box[key].get_payload(), '0\n') def test_add_mbox_or_mmdf_message(self): @@ -1667,18 +1669,23 @@ def test_initialize_with_unixfrom(self): msg = mailbox.Message(_sample_message) msg.set_unixfrom('From foo@bar blah') msg = mailbox.mboxMessage(msg) - self.assertEqual(msg.get_from(), 'foo@bar blah', msg.get_from()) + self.assertEqual(msg.get_from(), 'foo@bar blah') + self.assertEqual(msg.get_unixfrom(), 'From foo@bar blah') def test_from(self): # Get and set "From " line msg = mailbox.mboxMessage(_sample_message) self._check_from(msg) + self.assertIsNone(msg.get_unixfrom()) msg.set_from('foo bar') self.assertEqual(msg.get_from(), 'foo bar') + self.assertIsNone(msg.get_unixfrom()) msg.set_from('foo@bar', True) self._check_from(msg, 'foo@bar') + self.assertIsNone(msg.get_unixfrom()) msg.set_from('blah@temp', time.localtime()) self._check_from(msg, 'blah@temp') + self.assertIsNone(msg.get_unixfrom()) def test_flags(self): # Use get_flags(), set_flags(), add_flag(), remove_flag() @@ -1866,6 +1873,7 @@ def test_maildir_to_mboxmmdf(self): self.assertEqual(msg.get_flags(), result) self.assertEqual(msg.get_from(), 'MAILER-DAEMON %s' % time.asctime(time.gmtime(0.0))) + self.assertIsNone(msg.get_unixfrom()) msg_maildir.set_subdir('cur') self.assertEqual(class_(msg_maildir).get_flags(), 'RODFA') @@ -1914,10 +1922,12 @@ def test_mboxmmdf_to_mboxmmdf(self): msg_mboxMMDF = class_(_sample_message) msg_mboxMMDF.set_flags('RODFA') msg_mboxMMDF.set_from('foo@bar') + self.assertIsNone(msg_mboxMMDF.get_unixfrom()) for class2_ in (mailbox.mboxMessage, mailbox.MMDFMessage): msg2 = class2_(msg_mboxMMDF) self.assertEqual(msg2.get_flags(), 'RODFA') self.assertEqual(msg2.get_from(), 'foo@bar') + self.assertIsNone(msg2.get_unixfrom()) def test_mboxmmdf_to_mh(self): # Convert mboxMessage and MMDFMessage to MHMessage diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 6e17e010e7f355d..615568e6af21028 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -1,5 +1,5 @@ from test import support -from test.support import os_helper, requires_debug_ranges +from test.support import is_apple_mobile, os_helper, requires_debug_ranges from test.support.script_helper import assert_python_ok import array import io @@ -286,7 +286,7 @@ def test_recursion_limit(self): #if os.name == 'nt' and support.Py_DEBUG: if os.name == 'nt': MAX_MARSHAL_STACK_DEPTH = 1000 - elif sys.platform == 'wasi': + elif sys.platform == 'wasi' or is_apple_mobile: MAX_MARSHAL_STACK_DEPTH = 1500 else: MAX_MARSHAL_STACK_DEPTH = 2000 diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index d64aee71fc48b13..01bba0ac2eed5a4 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -96,14 +96,12 @@ def test_non_standard_types(self): # First try strict eq(self.db.guess_type('foo.xul', strict=True), (None, None)) eq(self.db.guess_extension('image/jpg', strict=True), None) - eq(self.db.guess_extension('image/webp', strict=True), None) # And then non-strict eq(self.db.guess_type('foo.xul', strict=False), ('text/xul', None)) eq(self.db.guess_type('foo.XUL', strict=False), ('text/xul', None)) eq(self.db.guess_type('foo.invalid', strict=False), (None, None)) eq(self.db.guess_extension('image/jpg', strict=False), '.jpg') eq(self.db.guess_extension('image/JPG', strict=False), '.jpg') - eq(self.db.guess_extension('image/webp', strict=False), '.webp') def test_filename_with_url_delimiters(self): # bpo-38449: URL delimiters cases should be handled also. @@ -183,6 +181,7 @@ def check_extensions(): self.assertEqual(mimetypes.guess_extension('application/xml'), '.xsl') self.assertEqual(mimetypes.guess_extension('audio/mpeg'), '.mp3') self.assertEqual(mimetypes.guess_extension('image/avif'), '.avif') + self.assertEqual(mimetypes.guess_extension('image/webp'), '.webp') self.assertEqual(mimetypes.guess_extension('image/jpeg'), '.jpg') self.assertEqual(mimetypes.guess_extension('image/tiff'), '.tiff') self.assertEqual(mimetypes.guess_extension('message/rfc822'), '.eml') diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index b89621e08577be0..ac759757d24659c 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -1,5 +1,5 @@ from test.support import ( - requires, _2G, _4G, gc_collect, cpython_only, is_emscripten + requires, _2G, _4G, gc_collect, cpython_only, is_emscripten, is_apple, ) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink @@ -1067,7 +1067,7 @@ def tearDown(self): unlink(TESTFN) def _make_test_file(self, num_zeroes, tail): - if sys.platform[:3] == 'win' or sys.platform == 'darwin': + if sys.platform[:3] == 'win' or is_apple: requires('largefile', 'test requires %s bytes and a long time to run' % str(0x180000000)) f = open(TESTFN, 'w+b') diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index a64d1ed79decd83..60b6326bfbad5e9 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1466,9 +1466,8 @@ def func(): ('branch', 'func', 4, 4), ('line', 'func', 5), ('line', 'meth', 1), - ('jump', 'func', 5, 5), - ('jump', 'func', 5, '[offset=114]'), - ('branch', 'func', '[offset=120]', '[offset=124]'), + ('jump', 'func', 5, '[offset=118]'), + ('branch', 'func', '[offset=122]', '[offset=126]'), ('line', 'get_events', 11)]) self.check_events(func, recorders = FLOW_AND_LINE_RECORDERS, expected = [ @@ -1482,9 +1481,8 @@ def func(): ('line', 'func', 5), ('line', 'meth', 1), ('return', 'meth', None), - ('jump', 'func', 5, 5), - ('jump', 'func', 5, '[offset=114]'), - ('branch', 'func', '[offset=120]', '[offset=124]'), + ('jump', 'func', 5, '[offset=118]'), + ('branch', 'func', '[offset=122]', '[offset=126]'), ('return', 'func', None), ('line', 'get_events', 11)]) diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index aefcb98f1c30eb0..9cb03e3cd5de8d7 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -981,6 +981,62 @@ def test_ismount(self): self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$")) self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$\\")) + def test_isreserved(self): + self.assertFalse(ntpath.isreserved('')) + self.assertFalse(ntpath.isreserved('.')) + self.assertFalse(ntpath.isreserved('..')) + self.assertFalse(ntpath.isreserved('/')) + self.assertFalse(ntpath.isreserved('/foo/bar')) + # A name that ends with a space or dot is reserved. + self.assertTrue(ntpath.isreserved('foo.')) + self.assertTrue(ntpath.isreserved('foo ')) + # ASCII control characters are reserved. + self.assertTrue(ntpath.isreserved('\foo')) + # Wildcard characters, colon, and pipe are reserved. + self.assertTrue(ntpath.isreserved('foo*bar')) + self.assertTrue(ntpath.isreserved('foo?bar')) + self.assertTrue(ntpath.isreserved('foo"bar')) + self.assertTrue(ntpath.isreserved('foo<bar')) + self.assertTrue(ntpath.isreserved('foo>bar')) + self.assertTrue(ntpath.isreserved('foo:bar')) + self.assertTrue(ntpath.isreserved('foo|bar')) + # Case-insensitive DOS-device names are reserved. + self.assertTrue(ntpath.isreserved('nul')) + self.assertTrue(ntpath.isreserved('aux')) + self.assertTrue(ntpath.isreserved('prn')) + self.assertTrue(ntpath.isreserved('con')) + self.assertTrue(ntpath.isreserved('conin$')) + self.assertTrue(ntpath.isreserved('conout$')) + # COM/LPT + 1-9 or + superscript 1-3 are reserved. + self.assertTrue(ntpath.isreserved('COM1')) + self.assertTrue(ntpath.isreserved('LPT9')) + self.assertTrue(ntpath.isreserved('com\xb9')) + self.assertTrue(ntpath.isreserved('com\xb2')) + self.assertTrue(ntpath.isreserved('lpt\xb3')) + # DOS-device name matching ignores characters after a dot or + # a colon and also ignores trailing spaces. + self.assertTrue(ntpath.isreserved('NUL.txt')) + self.assertTrue(ntpath.isreserved('PRN ')) + self.assertTrue(ntpath.isreserved('AUX .txt')) + self.assertTrue(ntpath.isreserved('COM1:bar')) + self.assertTrue(ntpath.isreserved('LPT9 :bar')) + # DOS-device names are only matched at the beginning + # of a path component. + self.assertFalse(ntpath.isreserved('bar.com9')) + self.assertFalse(ntpath.isreserved('bar.lpt9')) + # The entire path is checked, except for the drive. + self.assertTrue(ntpath.isreserved('c:/bar/baz/NUL')) + self.assertTrue(ntpath.isreserved('c:/NUL/bar/baz')) + self.assertFalse(ntpath.isreserved('//./NUL')) + # Bytes are supported. + self.assertFalse(ntpath.isreserved(b'')) + self.assertFalse(ntpath.isreserved(b'.')) + self.assertFalse(ntpath.isreserved(b'..')) + self.assertFalse(ntpath.isreserved(b'/')) + self.assertFalse(ntpath.isreserved(b'/foo/bar')) + self.assertTrue(ntpath.isreserved(b'foo.')) + self.assertTrue(ntpath.isreserved(b'nul')) + def assertEqualCI(self, s1, s2): """Assert that two strings are equal ignoring case differences.""" self.assertEqual(s1.lower(), s2.lower()) diff --git a/Lib/test/test_optimizer.py b/Lib/test/test_optimizer.py new file mode 100644 index 000000000000000..c8554c40df4b2de --- /dev/null +++ b/Lib/test/test_optimizer.py @@ -0,0 +1,78 @@ +import unittest +import types +from test.support import import_helper + + +_testinternalcapi = import_helper.import_module("_testinternalcapi") + + +class TestRareEventCounters(unittest.TestCase): + def test_set_class(self): + class A: + pass + class B: + pass + a = A() + + orig_counter = _testinternalcapi.get_rare_event_counters()["set_class"] + a.__class__ = B + self.assertEqual( + orig_counter + 1, + _testinternalcapi.get_rare_event_counters()["set_class"] + ) + + def test_set_bases(self): + class A: + pass + class B: + pass + class C(B): + pass + + orig_counter = _testinternalcapi.get_rare_event_counters()["set_bases"] + C.__bases__ = (A,) + self.assertEqual( + orig_counter + 1, + _testinternalcapi.get_rare_event_counters()["set_bases"] + ) + + def test_set_eval_frame_func(self): + orig_counter = _testinternalcapi.get_rare_event_counters()["set_eval_frame_func"] + _testinternalcapi.set_eval_frame_record([]) + self.assertEqual( + orig_counter + 1, + _testinternalcapi.get_rare_event_counters()["set_eval_frame_func"] + ) + _testinternalcapi.set_eval_frame_default() + + def test_builtin_dict(self): + orig_counter = _testinternalcapi.get_rare_event_counters()["builtin_dict"] + if isinstance(__builtins__, types.ModuleType): + builtins = __builtins__.__dict__ + else: + builtins = __builtins__ + builtins["FOO"] = 42 + self.assertEqual( + orig_counter + 1, + _testinternalcapi.get_rare_event_counters()["builtin_dict"] + ) + del builtins["FOO"] + + def test_func_modification(self): + def func(x=0): + pass + + for attribute in ( + "__code__", + "__defaults__", + "__kwdefaults__" + ): + orig_counter = _testinternalcapi.get_rare_event_counters()["func_modification"] + setattr(func, attribute, getattr(func, attribute)) + self.assertEqual( + orig_counter + 1, + _testinternalcapi.get_rare_event_counters()["func_modification"] + ) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 98b30d2108a1a12..2c8823ae47c726e 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2195,12 +2195,15 @@ def test_chmod(self): class TestInvalidFD(unittest.TestCase): singles = ["fchdir", "dup", "fdatasync", "fstat", "fstatvfs", "fsync", "tcgetpgrp", "ttyname"] + singles_fildes = {"fchdir", "fdatasync", "fsync"} #singles.append("close") #We omit close because it doesn't raise an exception on some platforms def get_single(f): def helper(self): if hasattr(os, f): self.check(getattr(os, f)) + if f in self.singles_fildes: + self.check_bool(getattr(os, f)) return helper for f in singles: locals()["test_"+f] = get_single(f) @@ -2214,8 +2217,16 @@ def check(self, f, *args, **kwargs): self.fail("%r didn't raise an OSError with a bad file descriptor" % f) + def check_bool(self, f, *args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) + for fd in False, True: + with self.assertRaises(RuntimeWarning): + f(fd, *args, **kwargs) + def test_fdopen(self): self.check(os.fdopen, encoding="utf-8") + self.check_bool(os.fdopen, encoding="utf-8") @unittest.skipUnless(hasattr(os, 'isatty'), 'test needs os.isatty()') def test_isatty(self): @@ -2277,11 +2288,14 @@ def test_fchown(self): def test_fpathconf(self): self.check(os.pathconf, "PC_NAME_MAX") self.check(os.fpathconf, "PC_NAME_MAX") + self.check_bool(os.pathconf, "PC_NAME_MAX") + self.check_bool(os.fpathconf, "PC_NAME_MAX") @unittest.skipUnless(hasattr(os, 'ftruncate'), 'test needs os.ftruncate()') def test_ftruncate(self): self.check(os.truncate, 0) self.check(os.ftruncate, 0) + self.check_bool(os.truncate, 0) @unittest.skipUnless(hasattr(os, 'lseek'), 'test needs os.lseek()') def test_lseek(self): @@ -3129,10 +3143,9 @@ def cleanup(): if support.verbose: print(" without access:", stat2) - # We cannot get st_dev/st_ino, so ensure those are 0 or else our test - # is not set up correctly - self.assertEqual(0, stat2.st_dev) - self.assertEqual(0, stat2.st_ino) + # We may not get st_dev/st_ino, so ensure those are 0 or match + self.assertIn(stat2.st_dev, (0, stat1.st_dev)) + self.assertIn(stat2.st_ino, (0, stat1.st_ino)) # st_mode and st_size should match (for a normal file, at least) self.assertEqual(stat1.st_mode, stat2.st_mode) @@ -3849,6 +3862,7 @@ def test_does_not_crash(self): self.assertGreaterEqual(size.columns, 0) self.assertGreaterEqual(size.lines, 0) + @support.requires_subprocess() def test_stty_match(self): """Check if stty returns the same results @@ -4537,15 +4551,49 @@ def test_dup2(self): self.assertEqual(os.dup2(fd, fd3, inheritable=False), fd3) self.assertFalse(os.get_inheritable(fd3)) - @unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()") - def test_openpty(self): - master_fd, slave_fd = os.openpty() - self.addCleanup(os.close, master_fd) - self.addCleanup(os.close, slave_fd) - self.assertEqual(os.get_inheritable(master_fd), False) - self.assertEqual(os.get_inheritable(slave_fd), False) +@unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()") +class PseudoterminalTests(unittest.TestCase): + def open_pty(self): + """Open a pty fd-pair, and schedule cleanup for it""" + main_fd, second_fd = os.openpty() + self.addCleanup(os.close, main_fd) + self.addCleanup(os.close, second_fd) + return main_fd, second_fd - @unittest.skipUnless(hasattr(os, 'spawnl'), "need os.openpty()") + def test_openpty(self): + main_fd, second_fd = self.open_pty() + self.assertEqual(os.get_inheritable(main_fd), False) + self.assertEqual(os.get_inheritable(second_fd), False) + + @unittest.skipUnless(hasattr(os, 'ptsname'), "need os.ptsname()") + @unittest.skipUnless(hasattr(os, 'O_RDWR'), "need os.O_RDWR") + @unittest.skipUnless(hasattr(os, 'O_NOCTTY'), "need os.O_NOCTTY") + def test_open_via_ptsname(self): + main_fd, second_fd = self.open_pty() + second_path = os.ptsname(main_fd) + reopened_second_fd = os.open(second_path, os.O_RDWR|os.O_NOCTTY) + self.addCleanup(os.close, reopened_second_fd) + os.write(reopened_second_fd, b'foo') + self.assertEqual(os.read(main_fd, 3), b'foo') + + @unittest.skipUnless(hasattr(os, 'posix_openpt'), "need os.posix_openpt()") + @unittest.skipUnless(hasattr(os, 'grantpt'), "need os.grantpt()") + @unittest.skipUnless(hasattr(os, 'unlockpt'), "need os.unlockpt()") + @unittest.skipUnless(hasattr(os, 'ptsname'), "need os.ptsname()") + @unittest.skipUnless(hasattr(os, 'O_RDWR'), "need os.O_RDWR") + @unittest.skipUnless(hasattr(os, 'O_NOCTTY'), "need os.O_NOCTTY") + def test_posix_pty_functions(self): + mother_fd = os.posix_openpt(os.O_RDWR|os.O_NOCTTY) + self.addCleanup(os.close, mother_fd) + os.grantpt(mother_fd) + os.unlockpt(mother_fd) + son_path = os.ptsname(mother_fd) + son_fd = os.open(son_path, os.O_RDWR|os.O_NOCTTY) + self.addCleanup(os.close, son_fd) + self.assertEqual(os.ptsname(mother_fd), os.ttyname(son_fd)) + + @unittest.skipUnless(hasattr(os, 'spawnl'), "need os.spawnl()") + @support.requires_subprocess() def test_pipe_spawnl(self): # gh-77046: On Windows, os.pipe() file descriptors must be created with # _O_NOINHERIT to make them non-inheritable. UCRT has no public API to @@ -4596,8 +4644,11 @@ def test_pipe_spawnl(self): with open(filename, "w") as fp: print(code, file=fp, end="") - cmd = [sys.executable, filename] - exitcode = os.spawnl(os.P_WAIT, cmd[0], *cmd) + executable = sys.executable + cmd = [executable, filename] + if os.name == "nt" and " " in cmd[0]: + cmd[0] = f'"{cmd[0]}"' + exitcode = os.spawnl(os.P_WAIT, executable, *cmd) self.assertEqual(exitcode, 0) diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index bdbe92369639ef8..2b166451243775b 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -19,6 +19,7 @@ from test.support import os_helper from test.support.os_helper import TESTFN, FakePath from test.test_pathlib import test_pathlib_abc +from test.test_pathlib.test_pathlib_abc import needs_posix, needs_windows, needs_symlinks try: import grp, pwd @@ -26,11 +27,6 @@ grp = pwd = None -only_nt = unittest.skipIf(os.name != 'nt', - 'test requires a Windows-compatible system') -only_posix = unittest.skipIf(os.name == 'nt', - 'test requires a POSIX-compatible system') - root_in_posix = False if hasattr(os, 'geteuid'): root_in_posix = (os.geteuid() == 0) @@ -193,7 +189,7 @@ def test_fspath_common(self): self._check_str(p.__fspath__(), ('a/b',)) self._check_str(os.fspath(p), ('a/b',)) - def test_bytes(self): + def test_bytes_exc_message(self): P = self.cls message = (r"argument should be a str or an os\.PathLike object " r"where __fspath__ returns a str, not 'bytes'") @@ -203,22 +199,6 @@ def test_bytes(self): P(b'a', 'b') with self.assertRaisesRegex(TypeError, message): P('a', b'b') - with self.assertRaises(TypeError): - P('a').joinpath(b'b') - with self.assertRaises(TypeError): - P('a') / b'b' - with self.assertRaises(TypeError): - b'a' / P('b') - with self.assertRaises(TypeError): - P('a').match(b'b') - with self.assertRaises(TypeError): - P('a').relative_to(b'b') - with self.assertRaises(TypeError): - P('a').with_name(b'b') - with self.assertRaises(TypeError): - P('a').with_stem(b'b') - with self.assertRaises(TypeError): - P('a').with_suffix(b'b') def test_as_bytes_common(self): sep = os.fsencode(self.sep) @@ -238,8 +218,10 @@ def test_eq_common(self): self.assertNotEqual(P(), {}) self.assertNotEqual(P(), int) - def test_equivalences(self): - for k, tuples in self.equivalences.items(): + def test_equivalences(self, equivalences=None): + if equivalences is None: + equivalences = self.equivalences + for k, tuples in equivalences.items(): canon = k.replace('/', self.sep) posix = k.replace(self.sep, '/') if canon != posix: @@ -329,13 +311,6 @@ def test_with_stem_empty(self): self.assertRaises(ValueError, P('a/b').with_stem, '') self.assertRaises(ValueError, P('a/b').with_stem, '.') - def test_with_suffix_empty(self): - # Path doesn't have a "filename" component. - P = self.cls - self.assertRaises(ValueError, P('').with_suffix, '.gz') - self.assertRaises(ValueError, P('.').with_suffix, '.gz') - self.assertRaises(ValueError, P('/').with_suffix, '.gz') - def test_relative_to_several_args(self): P = self.cls p = P('a/b') @@ -349,16 +324,19 @@ def test_is_relative_to_several_args(self): with self.assertWarns(DeprecationWarning): p.is_relative_to('a', 'b') + def test_is_reserved_deprecated(self): + P = self.cls + p = P('a/b') + with self.assertWarns(DeprecationWarning): + p.is_reserved() + def test_match_empty(self): P = self.cls self.assertRaises(ValueError, P('a').match, '') self.assertRaises(ValueError, P('a').match, '.') - -class PurePosixPathTest(PurePathTest): - cls = pathlib.PurePosixPath - - def test_parse_path(self): + @needs_posix + def test_parse_path_posix(self): check = self._check_parse_path # Collapsing of excess leading slashes, except for the double-slash # special case. @@ -370,25 +348,21 @@ def test_parse_path(self): check('c:\\a', '', '', ['c:\\a',]) check('\\a', '', '', ['\\a',]) - def test_root(self): - P = self.cls - self.assertEqual(P('/a/b').root, '/') - self.assertEqual(P('///a/b').root, '/') - # POSIX special case for two leading slashes. - self.assertEqual(P('//a/b').root, '//') - - def test_eq(self): + @needs_posix + def test_eq_posix(self): P = self.cls self.assertNotEqual(P('a/b'), P('A/b')) self.assertEqual(P('/a'), P('///a')) self.assertNotEqual(P('/a'), P('//a')) - def test_as_uri(self): + @needs_posix + def test_as_uri_posix(self): P = self.cls self.assertEqual(P('/').as_uri(), 'file:///') self.assertEqual(P('/a/b.c').as_uri(), 'file:///a/b.c') self.assertEqual(P('/a/b%#c').as_uri(), 'file:///a/b%25%23c') + @needs_posix def test_as_uri_non_ascii(self): from urllib.parse import quote_from_bytes P = self.cls @@ -399,61 +373,14 @@ def test_as_uri_non_ascii(self): self.assertEqual(P('/a/b\xe9').as_uri(), 'file:///a/b' + quote_from_bytes(os.fsencode('\xe9'))) - def test_match(self): - P = self.cls - self.assertFalse(P('A.py').match('a.PY')) - - def test_is_absolute(self): - P = self.cls - self.assertFalse(P().is_absolute()) - self.assertFalse(P('a').is_absolute()) - self.assertFalse(P('a/b/').is_absolute()) - self.assertTrue(P('/').is_absolute()) - self.assertTrue(P('/a').is_absolute()) - self.assertTrue(P('/a/b/').is_absolute()) - self.assertTrue(P('//a').is_absolute()) - self.assertTrue(P('//a/b').is_absolute()) - - def test_is_reserved(self): - P = self.cls - self.assertIs(False, P('').is_reserved()) - self.assertIs(False, P('/').is_reserved()) - self.assertIs(False, P('/foo/bar').is_reserved()) - self.assertIs(False, P('/dev/con/PRN/NUL').is_reserved()) - - def test_join(self): - P = self.cls - p = P('//a') - pp = p.joinpath('b') - self.assertEqual(pp, P('//a/b')) - pp = P('/a').joinpath('//c') - self.assertEqual(pp, P('//c')) - pp = P('//a').joinpath('/c') - self.assertEqual(pp, P('/c')) - - def test_div(self): - # Basically the same as joinpath(). - P = self.cls - p = P('//a') - pp = p / 'b' - self.assertEqual(pp, P('//a/b')) - pp = P('/a') / '//c' - self.assertEqual(pp, P('//c')) - pp = P('//a') / '/c' - self.assertEqual(pp, P('/c')) - + @needs_posix def test_parse_windows_path(self): P = self.cls p = P('c:', 'a', 'b') pp = P(pathlib.PureWindowsPath('c:\\a\\b')) self.assertEqual(p, pp) - -class PureWindowsPathTest(PurePathTest): - cls = pathlib.PureWindowsPath - - equivalences = PurePathTest.equivalences.copy() - equivalences.update({ + windows_equivalences = { './a:b': [ ('./a:b',) ], 'c:a': [ ('c:', 'a'), ('c:', 'a/'), ('.', 'c:', 'a') ], 'c:/a': [ @@ -464,9 +391,14 @@ class PureWindowsPathTest(PurePathTest): '//a/b/c': [ ('//a/b', 'c'), ('//a/b/', 'c'), ], - }) + } + + @needs_windows + def test_equivalences_windows(self): + self.test_equivalences(self.windows_equivalences) - def test_parse_path(self): + @needs_windows + def test_parse_path_windows(self): check = self._check_parse_path # First part is anchored. check('c:', 'c:', '', []) @@ -514,31 +446,8 @@ def test_parse_path(self): check('D:a/c:b', 'D:', '', ['a', 'c:b']) check('D:/a/c:b', 'D:', '\\', ['a', 'c:b']) - def test_str(self): - p = self.cls('a/b/c') - self.assertEqual(str(p), 'a\\b\\c') - p = self.cls('c:/a/b/c') - self.assertEqual(str(p), 'c:\\a\\b\\c') - p = self.cls('//a/b') - self.assertEqual(str(p), '\\\\a\\b\\') - p = self.cls('//a/b/c') - self.assertEqual(str(p), '\\\\a\\b\\c') - p = self.cls('//a/b/c/d') - self.assertEqual(str(p), '\\\\a\\b\\c\\d') - - def test_str_subclass(self): - self._check_str_subclass('.\\a:b') - self._check_str_subclass('c:') - self._check_str_subclass('c:a') - self._check_str_subclass('c:a\\b.txt') - self._check_str_subclass('c:\\') - self._check_str_subclass('c:\\a') - self._check_str_subclass('c:\\a\\b.txt') - self._check_str_subclass('\\\\some\\share') - self._check_str_subclass('\\\\some\\share\\a') - self._check_str_subclass('\\\\some\\share\\a\\b.txt') - - def test_eq(self): + @needs_windows + def test_eq_windows(self): P = self.cls self.assertEqual(P('c:a/b'), P('c:a/b')) self.assertEqual(P('c:a/b'), P('c:', 'a', 'b')) @@ -551,7 +460,8 @@ def test_eq(self): self.assertEqual(P('//Some/SHARE/a/B'), P('//somE/share/A/b')) self.assertEqual(P('\u0130'), P('i\u0307')) - def test_as_uri(self): + @needs_windows + def test_as_uri_windows(self): P = self.cls with self.assertRaises(ValueError): P('/a/b').as_uri() @@ -567,34 +477,8 @@ def test_as_uri(self): self.assertEqual(P('//some/share/a/b%#c\xe9').as_uri(), 'file://some/share/a/b%25%23c%C3%A9') - def test_match(self): - P = self.cls - # Absolute patterns. - self.assertTrue(P('c:/b.py').match('*:/*.py')) - self.assertTrue(P('c:/b.py').match('c:/*.py')) - self.assertFalse(P('d:/b.py').match('c:/*.py')) # wrong drive - self.assertFalse(P('b.py').match('/*.py')) - self.assertFalse(P('b.py').match('c:*.py')) - self.assertFalse(P('b.py').match('c:/*.py')) - self.assertFalse(P('c:b.py').match('/*.py')) - self.assertFalse(P('c:b.py').match('c:/*.py')) - self.assertFalse(P('/b.py').match('c:*.py')) - self.assertFalse(P('/b.py').match('c:/*.py')) - # UNC patterns. - self.assertTrue(P('//some/share/a.py').match('//*/*/*.py')) - self.assertTrue(P('//some/share/a.py').match('//some/share/*.py')) - self.assertFalse(P('//other/share/a.py').match('//some/share/*.py')) - self.assertFalse(P('//some/share/a/b.py').match('//some/share/*.py')) - # Case-insensitivity. - self.assertTrue(P('B.py').match('b.PY')) - self.assertTrue(P('c:/a/B.Py').match('C:/A/*.pY')) - self.assertTrue(P('//Some/Share/B.Py').match('//somE/sharE/*.pY')) - # Path anchor doesn't match pattern anchor - self.assertFalse(P('c:/b.py').match('/*.py')) # 'c:/' vs '/' - self.assertFalse(P('c:/b.py').match('c:*.py')) # 'c:/' vs 'c:' - self.assertFalse(P('//some/share/a.py').match('/*.py')) # '//some/share/' vs '/' - - def test_ordering_common(self): + @needs_windows + def test_ordering_windows(self): # Case-insensitivity. def assertOrderedEqual(a, b): self.assertLessEqual(a, b) @@ -611,511 +495,13 @@ def assertOrderedEqual(a, b): self.assertFalse(p < q) self.assertFalse(p > q) - def test_parts(self): - P = self.cls - p = P('c:a/b') - parts = p.parts - self.assertEqual(parts, ('c:', 'a', 'b')) - p = P('c:/a/b') - parts = p.parts - self.assertEqual(parts, ('c:\\', 'a', 'b')) - p = P('//a/b/c/d') - parts = p.parts - self.assertEqual(parts, ('\\\\a\\b\\', 'c', 'd')) - - def test_parent(self): - # Anchored - P = self.cls - p = P('z:a/b/c') - self.assertEqual(p.parent, P('z:a/b')) - self.assertEqual(p.parent.parent, P('z:a')) - self.assertEqual(p.parent.parent.parent, P('z:')) - self.assertEqual(p.parent.parent.parent.parent, P('z:')) - p = P('z:/a/b/c') - self.assertEqual(p.parent, P('z:/a/b')) - self.assertEqual(p.parent.parent, P('z:/a')) - self.assertEqual(p.parent.parent.parent, P('z:/')) - self.assertEqual(p.parent.parent.parent.parent, P('z:/')) - p = P('//a/b/c/d') - self.assertEqual(p.parent, P('//a/b/c')) - self.assertEqual(p.parent.parent, P('//a/b')) - self.assertEqual(p.parent.parent.parent, P('//a/b')) - - def test_parents(self): - # Anchored - P = self.cls - p = P('z:a/b/') - par = p.parents - self.assertEqual(len(par), 2) - self.assertEqual(par[0], P('z:a')) - self.assertEqual(par[1], P('z:')) - self.assertEqual(par[0:1], (P('z:a'),)) - self.assertEqual(par[:-1], (P('z:a'),)) - self.assertEqual(par[:2], (P('z:a'), P('z:'))) - self.assertEqual(par[1:], (P('z:'),)) - self.assertEqual(par[::2], (P('z:a'),)) - self.assertEqual(par[::-1], (P('z:'), P('z:a'))) - self.assertEqual(list(par), [P('z:a'), P('z:')]) - with self.assertRaises(IndexError): - par[2] - p = P('z:/a/b/') - par = p.parents - self.assertEqual(len(par), 2) - self.assertEqual(par[0], P('z:/a')) - self.assertEqual(par[1], P('z:/')) - self.assertEqual(par[0:1], (P('z:/a'),)) - self.assertEqual(par[0:-1], (P('z:/a'),)) - self.assertEqual(par[:2], (P('z:/a'), P('z:/'))) - self.assertEqual(par[1:], (P('z:/'),)) - self.assertEqual(par[::2], (P('z:/a'),)) - self.assertEqual(par[::-1], (P('z:/'), P('z:/a'),)) - self.assertEqual(list(par), [P('z:/a'), P('z:/')]) - with self.assertRaises(IndexError): - par[2] - p = P('//a/b/c/d') - par = p.parents - self.assertEqual(len(par), 2) - self.assertEqual(par[0], P('//a/b/c')) - self.assertEqual(par[1], P('//a/b')) - self.assertEqual(par[0:1], (P('//a/b/c'),)) - self.assertEqual(par[0:-1], (P('//a/b/c'),)) - self.assertEqual(par[:2], (P('//a/b/c'), P('//a/b'))) - self.assertEqual(par[1:], (P('//a/b'),)) - self.assertEqual(par[::2], (P('//a/b/c'),)) - self.assertEqual(par[::-1], (P('//a/b'), P('//a/b/c'))) - self.assertEqual(list(par), [P('//a/b/c'), P('//a/b')]) - with self.assertRaises(IndexError): - par[2] - - def test_drive(self): - P = self.cls - self.assertEqual(P('c:').drive, 'c:') - self.assertEqual(P('c:a/b').drive, 'c:') - self.assertEqual(P('c:/').drive, 'c:') - self.assertEqual(P('c:/a/b/').drive, 'c:') - self.assertEqual(P('//a/b').drive, '\\\\a\\b') - self.assertEqual(P('//a/b/').drive, '\\\\a\\b') - self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b') - self.assertEqual(P('./c:a').drive, '') - - def test_root(self): - P = self.cls - self.assertEqual(P('c:').root, '') - self.assertEqual(P('c:a/b').root, '') - self.assertEqual(P('c:/').root, '\\') - self.assertEqual(P('c:/a/b/').root, '\\') - self.assertEqual(P('//a/b').root, '\\') - self.assertEqual(P('//a/b/').root, '\\') - self.assertEqual(P('//a/b/c/d').root, '\\') - - def test_anchor(self): - P = self.cls - self.assertEqual(P('c:').anchor, 'c:') - self.assertEqual(P('c:a/b').anchor, 'c:') - self.assertEqual(P('c:/').anchor, 'c:\\') - self.assertEqual(P('c:/a/b/').anchor, 'c:\\') - self.assertEqual(P('//a/b').anchor, '\\\\a\\b\\') - self.assertEqual(P('//a/b/').anchor, '\\\\a\\b\\') - self.assertEqual(P('//a/b/c/d').anchor, '\\\\a\\b\\') - - def test_name(self): - P = self.cls - self.assertEqual(P('c:').name, '') - self.assertEqual(P('c:/').name, '') - self.assertEqual(P('c:a/b').name, 'b') - self.assertEqual(P('c:/a/b').name, 'b') - self.assertEqual(P('c:a/b.py').name, 'b.py') - self.assertEqual(P('c:/a/b.py').name, 'b.py') - self.assertEqual(P('//My.py/Share.php').name, '') - self.assertEqual(P('//My.py/Share.php/a/b').name, 'b') - - def test_suffix(self): - P = self.cls - self.assertEqual(P('c:').suffix, '') - self.assertEqual(P('c:/').suffix, '') - self.assertEqual(P('c:a/b').suffix, '') - self.assertEqual(P('c:/a/b').suffix, '') - self.assertEqual(P('c:a/b.py').suffix, '.py') - self.assertEqual(P('c:/a/b.py').suffix, '.py') - self.assertEqual(P('c:a/.hgrc').suffix, '') - self.assertEqual(P('c:/a/.hgrc').suffix, '') - self.assertEqual(P('c:a/.hg.rc').suffix, '.rc') - self.assertEqual(P('c:/a/.hg.rc').suffix, '.rc') - self.assertEqual(P('c:a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('c:/a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('c:a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('//My.py/Share.php').suffix, '') - self.assertEqual(P('//My.py/Share.php/a/b').suffix, '') - - def test_suffixes(self): - P = self.cls - self.assertEqual(P('c:').suffixes, []) - self.assertEqual(P('c:/').suffixes, []) - self.assertEqual(P('c:a/b').suffixes, []) - self.assertEqual(P('c:/a/b').suffixes, []) - self.assertEqual(P('c:a/b.py').suffixes, ['.py']) - self.assertEqual(P('c:/a/b.py').suffixes, ['.py']) - self.assertEqual(P('c:a/.hgrc').suffixes, []) - self.assertEqual(P('c:/a/.hgrc').suffixes, []) - self.assertEqual(P('c:a/.hg.rc').suffixes, ['.rc']) - self.assertEqual(P('c:/a/.hg.rc').suffixes, ['.rc']) - self.assertEqual(P('c:a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('c:/a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('//My.py/Share.php').suffixes, []) - self.assertEqual(P('//My.py/Share.php/a/b').suffixes, []) - self.assertEqual(P('c:a/Some name. Ending with a dot.').suffixes, []) - self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffixes, []) - - def test_stem(self): - P = self.cls - self.assertEqual(P('c:').stem, '') - self.assertEqual(P('c:.').stem, '') - self.assertEqual(P('c:..').stem, '..') - self.assertEqual(P('c:/').stem, '') - self.assertEqual(P('c:a/b').stem, 'b') - self.assertEqual(P('c:a/b.py').stem, 'b') - self.assertEqual(P('c:a/.hgrc').stem, '.hgrc') - self.assertEqual(P('c:a/.hg.rc').stem, '.hg') - self.assertEqual(P('c:a/b.tar.gz').stem, 'b.tar') - self.assertEqual(P('c:a/Some name. Ending with a dot.').stem, - 'Some name. Ending with a dot.') - - def test_with_name(self): - P = self.cls - self.assertEqual(P('c:a/b').with_name('d.xml'), P('c:a/d.xml')) - self.assertEqual(P('c:/a/b').with_name('d.xml'), P('c:/a/d.xml')) - self.assertEqual(P('c:a/Dot ending.').with_name('d.xml'), P('c:a/d.xml')) - self.assertEqual(P('c:/a/Dot ending.').with_name('d.xml'), P('c:/a/d.xml')) - self.assertRaises(ValueError, P('c:').with_name, 'd.xml') - self.assertRaises(ValueError, P('c:/').with_name, 'd.xml') - self.assertRaises(ValueError, P('//My/Share').with_name, 'd.xml') - self.assertEqual(str(P('a').with_name('d:')), '.\\d:') - self.assertEqual(str(P('a').with_name('d:e')), '.\\d:e') - self.assertEqual(P('c:a/b').with_name('d:'), P('c:a/d:')) - self.assertEqual(P('c:a/b').with_name('d:e'), P('c:a/d:e')) - self.assertRaises(ValueError, P('c:a/b').with_name, 'd:/e') - self.assertRaises(ValueError, P('c:a/b').with_name, '//My/Share') - - def test_with_stem(self): - P = self.cls - self.assertEqual(P('c:a/b').with_stem('d'), P('c:a/d')) - self.assertEqual(P('c:/a/b').with_stem('d'), P('c:/a/d')) - self.assertEqual(P('c:a/Dot ending.').with_stem('d'), P('c:a/d')) - self.assertEqual(P('c:/a/Dot ending.').with_stem('d'), P('c:/a/d')) - self.assertRaises(ValueError, P('c:').with_stem, 'd') - self.assertRaises(ValueError, P('c:/').with_stem, 'd') - self.assertRaises(ValueError, P('//My/Share').with_stem, 'd') - self.assertEqual(str(P('a').with_stem('d:')), '.\\d:') - self.assertEqual(str(P('a').with_stem('d:e')), '.\\d:e') - self.assertEqual(P('c:a/b').with_stem('d:'), P('c:a/d:')) - self.assertEqual(P('c:a/b').with_stem('d:e'), P('c:a/d:e')) - self.assertRaises(ValueError, P('c:a/b').with_stem, 'd:/e') - self.assertRaises(ValueError, P('c:a/b').with_stem, '//My/Share') - - def test_with_suffix(self): - P = self.cls - self.assertEqual(P('c:a/b').with_suffix('.gz'), P('c:a/b.gz')) - self.assertEqual(P('c:/a/b').with_suffix('.gz'), P('c:/a/b.gz')) - self.assertEqual(P('c:a/b.py').with_suffix('.gz'), P('c:a/b.gz')) - self.assertEqual(P('c:/a/b.py').with_suffix('.gz'), P('c:/a/b.gz')) - # Path doesn't have a "filename" component. - self.assertRaises(ValueError, P('').with_suffix, '.gz') - self.assertRaises(ValueError, P('.').with_suffix, '.gz') - self.assertRaises(ValueError, P('/').with_suffix, '.gz') - self.assertRaises(ValueError, P('//My/Share').with_suffix, '.gz') - # Invalid suffix. - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'gz') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '/') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\') - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '/.gz') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\.gz') - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:.gz') - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c/d') - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c\\d') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c/d') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c\\d') - - def test_relative_to(self): - P = self.cls - p = P('C:Foo/Bar') - self.assertEqual(p.relative_to(P('c:')), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:'), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:foO')), P('Bar')) - self.assertEqual(p.relative_to('c:foO'), P('Bar')) - self.assertEqual(p.relative_to('c:foO/'), P('Bar')) - self.assertEqual(p.relative_to(P('c:foO/baR')), P()) - self.assertEqual(p.relative_to('c:foO/baR'), P()) - self.assertEqual(p.relative_to(P('c:'), walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:foO'), walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:foO', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:foO/', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to(P('c:foO/baR'), walk_up=True), P()) - self.assertEqual(p.relative_to('c:foO/baR', walk_up=True), P()) - self.assertEqual(p.relative_to(P('C:Foo/Bar/Baz'), walk_up=True), P('..')) - self.assertEqual(p.relative_to(P('C:Foo/Baz'), walk_up=True), P('../Bar')) - self.assertEqual(p.relative_to(P('C:Baz/Bar'), walk_up=True), P('../../Foo/Bar')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P()) - self.assertRaises(ValueError, p.relative_to, '') - self.assertRaises(ValueError, p.relative_to, P('d:')) - self.assertRaises(ValueError, p.relative_to, P('/')) - self.assertRaises(ValueError, p.relative_to, P('Foo')) - self.assertRaises(ValueError, p.relative_to, P('/Foo')) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo')) - self.assertRaises(ValueError, p.relative_to, P('C:Foo/Bar/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:Foo/Baz')) - self.assertRaises(ValueError, p.relative_to, P(), walk_up=True) - self.assertRaises(ValueError, p.relative_to, '', walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo'), walk_up=True) - p = P('C:/Foo/Bar') - self.assertEqual(p.relative_to(P('c:/')), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:/'), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:/foO')), P('Bar')) - self.assertEqual(p.relative_to('c:/foO'), P('Bar')) - self.assertEqual(p.relative_to('c:/foO/'), P('Bar')) - self.assertEqual(p.relative_to(P('c:/foO/baR')), P()) - self.assertEqual(p.relative_to('c:/foO/baR'), P()) - self.assertEqual(p.relative_to(P('c:/'), walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:/', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:/foO'), walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:/foO', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:/foO/', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to(P('c:/foO/baR'), walk_up=True), P()) - self.assertEqual(p.relative_to('c:/foO/baR', walk_up=True), P()) - self.assertEqual(p.relative_to('C:/Baz', walk_up=True), P('../Foo/Bar')) - self.assertEqual(p.relative_to('C:/Foo/Bar/Baz', walk_up=True), P('..')) - self.assertEqual(p.relative_to('C:/Foo/Baz', walk_up=True), P('../Bar')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, 'c:') - self.assertRaises(ValueError, p.relative_to, P('c:')) - self.assertRaises(ValueError, p.relative_to, P('C:/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Bar/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:Foo')) - self.assertRaises(ValueError, p.relative_to, P('d:')) - self.assertRaises(ValueError, p.relative_to, P('d:/')) - self.assertRaises(ValueError, p.relative_to, P('/')) - self.assertRaises(ValueError, p.relative_to, P('/Foo')) - self.assertRaises(ValueError, p.relative_to, P('//C/Foo')) - self.assertRaises(ValueError, p.relative_to, 'c:', walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('c:'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('C:Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('d:/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('//C/Foo'), walk_up=True) - # UNC paths. - p = P('//Server/Share/Foo/Bar') - self.assertEqual(p.relative_to(P('//sErver/sHare')), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare'), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/'), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo')), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo'), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/'), P('Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar')), P()) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar'), P()) - self.assertEqual(p.relative_to(P('//sErver/sHare'), walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo'), walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar'), walk_up=True), P()) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar', walk_up=True), P()) - self.assertEqual(p.relative_to(P('//sErver/sHare/bar'), walk_up=True), P('../Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/bar', walk_up=True), P('../Foo/Bar')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo')) - self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo')) - self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo')) - self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo')) - self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo'), walk_up=True) - - def test_is_relative_to(self): - P = self.cls - p = P('C:Foo/Bar') - self.assertTrue(p.is_relative_to(P('c:'))) - self.assertTrue(p.is_relative_to('c:')) - self.assertTrue(p.is_relative_to(P('c:foO'))) - self.assertTrue(p.is_relative_to('c:foO')) - self.assertTrue(p.is_relative_to('c:foO/')) - self.assertTrue(p.is_relative_to(P('c:foO/baR'))) - self.assertTrue(p.is_relative_to('c:foO/baR')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P())) - self.assertFalse(p.is_relative_to('')) - self.assertFalse(p.is_relative_to(P('d:'))) - self.assertFalse(p.is_relative_to(P('/'))) - self.assertFalse(p.is_relative_to(P('Foo'))) - self.assertFalse(p.is_relative_to(P('/Foo'))) - self.assertFalse(p.is_relative_to(P('C:/Foo'))) - self.assertFalse(p.is_relative_to(P('C:Foo/Bar/Baz'))) - self.assertFalse(p.is_relative_to(P('C:Foo/Baz'))) - p = P('C:/Foo/Bar') - self.assertTrue(p.is_relative_to(P('c:/'))) - self.assertTrue(p.is_relative_to(P('c:/foO'))) - self.assertTrue(p.is_relative_to('c:/foO/')) - self.assertTrue(p.is_relative_to(P('c:/foO/baR'))) - self.assertTrue(p.is_relative_to('c:/foO/baR')) - # Unrelated paths. - self.assertFalse(p.is_relative_to('c:')) - self.assertFalse(p.is_relative_to(P('C:/Baz'))) - self.assertFalse(p.is_relative_to(P('C:/Foo/Bar/Baz'))) - self.assertFalse(p.is_relative_to(P('C:/Foo/Baz'))) - self.assertFalse(p.is_relative_to(P('C:Foo'))) - self.assertFalse(p.is_relative_to(P('d:'))) - self.assertFalse(p.is_relative_to(P('d:/'))) - self.assertFalse(p.is_relative_to(P('/'))) - self.assertFalse(p.is_relative_to(P('/Foo'))) - self.assertFalse(p.is_relative_to(P('//C/Foo'))) - # UNC paths. - p = P('//Server/Share/Foo/Bar') - self.assertTrue(p.is_relative_to(P('//sErver/sHare'))) - self.assertTrue(p.is_relative_to('//sErver/sHare')) - self.assertTrue(p.is_relative_to('//sErver/sHare/')) - self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo'))) - self.assertTrue(p.is_relative_to('//sErver/sHare/Foo')) - self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/')) - self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo/Bar'))) - self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/Bar')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('/Server/Share/Foo'))) - self.assertFalse(p.is_relative_to(P('c:/Server/Share/Foo'))) - self.assertFalse(p.is_relative_to(P('//z/Share/Foo'))) - self.assertFalse(p.is_relative_to(P('//Server/z/Foo'))) - - def test_is_absolute(self): - P = self.cls - # Under NT, only paths with both a drive and a root are absolute. - self.assertFalse(P().is_absolute()) - self.assertFalse(P('a').is_absolute()) - self.assertFalse(P('a/b/').is_absolute()) - self.assertFalse(P('/').is_absolute()) - self.assertFalse(P('/a').is_absolute()) - self.assertFalse(P('/a/b/').is_absolute()) - self.assertFalse(P('c:').is_absolute()) - self.assertFalse(P('c:a').is_absolute()) - self.assertFalse(P('c:a/b/').is_absolute()) - self.assertTrue(P('c:/').is_absolute()) - self.assertTrue(P('c:/a').is_absolute()) - self.assertTrue(P('c:/a/b/').is_absolute()) - # UNC paths are absolute by definition. - self.assertTrue(P('//').is_absolute()) - self.assertTrue(P('//a').is_absolute()) - self.assertTrue(P('//a/b').is_absolute()) - self.assertTrue(P('//a/b/').is_absolute()) - self.assertTrue(P('//a/b/c').is_absolute()) - self.assertTrue(P('//a/b/c/d').is_absolute()) - self.assertTrue(P('//?/UNC/').is_absolute()) - self.assertTrue(P('//?/UNC/spam').is_absolute()) - - def test_join(self): - P = self.cls - p = P('C:/a/b') - pp = p.joinpath('x/y') - self.assertEqual(pp, P('C:/a/b/x/y')) - pp = p.joinpath('/x/y') - self.assertEqual(pp, P('C:/x/y')) - # Joining with a different drive => the first path is ignored, even - # if the second path is relative. - pp = p.joinpath('D:x/y') - self.assertEqual(pp, P('D:x/y')) - pp = p.joinpath('D:/x/y') - self.assertEqual(pp, P('D:/x/y')) - pp = p.joinpath('//host/share/x/y') - self.assertEqual(pp, P('//host/share/x/y')) - # Joining with the same drive => the first path is appended to if - # the second path is relative. - pp = p.joinpath('c:x/y') - self.assertEqual(pp, P('C:/a/b/x/y')) - pp = p.joinpath('c:/x/y') - self.assertEqual(pp, P('C:/x/y')) - # Joining with files with NTFS data streams => the filename should - # not be parsed as a drive letter - pp = p.joinpath(P('./d:s')) - self.assertEqual(pp, P('C:/a/b/d:s')) - pp = p.joinpath(P('./dd:s')) - self.assertEqual(pp, P('C:/a/b/dd:s')) - pp = p.joinpath(P('E:d:s')) - self.assertEqual(pp, P('E:d:s')) - # Joining onto a UNC path with no root - pp = P('//').joinpath('server') - self.assertEqual(pp, P('//server')) - pp = P('//server').joinpath('share') - self.assertEqual(pp, P('//server/share')) - pp = P('//./BootPartition').joinpath('Windows') - self.assertEqual(pp, P('//./BootPartition/Windows')) - - def test_div(self): - # Basically the same as joinpath(). - P = self.cls - p = P('C:/a/b') - self.assertEqual(p / 'x/y', P('C:/a/b/x/y')) - self.assertEqual(p / 'x' / 'y', P('C:/a/b/x/y')) - self.assertEqual(p / '/x/y', P('C:/x/y')) - self.assertEqual(p / '/x' / 'y', P('C:/x/y')) - # Joining with a different drive => the first path is ignored, even - # if the second path is relative. - self.assertEqual(p / 'D:x/y', P('D:x/y')) - self.assertEqual(p / 'D:' / 'x/y', P('D:x/y')) - self.assertEqual(p / 'D:/x/y', P('D:/x/y')) - self.assertEqual(p / 'D:' / '/x/y', P('D:/x/y')) - self.assertEqual(p / '//host/share/x/y', P('//host/share/x/y')) - # Joining with the same drive => the first path is appended to if - # the second path is relative. - self.assertEqual(p / 'c:x/y', P('C:/a/b/x/y')) - self.assertEqual(p / 'c:/x/y', P('C:/x/y')) - # Joining with files with NTFS data streams => the filename should - # not be parsed as a drive letter - self.assertEqual(p / P('./d:s'), P('C:/a/b/d:s')) - self.assertEqual(p / P('./dd:s'), P('C:/a/b/dd:s')) - self.assertEqual(p / P('E:d:s'), P('E:d:s')) - - def test_is_reserved(self): - P = self.cls - self.assertIs(False, P('').is_reserved()) - self.assertIs(False, P('/').is_reserved()) - self.assertIs(False, P('/foo/bar').is_reserved()) - # UNC paths are never reserved. - self.assertIs(False, P('//my/share/nul/con/aux').is_reserved()) - # Case-insensitive DOS-device names are reserved. - self.assertIs(True, P('nul').is_reserved()) - self.assertIs(True, P('aux').is_reserved()) - self.assertIs(True, P('prn').is_reserved()) - self.assertIs(True, P('con').is_reserved()) - self.assertIs(True, P('conin$').is_reserved()) - self.assertIs(True, P('conout$').is_reserved()) - # COM/LPT + 1-9 or + superscript 1-3 are reserved. - self.assertIs(True, P('COM1').is_reserved()) - self.assertIs(True, P('LPT9').is_reserved()) - self.assertIs(True, P('com\xb9').is_reserved()) - self.assertIs(True, P('com\xb2').is_reserved()) - self.assertIs(True, P('lpt\xb3').is_reserved()) - # DOS-device name mataching ignores characters after a dot or - # a colon and also ignores trailing spaces. - self.assertIs(True, P('NUL.txt').is_reserved()) - self.assertIs(True, P('PRN ').is_reserved()) - self.assertIs(True, P('AUX .txt').is_reserved()) - self.assertIs(True, P('COM1:bar').is_reserved()) - self.assertIs(True, P('LPT9 :bar').is_reserved()) - # DOS-device names are only matched at the beginning - # of a path component. - self.assertIs(False, P('bar.com9').is_reserved()) - self.assertIs(False, P('bar.lpt9').is_reserved()) - # Only the last path component matters. - self.assertIs(True, P('c:/baz/con/NUL').is_reserved()) - self.assertIs(False, P('c:/NUL/con/baz').is_reserved()) + +class PurePosixPathTest(PurePathTest): + cls = pathlib.PurePosixPath + + +class PureWindowsPathTest(PurePathTest): + cls = pathlib.PureWindowsPath class PurePathSubclassTest(PurePathTest): @@ -1304,7 +690,7 @@ def test_chmod(self): self.assertEqual(p.stat().st_mode, new_mode) # On Windows, os.chmod does not follow symlinks (issue #15411) - @only_posix + @needs_posix @os_helper.skip_unless_working_chmod def test_chmod_follow_symlinks_true(self): p = self.cls(self.base) / 'linkA' @@ -1573,7 +959,7 @@ def test_mkdir_exist_ok_root(self): self.cls('/').resolve().mkdir(exist_ok=True) self.cls('/').resolve().mkdir(parents=True, exist_ok=True) - @only_nt # XXX: not sure how to test this on POSIX. + @needs_windows # XXX: not sure how to test this on POSIX. def test_mkdir_with_unknown_drive(self): for d in 'ZYXWVUTSRQPONMLKJIHGFEDCBA': p = self.cls(d + ':\\') @@ -1638,9 +1024,8 @@ def my_mkdir(path, mode=0o777): self.assertNotIn(str(p12), concurrently_created) self.assertTrue(p.exists()) + @needs_symlinks def test_symlink_to(self): - if not self.can_symlink: - self.skipTest("symlinks required") P = self.cls(self.base) target = P / 'fileA' # Symlinking a path target. @@ -1824,6 +1209,8 @@ def test_glob_empty_pattern(self): list(p.glob('')) with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): list(p.glob('.')) + with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): + list(p.glob('./')) def test_glob_many_open_files(self): depth = 30 @@ -1855,18 +1242,6 @@ def test_glob_above_recursion_limit(self): with set_recursion_limit(recursion_limit): list(base.glob('**/')) - def test_glob_recursive_no_trailing_slash(self): - P = self.cls - p = P(self.base) - with self.assertWarns(FutureWarning): - p.glob('**') - with self.assertWarns(FutureWarning): - p.glob('*/**') - with self.assertWarns(FutureWarning): - p.rglob('**') - with self.assertWarns(FutureWarning): - p.rglob('*/**') - def test_glob_pathlike(self): P = self.cls p = P(self.base) @@ -1883,12 +1258,8 @@ def test_rglob_pathlike(self): self.assertEqual(expect, set(p.rglob(P(pattern)))) self.assertEqual(expect, set(p.rglob(FakePath(pattern)))) - -@only_posix -class PosixPathTest(PathTest, PurePosixPathTest): - cls = pathlib.PosixPath - - def test_absolute(self): + @needs_posix + def test_absolute_posix(self): P = self.cls self.assertEqual(str(P('/').absolute()), '/') self.assertEqual(str(P('/a').absolute()), '/a') @@ -1903,6 +1274,7 @@ def test_absolute(self): is_emscripten or is_wasi, "umask is not implemented on Emscripten/WASI." ) + @needs_posix def test_open_mode(self): old_mask = os.umask(0) self.addCleanup(os.umask, old_mask) @@ -1917,6 +1289,7 @@ def test_open_mode(self): st = os.stat(self.pathmod.join(self.base, 'other_new_file')) self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) + @needs_posix def test_resolve_root(self): current_directory = os.getcwd() try: @@ -1930,6 +1303,7 @@ def test_resolve_root(self): is_emscripten or is_wasi, "umask is not implemented on Emscripten/WASI." ) + @needs_posix def test_touch_mode(self): old_mask = os.umask(0) self.addCleanup(os.umask, old_mask) @@ -1945,27 +1319,12 @@ def test_touch_mode(self): st = os.stat(self.pathmod.join(self.base, 'masked_new_file')) self.assertEqual(stat.S_IMODE(st.st_mode), 0o750) - def test_glob(self): - P = self.cls - p = P(self.base) - given = set(p.glob("FILEa")) - expect = set() if not os_helper.fs_is_case_insensitive(self.base) else given - self.assertEqual(given, expect) - self.assertEqual(set(p.glob("FILEa*")), set()) - - def test_rglob(self): - P = self.cls - p = P(self.base, "dirC") - given = set(p.rglob("FILEd")) - expect = set() if not os_helper.fs_is_case_insensitive(self.base) else given - self.assertEqual(given, expect) - self.assertEqual(set(p.rglob("FILEd*")), set()) - @unittest.skipUnless(hasattr(pwd, 'getpwall'), 'pwd module does not expose getpwall()') @unittest.skipIf(sys.platform == "vxworks", "no home directory on VxWorks") - def test_expanduser(self): + @needs_posix + def test_expanduser_posix(self): P = self.cls import_helper.import_module('pwd') import pwd @@ -2020,6 +1379,7 @@ def test_expanduser(self): @unittest.skipIf(sys.platform != "darwin", "Bad file descriptor in /dev/fd affects only macOS") + @needs_posix def test_handling_bad_descriptor(self): try: file_descriptors = list(pathlib.Path('/dev/fd').rglob("*"))[3:] @@ -2041,7 +1401,8 @@ def test_handling_bad_descriptor(self): self.fail("Bad file descriptor not handled.") raise - def test_from_uri(self): + @needs_posix + def test_from_uri_posix(self): P = self.cls self.assertEqual(P.from_uri('file:/foo/bar'), P('/foo/bar')) self.assertEqual(P.from_uri('file://foo/bar'), P('//foo/bar')) @@ -2054,17 +1415,14 @@ def test_from_uri(self): self.assertRaises(ValueError, P.from_uri, 'file:foo/bar') self.assertRaises(ValueError, P.from_uri, 'http://foo/bar') - def test_from_uri_pathname2url(self): + @needs_posix + def test_from_uri_pathname2url_posix(self): P = self.cls self.assertEqual(P.from_uri('file:' + pathname2url('/foo/bar')), P('/foo/bar')) self.assertEqual(P.from_uri('file:' + pathname2url('//foo/bar')), P('//foo/bar')) - -@only_nt -class WindowsPathTest(PathTest, PureWindowsPathTest): - cls = pathlib.WindowsPath - - def test_absolute(self): + @needs_windows + def test_absolute_windows(self): P = self.cls # Simple absolute paths. @@ -2109,23 +1467,8 @@ def test_absolute(self): self.assertEqual(str(P(other_drive).absolute()), other_cwd) self.assertEqual(str(P(other_drive + 'foo').absolute()), other_cwd + '\\foo') - def test_glob(self): - P = self.cls - p = P(self.base) - self.assertEqual(set(p.glob("FILEa")), { P(self.base, "fileA") }) - self.assertEqual(set(p.glob("*a\\")), { P(self.base, "dirA/") }) - self.assertEqual(set(p.glob("F*a")), { P(self.base, "fileA") }) - self.assertEqual(set(map(str, p.glob("FILEa"))), {f"{p}\\fileA"}) - self.assertEqual(set(map(str, p.glob("F*a"))), {f"{p}\\fileA"}) - - def test_rglob(self): - P = self.cls - p = P(self.base, "dirC") - self.assertEqual(set(p.rglob("FILEd")), { P(self.base, "dirC/dirD/fileD") }) - self.assertEqual(set(p.rglob("*\\")), { P(self.base, "dirC/dirD/") }) - self.assertEqual(set(map(str, p.rglob("FILEd"))), {f"{p}\\dirD\\fileD"}) - - def test_expanduser(self): + @needs_windows + def test_expanduser_windows(self): P = self.cls with os_helper.EnvironmentVarGuard() as env: env.pop('HOME', None) @@ -2178,7 +1521,8 @@ def check(): env['HOME'] = 'C:\\Users\\eve' check() - def test_from_uri(self): + @needs_windows + def test_from_uri_windows(self): P = self.cls # DOS drive paths self.assertEqual(P.from_uri('file:c:/path/to/file'), P('c:/path/to/file')) @@ -2199,22 +1543,35 @@ def test_from_uri(self): self.assertRaises(ValueError, P.from_uri, 'file:foo/bar') self.assertRaises(ValueError, P.from_uri, 'http://foo/bar') - def test_from_uri_pathname2url(self): + @needs_windows + def test_from_uri_pathname2url_windows(self): P = self.cls self.assertEqual(P.from_uri('file:' + pathname2url(r'c:\path\to\file')), P('c:/path/to/file')) self.assertEqual(P.from_uri('file:' + pathname2url(r'\\server\path\to\file')), P('//server/path/to/file')) - def test_owner(self): + @needs_windows + def test_owner_windows(self): P = self.cls with self.assertRaises(pathlib.UnsupportedOperation): P('c:/').owner() - def test_group(self): + @needs_windows + def test_group_windows(self): P = self.cls with self.assertRaises(pathlib.UnsupportedOperation): P('c:/').group() +@unittest.skipIf(os.name == 'nt', 'test requires a POSIX-compatible system') +class PosixPathTest(PathTest, PurePosixPathTest): + cls = pathlib.PosixPath + + +@unittest.skipIf(os.name != 'nt', 'test requires a Windows-compatible system') +class WindowsPathTest(PathTest, PureWindowsPathTest): + cls = pathlib.WindowsPath + + class PathSubclassTest(PathTest): class cls(pathlib.Path): pass diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 199718a8a69c5ad..1d30deca8f7a1bb 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -11,6 +11,27 @@ from test.support.os_helper import TESTFN +_tests_needing_posix = set() +_tests_needing_windows = set() +_tests_needing_symlinks = set() + + +def needs_posix(fn): + """Decorator that marks a test as requiring a POSIX-flavoured path class.""" + _tests_needing_posix.add(fn.__name__) + return fn + +def needs_windows(fn): + """Decorator that marks a test as requiring a Windows-flavoured path class.""" + _tests_needing_windows.add(fn.__name__) + return fn + +def needs_symlinks(fn): + """Decorator that marks a test as requiring a path class that supports symlinks.""" + _tests_needing_symlinks.add(fn.__name__) + return fn + + class UnsupportedOperationTest(unittest.TestCase): def test_is_notimplemented(self): self.assertTrue(issubclass(UnsupportedOperation, NotImplementedError)) @@ -115,6 +136,11 @@ class DummyPurePathTest(unittest.TestCase): base = f'/this/path/kills/fascists/{TESTFN}' def setUp(self): + name = self.id().split('.')[-1] + if name in _tests_needing_posix and self.cls.pathmod is not posixpath: + self.skipTest('requires POSIX-flavoured path class') + if name in _tests_needing_windows and self.cls.pathmod is posixpath: + self.skipTest('requires Windows-flavoured path class') p = self.cls('a') self.pathmod = p.pathmod self.sep = self.pathmod.sep @@ -129,6 +155,31 @@ def test_constructor_common(self): P('a/b/c') P('/a/b/c') + def test_bytes(self): + P = self.cls + with self.assertRaises(TypeError): + P(b'a') + with self.assertRaises(TypeError): + P(b'a', 'b') + with self.assertRaises(TypeError): + P('a', b'b') + with self.assertRaises(TypeError): + P('a').joinpath(b'b') + with self.assertRaises(TypeError): + P('a') / b'b' + with self.assertRaises(TypeError): + b'a' / P('b') + with self.assertRaises(TypeError): + P('a').match(b'b') + with self.assertRaises(TypeError): + P('a').relative_to(b'b') + with self.assertRaises(TypeError): + P('a').with_name(b'b') + with self.assertRaises(TypeError): + P('a').with_stem(b'b') + with self.assertRaises(TypeError): + P('a').with_suffix(b'b') + def _check_str_subclass(self, *args): # Issue #21127: it should be possible to construct a PurePath object # from a str subclass instance, and it then gets converted to @@ -148,6 +199,19 @@ def test_str_subclass_common(self): self._check_str_subclass('a/b.txt') self._check_str_subclass('/a/b.txt') + @needs_windows + def test_str_subclass_windows(self): + self._check_str_subclass('.\\a:b') + self._check_str_subclass('c:') + self._check_str_subclass('c:a') + self._check_str_subclass('c:a\\b.txt') + self._check_str_subclass('c:\\') + self._check_str_subclass('c:\\a') + self._check_str_subclass('c:\\a\\b.txt') + self._check_str_subclass('\\\\some\\share') + self._check_str_subclass('\\\\some\\share\\a') + self._check_str_subclass('\\\\some\\share\\a\\b.txt') + def test_with_segments_common(self): class P(self.cls): def __init__(self, *pathsegments, session_id): @@ -180,6 +244,55 @@ def test_join_common(self): pp = p.joinpath('/c') self.assertEqual(pp, P('/c')) + @needs_posix + def test_join_posix(self): + P = self.cls + p = P('//a') + pp = p.joinpath('b') + self.assertEqual(pp, P('//a/b')) + pp = P('/a').joinpath('//c') + self.assertEqual(pp, P('//c')) + pp = P('//a').joinpath('/c') + self.assertEqual(pp, P('/c')) + + @needs_windows + def test_join_windows(self): + P = self.cls + p = P('C:/a/b') + pp = p.joinpath('x/y') + self.assertEqual(pp, P('C:/a/b/x/y')) + pp = p.joinpath('/x/y') + self.assertEqual(pp, P('C:/x/y')) + # Joining with a different drive => the first path is ignored, even + # if the second path is relative. + pp = p.joinpath('D:x/y') + self.assertEqual(pp, P('D:x/y')) + pp = p.joinpath('D:/x/y') + self.assertEqual(pp, P('D:/x/y')) + pp = p.joinpath('//host/share/x/y') + self.assertEqual(pp, P('//host/share/x/y')) + # Joining with the same drive => the first path is appended to if + # the second path is relative. + pp = p.joinpath('c:x/y') + self.assertEqual(pp, P('C:/a/b/x/y')) + pp = p.joinpath('c:/x/y') + self.assertEqual(pp, P('C:/x/y')) + # Joining with files with NTFS data streams => the filename should + # not be parsed as a drive letter + pp = p.joinpath(P('./d:s')) + self.assertEqual(pp, P('C:/a/b/d:s')) + pp = p.joinpath(P('./dd:s')) + self.assertEqual(pp, P('C:/a/b/dd:s')) + pp = p.joinpath(P('E:d:s')) + self.assertEqual(pp, P('E:d:s')) + # Joining onto a UNC path with no root + pp = P('//').joinpath('server') + self.assertEqual(pp, P('//server')) + pp = P('//server').joinpath('share') + self.assertEqual(pp, P('//server/share')) + pp = P('//./BootPartition').joinpath('Windows') + self.assertEqual(pp, P('//./BootPartition/Windows')) + def test_div_common(self): # Basically the same as joinpath(). P = self.cls @@ -196,6 +309,44 @@ def test_div_common(self): pp = p/ '/c' self.assertEqual(pp, P('/c')) + @needs_posix + def test_div_posix(self): + # Basically the same as joinpath(). + P = self.cls + p = P('//a') + pp = p / 'b' + self.assertEqual(pp, P('//a/b')) + pp = P('/a') / '//c' + self.assertEqual(pp, P('//c')) + pp = P('//a') / '/c' + self.assertEqual(pp, P('/c')) + + @needs_windows + def test_div_windows(self): + # Basically the same as joinpath(). + P = self.cls + p = P('C:/a/b') + self.assertEqual(p / 'x/y', P('C:/a/b/x/y')) + self.assertEqual(p / 'x' / 'y', P('C:/a/b/x/y')) + self.assertEqual(p / '/x/y', P('C:/x/y')) + self.assertEqual(p / '/x' / 'y', P('C:/x/y')) + # Joining with a different drive => the first path is ignored, even + # if the second path is relative. + self.assertEqual(p / 'D:x/y', P('D:x/y')) + self.assertEqual(p / 'D:' / 'x/y', P('D:x/y')) + self.assertEqual(p / 'D:/x/y', P('D:/x/y')) + self.assertEqual(p / 'D:' / '/x/y', P('D:/x/y')) + self.assertEqual(p / '//host/share/x/y', P('//host/share/x/y')) + # Joining with the same drive => the first path is appended to if + # the second path is relative. + self.assertEqual(p / 'c:x/y', P('C:/a/b/x/y')) + self.assertEqual(p / 'c:/x/y', P('C:/x/y')) + # Joining with files with NTFS data streams => the filename should + # not be parsed as a drive letter + self.assertEqual(p / P('./d:s'), P('C:/a/b/d:s')) + self.assertEqual(p / P('./dd:s'), P('C:/a/b/dd:s')) + self.assertEqual(p / P('E:d:s'), P('E:d:s')) + def _check_str(self, expected, args): p = self.cls(*args) self.assertEqual(str(p), expected.replace('/', self.sep)) @@ -206,6 +357,19 @@ def test_str_common(self): self._check_str(pathstr, (pathstr,)) # Other tests for str() are in test_equivalences(). + @needs_windows + def test_str_windows(self): + p = self.cls('a/b/c') + self.assertEqual(str(p), 'a\\b\\c') + p = self.cls('c:/a/b/c') + self.assertEqual(str(p), 'c:\\a\\b\\c') + p = self.cls('//a/b') + self.assertEqual(str(p), '\\\\a\\b\\') + p = self.cls('//a/b/c') + self.assertEqual(str(p), '\\\\a\\b\\c') + p = self.cls('//a/b/c/d') + self.assertEqual(str(p), '\\\\a\\b\\c\\d') + def test_as_posix_common(self): P = self.cls for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): @@ -249,29 +413,8 @@ def test_match_common(self): self.assertFalse(P('/ab.py').match('/a/*.py')) self.assertFalse(P('/a/b/c.py').match('/a/*.py')) # Multi-part glob-style pattern. - self.assertTrue(P('a').match('**')) - self.assertTrue(P('c.py').match('**')) - self.assertTrue(P('a/b/c.py').match('**')) - self.assertTrue(P('/a/b/c.py').match('**')) - self.assertTrue(P('/a/b/c.py').match('/**')) - self.assertTrue(P('/a/b/c.py').match('/a/**')) - self.assertTrue(P('/a/b/c.py').match('**/*.py')) - self.assertTrue(P('/a/b/c.py').match('/**/*.py')) + self.assertFalse(P('/a/b/c.py').match('/**/*.py')) self.assertTrue(P('/a/b/c.py').match('/a/**/*.py')) - self.assertTrue(P('/a/b/c.py').match('/a/b/**/*.py')) - self.assertTrue(P('/a/b/c.py').match('/**/**/**/**/*.py')) - self.assertFalse(P('c.py').match('**/a.py')) - self.assertFalse(P('c.py').match('c/**')) - self.assertFalse(P('a/b/c.py').match('**/a')) - self.assertFalse(P('a/b/c.py').match('**/a/b')) - self.assertFalse(P('a/b/c.py').match('**/a/b/c')) - self.assertFalse(P('a/b/c.py').match('**/a/b/c.')) - self.assertFalse(P('a/b/c.py').match('**/a/b/c./**')) - self.assertFalse(P('a/b/c.py').match('**/a/b/c./**')) - self.assertFalse(P('a/b/c.py').match('/a/b/c.py/**')) - self.assertFalse(P('a/b/c.py').match('/**/a/b/c.py')) - self.assertRaises(ValueError, P('a').match, '**a/b/c') - self.assertRaises(ValueError, P('a').match, 'a/b/c**') # Case-sensitive flag self.assertFalse(P('A.py').match('a.PY', case_sensitive=True)) self.assertTrue(P('A.py').match('a.PY', case_sensitive=False)) @@ -279,9 +422,115 @@ def test_match_common(self): self.assertTrue(P('/a/b/c.py').match('/A/*/*.Py', case_sensitive=False)) # Matching against empty path self.assertFalse(P('').match('*')) - self.assertTrue(P('').match('**')) + self.assertFalse(P('').match('**')) self.assertFalse(P('').match('**/*')) + @needs_posix + def test_match_posix(self): + P = self.cls + self.assertFalse(P('A.py').match('a.PY')) + + @needs_windows + def test_match_windows(self): + P = self.cls + # Absolute patterns. + self.assertTrue(P('c:/b.py').match('*:/*.py')) + self.assertTrue(P('c:/b.py').match('c:/*.py')) + self.assertFalse(P('d:/b.py').match('c:/*.py')) # wrong drive + self.assertFalse(P('b.py').match('/*.py')) + self.assertFalse(P('b.py').match('c:*.py')) + self.assertFalse(P('b.py').match('c:/*.py')) + self.assertFalse(P('c:b.py').match('/*.py')) + self.assertFalse(P('c:b.py').match('c:/*.py')) + self.assertFalse(P('/b.py').match('c:*.py')) + self.assertFalse(P('/b.py').match('c:/*.py')) + # UNC patterns. + self.assertTrue(P('//some/share/a.py').match('//*/*/*.py')) + self.assertTrue(P('//some/share/a.py').match('//some/share/*.py')) + self.assertFalse(P('//other/share/a.py').match('//some/share/*.py')) + self.assertFalse(P('//some/share/a/b.py').match('//some/share/*.py')) + # Case-insensitivity. + self.assertTrue(P('B.py').match('b.PY')) + self.assertTrue(P('c:/a/B.Py').match('C:/A/*.pY')) + self.assertTrue(P('//Some/Share/B.Py').match('//somE/sharE/*.pY')) + # Path anchor doesn't match pattern anchor + self.assertFalse(P('c:/b.py').match('/*.py')) # 'c:/' vs '/' + self.assertFalse(P('c:/b.py').match('c:*.py')) # 'c:/' vs 'c:' + self.assertFalse(P('//some/share/a.py').match('/*.py')) # '//some/share/' vs '/' + + def test_full_match_common(self): + P = self.cls + # Simple relative pattern. + self.assertTrue(P('b.py').full_match('b.py')) + self.assertFalse(P('a/b.py').full_match('b.py')) + self.assertFalse(P('/a/b.py').full_match('b.py')) + self.assertFalse(P('a.py').full_match('b.py')) + self.assertFalse(P('b/py').full_match('b.py')) + self.assertFalse(P('/a.py').full_match('b.py')) + self.assertFalse(P('b.py/c').full_match('b.py')) + # Wildcard relative pattern. + self.assertTrue(P('b.py').full_match('*.py')) + self.assertFalse(P('a/b.py').full_match('*.py')) + self.assertFalse(P('/a/b.py').full_match('*.py')) + self.assertFalse(P('b.pyc').full_match('*.py')) + self.assertFalse(P('b./py').full_match('*.py')) + self.assertFalse(P('b.py/c').full_match('*.py')) + # Multi-part relative pattern. + self.assertTrue(P('ab/c.py').full_match('a*/*.py')) + self.assertFalse(P('/d/ab/c.py').full_match('a*/*.py')) + self.assertFalse(P('a.py').full_match('a*/*.py')) + self.assertFalse(P('/dab/c.py').full_match('a*/*.py')) + self.assertFalse(P('ab/c.py/d').full_match('a*/*.py')) + # Absolute pattern. + self.assertTrue(P('/b.py').full_match('/*.py')) + self.assertFalse(P('b.py').full_match('/*.py')) + self.assertFalse(P('a/b.py').full_match('/*.py')) + self.assertFalse(P('/a/b.py').full_match('/*.py')) + # Multi-part absolute pattern. + self.assertTrue(P('/a/b.py').full_match('/a/*.py')) + self.assertFalse(P('/ab.py').full_match('/a/*.py')) + self.assertFalse(P('/a/b/c.py').full_match('/a/*.py')) + # Multi-part glob-style pattern. + self.assertTrue(P('a').full_match('**')) + self.assertTrue(P('c.py').full_match('**')) + self.assertTrue(P('a/b/c.py').full_match('**')) + self.assertTrue(P('/a/b/c.py').full_match('**')) + self.assertTrue(P('/a/b/c.py').full_match('/**')) + self.assertTrue(P('/a/b/c.py').full_match('/a/**')) + self.assertTrue(P('/a/b/c.py').full_match('**/*.py')) + self.assertTrue(P('/a/b/c.py').full_match('/**/*.py')) + self.assertTrue(P('/a/b/c.py').full_match('/a/**/*.py')) + self.assertTrue(P('/a/b/c.py').full_match('/a/b/**/*.py')) + self.assertTrue(P('/a/b/c.py').full_match('/**/**/**/**/*.py')) + self.assertFalse(P('c.py').full_match('**/a.py')) + self.assertFalse(P('c.py').full_match('c/**')) + self.assertFalse(P('a/b/c.py').full_match('**/a')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b/c')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b/c.')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b/c./**')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b/c./**')) + self.assertFalse(P('a/b/c.py').full_match('/a/b/c.py/**')) + self.assertFalse(P('a/b/c.py').full_match('/**/a/b/c.py')) + self.assertRaises(ValueError, P('a').full_match, '**a/b/c') + self.assertRaises(ValueError, P('a').full_match, 'a/b/c**') + # Case-sensitive flag + self.assertFalse(P('A.py').full_match('a.PY', case_sensitive=True)) + self.assertTrue(P('A.py').full_match('a.PY', case_sensitive=False)) + self.assertFalse(P('c:/a/B.Py').full_match('C:/A/*.pY', case_sensitive=True)) + self.assertTrue(P('/a/b/c.py').full_match('/A/*/*.Py', case_sensitive=False)) + # Matching against empty path + self.assertFalse(P('').full_match('*')) + self.assertTrue(P('').full_match('**')) + self.assertFalse(P('').full_match('**/*')) + # Matching with empty pattern + self.assertTrue(P('').full_match('')) + self.assertTrue(P('.').full_match('.')) + self.assertFalse(P('/').full_match('')) + self.assertFalse(P('/').full_match('.')) + self.assertFalse(P('foo').full_match('')) + self.assertFalse(P('foo').full_match('.')) + def test_parts_common(self): # `parts` returns a tuple. sep = self.sep @@ -294,6 +543,19 @@ def test_parts_common(self): parts = p.parts self.assertEqual(parts, (sep, 'a', 'b')) + @needs_windows + def test_parts_windows(self): + P = self.cls + p = P('c:a/b') + parts = p.parts + self.assertEqual(parts, ('c:', 'a', 'b')) + p = P('c:/a/b') + parts = p.parts + self.assertEqual(parts, ('c:\\', 'a', 'b')) + p = P('//a/b/c/d') + parts = p.parts + self.assertEqual(parts, ('\\\\a\\b\\', 'c', 'd')) + def test_parent_common(self): # Relative P = self.cls @@ -309,6 +571,25 @@ def test_parent_common(self): self.assertEqual(p.parent.parent.parent, P('/')) self.assertEqual(p.parent.parent.parent.parent, P('/')) + @needs_windows + def test_parent_windows(self): + # Anchored + P = self.cls + p = P('z:a/b/c') + self.assertEqual(p.parent, P('z:a/b')) + self.assertEqual(p.parent.parent, P('z:a')) + self.assertEqual(p.parent.parent.parent, P('z:')) + self.assertEqual(p.parent.parent.parent.parent, P('z:')) + p = P('z:/a/b/c') + self.assertEqual(p.parent, P('z:/a/b')) + self.assertEqual(p.parent.parent, P('z:/a')) + self.assertEqual(p.parent.parent.parent, P('z:/')) + self.assertEqual(p.parent.parent.parent.parent, P('z:/')) + p = P('//a/b/c/d') + self.assertEqual(p.parent, P('//a/b/c')) + self.assertEqual(p.parent.parent, P('//a/b')) + self.assertEqual(p.parent.parent.parent, P('//a/b')) + def test_parents_common(self): # Relative P = self.cls @@ -356,12 +637,71 @@ def test_parents_common(self): with self.assertRaises(IndexError): par[3] + @needs_windows + def test_parents_windows(self): + # Anchored + P = self.cls + p = P('z:a/b/') + par = p.parents + self.assertEqual(len(par), 2) + self.assertEqual(par[0], P('z:a')) + self.assertEqual(par[1], P('z:')) + self.assertEqual(par[0:1], (P('z:a'),)) + self.assertEqual(par[:-1], (P('z:a'),)) + self.assertEqual(par[:2], (P('z:a'), P('z:'))) + self.assertEqual(par[1:], (P('z:'),)) + self.assertEqual(par[::2], (P('z:a'),)) + self.assertEqual(par[::-1], (P('z:'), P('z:a'))) + self.assertEqual(list(par), [P('z:a'), P('z:')]) + with self.assertRaises(IndexError): + par[2] + p = P('z:/a/b/') + par = p.parents + self.assertEqual(len(par), 2) + self.assertEqual(par[0], P('z:/a')) + self.assertEqual(par[1], P('z:/')) + self.assertEqual(par[0:1], (P('z:/a'),)) + self.assertEqual(par[0:-1], (P('z:/a'),)) + self.assertEqual(par[:2], (P('z:/a'), P('z:/'))) + self.assertEqual(par[1:], (P('z:/'),)) + self.assertEqual(par[::2], (P('z:/a'),)) + self.assertEqual(par[::-1], (P('z:/'), P('z:/a'),)) + self.assertEqual(list(par), [P('z:/a'), P('z:/')]) + with self.assertRaises(IndexError): + par[2] + p = P('//a/b/c/d') + par = p.parents + self.assertEqual(len(par), 2) + self.assertEqual(par[0], P('//a/b/c')) + self.assertEqual(par[1], P('//a/b')) + self.assertEqual(par[0:1], (P('//a/b/c'),)) + self.assertEqual(par[0:-1], (P('//a/b/c'),)) + self.assertEqual(par[:2], (P('//a/b/c'), P('//a/b'))) + self.assertEqual(par[1:], (P('//a/b'),)) + self.assertEqual(par[::2], (P('//a/b/c'),)) + self.assertEqual(par[::-1], (P('//a/b'), P('//a/b/c'))) + self.assertEqual(list(par), [P('//a/b/c'), P('//a/b')]) + with self.assertRaises(IndexError): + par[2] + def test_drive_common(self): P = self.cls self.assertEqual(P('a/b').drive, '') self.assertEqual(P('/a/b').drive, '') self.assertEqual(P('').drive, '') + @needs_windows + def test_drive_windows(self): + P = self.cls + self.assertEqual(P('c:').drive, 'c:') + self.assertEqual(P('c:a/b').drive, 'c:') + self.assertEqual(P('c:/').drive, 'c:') + self.assertEqual(P('c:/a/b/').drive, 'c:') + self.assertEqual(P('//a/b').drive, '\\\\a\\b') + self.assertEqual(P('//a/b/').drive, '\\\\a\\b') + self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b') + self.assertEqual(P('./c:a').drive, '') + def test_root_common(self): P = self.cls sep = self.sep @@ -370,6 +710,24 @@ def test_root_common(self): self.assertEqual(P('/').root, sep) self.assertEqual(P('/a/b').root, sep) + @needs_posix + def test_root_posix(self): + P = self.cls + self.assertEqual(P('/a/b').root, '/') + # POSIX special case for two leading slashes. + self.assertEqual(P('//a/b').root, '//') + + @needs_windows + def test_root_windows(self): + P = self.cls + self.assertEqual(P('c:').root, '') + self.assertEqual(P('c:a/b').root, '') + self.assertEqual(P('c:/').root, '\\') + self.assertEqual(P('c:/a/b/').root, '\\') + self.assertEqual(P('//a/b').root, '\\') + self.assertEqual(P('//a/b/').root, '\\') + self.assertEqual(P('//a/b/c/d').root, '\\') + def test_anchor_common(self): P = self.cls sep = self.sep @@ -378,6 +736,17 @@ def test_anchor_common(self): self.assertEqual(P('/').anchor, sep) self.assertEqual(P('/a/b').anchor, sep) + @needs_windows + def test_anchor_windows(self): + P = self.cls + self.assertEqual(P('c:').anchor, 'c:') + self.assertEqual(P('c:a/b').anchor, 'c:') + self.assertEqual(P('c:/').anchor, 'c:\\') + self.assertEqual(P('c:/a/b/').anchor, 'c:\\') + self.assertEqual(P('//a/b').anchor, '\\\\a\\b\\') + self.assertEqual(P('//a/b/').anchor, '\\\\a\\b\\') + self.assertEqual(P('//a/b/c/d').anchor, '\\\\a\\b\\') + def test_name_empty(self): P = self.cls self.assertEqual(P('').name, '') @@ -392,6 +761,18 @@ def test_name_common(self): self.assertEqual(P('a/b.py').name, 'b.py') self.assertEqual(P('/a/b.py').name, 'b.py') + @needs_windows + def test_name_windows(self): + P = self.cls + self.assertEqual(P('c:').name, '') + self.assertEqual(P('c:/').name, '') + self.assertEqual(P('c:a/b').name, 'b') + self.assertEqual(P('c:/a/b').name, 'b') + self.assertEqual(P('c:a/b.py').name, 'b.py') + self.assertEqual(P('c:/a/b.py').name, 'b.py') + self.assertEqual(P('//My.py/Share.php').name, '') + self.assertEqual(P('//My.py/Share.php/a/b').name, 'b') + def test_suffix_common(self): P = self.cls self.assertEqual(P('').suffix, '') @@ -412,6 +793,26 @@ def test_suffix_common(self): self.assertEqual(P('a/Some name. Ending with a dot.').suffix, '') self.assertEqual(P('/a/Some name. Ending with a dot.').suffix, '') + @needs_windows + def test_suffix_windows(self): + P = self.cls + self.assertEqual(P('c:').suffix, '') + self.assertEqual(P('c:/').suffix, '') + self.assertEqual(P('c:a/b').suffix, '') + self.assertEqual(P('c:/a/b').suffix, '') + self.assertEqual(P('c:a/b.py').suffix, '.py') + self.assertEqual(P('c:/a/b.py').suffix, '.py') + self.assertEqual(P('c:a/.hgrc').suffix, '') + self.assertEqual(P('c:/a/.hgrc').suffix, '') + self.assertEqual(P('c:a/.hg.rc').suffix, '.rc') + self.assertEqual(P('c:/a/.hg.rc').suffix, '.rc') + self.assertEqual(P('c:a/b.tar.gz').suffix, '.gz') + self.assertEqual(P('c:/a/b.tar.gz').suffix, '.gz') + self.assertEqual(P('c:a/Some name. Ending with a dot.').suffix, '') + self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffix, '') + self.assertEqual(P('//My.py/Share.php').suffix, '') + self.assertEqual(P('//My.py/Share.php/a/b').suffix, '') + def test_suffixes_common(self): P = self.cls self.assertEqual(P('').suffixes, []) @@ -431,6 +832,26 @@ def test_suffixes_common(self): self.assertEqual(P('a/Some name. Ending with a dot.').suffixes, []) self.assertEqual(P('/a/Some name. Ending with a dot.').suffixes, []) + @needs_windows + def test_suffixes_windows(self): + P = self.cls + self.assertEqual(P('c:').suffixes, []) + self.assertEqual(P('c:/').suffixes, []) + self.assertEqual(P('c:a/b').suffixes, []) + self.assertEqual(P('c:/a/b').suffixes, []) + self.assertEqual(P('c:a/b.py').suffixes, ['.py']) + self.assertEqual(P('c:/a/b.py').suffixes, ['.py']) + self.assertEqual(P('c:a/.hgrc').suffixes, []) + self.assertEqual(P('c:/a/.hgrc').suffixes, []) + self.assertEqual(P('c:a/.hg.rc').suffixes, ['.rc']) + self.assertEqual(P('c:/a/.hg.rc').suffixes, ['.rc']) + self.assertEqual(P('c:a/b.tar.gz').suffixes, ['.tar', '.gz']) + self.assertEqual(P('c:/a/b.tar.gz').suffixes, ['.tar', '.gz']) + self.assertEqual(P('//My.py/Share.php').suffixes, []) + self.assertEqual(P('//My.py/Share.php/a/b').suffixes, []) + self.assertEqual(P('c:a/Some name. Ending with a dot.').suffixes, []) + self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffixes, []) + def test_stem_empty(self): P = self.cls self.assertEqual(P('').stem, '') @@ -448,6 +869,20 @@ def test_stem_common(self): self.assertEqual(P('a/Some name. Ending with a dot.').stem, 'Some name. Ending with a dot.') + @needs_windows + def test_stem_windows(self): + P = self.cls + self.assertEqual(P('c:').stem, '') + self.assertEqual(P('c:.').stem, '') + self.assertEqual(P('c:..').stem, '..') + self.assertEqual(P('c:/').stem, '') + self.assertEqual(P('c:a/b').stem, 'b') + self.assertEqual(P('c:a/b.py').stem, 'b') + self.assertEqual(P('c:a/.hgrc').stem, '.hgrc') + self.assertEqual(P('c:a/.hg.rc').stem, '.hg') + self.assertEqual(P('c:a/b.tar.gz').stem, 'b.tar') + self.assertEqual(P('c:a/Some name. Ending with a dot.').stem, + 'Some name. Ending with a dot.') def test_with_name_common(self): P = self.cls self.assertEqual(P('a/b').with_name('d.xml'), P('a/d.xml')) @@ -457,6 +892,23 @@ def test_with_name_common(self): self.assertEqual(P('a/Dot ending.').with_name('d.xml'), P('a/d.xml')) self.assertEqual(P('/a/Dot ending.').with_name('d.xml'), P('/a/d.xml')) + @needs_windows + def test_with_name_windows(self): + P = self.cls + self.assertEqual(P('c:a/b').with_name('d.xml'), P('c:a/d.xml')) + self.assertEqual(P('c:/a/b').with_name('d.xml'), P('c:/a/d.xml')) + self.assertEqual(P('c:a/Dot ending.').with_name('d.xml'), P('c:a/d.xml')) + self.assertEqual(P('c:/a/Dot ending.').with_name('d.xml'), P('c:/a/d.xml')) + self.assertRaises(ValueError, P('c:').with_name, 'd.xml') + self.assertRaises(ValueError, P('c:/').with_name, 'd.xml') + self.assertRaises(ValueError, P('//My/Share').with_name, 'd.xml') + self.assertEqual(str(P('a').with_name('d:')), '.\\d:') + self.assertEqual(str(P('a').with_name('d:e')), '.\\d:e') + self.assertEqual(P('c:a/b').with_name('d:'), P('c:a/d:')) + self.assertEqual(P('c:a/b').with_name('d:e'), P('c:a/d:e')) + self.assertRaises(ValueError, P('c:a/b').with_name, 'd:/e') + self.assertRaises(ValueError, P('c:a/b').with_name, '//My/Share') + def test_with_name_empty(self): P = self.cls self.assertEqual(P('').with_name('d.xml'), P('d.xml')) @@ -481,6 +933,23 @@ def test_with_stem_common(self): self.assertEqual(P('a/Dot ending.').with_stem('d'), P('a/d')) self.assertEqual(P('/a/Dot ending.').with_stem('d'), P('/a/d')) + @needs_windows + def test_with_stem_windows(self): + P = self.cls + self.assertEqual(P('c:a/b').with_stem('d'), P('c:a/d')) + self.assertEqual(P('c:/a/b').with_stem('d'), P('c:/a/d')) + self.assertEqual(P('c:a/Dot ending.').with_stem('d'), P('c:a/d')) + self.assertEqual(P('c:/a/Dot ending.').with_stem('d'), P('c:/a/d')) + self.assertRaises(ValueError, P('c:').with_stem, 'd') + self.assertRaises(ValueError, P('c:/').with_stem, 'd') + self.assertRaises(ValueError, P('//My/Share').with_stem, 'd') + self.assertEqual(str(P('a').with_stem('d:')), '.\\d:') + self.assertEqual(str(P('a').with_stem('d:e')), '.\\d:e') + self.assertEqual(P('c:a/b').with_stem('d:'), P('c:a/d:')) + self.assertEqual(P('c:a/b').with_stem('d:e'), P('c:a/d:e')) + self.assertRaises(ValueError, P('c:a/b').with_stem, 'd:/e') + self.assertRaises(ValueError, P('c:a/b').with_stem, '//My/Share') + def test_with_stem_empty(self): P = self.cls self.assertEqual(P('').with_stem('d'), P('d')) @@ -505,12 +974,36 @@ def test_with_suffix_common(self): self.assertEqual(P('a/b.py').with_suffix(''), P('a/b')) self.assertEqual(P('/a/b').with_suffix(''), P('/a/b')) + @needs_windows + def test_with_suffix_windows(self): + P = self.cls + self.assertEqual(P('c:a/b').with_suffix('.gz'), P('c:a/b.gz')) + self.assertEqual(P('c:/a/b').with_suffix('.gz'), P('c:/a/b.gz')) + self.assertEqual(P('c:a/b.py').with_suffix('.gz'), P('c:a/b.gz')) + self.assertEqual(P('c:/a/b.py').with_suffix('.gz'), P('c:/a/b.gz')) + # Path doesn't have a "filename" component. + self.assertRaises(ValueError, P('').with_suffix, '.gz') + self.assertRaises(ValueError, P('.').with_suffix, '.gz') + self.assertRaises(ValueError, P('/').with_suffix, '.gz') + self.assertRaises(ValueError, P('//My/Share').with_suffix, '.gz') + # Invalid suffix. + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'gz') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '/') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\') + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '/.gz') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\.gz') + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:.gz') + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c/d') + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c\\d') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c/d') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c\\d') + def test_with_suffix_empty(self): P = self.cls # Path doesn't have a "filename" component. - self.assertEqual(P('').with_suffix('.gz'), P('.gz')) - self.assertEqual(P('.').with_suffix('.gz'), P('..gz')) - self.assertEqual(P('/').with_suffix('.gz'), P('/.gz')) + self.assertRaises(ValueError, P('').with_suffix, '.gz') + self.assertRaises(ValueError, P('/').with_suffix, '.gz') def test_with_suffix_seps(self): P = self.cls @@ -599,6 +1092,112 @@ def test_relative_to_common(self): self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) + @needs_windows + def test_relative_to_windows(self): + P = self.cls + p = P('C:Foo/Bar') + self.assertEqual(p.relative_to(P('c:')), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:'), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:foO')), P('Bar')) + self.assertEqual(p.relative_to('c:foO'), P('Bar')) + self.assertEqual(p.relative_to('c:foO/'), P('Bar')) + self.assertEqual(p.relative_to(P('c:foO/baR')), P()) + self.assertEqual(p.relative_to('c:foO/baR'), P()) + self.assertEqual(p.relative_to(P('c:'), walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:foO'), walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:foO', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:foO/', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to(P('c:foO/baR'), walk_up=True), P()) + self.assertEqual(p.relative_to('c:foO/baR', walk_up=True), P()) + self.assertEqual(p.relative_to(P('C:Foo/Bar/Baz'), walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('C:Foo/Baz'), walk_up=True), P('../Bar')) + self.assertEqual(p.relative_to(P('C:Baz/Bar'), walk_up=True), P('../../Foo/Bar')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P()) + self.assertRaises(ValueError, p.relative_to, '') + self.assertRaises(ValueError, p.relative_to, P('d:')) + self.assertRaises(ValueError, p.relative_to, P('/')) + self.assertRaises(ValueError, p.relative_to, P('Foo')) + self.assertRaises(ValueError, p.relative_to, P('/Foo')) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo')) + self.assertRaises(ValueError, p.relative_to, P('C:Foo/Bar/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:Foo/Baz')) + self.assertRaises(ValueError, p.relative_to, P(), walk_up=True) + self.assertRaises(ValueError, p.relative_to, '', walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo'), walk_up=True) + p = P('C:/Foo/Bar') + self.assertEqual(p.relative_to(P('c:/')), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:/'), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:/foO')), P('Bar')) + self.assertEqual(p.relative_to('c:/foO'), P('Bar')) + self.assertEqual(p.relative_to('c:/foO/'), P('Bar')) + self.assertEqual(p.relative_to(P('c:/foO/baR')), P()) + self.assertEqual(p.relative_to('c:/foO/baR'), P()) + self.assertEqual(p.relative_to(P('c:/'), walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:/', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:/foO'), walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:/foO', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:/foO/', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to(P('c:/foO/baR'), walk_up=True), P()) + self.assertEqual(p.relative_to('c:/foO/baR', walk_up=True), P()) + self.assertEqual(p.relative_to('C:/Baz', walk_up=True), P('../Foo/Bar')) + self.assertEqual(p.relative_to('C:/Foo/Bar/Baz', walk_up=True), P('..')) + self.assertEqual(p.relative_to('C:/Foo/Baz', walk_up=True), P('../Bar')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, 'c:') + self.assertRaises(ValueError, p.relative_to, P('c:')) + self.assertRaises(ValueError, p.relative_to, P('C:/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Bar/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:Foo')) + self.assertRaises(ValueError, p.relative_to, P('d:')) + self.assertRaises(ValueError, p.relative_to, P('d:/')) + self.assertRaises(ValueError, p.relative_to, P('/')) + self.assertRaises(ValueError, p.relative_to, P('/Foo')) + self.assertRaises(ValueError, p.relative_to, P('//C/Foo')) + self.assertRaises(ValueError, p.relative_to, 'c:', walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('c:'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('C:Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('d:/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('//C/Foo'), walk_up=True) + # UNC paths. + p = P('//Server/Share/Foo/Bar') + self.assertEqual(p.relative_to(P('//sErver/sHare')), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare'), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/'), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo')), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo'), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/'), P('Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar')), P()) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar'), P()) + self.assertEqual(p.relative_to(P('//sErver/sHare'), walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo'), walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar'), walk_up=True), P()) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar', walk_up=True), P()) + self.assertEqual(p.relative_to(P('//sErver/sHare/bar'), walk_up=True), P('../Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/bar', walk_up=True), P('../Foo/Bar')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo')) + self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo')) + self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo')) + self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo')) + self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo'), walk_up=True) + def test_is_relative_to_common(self): P = self.cls p = P('a/b') @@ -631,6 +1230,98 @@ def test_is_relative_to_common(self): self.assertFalse(p.is_relative_to('')) self.assertFalse(p.is_relative_to(P('a'))) + @needs_windows + def test_is_relative_to_windows(self): + P = self.cls + p = P('C:Foo/Bar') + self.assertTrue(p.is_relative_to(P('c:'))) + self.assertTrue(p.is_relative_to('c:')) + self.assertTrue(p.is_relative_to(P('c:foO'))) + self.assertTrue(p.is_relative_to('c:foO')) + self.assertTrue(p.is_relative_to('c:foO/')) + self.assertTrue(p.is_relative_to(P('c:foO/baR'))) + self.assertTrue(p.is_relative_to('c:foO/baR')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P())) + self.assertFalse(p.is_relative_to('')) + self.assertFalse(p.is_relative_to(P('d:'))) + self.assertFalse(p.is_relative_to(P('/'))) + self.assertFalse(p.is_relative_to(P('Foo'))) + self.assertFalse(p.is_relative_to(P('/Foo'))) + self.assertFalse(p.is_relative_to(P('C:/Foo'))) + self.assertFalse(p.is_relative_to(P('C:Foo/Bar/Baz'))) + self.assertFalse(p.is_relative_to(P('C:Foo/Baz'))) + p = P('C:/Foo/Bar') + self.assertTrue(p.is_relative_to(P('c:/'))) + self.assertTrue(p.is_relative_to(P('c:/foO'))) + self.assertTrue(p.is_relative_to('c:/foO/')) + self.assertTrue(p.is_relative_to(P('c:/foO/baR'))) + self.assertTrue(p.is_relative_to('c:/foO/baR')) + # Unrelated paths. + self.assertFalse(p.is_relative_to('c:')) + self.assertFalse(p.is_relative_to(P('C:/Baz'))) + self.assertFalse(p.is_relative_to(P('C:/Foo/Bar/Baz'))) + self.assertFalse(p.is_relative_to(P('C:/Foo/Baz'))) + self.assertFalse(p.is_relative_to(P('C:Foo'))) + self.assertFalse(p.is_relative_to(P('d:'))) + self.assertFalse(p.is_relative_to(P('d:/'))) + self.assertFalse(p.is_relative_to(P('/'))) + self.assertFalse(p.is_relative_to(P('/Foo'))) + self.assertFalse(p.is_relative_to(P('//C/Foo'))) + # UNC paths. + p = P('//Server/Share/Foo/Bar') + self.assertTrue(p.is_relative_to(P('//sErver/sHare'))) + self.assertTrue(p.is_relative_to('//sErver/sHare')) + self.assertTrue(p.is_relative_to('//sErver/sHare/')) + self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo'))) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo')) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/')) + self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo/Bar'))) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/Bar')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('/Server/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('c:/Server/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('//z/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('//Server/z/Foo'))) + + @needs_posix + def test_is_absolute_posix(self): + P = self.cls + self.assertFalse(P('').is_absolute()) + self.assertFalse(P('a').is_absolute()) + self.assertFalse(P('a/b/').is_absolute()) + self.assertTrue(P('/').is_absolute()) + self.assertTrue(P('/a').is_absolute()) + self.assertTrue(P('/a/b/').is_absolute()) + self.assertTrue(P('//a').is_absolute()) + self.assertTrue(P('//a/b').is_absolute()) + + @needs_windows + def test_is_absolute_windows(self): + P = self.cls + # Under NT, only paths with both a drive and a root are absolute. + self.assertFalse(P().is_absolute()) + self.assertFalse(P('a').is_absolute()) + self.assertFalse(P('a/b/').is_absolute()) + self.assertFalse(P('/').is_absolute()) + self.assertFalse(P('/a').is_absolute()) + self.assertFalse(P('/a/b/').is_absolute()) + self.assertFalse(P('c:').is_absolute()) + self.assertFalse(P('c:a').is_absolute()) + self.assertFalse(P('c:a/b/').is_absolute()) + self.assertTrue(P('c:/').is_absolute()) + self.assertTrue(P('c:/a').is_absolute()) + self.assertTrue(P('c:/a/b/').is_absolute()) + # UNC paths are absolute by definition. + self.assertTrue(P('//').is_absolute()) + self.assertTrue(P('//a').is_absolute()) + self.assertTrue(P('//a/b').is_absolute()) + self.assertTrue(P('//a/b/').is_absolute()) + self.assertTrue(P('//a/b/c').is_absolute()) + self.assertTrue(P('//a/b/c/d').is_absolute()) + self.assertTrue(P('//?/UNC/').is_absolute()) + self.assertTrue(P('//?/UNC/spam').is_absolute()) + # # Tests for the virtual classes. @@ -836,6 +1527,9 @@ class DummyPathTest(DummyPurePathTest): def setUp(self): super().setUp() + name = self.id().split('.')[-1] + if name in _tests_needing_symlinks and not self.can_symlink: + self.skipTest('requires symlinks') pathmod = self.cls.pathmod p = self.cls(self.base) p.mkdir(parents=True) @@ -993,9 +1687,8 @@ def test_iterdir(self): expected += ['linkA', 'linkB', 'brokenLink', 'brokenLinkLoop'] self.assertEqual(paths, { P(self.base, q) for q in expected }) + @needs_symlinks def test_iterdir_symlink(self): - if not self.can_symlink: - self.skipTest("symlinks required") # __iter__ on a symlink to a directory. P = self.cls p = P(self.base, 'linkB') @@ -1044,13 +1737,31 @@ def _check(glob, expected): else: _check(p.glob("*/"), ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"]) + @needs_posix + def test_glob_posix(self): + P = self.cls + p = P(self.base) + given = set(p.glob("FILEa")) + expect = set() + self.assertEqual(given, expect) + self.assertEqual(set(p.glob("FILEa*")), set()) + + @needs_windows + def test_glob_windows(self): + P = self.cls + p = P(self.base) + self.assertEqual(set(p.glob("FILEa")), { P(self.base, "fileA") }) + self.assertEqual(set(p.glob("*a\\")), { P(self.base, "dirA/") }) + self.assertEqual(set(p.glob("F*a")), { P(self.base, "fileA") }) + self.assertEqual(set(map(str, p.glob("FILEa"))), {f"{p}\\fileA"}) + self.assertEqual(set(map(str, p.glob("F*a"))), {f"{p}\\fileA"}) + def test_glob_empty_pattern(self): - def _check(glob, expected): - self.assertEqual(set(glob), { P(self.base, q) for q in expected }) P = self.cls p = P(self.base) - _check(p.glob(""), [""]) - _check(p.glob("."), ["."]) + self.assertEqual(list(p.glob("")), [p]) + self.assertEqual(list(p.glob(".")), [p / "."]) + self.assertEqual(list(p.glob("./")), [p / "./"]) def test_glob_case_sensitive(self): P = self.cls @@ -1064,9 +1775,8 @@ def _check(path, pattern, case_sensitive, expected): _check(path, "dirb/file*", True, []) _check(path, "dirb/file*", False, ["dirB/fileB"]) + @needs_symlinks def test_glob_follow_symlinks_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") def _check(path, glob, expected): actual = {path for path in path.glob(glob, follow_symlinks=True) if path.parts.count("linkD") <= 1} # exclude symlink loop. @@ -1080,21 +1790,30 @@ def _check(path, glob, expected): _check(p, "*/fileB", ["dirB/fileB", "linkB/fileB"]) _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"]) _check(p, "dir*/*/..", ["dirC/dirD/..", "dirA/linkC/..", "dirB/linkD/.."]) + _check(p, "dir*/**", [ + "dirA/", "dirA/linkC", "dirA/linkC/fileB", "dirA/linkC/linkD", "dirA/linkC/linkD/fileB", + "dirB/", "dirB/fileB", "dirB/linkD", "dirB/linkD/fileB", + "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt", + "dirE/"]) _check(p, "dir*/**/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", "dirC/", "dirC/dirD/", "dirE/"]) _check(p, "dir*/**/..", ["dirA/..", "dirA/linkC/..", "dirB/..", "dirB/linkD/..", "dirA/linkC/linkD/..", "dirC/..", "dirC/dirD/..", "dirE/.."]) + _check(p, "dir*/*/**", [ + "dirA/linkC/", "dirA/linkC/linkD", "dirA/linkC/fileB", "dirA/linkC/linkD/fileB", + "dirB/linkD/", "dirB/linkD/fileB", + "dirC/dirD/", "dirC/dirD/fileD"]) _check(p, "dir*/*/**/", ["dirA/linkC/", "dirA/linkC/linkD/", "dirB/linkD/", "dirC/dirD/"]) _check(p, "dir*/*/**/..", ["dirA/linkC/..", "dirA/linkC/linkD/..", "dirB/linkD/..", "dirC/dirD/.."]) _check(p, "dir*/**/fileC", ["dirC/fileC"]) _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) + _check(p, "*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) _check(p, "*/dirD/**/", ["dirC/dirD/"]) + @needs_symlinks def test_glob_no_follow_symlinks_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") def _check(path, glob, expected): actual = {path for path in path.glob(glob, follow_symlinks=False)} self.assertEqual(actual, { P(self.base, q) for q in expected }) @@ -1107,12 +1826,20 @@ def _check(path, glob, expected): _check(p, "*/fileB", ["dirB/fileB"]) _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/"]) _check(p, "dir*/*/..", ["dirC/dirD/.."]) + _check(p, "dir*/**", [ + "dirA/", "dirA/linkC", + "dirB/", "dirB/fileB", "dirB/linkD", + "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt", + "dirE/"]) _check(p, "dir*/**/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"]) _check(p, "dir*/**/..", ["dirA/..", "dirB/..", "dirC/..", "dirC/dirD/..", "dirE/.."]) + _check(p, "dir*/*/**", ["dirC/dirD/", "dirC/dirD/fileD"]) _check(p, "dir*/*/**/", ["dirC/dirD/"]) _check(p, "dir*/*/**/..", ["dirC/dirD/.."]) _check(p, "dir*/**/fileC", ["dirC/fileC"]) + _check(p, "dir*/*/../dirD/**", ["dirC/dirD/../dirD/", "dirC/dirD/../dirD/fileD"]) _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) + _check(p, "*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) _check(p, "*/dirD/**/", ["dirC/dirD/"]) def test_rglob_common(self): @@ -1149,18 +1876,37 @@ def _check(glob, expected): "dirC/dirD", "dirC/dirD/fileD"]) _check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"]) _check(p.rglob("**/file*"), ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p.rglob("dir*/**"), ["dirC/dirD/", "dirC/dirD/fileD"]) _check(p.rglob("dir*/**/"), ["dirC/dirD/"]) _check(p.rglob("*/*"), ["dirC/dirD/fileD"]) _check(p.rglob("*/"), ["dirC/dirD/"]) _check(p.rglob(""), ["dirC/", "dirC/dirD/"]) + _check(p.rglob("**"), [ + "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt"]) _check(p.rglob("**/"), ["dirC/", "dirC/dirD/"]) # gh-91616, a re module regression _check(p.rglob("*.txt"), ["dirC/novel.txt"]) _check(p.rglob("*.*"), ["dirC/novel.txt"]) + @needs_posix + def test_rglob_posix(self): + P = self.cls + p = P(self.base, "dirC") + given = set(p.rglob("FILEd")) + expect = set() + self.assertEqual(given, expect) + self.assertEqual(set(p.rglob("FILEd*")), set()) + + @needs_windows + def test_rglob_windows(self): + P = self.cls + p = P(self.base, "dirC") + self.assertEqual(set(p.rglob("FILEd")), { P(self.base, "dirC/dirD/fileD") }) + self.assertEqual(set(p.rglob("*\\")), { P(self.base, "dirC/dirD/") }) + self.assertEqual(set(map(str, p.rglob("FILEd"))), {f"{p}\\dirD\\fileD"}) + + @needs_symlinks def test_rglob_follow_symlinks_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") def _check(path, glob, expected): actual = {path for path in path.rglob(glob, follow_symlinks=True) if path.parts.count("linkD") <= 1} # exclude symlink loop. @@ -1191,9 +1937,8 @@ def _check(path, glob, expected): _check(p, "*.txt", ["dirC/novel.txt"]) _check(p, "*.*", ["dirC/novel.txt"]) + @needs_symlinks def test_rglob_no_follow_symlinks_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") def _check(path, glob, expected): actual = {path for path in path.rglob(glob, follow_symlinks=False)} self.assertEqual(actual, { P(self.base, q) for q in expected }) @@ -1217,10 +1962,9 @@ def _check(path, glob, expected): _check(p, "*.txt", ["dirC/novel.txt"]) _check(p, "*.*", ["dirC/novel.txt"]) + @needs_symlinks def test_rglob_symlink_loop(self): # Don't get fooled by symlink loops (Issue #26012). - if not self.can_symlink: - self.skipTest("symlinks required") P = self.cls p = P(self.base) given = set(p.rglob('*')) @@ -1250,10 +1994,9 @@ def test_glob_dotdot(self): self.assertEqual(set(p.glob("xyzzy/..")), set()) self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(self.base, *[".."] * 50)}) + @needs_symlinks def test_glob_permissions(self): # See bpo-38894 - if not self.can_symlink: - self.skipTest("symlinks required") P = self.cls base = P(self.base) / 'permissions' base.mkdir() @@ -1270,19 +2013,17 @@ def test_glob_permissions(self): self.assertEqual(len(set(base.glob("*/fileC"))), 50) self.assertEqual(len(set(base.glob("*/file*"))), 50) + @needs_symlinks def test_glob_long_symlink(self): # See gh-87695 - if not self.can_symlink: - self.skipTest("symlinks required") base = self.cls(self.base) / 'long_symlink' base.mkdir() bad_link = base / 'bad_link' bad_link.symlink_to("bad" * 200) self.assertEqual(sorted(base.glob('**/*')), [bad_link]) + @needs_symlinks def test_readlink(self): - if not self.can_symlink: - self.skipTest("symlinks required") P = self.cls(self.base) self.assertEqual((P / 'linkA').readlink(), self.cls('fileA')) self.assertEqual((P / 'brokenLink').readlink(), @@ -1306,9 +2047,8 @@ def _check_resolve(self, p, expected, strict=True): # This can be used to check both relative and absolute resolutions. _check_resolve_relative = _check_resolve_absolute = _check_resolve + @needs_symlinks def test_resolve_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") P = self.cls p = P(self.base, 'foo') with self.assertRaises(OSError) as cm: @@ -1367,10 +2107,9 @@ def test_resolve_common(self): # resolves to 'dirB/..' first before resolving to parent of dirB. self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) + @needs_symlinks def test_resolve_dot(self): # See http://web.archive.org/web/20200623062557/https://bitbucket.org/pitrou/pathlib/issues/9/ - if not self.can_symlink: - self.skipTest("symlinks required") pathmod = self.pathmod p = self.cls(self.base) p.joinpath('0').symlink_to('.', target_is_directory=True) @@ -1389,11 +2128,9 @@ def _check_symlink_loop(self, *args): path.resolve(strict=True) self.assertEqual(cm.exception.errno, errno.ELOOP) + @needs_posix + @needs_symlinks def test_resolve_loop(self): - if not self.can_symlink: - self.skipTest("symlinks required") - if self.cls.pathmod is not posixpath: - self.skipTest("symlink loops work differently with concrete Windows paths") # Loops with relative symlinks. self.cls(self.base, 'linkX').symlink_to('linkX/inside') self._check_symlink_loop(self.base, 'linkX') @@ -1435,9 +2172,8 @@ def test_stat(self): self.assertEqual(statA.st_dev, statC.st_dev) # other attributes not used by pathlib. + @needs_symlinks def test_stat_no_follow_symlinks(self): - if not self.can_symlink: - self.skipTest("symlinks required") p = self.cls(self.base) / 'linkA' st = p.stat() self.assertNotEqual(st, p.stat(follow_symlinks=False)) @@ -1447,9 +2183,8 @@ def test_stat_no_follow_symlinks_nosymlink(self): st = p.stat() self.assertEqual(st, p.stat(follow_symlinks=False)) + @needs_symlinks def test_lstat(self): - if not self.can_symlink: - self.skipTest("symlinks required") p = self.cls(self.base)/ 'linkA' st = p.stat() self.assertNotEqual(st, p.lstat()) @@ -1582,9 +2317,6 @@ def test_is_char_device_false(self): self.assertIs((P / 'fileA\x00').is_char_device(), False) def _check_complex_symlinks(self, link0_target): - if not self.can_symlink: - self.skipTest("symlinks required") - # Test solving a non-looping chain of symlinks (issue #19887). pathmod = self.pathmod P = self.cls(self.base) @@ -1630,12 +2362,15 @@ def _check_complex_symlinks(self, link0_target): finally: os.chdir(old_path) + @needs_symlinks def test_complex_symlinks_absolute(self): self._check_complex_symlinks(self.base) + @needs_symlinks def test_complex_symlinks_relative(self): self._check_complex_symlinks('.') + @needs_symlinks def test_complex_symlinks_relative_dot_dot(self): self._check_complex_symlinks(self.pathmod.join('dirA', '..')) @@ -1751,9 +2486,8 @@ def test_walk_bottom_up(self): raise AssertionError(f"Unexpected path: {path}") self.assertTrue(seen_testfn) + @needs_symlinks def test_walk_follow_symlinks(self): - if not self.can_symlink: - self.skipTest("symlinks required") self.setUpWalk() walk_it = self.walk_path.walk(follow_symlinks=True) for root, dirs, files in walk_it: @@ -1764,9 +2498,8 @@ def test_walk_follow_symlinks(self): else: self.fail("Didn't follow symlink with follow_symlinks=True") + @needs_symlinks def test_walk_symlink_location(self): - if not self.can_symlink: - self.skipTest("symlinks required") self.setUpWalk() # Tests whether symlinks end up in filenames or dirnames depending # on the `follow_symlinks` argument. diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 03487aa6ffd81f4..2b0795cdad707e1 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -779,58 +779,62 @@ def test_pdb_where_command(): (Pdb) continue """ -def test_pdb_interact_command(): - """Test interact command - >>> g = 0 - >>> dict_g = {} +# skip this test if sys.flags.no_site = True; +# exit() isn't defined unless there's a site module. +if not sys.flags.no_site: + def test_pdb_interact_command(): + """Test interact command - >>> def test_function(): - ... x = 1 - ... lst_local = [] - ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + >>> g = 0 + >>> dict_g = {} - >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE - ... 'interact', - ... 'x', - ... 'g', - ... 'x = 2', - ... 'g = 3', - ... 'dict_g["a"] = True', - ... 'lst_local.append(x)', - ... 'exit()', - ... 'p x', - ... 'p g', - ... 'p dict_g', - ... 'p lst_local', - ... 'continue', - ... ]): - ... test_function() - --Return-- - > <doctest test.test_pdb.test_pdb_interact_command[2]>(4)test_function()->None - -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - (Pdb) interact - *pdb interact start* - ... x - 1 - ... g - 0 - ... x = 2 - ... g = 3 - ... dict_g["a"] = True - ... lst_local.append(x) - ... exit() - *exit from pdb interact command* - (Pdb) p x - 1 - (Pdb) p g - 0 - (Pdb) p dict_g - {'a': True} - (Pdb) p lst_local - [2] - (Pdb) continue - """ + >>> def test_function(): + ... x = 1 + ... lst_local = [] + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + + >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'interact', + ... 'x', + ... 'g', + ... 'x = 2', + ... 'g = 3', + ... 'dict_g["a"] = True', + ... 'lst_local.append(x)', + ... 'exit()', + ... 'p x', + ... 'p g', + ... 'p dict_g', + ... 'p lst_local', + ... 'continue', + ... ]): + ... test_function() + --Return-- + > <doctest test.test_pdb.test_pdb_interact_command[2]>(4)test_function()->None + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) interact + *pdb interact start* + ... x + 1 + ... g + 0 + ... x = 2 + ... g = 3 + ... dict_g["a"] = True + ... lst_local.append(x) + ... exit() + *exit from pdb interact command* + (Pdb) p x + 1 + (Pdb) p g + 0 + (Pdb) p dict_g + {'a': True} + (Pdb) p lst_local + [2] + (Pdb) continue + """ def test_convenience_variables(): """Test convenience variables @@ -2661,7 +2665,7 @@ def quux(): pass """.encode(), 'bœr', - ('bœr', 4), + ('bœr', 5), ) def test_find_function_found_with_encoding_cookie(self): @@ -2678,7 +2682,7 @@ def quux(): pass """.encode('iso-8859-15'), 'bœr', - ('bœr', 5), + ('bœr', 6), ) def test_find_function_found_with_bom(self): @@ -2688,9 +2692,34 @@ def bœr(): pass """.encode(), 'bœr', - ('bœr', 1), + ('bœr', 2), ) + def test_find_function_first_executable_line(self): + code = textwrap.dedent("""\ + def foo(): pass + + def bar(): + pass # line 4 + + def baz(): + # comment + pass # line 8 + + def mul(): + # code on multiple lines + code = compile( # line 12 + 'def f()', + '<string>', + 'exec', + ) + """).encode() + + self._assert_find_function(code, 'foo', ('foo', 1)) + self._assert_find_function(code, 'bar', ('bar', 4)) + self._assert_find_function(code, 'baz', ('baz', 8)) + self._assert_find_function(code, 'mul', ('mul', 12)) + def test_issue7964(self): # open the file as binary so we can force \r\n newline with open(os_helper.TESTFN, 'wb') as f: @@ -3056,6 +3085,87 @@ def test_blocks_at_first_code_line(self): self.assertTrue(any("__main__.py(4)<module>()" in l for l in stdout.splitlines()), stdout) + def test_file_modified_after_execution(self): + script = """ + print("hello") + """ + + commands = """ + filename = $_frame.f_code.co_filename + f = open(filename, "w") + f.write("print('goodbye')") + f.close() + ll + """ + + stdout, stderr = self.run_pdb_script(script, commands) + self.assertIn("WARNING:", stdout) + self.assertIn("was edited", stdout) + + def test_file_modified_after_execution_with_multiple_instances(self): + script = """ + import pdb; pdb.Pdb().set_trace() + with open(__file__, "w") as f: + f.write("print('goodbye')\\n" * 5) + import pdb; pdb.Pdb().set_trace() + """ + + commands = """ + continue + continue + """ + + filename = 'main.py' + with open(filename, 'w') as f: + f.write(textwrap.dedent(script)) + self.addCleanup(os_helper.unlink, filename) + self.addCleanup(os_helper.rmtree, '__pycache__') + cmd = [sys.executable, filename] + with subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE, + stderr=subprocess.STDOUT, + env = {**os.environ, 'PYTHONIOENCODING': 'utf-8'}, + ) as proc: + stdout, _ = proc.communicate(str.encode(commands)) + stdout = stdout and bytes.decode(stdout) + + self.assertEqual(proc.returncode, 0) + self.assertIn("WARNING:", stdout) + self.assertIn("was edited", stdout) + + def test_file_modified_after_execution_with_restart(self): + script = """ + import random + # Any code with a source to step into so this script is not checked + # for changes when it's being changed + random.randint(1, 4) + print("hello") + """ + + commands = """ + ll + n + s + filename = $_frame.f_back.f_code.co_filename + def change_file(content, filename): + with open(filename, "w") as f: + f.write(f"print({content})") + + change_file('world', filename) + restart + ll + """ + + stdout, stderr = self.run_pdb_script(script, commands) + # Make sure the code is running correctly and the file is edited + self.assertIn("hello", stdout) + self.assertIn("world", stdout) + # The file was edited, but restart should clear the state and consider + # the file as up to date + self.assertNotIn("WARNING:", stdout) + def test_relative_imports(self): self.module_name = 't_main' os_helper.rmtree(self.module_name) diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 76a6f25c34bbd35..2ea186c85c8823e 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -1150,10 +1150,11 @@ def get_insts(lno1, lno2, op1, op2): lno1, lno2 = (4, 5) with self.subTest(lno = (lno1, lno2), ops = (op1, op2)): insts = get_insts(lno1, lno2, op1, op2) + op = 'JUMP' if 'JUMP' in (op1, op2) else 'JUMP_NO_INTERRUPT' expected_insts = [ ('LOAD_NAME', 0, 10), ('NOP', 0, 4), - (op2, 0, 5), + (op, 0, 5), ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index f6405d6dd44ef6f..5e187e5189d1179 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -402,7 +402,9 @@ def recurse(deep): check_unpickler(recurse(1), 32, 20) check_unpickler(recurse(20), 32, 20) check_unpickler(recurse(50), 64, 60) - check_unpickler(recurse(100), 128, 140) + if not (support.is_wasi and support.Py_DEBUG): + # stack depth too shallow in pydebug WASI. + check_unpickler(recurse(100), 128, 140) u = unpickler(io.BytesIO(pickle.dumps('a', 0)), encoding='ASCII', errors='strict') @@ -565,7 +567,8 @@ def test_exceptions(self): RecursionError, EncodingWarning, BaseExceptionGroup, - ExceptionGroup): + ExceptionGroup, + IncompleteInputError): continue if exc is not OSError and issubclass(exc, OSError): self.assertEqual(reverse_mapping('builtins', name), diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 216973350319fe6..648e18d0150ef08 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -472,7 +472,8 @@ def test_macos(self): 'root:xnu-4570.71.2~1/RELEASE_X86_64'), 'x86_64', 'i386') arch = ('64bit', '') - with mock.patch.object(platform, 'uname', return_value=uname), \ + with mock.patch.object(sys, "platform", "darwin"), \ + mock.patch.object(platform, 'uname', return_value=uname), \ mock.patch.object(platform, 'architecture', return_value=arch): for mac_ver, expected_terse, expected in [ # darwin: mac_ver() returns empty strings diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 9c382ace806e0f8..a45f620e18dc1d0 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1,7 +1,7 @@ "Test posix functions" from test import support -from test.support import import_helper +from test.support import is_apple from test.support import os_helper from test.support import warnings_helper from test.support.script_helper import assert_python_ok @@ -781,9 +781,10 @@ def check_stat(uid, gid): check_stat(uid, gid) self.assertRaises(OSError, chown_func, first_param, 0, -1) check_stat(uid, gid) - if 0 not in os.getgroups(): - self.assertRaises(OSError, chown_func, first_param, -1, 0) - check_stat(uid, gid) + if hasattr(os, 'getgroups'): + if 0 not in os.getgroups(): + self.assertRaises(OSError, chown_func, first_param, -1, 0) + check_stat(uid, gid) # test illegal types for t in str, float: self.assertRaises(TypeError, chown_func, first_param, t(uid), gid) @@ -1256,8 +1257,8 @@ def test_sched_priority(self): self.assertIsInstance(lo, int) self.assertIsInstance(hi, int) self.assertGreaterEqual(hi, lo) - # OSX evidently just returns 15 without checking the argument. - if sys.platform != "darwin": + # Apple plaforms return 15 without checking the argument. + if not is_apple: self.assertRaises(OSError, posix.sched_get_priority_min, -23) self.assertRaises(OSError, posix.sched_get_priority_max, -23) @@ -1513,6 +1514,13 @@ def test_stat_dir_fd(self): self.assertRaises(OverflowError, posix.stat, name, dir_fd=10**20) + for fd in False, True: + with self.assertWarnsRegex(RuntimeWarning, + 'bool is used as a file descriptor') as cm: + with self.assertRaises(OSError): + posix.stat('nonexisting', dir_fd=fd) + self.assertEqual(cm.filename, __file__) + @unittest.skipUnless(os.utime in os.supports_dir_fd, "test needs dir_fd support in os.utime()") def test_utime_dir_fd(self): with self.prepare_file() as (dir_fd, name, fullname): @@ -2028,11 +2036,13 @@ def test_dup2(self): @unittest.skipUnless(hasattr(os, 'posix_spawn'), "test needs os.posix_spawn") +@support.requires_subprocess() class TestPosixSpawn(unittest.TestCase, _PosixSpawnMixin): spawn_func = getattr(posix, 'posix_spawn', None) @unittest.skipUnless(hasattr(os, 'posix_spawnp'), "test needs os.posix_spawnp") +@support.requires_subprocess() class TestPosixSpawnP(unittest.TestCase, _PosixSpawnMixin): spawn_func = getattr(posix, 'posix_spawnp', None) diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index f31a68c5d84e037..3f2bac0155fd9e8 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -1,11 +1,17 @@ -from test.support import verbose, reap_children +import sys +import unittest +from test.support import ( + is_apple_mobile, is_emscripten, is_wasi, reap_children, verbose +) from test.support.import_helper import import_module +from test.support.os_helper import TESTFN, unlink -# Skip these tests if termios or fcntl are not available +# Skip these tests if termios is not available import_module('termios') -# fcntl is a proxy for not being one of the wasm32 platforms even though we -# don't use this module... a proper check for what crashes those is needed. -import_module("fcntl") + +# Skip tests on WASM platforms, plus iOS/tvOS/watchOS +if is_apple_mobile or is_emscripten or is_wasi: + raise unittest.SkipTest(f"pty tests not required on {sys.platform}") import errno import os @@ -16,7 +22,6 @@ import signal import socket import io # readline -import unittest import warnings TEST_STRING_1 = b"I wish to buy a fish license.\n" @@ -292,7 +297,26 @@ def test_master_read(self): self.assertEqual(data, b"") def test_spawn_doesnt_hang(self): - pty.spawn([sys.executable, '-c', 'print("hi there")']) + self.addCleanup(unlink, TESTFN) + with open(TESTFN, 'wb') as f: + STDOUT_FILENO = 1 + dup_stdout = os.dup(STDOUT_FILENO) + os.dup2(f.fileno(), STDOUT_FILENO) + buf = b'' + def master_read(fd): + nonlocal buf + data = os.read(fd, 1024) + buf += data + return data + try: + pty.spawn([sys.executable, '-c', 'print("hi there")'], + master_read) + finally: + os.dup2(dup_stdout, STDOUT_FILENO) + os.close(dup_stdout) + self.assertEqual(buf, b'hi there\r\n') + with open(TESTFN, 'rb') as f: + self.assertEqual(f.read(), b'hi there\r\n') class SmallPtyTests(unittest.TestCase): """These tests don't spawn children or hang.""" diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index 33113a72e6b6a9d..e3d4d566cdda48a 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -241,6 +241,384 @@ def test_shrinking_queue(self): with self.assertRaises(self.queue.Full): q.put_nowait(4) + def test_shutdown_empty(self): + q = self.type2test() + q.shutdown() + with self.assertRaises(self.queue.ShutDown): + q.put("data") + with self.assertRaises(self.queue.ShutDown): + q.get() + + def test_shutdown_nonempty(self): + q = self.type2test() + q.put("data") + q.shutdown() + q.get() + with self.assertRaises(self.queue.ShutDown): + q.get() + + def test_shutdown_immediate(self): + q = self.type2test() + q.put("data") + q.shutdown(immediate=True) + with self.assertRaises(self.queue.ShutDown): + q.get() + + def test_shutdown_allowed_transitions(self): + # allowed transitions would be from alive via shutdown to immediate + q = self.type2test() + self.assertFalse(q.is_shutdown) + + q.shutdown() + self.assertTrue(q.is_shutdown) + + q.shutdown(immediate=True) + self.assertTrue(q.is_shutdown) + + q.shutdown(immediate=False) + + def _shutdown_all_methods_in_one_thread(self, immediate): + q = self.type2test(2) + q.put("L") + q.put_nowait("O") + q.shutdown(immediate) + + with self.assertRaises(self.queue.ShutDown): + q.put("E") + with self.assertRaises(self.queue.ShutDown): + q.put_nowait("W") + if immediate: + with self.assertRaises(self.queue.ShutDown): + q.get() + with self.assertRaises(self.queue.ShutDown): + q.get_nowait() + with self.assertRaises(ValueError): + q.task_done() + q.join() + else: + self.assertIn(q.get(), "LO") + q.task_done() + self.assertIn(q.get(), "LO") + q.task_done() + q.join() + # on shutdown(immediate=False) + # when queue is empty, should raise ShutDown Exception + with self.assertRaises(self.queue.ShutDown): + q.get() # p.get(True) + with self.assertRaises(self.queue.ShutDown): + q.get_nowait() # p.get(False) + with self.assertRaises(self.queue.ShutDown): + q.get(True, 1.0) + + def test_shutdown_all_methods_in_one_thread(self): + return self._shutdown_all_methods_in_one_thread(False) + + def test_shutdown_immediate_all_methods_in_one_thread(self): + return self._shutdown_all_methods_in_one_thread(True) + + def _write_msg_thread(self, q, n, results, delay, + i_when_exec_shutdown, + event_start, event_end): + event_start.wait() + for i in range(1, n+1): + try: + q.put((i, "YDLO")) + results.append(True) + except self.queue.ShutDown: + results.append(False) + # triggers shutdown of queue + if i == i_when_exec_shutdown: + event_end.set() + time.sleep(delay) + # end of all puts + q.join() + + def _read_msg_thread(self, q, nb, results, delay, event_start): + event_start.wait() + block = True + while nb: + time.sleep(delay) + try: + # Get at least one message + q.get(block) + block = False + q.task_done() + results.append(True) + nb -= 1 + except self.queue.ShutDown: + results.append(False) + nb -= 1 + except self.queue.Empty: + pass + q.join() + + def _shutdown_thread(self, q, event_end, immediate): + event_end.wait() + q.shutdown(immediate) + q.join() + + def _join_thread(self, q, delay, event_start): + event_start.wait() + time.sleep(delay) + q.join() + + def _shutdown_all_methods_in_many_threads(self, immediate): + q = self.type2test() + ps = [] + ev_start = threading.Event() + ev_exec_shutdown = threading.Event() + res_puts = [] + res_gets = [] + delay = 1e-4 + read_process = 4 + nb_msgs = read_process * 16 + nb_msgs_r = nb_msgs // read_process + when_exec_shutdown = nb_msgs // 2 + lprocs = ( + (self._write_msg_thread, 1, (q, nb_msgs, res_puts, delay, + when_exec_shutdown, + ev_start, ev_exec_shutdown)), + (self._read_msg_thread, read_process, (q, nb_msgs_r, + res_gets, delay*2, + ev_start)), + (self._join_thread, 2, (q, delay*2, ev_start)), + (self._shutdown_thread, 1, (q, ev_exec_shutdown, immediate)), + ) + # start all threds + for func, n, args in lprocs: + for i in range(n): + ps.append(threading.Thread(target=func, args=args)) + ps[-1].start() + # set event in order to run q.shutdown() + ev_start.set() + + if not immediate: + assert(len(res_gets) == len(res_puts)) + assert(res_gets.count(True) == res_puts.count(True)) + else: + assert(len(res_gets) <= len(res_puts)) + assert(res_gets.count(True) <= res_puts.count(True)) + + for thread in ps[1:]: + thread.join() + + def test_shutdown_all_methods_in_many_threads(self): + return self._shutdown_all_methods_in_many_threads(False) + + def test_shutdown_immediate_all_methods_in_many_threads(self): + return self._shutdown_all_methods_in_many_threads(True) + + def _get(self, q, go, results, shutdown=False): + go.wait() + try: + msg = q.get() + results.append(not shutdown) + return not shutdown + except self.queue.ShutDown: + results.append(shutdown) + return shutdown + + def _get_shutdown(self, q, go, results): + return self._get(q, go, results, True) + + def _get_task_done(self, q, go, results): + go.wait() + try: + msg = q.get() + q.task_done() + results.append(True) + return msg + except self.queue.ShutDown: + results.append(False) + return False + + def _put(self, q, msg, go, results, shutdown=False): + go.wait() + try: + q.put(msg) + results.append(not shutdown) + return not shutdown + except self.queue.ShutDown: + results.append(shutdown) + return shutdown + + def _put_shutdown(self, q, msg, go, results): + return self._put(q, msg, go, results, True) + + def _join(self, q, results, shutdown=False): + try: + q.join() + results.append(not shutdown) + return not shutdown + except self.queue.ShutDown: + results.append(shutdown) + return shutdown + + def _join_shutdown(self, q, results): + return self._join(q, results, True) + + def _shutdown_get(self, immediate): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + q.put("D") + # queue full + + if immediate: + thrds = ( + (self._get_shutdown, (q, go, results)), + (self._get_shutdown, (q, go, results)), + ) + else: + thrds = ( + # on shutdown(immediate=False) + # one of these threads shoud raise Shutdown + (self._get, (q, go, results)), + (self._get, (q, go, results)), + (self._get, (q, go, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + q.shutdown(immediate) + go.set() + for t in threads: + t.join() + if immediate: + self.assertListEqual(results, [True, True]) + else: + self.assertListEqual(sorted(results), [False] + [True]*(len(thrds)-1)) + + def test_shutdown_get(self): + return self._shutdown_get(False) + + def test_shutdown_immediate_get(self): + return self._shutdown_get(True) + + def _shutdown_put(self, immediate): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + q.put("D") + # queue fulled + + thrds = ( + (self._put_shutdown, (q, "E", go, results)), + (self._put_shutdown, (q, "W", go, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + q.shutdown() + go.set() + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + def test_shutdown_put(self): + return self._shutdown_put(False) + + def test_shutdown_immediate_put(self): + return self._shutdown_put(True) + + def _shutdown_join(self, immediate): + q = self.type2test() + results = [] + q.put("Y") + go = threading.Event() + nb = q.qsize() + + thrds = ( + (self._join, (q, results)), + (self._join, (q, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + if not immediate: + res = [] + for i in range(nb): + threads.append(threading.Thread(target=self._get_task_done, args=(q, go, res))) + threads[-1].start() + q.shutdown(immediate) + go.set() + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + def test_shutdown_immediate_join(self): + return self._shutdown_join(True) + + def test_shutdown_join(self): + return self._shutdown_join(False) + + def _shutdown_put_join(self, immediate): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + nb = q.qsize() + # queue not fulled + + thrds = ( + (self._put_shutdown, (q, "E", go, results)), + (self._join, (q, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + self.assertEqual(q.unfinished_tasks, nb) + for i in range(nb): + t = threading.Thread(target=q.task_done) + t.start() + threads.append(t) + q.shutdown(immediate) + go.set() + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + def test_shutdown_immediate_put_join(self): + return self._shutdown_put_join(True) + + def test_shutdown_put_join(self): + return self._shutdown_put_join(False) + + def test_shutdown_get_task_done_join(self): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + q.put("D") + self.assertEqual(q.unfinished_tasks, q.qsize()) + + thrds = ( + (self._get_task_done, (q, go, results)), + (self._get_task_done, (q, go, results)), + (self._join, (q, results)), + (self._join, (q, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + go.set() + q.shutdown(False) + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + class QueueTest(BaseQueueTestMixin): def setUp(self): diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index 57fe859e366b5b7..9d76764c75be3eb 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -12,7 +12,8 @@ import textwrap import unittest import warnings -from test.support import no_tracing, verbose, requires_subprocess, requires_resource +from test.support import (infinite_recursion, no_tracing, verbose, + requires_subprocess, requires_resource) from test.support.import_helper import forget, make_legacy_pyc, unload from test.support.os_helper import create_empty_file, temp_dir from test.support.script_helper import make_script, make_zip_script @@ -743,7 +744,8 @@ def test_main_recursion_error(self): "runpy.run_path(%r)\n") % dummy_dir script_name = self._make_test_script(script_dir, mod_name, source) zip_name, fname = make_zip_script(script_dir, 'test_zip', script_name) - self.assertRaises(RecursionError, run_path, zip_name) + with infinite_recursion(25): + self.assertRaises(RecursionError, run_path, zip_name) def test_encoding(self): with temp_dir() as script_dir: diff --git a/Lib/test/test_selectors.py b/Lib/test/test_selectors.py index 677349c2bfca93d..643775597c56c67 100644 --- a/Lib/test/test_selectors.py +++ b/Lib/test/test_selectors.py @@ -6,8 +6,7 @@ import socket import sys from test import support -from test.support import os_helper -from test.support import socket_helper +from test.support import is_apple, os_helper, socket_helper from time import sleep import unittest import unittest.mock @@ -526,7 +525,7 @@ def test_above_fd_setsize(self): try: fds = s.select() except OSError as e: - if e.errno == errno.EINVAL and sys.platform == 'darwin': + if e.errno == errno.EINVAL and is_apple: # unexplainable errors on macOS don't need to fail the test self.skipTest("Invalid argument error calling poll()") raise diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 8edd75e9907ec02..d96dad4eb9475d2 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -2148,6 +2148,7 @@ def check_chown(path, uid=None, gid=None): check_chown(dirname, uid, gid) +@support.requires_subprocess() class TestWhich(BaseTest, unittest.TestCase): def setUp(self): @@ -3181,6 +3182,7 @@ def test_bad_environ(self): self.assertGreaterEqual(size.lines, 0) @unittest.skipUnless(os.isatty(sys.__stdout__.fileno()), "not on tty") + @support.requires_subprocess() @unittest.skipUnless(hasattr(os, 'get_terminal_size'), 'need os.get_terminal_size()') def test_stty_match(self): diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 637a0ca3b369726..61fb047caf6dab6 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -13,9 +13,10 @@ import time import unittest from test import support -from test.support import os_helper +from test.support import ( + is_apple, is_apple_mobile, os_helper, threading_helper +) from test.support.script_helper import assert_python_ok, spawn_python -from test.support import threading_helper try: import _testcapi except ImportError: @@ -832,7 +833,7 @@ def test_itimer_real(self): self.assertEqual(self.hndl_called, True) # Issue 3864, unknown if this affects earlier versions of freebsd also - @unittest.skipIf(sys.platform in ('netbsd5',), + @unittest.skipIf(sys.platform in ('netbsd5',) or is_apple_mobile, 'itimer not reliable (does not mix well with threading) on some BSDs.') def test_itimer_virtual(self): self.itimer = signal.ITIMER_VIRTUAL @@ -1344,7 +1345,7 @@ def handler(signum, frame): # Python handler self.assertEqual(len(sigs), N, "Some signals were lost") - @unittest.skipIf(sys.platform == "darwin", "crashes due to system bug (FB13453490)") + @unittest.skipIf(is_apple, "crashes due to system bug (FB13453490)") @unittest.skipUnless(hasattr(signal, "SIGUSR1"), "test needs SIGUSR1") @threading_helper.requires_working_threading() diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 231448c75f01db0..179642349920627 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1,9 +1,8 @@ import unittest from test import support -from test.support import os_helper -from test.support import socket_helper -from test.support import threading_helper -from test.support import refleak_helper +from test.support import ( + is_apple, os_helper, refleak_helper, socket_helper, threading_helper +) import _thread as thread import array @@ -1196,8 +1195,11 @@ def testGetServBy(self): # Find one service that exists, then check all the related interfaces. # I've ordered this by protocols that have both a tcp and udp # protocol, at least for modern Linuxes. - if (sys.platform.startswith(('freebsd', 'netbsd', 'gnukfreebsd')) - or sys.platform in ('linux', 'darwin')): + if ( + sys.platform.startswith(('freebsd', 'netbsd', 'gnukfreebsd')) + or sys.platform == 'linux' + or is_apple + ): # avoid the 'echo' service on this platform, as there is an # assumption breaking non-standard port/protocol entry services = ('daytime', 'qotd', 'domain') @@ -3708,7 +3710,7 @@ def testFDPassCMSG_LEN(self): def _testFDPassCMSG_LEN(self): self.createAndSendFDs(1) - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") @unittest.skipIf(AIX, "skipping, see issue #22397") @requireAttrs(socket, "CMSG_SPACE") def testFDPassSeparate(self): @@ -3719,7 +3721,7 @@ def testFDPassSeparate(self): maxcmsgs=2) @testFDPassSeparate.client_skip - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") @unittest.skipIf(AIX, "skipping, see issue #22397") def _testFDPassSeparate(self): fd0, fd1 = self.newFDs(2) @@ -3732,7 +3734,7 @@ def _testFDPassSeparate(self): array.array("i", [fd1]))]), len(MSG)) - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") @unittest.skipIf(AIX, "skipping, see issue #22397") @requireAttrs(socket, "CMSG_SPACE") def testFDPassSeparateMinSpace(self): @@ -3746,7 +3748,7 @@ def testFDPassSeparateMinSpace(self): maxcmsgs=2, ignoreflags=socket.MSG_CTRUNC) @testFDPassSeparateMinSpace.client_skip - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") @unittest.skipIf(AIX, "skipping, see issue #22397") def _testFDPassSeparateMinSpace(self): fd0, fd1 = self.newFDs(2) @@ -3770,7 +3772,7 @@ def sendAncillaryIfPossible(self, msg, ancdata): nbytes = self.sendmsgToServer([msg]) self.assertEqual(nbytes, len(msg)) - @unittest.skipIf(sys.platform == "darwin", "see issue #24725") + @unittest.skipIf(is_apple, "skipping, see issue #12958") def testFDPassEmpty(self): # Try to pass an empty FD array. Can receive either no array # or an empty array. diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index f3efe0f52f4fd73..588272448bbfda6 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -31,7 +31,7 @@ from test.support import ( SHORT_TIMEOUT, check_disallow_instantiation, requires_subprocess, - is_emscripten, is_wasi + is_apple, is_emscripten, is_wasi ) from test.support import gc_collect from test.support import threading_helper @@ -667,7 +667,7 @@ def test_open_with_path_like_object(self): cx.execute(self._sql) @unittest.skipIf(sys.platform == "win32", "skipped on Windows") - @unittest.skipIf(sys.platform == "darwin", "skipped on macOS") + @unittest.skipIf(is_apple, "skipped on Apple platforms") @unittest.skipIf(is_emscripten or is_wasi, "not supported on Emscripten/WASI") @unittest.skipUnless(TESTFN_UNDECODABLE, "only works if there are undecodable paths") def test_open_with_undecodable_path(self): @@ -713,7 +713,7 @@ def test_open_uri_readonly(self): cx.execute(self._sql) @unittest.skipIf(sys.platform == "win32", "skipped on Windows") - @unittest.skipIf(sys.platform == "darwin", "skipped on macOS") + @unittest.skipIf(is_apple, "skipped on Apple platforms") @unittest.skipIf(is_emscripten or is_wasi, "not supported on Emscripten/WASI") @unittest.skipUnless(TESTFN_UNDECODABLE, "only works if there are undecodable paths") def test_open_undecodable_uri(self): diff --git a/Lib/test/test_sqlite3/test_dump.py b/Lib/test/test_sqlite3/test_dump.py index 2e1f0b80c10f46e..7261b7f0dc93d0a 100644 --- a/Lib/test/test_sqlite3/test_dump.py +++ b/Lib/test/test_sqlite3/test_dump.py @@ -54,6 +54,76 @@ def test_table_dump(self): [self.assertEqual(expected_sqls[i], actual_sqls[i]) for i in range(len(expected_sqls))] + def test_table_dump_filter(self): + all_table_sqls = [ + """CREATE TABLE "some_table_2" ("id_1" INTEGER);""", + """INSERT INTO "some_table_2" VALUES(3);""", + """INSERT INTO "some_table_2" VALUES(4);""", + """CREATE TABLE "test_table_1" ("id_2" INTEGER);""", + """INSERT INTO "test_table_1" VALUES(1);""", + """INSERT INTO "test_table_1" VALUES(2);""", + ] + all_views_sqls = [ + """CREATE VIEW "view_1" AS SELECT * FROM "some_table_2";""", + """CREATE VIEW "view_2" AS SELECT * FROM "test_table_1";""", + ] + # Create database structure. + for sql in [*all_table_sqls, *all_views_sqls]: + self.cu.execute(sql) + # %_table_% matches all tables. + dump_sqls = list(self.cx.iterdump(filter="%_table_%")) + self.assertEqual( + dump_sqls, + ["BEGIN TRANSACTION;", *all_table_sqls, "COMMIT;"], + ) + # view_% matches all views. + dump_sqls = list(self.cx.iterdump(filter="view_%")) + self.assertEqual( + dump_sqls, + ["BEGIN TRANSACTION;", *all_views_sqls, "COMMIT;"], + ) + # %_1 matches tables and views with the _1 suffix. + dump_sqls = list(self.cx.iterdump(filter="%_1")) + self.assertEqual( + dump_sqls, + [ + "BEGIN TRANSACTION;", + """CREATE TABLE "test_table_1" ("id_2" INTEGER);""", + """INSERT INTO "test_table_1" VALUES(1);""", + """INSERT INTO "test_table_1" VALUES(2);""", + """CREATE VIEW "view_1" AS SELECT * FROM "some_table_2";""", + "COMMIT;" + ], + ) + # some_% matches some_table_2. + dump_sqls = list(self.cx.iterdump(filter="some_%")) + self.assertEqual( + dump_sqls, + [ + "BEGIN TRANSACTION;", + """CREATE TABLE "some_table_2" ("id_1" INTEGER);""", + """INSERT INTO "some_table_2" VALUES(3);""", + """INSERT INTO "some_table_2" VALUES(4);""", + "COMMIT;" + ], + ) + # Only single object. + dump_sqls = list(self.cx.iterdump(filter="view_2")) + self.assertEqual( + dump_sqls, + [ + "BEGIN TRANSACTION;", + """CREATE VIEW "view_2" AS SELECT * FROM "test_table_1";""", + "COMMIT;" + ], + ) + # % matches all objects. + dump_sqls = list(self.cx.iterdump(filter="%")) + self.assertEqual( + dump_sqls, + ["BEGIN TRANSACTION;", *all_table_sqls, *all_views_sqls, "COMMIT;"], + ) + def test_dump_autoincrement(self): expected = [ 'CREATE TABLE "t1" (id integer primary key autoincrement);', diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 3fdfa2960503b8f..1b18230d83577dc 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -2206,14 +2206,15 @@ def _test_get_server_certificate(test, host, port, cert=None): sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) def _test_get_server_certificate_fail(test, host, port): - try: - pem = ssl.get_server_certificate((host, port), ca_certs=CERTFILE) - except ssl.SSLError as x: - #should fail - if support.verbose: - sys.stdout.write("%s\n" % x) - else: - test.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) + with warnings_helper.check_no_resource_warning(test): + try: + pem = ssl.get_server_certificate((host, port), ca_certs=CERTFILE) + except ssl.SSLError as x: + #should fail + if support.verbose: + sys.stdout.write("%s\n" % x) + else: + test.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) from test.ssl_servers import make_https_server @@ -3026,6 +3027,16 @@ def test_check_hostname_idn(self): server_hostname="python.example.org") as s: with self.assertRaises(ssl.CertificateError): s.connect((HOST, server.port)) + with ThreadedEchoServer(context=server_context, chatty=True) as server: + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(UnicodeError): + context.wrap_socket(socket.socket(), + server_hostname='.pythontest.net') + with ThreadedEchoServer(context=server_context, chatty=True) as server: + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(UnicodeDecodeError): + context.wrap_socket(socket.socket(), + server_hostname=b'k\xf6nig.idn.pythontest.net') def test_wrong_cert_tls12(self): """Connecting when the server rejects the client's certificate @@ -4983,7 +4994,8 @@ def call_after_accept(conn_to_client): self.assertIsNone(wrap_error.library, msg="attr must exist") finally: # gh-108342: Explicitly break the reference cycle - wrap_error = None + with warnings_helper.check_no_resource_warning(self): + wrap_error = None server = None def test_https_client_non_tls_response_ignored(self): @@ -5032,7 +5044,8 @@ def call_after_accept(conn_to_client): # socket; that fails if the connection is broken. It may seem pointless # to test this. It serves as an illustration of something that we never # want to happen... properly not happening. - with self.assertRaises(OSError): + with warnings_helper.check_no_resource_warning(self), \ + self.assertRaises(OSError): connection.request("HEAD", "/test", headers={"Host": "localhost"}) response = connection.getresponse() diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 4976ac3642bbe46..8bd373976426ef3 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -9,6 +9,13 @@ from _testcapi import get_feature_macros feature_macros = get_feature_macros() + +# Stable ABI is incompatible with Py_TRACE_REFS builds due to PyObject +# layout differences. +# See https://github.com/python/cpython/issues/88299#issuecomment-1113366226 +if feature_macros['Py_TRACE_REFS']: + raise unittest.SkipTest("incompatible with Py_TRACE_REFS.") + ctypes_test = import_module('ctypes') class TestStableABIAvailability(unittest.TestCase): @@ -254,6 +261,7 @@ def test_windows_feature_macros(self): "PyExc_IOError", "PyExc_ImportError", "PyExc_ImportWarning", + "PyExc_IncompleteInputError", "PyExc_IndentationError", "PyExc_IndexError", "PyExc_InterruptedError", @@ -364,6 +372,7 @@ def test_windows_feature_macros(self): "PyList_Append", "PyList_AsTuple", "PyList_GetItem", + "PyList_GetItemRef", "PyList_GetSlice", "PyList_Insert", "PyList_New", @@ -441,7 +450,9 @@ def test_windows_feature_macros(self): "PyModule_AddObjectRef", "PyModule_AddStringConstant", "PyModule_AddType", + "PyModule_Create2", "PyModule_ExecDef", + "PyModule_FromDefAndSpec2", "PyModule_GetDef", "PyModule_GetDict", "PyModule_GetFilename", @@ -911,6 +922,13 @@ def test_windows_feature_macros(self): "_Py_TrueStruct", "_Py_VaBuildValue_SizeT", ) +if feature_macros['HAVE_FORK']: + SYMBOL_NAMES += ( + 'PyOS_AfterFork', + 'PyOS_AfterFork_Child', + 'PyOS_AfterFork_Parent', + 'PyOS_BeforeFork', + ) if feature_macros['MS_WINDOWS']: SYMBOL_NAMES += ( 'PyErr_SetExcFromWindowsErr', @@ -926,17 +944,6 @@ def test_windows_feature_macros(self): 'PyUnicode_DecodeMBCSStateful', 'PyUnicode_EncodeCodePage', ) -if feature_macros['HAVE_FORK']: - SYMBOL_NAMES += ( - 'PyOS_AfterFork', - 'PyOS_AfterFork_Child', - 'PyOS_AfterFork_Parent', - 'PyOS_BeforeFork', - ) -if feature_macros['USE_STACKCHECK']: - SYMBOL_NAMES += ( - 'PyOS_CheckStack', - ) if feature_macros['PY_HAVE_THREAD_NATIVE_ID']: SYMBOL_NAMES += ( 'PyThread_get_thread_native_id', @@ -946,14 +953,23 @@ def test_windows_feature_macros(self): '_Py_NegativeRefcount', '_Py_RefTotal', ) +if feature_macros['Py_TRACE_REFS']: + SYMBOL_NAMES += ( + ) +if feature_macros['USE_STACKCHECK']: + SYMBOL_NAMES += ( + 'PyOS_CheckStack', + ) EXPECTED_FEATURE_MACROS = set(['HAVE_FORK', 'MS_WINDOWS', 'PY_HAVE_THREAD_NATIVE_ID', 'Py_REF_DEBUG', + 'Py_TRACE_REFS', 'USE_STACKCHECK']) WINDOWS_FEATURE_MACROS = {'HAVE_FORK': False, 'MS_WINDOWS': True, 'PY_HAVE_THREAD_NATIVE_ID': True, 'Py_REF_DEBUG': 'maybe', + 'Py_TRACE_REFS': 'maybe', 'USE_STACKCHECK': 'maybe'} diff --git a/Lib/test/test_stat.py b/Lib/test/test_stat.py index d6b6dd6e7417002..49013a4bcd8af6b 100644 --- a/Lib/test/test_stat.py +++ b/Lib/test/test_stat.py @@ -2,8 +2,7 @@ import os import socket import sys -from test.support import os_helper -from test.support import socket_helper +from test.support import is_apple, os_helper, socket_helper from test.support.import_helper import import_fresh_module from test.support.os_helper import TESTFN @@ -247,7 +246,7 @@ def test_flags_consistent(self): for flag in self.file_flags: if flag.startswith("UF"): self.assertTrue(getattr(self.statmod, flag) & self.statmod.UF_SETTABLE, f"{flag} not in UF_SETTABLE") - elif sys.platform == 'darwin' and self.statmod is c_stat and flag == 'SF_DATALESS': + elif is_apple and self.statmod is c_stat and flag == 'SF_DATALESS': self.assertTrue(self.statmod.SF_DATALESS & self.statmod.SF_SYNTHETIC, "SF_DATALESS not in SF_SYNTHETIC") self.assertFalse(self.statmod.SF_DATALESS & self.statmod.SF_SETTABLE, "SF_DATALESS in SF_SETTABLE") else: diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 6c87dfabad9f0f7..71671a5a984256d 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1392,6 +1392,7 @@ def setUp(self): self.longdigit = sys.int_info.sizeof_digit import _testinternalcapi self.gc_headsize = _testinternalcapi.SIZEOF_PYGC_HEAD + self.managed_pre_header_size = _testinternalcapi.SIZEOF_MANAGED_PRE_HEADER check_sizeof = test.support.check_sizeof @@ -1427,7 +1428,7 @@ class OverflowSizeof(int): def __sizeof__(self): return int(self) self.assertEqual(sys.getsizeof(OverflowSizeof(sys.maxsize)), - sys.maxsize + self.gc_headsize*2) + sys.maxsize + self.gc_headsize + self.managed_pre_header_size) with self.assertRaises(OverflowError): sys.getsizeof(OverflowSizeof(sys.maxsize + 1)) with self.assertRaises(ValueError): @@ -1650,7 +1651,7 @@ def delx(self): del self.__x # type # static type: PyTypeObject fmt = 'P2nPI13Pl4Pn9Pn12PIPc' - s = vsize('2P' + fmt) + s = vsize(fmt) check(int, s) # class s = vsize(fmt + # PyTypeObject diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index ae6e192a7ab6eff..125f40227118f63 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -7,7 +7,7 @@ import gc from functools import wraps import asyncio -from test.support import import_helper +from test.support import import_helper, requires_subprocess import contextlib import os import tempfile @@ -1810,6 +1810,7 @@ def compare_events(self, line_offset, events, expected_events): def make_tracer(): return Tracer(trace_opcode_events=True) + @requires_subprocess() def test_trace_opcodes_after_settrace(self): """Make sure setting f_trace_opcodes after starting trace works even if it's the first time f_trace_opcodes is being set. GH-103615""" diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index be609a0abd29c87..bb87bf00dc2d1a7 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -154,17 +154,21 @@ def test_posix_venv_scheme(self): 'python%d.%d' % sys.version_info[:2], 'site-packages') - # Resolve the paths in prefix - binpath = os.path.join(sys.prefix, binpath) - incpath = os.path.join(sys.prefix, incpath) - libpath = os.path.join(sys.prefix, libpath) + # Resolve the paths in an imaginary venv/ directory + binpath = os.path.join('venv', binpath) + incpath = os.path.join('venv', incpath) + libpath = os.path.join('venv', libpath) - self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='posix_venv')) - self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='posix_venv')) + # Mimic the venv module, set all bases to the venv directory + bases = ('base', 'platbase', 'installed_base', 'installed_platbase') + vars = {base: 'venv' for base in bases} + + self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='posix_venv', vars=vars)) + self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='posix_venv', vars=vars)) # The include directory on POSIX isn't exactly the same as before, # but it is "within" - sysconfig_includedir = sysconfig.get_path('include', scheme='posix_venv') + sysconfig_includedir = sysconfig.get_path('include', scheme='posix_venv', vars=vars) self.assertTrue(sysconfig_includedir.startswith(incpath + os.sep)) def test_nt_venv_scheme(self): @@ -174,14 +178,19 @@ def test_nt_venv_scheme(self): incpath = 'Include' libpath = os.path.join('Lib', 'site-packages') - # Resolve the paths in prefix - binpath = os.path.join(sys.prefix, binpath) - incpath = os.path.join(sys.prefix, incpath) - libpath = os.path.join(sys.prefix, libpath) + # Resolve the paths in an imaginary venv\ directory + venv = 'venv' + binpath = os.path.join(venv, binpath) + incpath = os.path.join(venv, incpath) + libpath = os.path.join(venv, libpath) + + # Mimic the venv module, set all bases to the venv directory + bases = ('base', 'platbase', 'installed_base', 'installed_platbase') + vars = {base: 'venv' for base in bases} - self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='nt_venv')) - self.assertEqual(incpath, sysconfig.get_path('include', scheme='nt_venv')) - self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='nt_venv')) + self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='nt_venv', vars=vars)) + self.assertEqual(incpath, sysconfig.get_path('include', scheme='nt_venv', vars=vars)) + self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='nt_venv', vars=vars)) def test_venv_scheme(self): if sys.platform == 'win32': diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index da5009126b3815b..51f070e96047a6e 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -15,6 +15,7 @@ import unittest.mock import tarfile +from test import archiver_tests from test import support from test.support import os_helper from test.support import script_helper @@ -4135,6 +4136,38 @@ def valueerror_filter(tarinfo, path): self.expect_exception(TypeError) # errorlevel is not int +class OverwriteTests(archiver_tests.OverwriteTests, unittest.TestCase): + testdir = os.path.join(TEMPDIR, "testoverwrite") + + @classmethod + def setUpClass(cls): + p = cls.ar_with_file = os.path.join(TEMPDIR, 'tar-with-file.tar') + cls.addClassCleanup(os_helper.unlink, p) + with tarfile.open(p, 'w') as tar: + t = tarfile.TarInfo('test') + t.size = 10 + tar.addfile(t, io.BytesIO(b'newcontent')) + + p = cls.ar_with_dir = os.path.join(TEMPDIR, 'tar-with-dir.tar') + cls.addClassCleanup(os_helper.unlink, p) + with tarfile.open(p, 'w') as tar: + tar.addfile(tar.gettarinfo(os.curdir, 'test')) + + p = os.path.join(TEMPDIR, 'tar-with-implicit-dir.tar') + cls.ar_with_implicit_dir = p + cls.addClassCleanup(os_helper.unlink, p) + with tarfile.open(p, 'w') as tar: + t = tarfile.TarInfo('test/file') + t.size = 10 + tar.addfile(t, io.BytesIO(b'newcontent')) + + def open(self, path): + return tarfile.open(path, 'r') + + def extractall(self, ar): + ar.extractall(self.testdir, filter='fully_trusted') + + def setUpModule(): os_helper.unlink(TEMPDIR) os.makedirs(TEMPDIR) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index dbdc46fff1e313c..1ab223b81e939e5 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -171,11 +171,21 @@ def test_args_argument(self): t.start() t.join() - @cpython_only - def test_disallow_instantiation(self): - # Ensure that the type disallows instantiation (bpo-43916) - lock = threading.Lock() - test.support.check_disallow_instantiation(self, type(lock)) + def test_lock_no_args(self): + threading.Lock() # works + self.assertRaises(TypeError, threading.Lock, 1) + self.assertRaises(TypeError, threading.Lock, a=1) + self.assertRaises(TypeError, threading.Lock, 1, 2, a=1, b=2) + + def test_lock_no_subclass(self): + # Intentionally disallow subclasses of threading.Lock because they have + # never been allowed, so why start now just because the type is public? + with self.assertRaises(TypeError): + class MyLock(threading.Lock): pass + + def test_lock_or_none(self): + import types + self.assertIsInstance(threading.Lock | None, types.UnionType) # Create a bunch of threads, let each do some work, wait until all are # done. diff --git a/Lib/test/test_tkinter/support.py b/Lib/test/test_tkinter/support.py index a37705f0ae6febb..ebb9e00ff91bf0e 100644 --- a/Lib/test/test_tkinter/support.py +++ b/Lib/test/test_tkinter/support.py @@ -14,7 +14,7 @@ def setUpClass(cls): # Some window managers can maximize new windows. cls.root.wm_state('normal') try: - cls.root.wm_attributes('-zoomed', False) + cls.root.wm_attributes(zoomed=False) except tkinter.TclError: pass diff --git a/Lib/test/test_tkinter/test_misc.py b/Lib/test/test_tkinter/test_misc.py index dc8a810235fc9bd..81a20b698a72eb4 100644 --- a/Lib/test/test_tkinter/test_misc.py +++ b/Lib/test/test_tkinter/test_misc.py @@ -437,6 +437,61 @@ def test_info_patchlevel(self): self.assertTrue(str(vi).startswith(f'{vi.major}.{vi.minor}')) +class WmTest(AbstractTkTest, unittest.TestCase): + + def test_wm_attribute(self): + w = self.root + attributes = w.wm_attributes(return_python_dict=True) + self.assertIsInstance(attributes, dict) + attributes2 = w.wm_attributes() + self.assertIsInstance(attributes2, tuple) + self.assertEqual(attributes2[::2], + tuple('-' + k for k in attributes)) + self.assertEqual(attributes2[1::2], tuple(attributes.values())) + # silently deprecated + attributes3 = w.wm_attributes(None) + if self.wantobjects: + self.assertEqual(attributes3, attributes2) + else: + self.assertIsInstance(attributes3, str) + + for name in attributes: + self.assertEqual(w.wm_attributes(name), attributes[name]) + # silently deprecated + for name in attributes: + self.assertEqual(w.wm_attributes('-' + name), attributes[name]) + + self.assertIn('alpha', attributes) + self.assertIn('fullscreen', attributes) + self.assertIn('topmost', attributes) + if w._windowingsystem == "win32": + self.assertIn('disabled', attributes) + self.assertIn('toolwindow', attributes) + self.assertIn('transparentcolor', attributes) + if w._windowingsystem == "aqua": + self.assertIn('modified', attributes) + self.assertIn('notify', attributes) + self.assertIn('titlepath', attributes) + self.assertIn('transparent', attributes) + if w._windowingsystem == "x11": + self.assertIn('type', attributes) + self.assertIn('zoomed', attributes) + + w.wm_attributes(alpha=0.5) + self.assertEqual(w.wm_attributes('alpha'), + 0.5 if self.wantobjects else '0.5') + w.wm_attributes(alpha=1.0) + self.assertEqual(w.wm_attributes('alpha'), + 1.0 if self.wantobjects else '1.0') + # silently deprecated + w.wm_attributes('-alpha', 0.5) + self.assertEqual(w.wm_attributes('alpha'), + 0.5 if self.wantobjects else '0.5') + w.wm_attributes(alpha=1.0) + self.assertEqual(w.wm_attributes('alpha'), + 1.0 if self.wantobjects else '1.0') + + class BindTest(AbstractTkTest, unittest.TestCase): def setUp(self): @@ -706,6 +761,101 @@ def test3(e): pass self.assertCommandExist(funcid2) self.assertCommandExist(funcid3) + def _test_tag_bind(self, w): + tag = 'sel' + event = '<Control-Alt-Key-a>' + w.pack() + self.assertRaises(TypeError, w.tag_bind) + tag_bind = w._tag_bind if isinstance(w, tkinter.Text) else w.tag_bind + if isinstance(w, tkinter.Text): + self.assertRaises(TypeError, w.tag_bind, tag) + self.assertRaises(TypeError, w.tag_bind, tag, event) + self.assertEqual(tag_bind(tag), ()) + self.assertEqual(tag_bind(tag, event), '') + def test1(e): pass + def test2(e): pass + + funcid = w.tag_bind(tag, event, test1) + self.assertEqual(tag_bind(tag), (event,)) + script = tag_bind(tag, event) + self.assertIn(funcid, script) + self.assertCommandExist(funcid) + + funcid2 = w.tag_bind(tag, event, test2, add=True) + script = tag_bind(tag, event) + self.assertIn(funcid, script) + self.assertIn(funcid2, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + def _test_tag_unbind(self, w): + tag = 'sel' + event = '<Control-Alt-Key-b>' + w.pack() + tag_bind = w._tag_bind if isinstance(w, tkinter.Text) else w.tag_bind + self.assertEqual(tag_bind(tag), ()) + self.assertEqual(tag_bind(tag, event), '') + def test1(e): pass + def test2(e): pass + + funcid = w.tag_bind(tag, event, test1) + funcid2 = w.tag_bind(tag, event, test2, add=True) + + self.assertRaises(TypeError, w.tag_unbind, tag) + w.tag_unbind(tag, event) + self.assertEqual(tag_bind(tag, event), '') + self.assertEqual(tag_bind(tag), ()) + + def _test_tag_bind_rebind(self, w): + tag = 'sel' + event = '<Control-Alt-Key-d>' + w.pack() + tag_bind = w._tag_bind if isinstance(w, tkinter.Text) else w.tag_bind + self.assertEqual(tag_bind(tag), ()) + self.assertEqual(tag_bind(tag, event), '') + def test1(e): pass + def test2(e): pass + def test3(e): pass + + funcid = w.tag_bind(tag, event, test1) + funcid2 = w.tag_bind(tag, event, test2, add=True) + script = tag_bind(tag, event) + self.assertIn(funcid2, script) + self.assertIn(funcid, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + funcid3 = w.tag_bind(tag, event, test3) + script = tag_bind(tag, event) + self.assertNotIn(funcid, script) + self.assertNotIn(funcid2, script) + self.assertIn(funcid3, script) + self.assertCommandExist(funcid3) + + def test_canvas_tag_bind(self): + c = tkinter.Canvas(self.frame) + self._test_tag_bind(c) + + def test_canvas_tag_unbind(self): + c = tkinter.Canvas(self.frame) + self._test_tag_unbind(c) + + def test_canvas_tag_bind_rebind(self): + c = tkinter.Canvas(self.frame) + self._test_tag_bind_rebind(c) + + def test_text_tag_bind(self): + t = tkinter.Text(self.frame) + self._test_tag_bind(t) + + def test_text_tag_unbind(self): + t = tkinter.Text(self.frame) + self._test_tag_unbind(t) + + def test_text_tag_bind_rebind(self): + t = tkinter.Text(self.frame) + self._test_tag_bind_rebind(t) + def test_bindtags(self): f = self.frame self.assertEqual(self.root.bindtags(), ('.', 'Tk', 'all')) diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index 295df78a17374a3..58572c6f4d31578 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -79,6 +79,19 @@ class C: _clear_type_cache() + def test_per_class_limit(self): + class C: + x = 0 + + type_assign_version(C) + orig_version = type_get_version(C) + for i in range(1001): + C.x = i + type_assign_version(C) + + new_version = type_get_version(C) + self.assertEqual(new_version, 0) + @support.cpython_only class TypeCacheWithSpecializationTests(unittest.TestCase): diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index b684af4f33ed71d..c3a092f3af30097 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -4323,6 +4323,16 @@ class C(B[int]): c.bar = 'abc' self.assertEqual(c.__dict__, {'bar': 'abc'}) + def test_setattr_exceptions(self): + class Immutable[T]: + def __setattr__(self, key, value): + raise RuntimeError("immutable") + + # gh-115165: This used to cause RuntimeError to be raised + # when we tried to set `__orig_class__` on the `Immutable` instance + # returned by the `Immutable[int]()` call + self.assertIsInstance(Immutable[int](), Immutable) + def test_subscripted_generics_as_proxies(self): T = TypeVar('T') class C(Generic[T]): @@ -4920,6 +4930,75 @@ class B(Generic[S]): ... class C(List[int], B): ... self.assertEqual(C.__mro__, (C, list, B, Generic, object)) + def test_multiple_inheritance_non_type_with___mro_entries__(self): + class GoodEntries: + def __mro_entries__(self, bases): + return (object,) + + class A(List[int], GoodEntries()): ... + + self.assertEqual(A.__mro__, (A, list, Generic, object)) + + def test_multiple_inheritance_non_type_without___mro_entries__(self): + # Error should be from the type machinery, not from typing.py + with self.assertRaisesRegex(TypeError, r"^bases must be types"): + class A(List[int], object()): ... + + def test_multiple_inheritance_non_type_bad___mro_entries__(self): + class BadEntries: + def __mro_entries__(self, bases): + return None + + # Error should be from the type machinery, not from typing.py + with self.assertRaisesRegex( + TypeError, + r"^__mro_entries__ must return a tuple", + ): + class A(List[int], BadEntries()): ... + + def test_multiple_inheritance___mro_entries___returns_non_type(self): + class BadEntries: + def __mro_entries__(self, bases): + return (object(),) + + # Error should be from the type machinery, not from typing.py + with self.assertRaisesRegex( + TypeError, + r"^bases must be types", + ): + class A(List[int], BadEntries()): ... + + def test_multiple_inheritance_with_genericalias(self): + class A(typing.Sized, list[int]): ... + + self.assertEqual( + A.__mro__, + (A, collections.abc.Sized, Generic, list, object), + ) + + def test_multiple_inheritance_with_genericalias_2(self): + T = TypeVar("T") + + class BaseSeq(typing.Sequence[T]): ... + class MySeq(List[T], BaseSeq[T]): ... + + self.assertEqual( + MySeq.__mro__, + ( + MySeq, + list, + BaseSeq, + collections.abc.Sequence, + collections.abc.Reversible, + collections.abc.Collection, + collections.abc.Sized, + collections.abc.Iterable, + collections.abc.Container, + Generic, + object, + ), + ) + def test_init_subclass_super_called(self): class FinalException(Exception): pass @@ -8492,6 +8571,17 @@ def test_instantiate_generic(self): self.assertEqual(MyCount([4, 4, 5]), {4: 2, 5: 1}) self.assertEqual(MyCount[int]([4, 4, 5]), {4: 2, 5: 1}) + def test_instantiate_immutable(self): + class C: + def __setattr__(self, key, value): + raise Exception("should be ignored") + + A = Annotated[C, "a decoration"] + # gh-115165: This used to cause RuntimeError to be raised + # when we tried to set `__orig_class__` on the `C` instance + # returned by the `A()` call + self.assertIsInstance(A(), C) + def test_cannot_instantiate_forward(self): A = Annotated["int", (5, 6)] with self.assertRaises(TypeError): diff --git a/Lib/test/test_unicode_file_functions.py b/Lib/test/test_unicode_file_functions.py index 47619c8807bafe3..25c16e3a0b7e430 100644 --- a/Lib/test/test_unicode_file_functions.py +++ b/Lib/test/test_unicode_file_functions.py @@ -5,7 +5,7 @@ import unittest import warnings from unicodedata import normalize -from test.support import os_helper +from test.support import is_apple, os_helper from test import support @@ -23,13 +23,13 @@ '10_\u1fee\u1ffd', ] -# Mac OS X decomposes Unicode names, using Normal Form D. +# Apple platforms decompose Unicode names, using Normal Form D. # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html # "However, most volume formats do not follow the exact specification for # these normal forms. For example, HFS Plus uses a variant of Normal Form D # in which U+2000 through U+2FFF, U+F900 through U+FAFF, and U+2F800 through # U+2FAFF are not decomposed." -if sys.platform != 'darwin': +if not is_apple: filenames.extend([ # Specific code points: NFC(fn), NFD(fn), NFKC(fn) and NFKD(fn) all different '11_\u0385\u03d3\u03d4', @@ -119,11 +119,11 @@ def test_open(self): os.stat(name) self._apply_failure(os.listdir, name, self._listdir_failure) - # Skip the test on darwin, because darwin does normalize the filename to + # Skip the test on Apple platforms, because they don't normalize the filename to # NFD (a variant of Unicode NFD form). Normalize the filename to NFC, NFKC, # NFKD in Python is useless, because darwin will normalize it later and so # open(), os.stat(), etc. don't raise any exception. - @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X') + @unittest.skipIf(is_apple, 'irrelevant test on Apple platforms') @unittest.skipIf( support.is_emscripten or support.is_wasi, "test fails on Emscripten/WASI when host platform is macOS." @@ -142,10 +142,10 @@ def test_normalize(self): self._apply_failure(os.remove, name) self._apply_failure(os.listdir, name) - # Skip the test on darwin, because darwin uses a normalization different + # Skip the test on Apple platforms, because they use a normalization different # than Python NFD normalization: filenames are different even if we use # Python NFD normalization. - @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X') + @unittest.skipIf(is_apple, 'irrelevant test on Apple platforms') def test_listdir(self): sf0 = set(self.files) with warnings.catch_warnings(): diff --git a/Lib/test/test_unittest/test_discovery.py b/Lib/test/test_unittest/test_discovery.py index dcb72d73efceabb..004898ed4318348 100644 --- a/Lib/test/test_unittest/test_discovery.py +++ b/Lib/test/test_unittest/test_discovery.py @@ -571,7 +571,7 @@ def _get_module_from_name(name): result = unittest.TestResult() suite.run(result) self.assertEqual(len(result.skipped), 1) - self.assertEqual(result.testsRun, 0) + self.assertEqual(result.testsRun, 1) self.assertEqual(import_calls, ['my_package']) # Check picklability diff --git a/Lib/test/test_unittest/test_skipping.py b/Lib/test/test_unittest/test_skipping.py index 1a6af06d32b4339..f146dcac18ecc09 100644 --- a/Lib/test/test_unittest/test_skipping.py +++ b/Lib/test/test_unittest/test_skipping.py @@ -103,16 +103,16 @@ def test_dont_skip(self): pass result = LoggingResult(events) self.assertIs(suite.run(result), result) self.assertEqual(len(result.skipped), 1) - expected = ['addSkip', 'stopTest', 'startTest', - 'addSuccess', 'stopTest'] + expected = ['startTest', 'addSkip', 'stopTest', + 'startTest', 'addSuccess', 'stopTest'] self.assertEqual(events, expected) - self.assertEqual(result.testsRun, 1) + self.assertEqual(result.testsRun, 2) self.assertEqual(result.skipped, [(test_do_skip, "testing")]) self.assertTrue(result.wasSuccessful()) events = [] result = test_do_skip.run() - self.assertEqual(events, ['startTestRun', 'addSkip', + self.assertEqual(events, ['startTestRun', 'startTest', 'addSkip', 'stopTest', 'stopTestRun']) self.assertEqual(result.skipped, [(test_do_skip, "testing")]) @@ -135,13 +135,13 @@ def test_1(self): test = Foo("test_1") suite = unittest.TestSuite([test]) self.assertIs(suite.run(result), result) - self.assertEqual(events, ['addSkip', 'stopTest']) + self.assertEqual(events, ['startTest', 'addSkip', 'stopTest']) self.assertEqual(result.skipped, [(test, "testing")]) self.assertEqual(record, []) events = [] result = test.run() - self.assertEqual(events, ['startTestRun', 'addSkip', + self.assertEqual(events, ['startTestRun', 'startTest', 'addSkip', 'stopTest', 'stopTestRun']) self.assertEqual(result.skipped, [(test, "testing")]) self.assertEqual(record, []) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 99c9e24994732fa..fa528a675892b5e 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1,6 +1,7 @@ import unittest from test import support from test.support import os_helper +from test.support import requires_subprocess from test.support import warnings_helper from test import test_urllib from unittest import mock @@ -998,6 +999,7 @@ def test_http_body_fileobj(self): file_obj.close() + @requires_subprocess() def test_http_body_pipe(self): # A file reading from a pipe. # A pipe cannot be seek'ed. There is no way to determine the diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 6dda00efd7bbb69..ba31beb81e80b09 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -19,8 +19,8 @@ import tempfile from test.support import (captured_stdout, captured_stderr, skip_if_broken_multiprocessing_synchronize, verbose, - requires_subprocess, is_emscripten, is_wasi, - requires_venv_with_pip, TEST_HOME_DIR, + requires_subprocess, is_apple_mobile, is_emscripten, + is_wasi, requires_venv_with_pip, TEST_HOME_DIR, requires_resource, copy_python_src_ignore) from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree) import unittest @@ -39,8 +39,10 @@ or sys._base_executable != sys.executable, 'cannot run venv.create from within a venv on this platform') -if is_emscripten or is_wasi: - raise unittest.SkipTest("venv is not available on Emscripten/WASI.") +# Skip tests on WASM platforms, plus iOS/tvOS/watchOS +if is_apple_mobile or is_emscripten or is_wasi: + raise unittest.SkipTest(f"venv tests not required on {sys.platform}") + @requires_subprocess() def check_output(cmd, encoding=None): diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index ca481c57c3d9724..8c074cb28a87e38 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -307,6 +307,7 @@ def test_get(self): webbrowser.get('fakebrowser') self.assertIsNotNone(webbrowser._tryorder) + @unittest.skipIf(" " in sys.executable, "test assumes no space in path (GH-114452)") def test_synthesize(self): webbrowser = import_helper.import_fresh_module('webbrowser') name = os.path.basename(sys.executable).lower() diff --git a/Lib/test/test_winconsoleio.py b/Lib/test/test_winconsoleio.py index 72ff9606908ed5a..209e4464e1a5c0c 100644 --- a/Lib/test/test_winconsoleio.py +++ b/Lib/test/test_winconsoleio.py @@ -6,7 +6,7 @@ import sys import tempfile import unittest -from test.support import os_helper +from test.support import os_helper, requires_resource if sys.platform != 'win32': raise unittest.SkipTest("test only relevant on win32") @@ -150,6 +150,7 @@ def assertStdinRoundTrip(self, text): sys.stdin = old_stdin self.assertEqual(actual, text) + @requires_resource('console') def test_input(self): # ASCII self.assertStdinRoundTrip('abc123') @@ -164,6 +165,7 @@ def test_input_nonbmp(self): # Non-BMP self.assertStdinRoundTrip('\U00100000\U0010ffff\U0010fffd') + @requires_resource('console') def test_partial_reads(self): # Test that reading less than 1 full character works when stdin # contains multibyte UTF-8 sequences @@ -199,6 +201,7 @@ def test_partial_surrogate_reads(self): self.assertEqual(actual, expected, 'stdin.read({})'.format(read_count)) + @requires_resource('console') def test_ctrl_z(self): with open('CONIN$', 'rb', buffering=0) as stdin: source = '\xC4\x1A\r\n'.encode('utf-16-le') diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index b9e7937b0bbc00a..a435ec7822ea0cb 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -536,7 +536,9 @@ def test_iterparse(self): iterparse = ET.iterparse context = iterparse(SIMPLE_XMLFILE) + self.assertIsNone(context.root) action, elem = next(context) + self.assertIsNone(context.root) self.assertEqual((action, elem.tag), ('end', 'element')) self.assertEqual([(action, elem.tag) for action, elem in context], [ ('end', 'element'), @@ -553,6 +555,17 @@ def test_iterparse(self): ('end', '{namespace}root'), ]) + with open(SIMPLE_XMLFILE, 'rb') as source: + context = iterparse(source) + action, elem = next(context) + self.assertEqual((action, elem.tag), ('end', 'element')) + self.assertEqual([(action, elem.tag) for action, elem in context], [ + ('end', 'element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + self.assertEqual(context.root.tag, 'root') + events = () context = iterparse(SIMPLE_XMLFILE, events) self.assertEqual([(action, elem.tag) for action, elem in context], []) @@ -644,12 +657,81 @@ def test_iterparse(self): # Not exhausting the iterator still closes the resource (bpo-43292) with warnings_helper.check_no_resource_warning(self): - it = iterparse(TESTFN) + it = iterparse(SIMPLE_XMLFILE) del it + with warnings_helper.check_no_resource_warning(self): + it = iterparse(SIMPLE_XMLFILE) + it.close() + del it + + with warnings_helper.check_no_resource_warning(self): + it = iterparse(SIMPLE_XMLFILE) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + del it, elem + + with warnings_helper.check_no_resource_warning(self): + it = iterparse(SIMPLE_XMLFILE) + action, elem = next(it) + it.close() + self.assertEqual((action, elem.tag), ('end', 'element')) + del it, elem + with self.assertRaises(FileNotFoundError): iterparse("nonexistent") + def test_iterparse_close(self): + iterparse = ET.iterparse + + it = iterparse(SIMPLE_XMLFILE) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + it = iterparse(SIMPLE_XMLFILE) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + it = iterparse(SIMPLE_XMLFILE) + list(it) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + list(it) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + def test_writefile(self): elem = ET.Element("tag") elem.text = "text" @@ -3042,8 +3124,7 @@ def test_basic(self): # With an explicit parser too (issue #9708) sourcefile = serialize(doc, to_string=False) parser = ET.XMLParser(target=ET.TreeBuilder()) - self.assertEqual(next(ET.iterparse(sourcefile, parser=parser))[0], - 'end') + self.assertEqual(next(ET.iterparse(sourcefile, parser=parser))[0], 'end') tree = ET.ElementTree(None) self.assertRaises(AttributeError, tree.iter) diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 9bdb08aeabb7817..087fa8d65cc3368 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -18,6 +18,7 @@ from tempfile import TemporaryFile from random import randint, random, randbytes +from test import archiver_tests from test.support import script_helper from test.support import ( findfile, requires_zlib, requires_bz2, requires_lzma, @@ -1687,6 +1688,33 @@ def _test_extract_hackers_arcnames(self, hacknames): unlink(TESTFN2) +class OverwriteTests(archiver_tests.OverwriteTests, unittest.TestCase): + testdir = TESTFN + + @classmethod + def setUpClass(cls): + p = cls.ar_with_file = TESTFN + '-with-file.zip' + cls.addClassCleanup(unlink, p) + with zipfile.ZipFile(p, 'w') as zipfp: + zipfp.writestr('test', b'newcontent') + + p = cls.ar_with_dir = TESTFN + '-with-dir.zip' + cls.addClassCleanup(unlink, p) + with zipfile.ZipFile(p, 'w') as zipfp: + zipfp.mkdir('test') + + p = cls.ar_with_implicit_dir = TESTFN + '-with-implicit-dir.zip' + cls.addClassCleanup(unlink, p) + with zipfile.ZipFile(p, 'w') as zipfp: + zipfp.writestr('test/file', b'newcontent') + + def open(self, path): + return zipfile.ZipFile(path, 'r') + + def extractall(self, ar): + ar.extractall(self.testdir) + + class OtherTests(unittest.TestCase): def test_open_via_zip_info(self): # Create the ZIP archive @@ -2959,7 +2987,7 @@ def test_create_directory_with_write(self): directory = os.path.join(TESTFN2, "directory2") os.mkdir(directory) - mode = os.stat(directory).st_mode + mode = os.stat(directory).st_mode & 0xFFFF zf.write(directory, arcname="directory2/") zinfo = zf.filelist[1] self.assertEqual(zinfo.filename, "directory2/") diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index 18eab5b33540c9e..8414721555731eb 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -20,7 +20,7 @@ from test.support import MISSING_C_DOCSTRINGS from test.test_zoneinfo import _support as test_support from test.test_zoneinfo._support import OS_ENV_LOCK, TZPATH_TEST_LOCK, ZoneInfoTestBase -from test.support.import_helper import import_module +from test.support.import_helper import import_module, CleanImport lzma = import_module('lzma') py_zoneinfo, c_zoneinfo = test_support.get_modules() @@ -1720,13 +1720,26 @@ def test_env_variable_relative_paths(self): with self.subTest("warning", path_var=path_var): # Note: Per PEP 615 the warning is implementation-defined # behavior, other implementations need not warn. - with self.assertWarns(self.module.InvalidTZPathWarning): + with self.assertWarns(self.module.InvalidTZPathWarning) as w: self.module.reset_tzpath() + self.assertEqual(w.warnings[0].filename, __file__) tzpath = self.module.TZPATH with self.subTest("filtered", path_var=path_var): self.assertSequenceEqual(tzpath, expected_paths) + def test_env_variable_relative_paths_warning_location(self): + path_var = "path/to/somewhere" + + with self.python_tzpath_context(path_var): + with CleanImport("zoneinfo", "zoneinfo._tzpath"): + with self.assertWarns(RuntimeWarning) as w: + import zoneinfo + InvalidTZPathWarning = zoneinfo.InvalidTZPathWarning + self.assertIsInstance(w.warnings[0].message, InvalidTZPathWarning) + # It should represent the current file: + self.assertEqual(w.warnings[0].filename, __file__) + def test_reset_tzpath_kwarg(self): self.module.reset_tzpath(to=[f"{DRIVE}/a/b/c"]) diff --git a/Lib/threading.py b/Lib/threading.py index ecf799bc26ab061..b6ff00acadd58fe 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -3,9 +3,7 @@ import os as _os import sys as _sys import _thread -import functools import warnings -import _weakref from time import monotonic as _time from _weakrefset import WeakSet @@ -37,6 +35,7 @@ _start_joinable_thread = _thread.start_joinable_thread _daemon_threads_allowed = _thread.daemon_threads_allowed _allocate_lock = _thread.allocate_lock +_LockType = _thread.LockType _set_sentinel = _thread._set_sentinel get_ident = _thread.get_ident _is_main_interpreter = _thread._is_main_interpreter @@ -115,7 +114,7 @@ def gettrace(): # Synchronization classes -Lock = _allocate_lock +Lock = _LockType def RLock(*args, **kwargs): """Factory function that returns a new reentrant lock. @@ -950,7 +949,6 @@ def _after_fork(self, new_ident=None): # This thread is alive. self._ident = new_ident if self._handle is not None: - self._handle.after_fork_alive() assert self._handle.ident == new_ident # bpo-42350: If the fork happens when the thread is already stopped # (ex: after threading._shutdown() has been called), _tstate_lock @@ -966,9 +964,7 @@ def _after_fork(self, new_ident=None): self._is_stopped = True self._tstate_lock = None self._join_lock = None - if self._handle is not None: - self._handle.after_fork_dead() - self._handle = None + self._handle = None def __repr__(self): assert self._initialized, "Thread.__init__() was not called" @@ -1630,8 +1626,7 @@ def _register_atexit(func, *arg, **kwargs): if _SHUTTING_DOWN: raise RuntimeError("can't register atexit after shutdown") - call = functools.partial(func, *arg, **kwargs) - _threading_atexits.append(call) + _threading_atexits.append(lambda: func(*arg, **kwargs)) from _thread import stack_size diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index e0db41dd915ece9..2be9da2cfb92993 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -1537,16 +1537,19 @@ def unbind(self, sequence, funcid=None): Otherwise destroy the current binding for SEQUENCE, leaving SEQUENCE unbound. """ + self._unbind(('bind', self._w, sequence), funcid) + + def _unbind(self, what, funcid=None): if funcid is None: - self.tk.call('bind', self._w, sequence, '') + self.tk.call(*what, '') else: - lines = self.tk.call('bind', self._w, sequence).split('\n') + lines = self.tk.call(what).split('\n') prefix = f'if {{"[{funcid} ' keep = '\n'.join(line for line in lines if not line.startswith(prefix)) if not keep.strip(): keep = '' - self.tk.call('bind', self._w, sequence, keep) + self.tk.call(*what, keep) self.deletecommand(funcid) def bind_all(self, sequence=None, func=None, add=None): @@ -1558,7 +1561,7 @@ def bind_all(self, sequence=None, func=None, add=None): def unbind_all(self, sequence): """Unbind for all widgets for event SEQUENCE all functions.""" - self.tk.call('bind', 'all' , sequence, '') + self._root()._unbind(('bind', 'all', sequence)) def bind_class(self, className, sequence=None, func=None, add=None): """Bind to widgets with bindtag CLASSNAME at event @@ -1573,7 +1576,7 @@ def bind_class(self, className, sequence=None, func=None, add=None): def unbind_class(self, className, sequence): """Unbind for all widgets with bindtag CLASSNAME for event SEQUENCE all functions.""" - self.tk.call('bind', className , sequence, '') + self._root()._unbind(('bind', className, sequence)) def mainloop(self, n=0): """Call the mainloop of Tk.""" @@ -2105,26 +2108,39 @@ def wm_aspect(self, aspect = wm_aspect - def wm_attributes(self, *args): - """This subcommand returns or sets platform specific attributes - - The first form returns a list of the platform specific flags and - their values. The second form returns the value for the specific - option. The third form sets one or more of the values. The values - are as follows: - - On Windows, -disabled gets or sets whether the window is in a - disabled state. -toolwindow gets or sets the style of the window - to toolwindow (as defined in the MSDN). -topmost gets or sets - whether this is a topmost window (displays above all other - windows). + def wm_attributes(self, *args, return_python_dict=False, **kwargs): + """Return or sets platform specific attributes. - On Macintosh, XXXXX + When called with a single argument return_python_dict=True, + return a dict of the platform specific attributes and their values. + When called without arguments or with a single argument + return_python_dict=False, return a tuple containing intermixed + attribute names with the minus prefix and their values. - On Unix, there are currently no special attribute values. + When called with a single string value, return the value for the + specific option. When called with keyword arguments, set the + corresponding attributes. """ - args = ('wm', 'attributes', self._w) + args - return self.tk.call(args) + if not kwargs: + if not args: + res = self.tk.call('wm', 'attributes', self._w) + if return_python_dict: + return _splitdict(self.tk, res) + else: + return self.tk.splitlist(res) + if len(args) == 1 and args[0] is not None: + option = args[0] + if option[0] == '-': + # TODO: deprecate + option = option[1:] + return self.tk.call('wm', 'attributes', self._w, '-' + option) + # TODO: deprecate + return self.tk.call('wm', 'attributes', self._w, *args) + elif args: + raise TypeError('wm_attribute() options have been specified as ' + 'positional and keyword arguments') + else: + self.tk.call('wm', 'attributes', self._w, *self._options(kwargs)) attributes = wm_attributes @@ -2885,9 +2901,7 @@ def bbox(self, *args): def tag_unbind(self, tagOrId, sequence, funcid=None): """Unbind for all items with TAGORID for event SEQUENCE the function identified with FUNCID.""" - self.tk.call(self._w, 'bind', tagOrId, sequence, '') - if funcid: - self.deletecommand(funcid) + self._unbind((self._w, 'bind', tagOrId, sequence), funcid) def tag_bind(self, tagOrId, sequence=None, func=None, add=None): """Bind to all items with TAGORID at event SEQUENCE a call to function FUNC. @@ -3997,9 +4011,7 @@ def tag_add(self, tagName, index1, *args): def tag_unbind(self, tagName, sequence, funcid=None): """Unbind for all characters with TAGNAME for event SEQUENCE the function identified with FUNCID.""" - self.tk.call(self._w, 'tag', 'bind', tagName, sequence, '') - if funcid: - self.deletecommand(funcid) + return self._unbind((self._w, 'tag', 'bind', tagName, sequence), funcid) def tag_bind(self, tagName, sequence, func, add=None): """Bind to all characters with TAGNAME at event SEQUENCE a call to function FUNC. @@ -4010,6 +4022,11 @@ def tag_bind(self, tagName, sequence, func, add=None): return self._bind((self._w, 'tag', 'bind', tagName), sequence, func, add) + def _tag_bind(self, tagName, sequence=None, func=None, add=None): + # For tests only + return self._bind((self._w, 'tag', 'bind', tagName), + sequence, func, add) + def tag_cget(self, tagName, option): """Return the value of OPTION for tag TAGNAME.""" if option[:1] != '-': diff --git a/Lib/tkinter/simpledialog.py b/Lib/tkinter/simpledialog.py index 538bbfc318d7044..0f0dc66460f7987 100644 --- a/Lib/tkinter/simpledialog.py +++ b/Lib/tkinter/simpledialog.py @@ -262,7 +262,7 @@ def _setup_dialog(w): w.tk.call("::tk::unsupported::MacWindowStyle", "style", w, "moveableModal", "") elif w._windowingsystem == "x11": - w.wm_attributes("-type", "dialog") + w.wm_attributes(type="dialog") # -------------------------------------------------------------------- # convenience dialogues diff --git a/Lib/typing.py b/Lib/typing.py index d278b4effc7ebaf..914ddeaf504cd0e 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1127,7 +1127,9 @@ def __call__(self, *args, **kwargs): result = self.__origin__(*args, **kwargs) try: result.__orig_class__ = self - except AttributeError: + # Some objects raise TypeError (or something even more exotic) + # if you try to set attributes on them; we guard against that here + except Exception: pass return result @@ -1135,9 +1137,29 @@ def __mro_entries__(self, bases): res = [] if self.__origin__ not in bases: res.append(self.__origin__) + + # Check if any base that occurs after us in `bases` is either itself a + # subclass of Generic, or something which will add a subclass of Generic + # to `__bases__` via its `__mro_entries__`. If not, add Generic + # ourselves. The goal is to ensure that Generic (or a subclass) will + # appear exactly once in the final bases tuple. If we let it appear + # multiple times, we risk "can't form a consistent MRO" errors. i = bases.index(self) for b in bases[i+1:]: - if isinstance(b, _BaseGenericAlias) or issubclass(b, Generic): + if isinstance(b, _BaseGenericAlias): + break + if not isinstance(b, type): + meth = getattr(b, "__mro_entries__", None) + new_bases = meth(bases) if meth else None + if ( + isinstance(new_bases, tuple) and + any( + isinstance(b2, type) and issubclass(b2, Generic) + for b2 in new_bases + ) + ): + break + elif issubclass(b, Generic): break else: res.append(Generic) diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index 811557498bb30ed..001b640dc43ad69 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -606,6 +606,7 @@ def run(self, result=None): else: stopTestRun = None + result.startTest(self) try: testMethod = getattr(self, self._testMethodName) if (getattr(self.__class__, "__unittest_skip__", False) or @@ -616,9 +617,6 @@ def run(self, result=None): _addSkip(result, self, skip_why) return result - # Increase the number of tests only if it hasn't been skipped - result.startTest(self) - expecting_failure = ( getattr(self, "__unittest_expecting_failure__", False) or getattr(testMethod, "__unittest_expecting_failure__", False) diff --git a/Lib/unittest/result.py b/Lib/unittest/result.py index 9e56f658027f4de..3ace0a5b7bf2efb 100644 --- a/Lib/unittest/result.py +++ b/Lib/unittest/result.py @@ -97,12 +97,10 @@ def _restoreStdout(self): sys.stdout = self._original_stdout sys.stderr = self._original_stderr - if self._stdout_buffer is not None: - self._stdout_buffer.seek(0) - self._stdout_buffer.truncate() - if self._stderr_buffer is not None: - self._stderr_buffer.seek(0) - self._stderr_buffer.truncate() + self._stdout_buffer.seek(0) + self._stdout_buffer.truncate() + self._stderr_buffer.seek(0) + self._stderr_buffer.truncate() def stopTestRun(self): """Called once after all tests are executed. diff --git a/Lib/venv/scripts/nt/activate.bat b/Lib/venv/scripts/nt/activate.bat index 2c98122362a060f..dd5ea8eb67b90a7 100644 --- a/Lib/venv/scripts/nt/activate.bat +++ b/Lib/venv/scripts/nt/activate.bat @@ -15,8 +15,8 @@ if not defined PROMPT set PROMPT=$P$G if defined _OLD_VIRTUAL_PROMPT set PROMPT=%_OLD_VIRTUAL_PROMPT% if defined _OLD_VIRTUAL_PYTHONHOME set PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME% -set _OLD_VIRTUAL_PROMPT=%PROMPT% -set PROMPT=(__VENV_PROMPT__) %PROMPT% +set "_OLD_VIRTUAL_PROMPT=%PROMPT%" +set "PROMPT=(__VENV_PROMPT__) %PROMPT%" if defined PYTHONHOME set _OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME% set PYTHONHOME= diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 42574eefd81bebe..a37fead41b750e4 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -99,6 +99,7 @@ import collections import collections.abc import contextlib +import weakref from . import ElementPath @@ -1223,13 +1224,14 @@ def iterparse(source, events=None, parser=None): # parser argument of iterparse is removed, this can be killed. pullparser = XMLPullParser(events=events, _parser=parser) - def iterator(source): + if not hasattr(source, "read"): + source = open(source, "rb") + close_source = True + else: close_source = False + + def iterator(source): try: - if not hasattr(source, "read"): - source = open(source, "rb") - close_source = True - yield None while True: yield from pullparser.read_events() # load event buffer @@ -1239,18 +1241,30 @@ def iterator(source): pullparser.feed(data) root = pullparser._close_and_return_root() yield from pullparser.read_events() - it.root = root + it = wr() + if it is not None: + it.root = root finally: if close_source: source.close() + gen = iterator(source) class IterParseIterator(collections.abc.Iterator): - __next__ = iterator(source).__next__ + __next__ = gen.__next__ + def close(self): + if close_source: + source.close() + gen.close() + + def __del__(self): + # TODO: Emit a ResourceWarning if it was not explicitly closed. + # (When the close() method will be supported in all maintained Python versions.) + if close_source: + source.close() + it = IterParseIterator() it.root = None - del iterator, IterParseIterator - - next(it) + wr = weakref.ref(it) return it diff --git a/Lib/zoneinfo/_tzpath.py b/Lib/zoneinfo/_tzpath.py index 4985dce2dc36d0e..5db17bea045d8c8 100644 --- a/Lib/zoneinfo/_tzpath.py +++ b/Lib/zoneinfo/_tzpath.py @@ -2,7 +2,7 @@ import sysconfig -def reset_tzpath(to=None): +def _reset_tzpath(to=None, stacklevel=4): global TZPATH tzpaths = to @@ -18,17 +18,22 @@ def reset_tzpath(to=None): base_tzpath = tzpaths else: env_var = os.environ.get("PYTHONTZPATH", None) - if env_var is not None: - base_tzpath = _parse_python_tzpath(env_var) - else: - base_tzpath = _parse_python_tzpath( - sysconfig.get_config_var("TZPATH") - ) + if env_var is None: + env_var = sysconfig.get_config_var("TZPATH") + base_tzpath = _parse_python_tzpath(env_var, stacklevel) TZPATH = tuple(base_tzpath) -def _parse_python_tzpath(env_var): +def reset_tzpath(to=None): + """Reset global TZPATH.""" + # We need `_reset_tzpath` helper function because it produces a warning, + # it is used as both a module-level call and a public API. + # This is how we equalize the stacklevel for both calls. + _reset_tzpath(to) + + +def _parse_python_tzpath(env_var, stacklevel): if not env_var: return () @@ -45,6 +50,7 @@ def _parse_python_tzpath(env_var): "Invalid paths specified in PYTHONTZPATH environment variable. " + msg, InvalidTZPathWarning, + stacklevel=stacklevel, ) return new_tzpath @@ -172,4 +178,4 @@ class InvalidTZPathWarning(RuntimeWarning): TZPATH = () -reset_tzpath() +_reset_tzpath(stacklevel=5) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 32de56bcf130863..0af90563cbbb2b7 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 3.0.11", - url="https://www.openssl.org/source/openssl-3.0.11.tar.gz", - checksum='b3425d3bb4a2218d0697eb41f7fc0cdede016ed19ca49d168b78e8d947887f55', + name="OpenSSL 3.0.13", + url="https://www.openssl.org/source/openssl-3.0.13.tar.gz", + checksum='88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313', buildrecipe=build_universal_openssl, configure=None, install=None, @@ -359,9 +359,9 @@ def library_recipes(): ), ), dict( - name="SQLite 3.44.2", - url="https://sqlite.org/2023/sqlite-autoconf-3440200.tar.gz", - checksum="c02f40fd4f809ced95096250adc5764a", + name="SQLite 3.45.1", + url="https://sqlite.org/2024/sqlite-autoconf-3450100.tar.gz", + checksum="cd9c27841b7a5932c9897651e20b86c701dd740556989b01ca596fcfa3d49a0a", extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' diff --git a/Makefile.pre.in b/Makefile.pre.in index 29033a8b64eb334..158b28a881be385 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -405,6 +405,7 @@ PYTHON_OBJS= \ Python/ast_opt.o \ Python/ast_unparse.o \ Python/bltinmodule.o \ + Python/brc.o \ Python/ceval.o \ Python/codecs.o \ Python/compile.o \ @@ -433,12 +434,14 @@ PYTHON_OBJS= \ Python/initconfig.o \ Python/instrumentation.o \ Python/intrinsics.o \ + Python/jit.o \ Python/legacy_tracing.o \ Python/lock.o \ Python/marshal.o \ Python/modsupport.o \ Python/mysnprintf.o \ Python/mystrtoul.o \ + Python/object_stack.o \ Python/optimizer.o \ Python/optimizer_analysis.o \ Python/parking_lot.o \ @@ -934,6 +937,262 @@ python.html: $(srcdir)/Tools/wasm/python.html python.worker.js python.worker.js: $(srcdir)/Tools/wasm/python.worker.js @cp $(srcdir)/Tools/wasm/python.worker.js $@ +############################################################################ +# Header files + +PYTHON_HEADERS= \ + $(srcdir)/Include/Python.h \ + $(srcdir)/Include/abstract.h \ + $(srcdir)/Include/bltinmodule.h \ + $(srcdir)/Include/boolobject.h \ + $(srcdir)/Include/bytearrayobject.h \ + $(srcdir)/Include/bytesobject.h \ + $(srcdir)/Include/ceval.h \ + $(srcdir)/Include/codecs.h \ + $(srcdir)/Include/compile.h \ + $(srcdir)/Include/complexobject.h \ + $(srcdir)/Include/descrobject.h \ + $(srcdir)/Include/dictobject.h \ + $(srcdir)/Include/dynamic_annotations.h \ + $(srcdir)/Include/enumobject.h \ + $(srcdir)/Include/errcode.h \ + $(srcdir)/Include/exports.h \ + $(srcdir)/Include/fileobject.h \ + $(srcdir)/Include/fileutils.h \ + $(srcdir)/Include/floatobject.h \ + $(srcdir)/Include/frameobject.h \ + $(srcdir)/Include/genericaliasobject.h \ + $(srcdir)/Include/import.h \ + $(srcdir)/Include/interpreteridobject.h \ + $(srcdir)/Include/intrcheck.h \ + $(srcdir)/Include/iterobject.h \ + $(srcdir)/Include/listobject.h \ + $(srcdir)/Include/longobject.h \ + $(srcdir)/Include/marshal.h \ + $(srcdir)/Include/memoryobject.h \ + $(srcdir)/Include/methodobject.h \ + $(srcdir)/Include/modsupport.h \ + $(srcdir)/Include/moduleobject.h \ + $(srcdir)/Include/object.h \ + $(srcdir)/Include/objimpl.h \ + $(srcdir)/Include/opcode.h \ + $(srcdir)/Include/opcode_ids.h \ + $(srcdir)/Include/osdefs.h \ + $(srcdir)/Include/osmodule.h \ + $(srcdir)/Include/patchlevel.h \ + $(srcdir)/Include/pyatomic.h \ + $(srcdir)/Include/pybuffer.h \ + $(srcdir)/Include/pycapsule.h \ + $(srcdir)/Include/pydtrace.h \ + $(srcdir)/Include/pyerrors.h \ + $(srcdir)/Include/pyexpat.h \ + $(srcdir)/Include/pyframe.h \ + $(srcdir)/Include/pyhash.h \ + $(srcdir)/Include/pylifecycle.h \ + $(srcdir)/Include/pymacconfig.h \ + $(srcdir)/Include/pymacro.h \ + $(srcdir)/Include/pymath.h \ + $(srcdir)/Include/pymem.h \ + $(srcdir)/Include/pyport.h \ + $(srcdir)/Include/pystate.h \ + $(srcdir)/Include/pystats.h \ + $(srcdir)/Include/pystrcmp.h \ + $(srcdir)/Include/pystrtod.h \ + $(srcdir)/Include/pythonrun.h \ + $(srcdir)/Include/pythread.h \ + $(srcdir)/Include/pytypedefs.h \ + $(srcdir)/Include/rangeobject.h \ + $(srcdir)/Include/setobject.h \ + $(srcdir)/Include/sliceobject.h \ + $(srcdir)/Include/structmember.h \ + $(srcdir)/Include/structseq.h \ + $(srcdir)/Include/sysmodule.h \ + $(srcdir)/Include/traceback.h \ + $(srcdir)/Include/tupleobject.h \ + $(srcdir)/Include/typeslots.h \ + $(srcdir)/Include/unicodeobject.h \ + $(srcdir)/Include/warnings.h \ + $(srcdir)/Include/weakrefobject.h \ + \ + pyconfig.h \ + $(PARSER_HEADERS) \ + \ + $(srcdir)/Include/cpython/abstract.h \ + $(srcdir)/Include/cpython/bytearrayobject.h \ + $(srcdir)/Include/cpython/bytesobject.h \ + $(srcdir)/Include/cpython/cellobject.h \ + $(srcdir)/Include/cpython/ceval.h \ + $(srcdir)/Include/cpython/classobject.h \ + $(srcdir)/Include/cpython/code.h \ + $(srcdir)/Include/cpython/compile.h \ + $(srcdir)/Include/cpython/complexobject.h \ + $(srcdir)/Include/cpython/context.h \ + $(srcdir)/Include/cpython/descrobject.h \ + $(srcdir)/Include/cpython/dictobject.h \ + $(srcdir)/Include/cpython/fileobject.h \ + $(srcdir)/Include/cpython/fileutils.h \ + $(srcdir)/Include/cpython/floatobject.h \ + $(srcdir)/Include/cpython/frameobject.h \ + $(srcdir)/Include/cpython/funcobject.h \ + $(srcdir)/Include/cpython/genobject.h \ + $(srcdir)/Include/cpython/import.h \ + $(srcdir)/Include/cpython/initconfig.h \ + $(srcdir)/Include/cpython/interpreteridobject.h \ + $(srcdir)/Include/cpython/listobject.h \ + $(srcdir)/Include/cpython/longintrepr.h \ + $(srcdir)/Include/cpython/longobject.h \ + $(srcdir)/Include/cpython/memoryobject.h \ + $(srcdir)/Include/cpython/methodobject.h \ + $(srcdir)/Include/cpython/object.h \ + $(srcdir)/Include/cpython/objimpl.h \ + $(srcdir)/Include/cpython/odictobject.h \ + $(srcdir)/Include/cpython/optimizer.h \ + $(srcdir)/Include/cpython/picklebufobject.h \ + $(srcdir)/Include/cpython/pthread_stubs.h \ + $(srcdir)/Include/cpython/pyatomic.h \ + $(srcdir)/Include/cpython/pyatomic_gcc.h \ + $(srcdir)/Include/cpython/pyatomic_std.h \ + $(srcdir)/Include/cpython/pyctype.h \ + $(srcdir)/Include/cpython/pydebug.h \ + $(srcdir)/Include/cpython/pyerrors.h \ + $(srcdir)/Include/cpython/pyfpe.h \ + $(srcdir)/Include/cpython/pyframe.h \ + $(srcdir)/Include/cpython/pyhash.h \ + $(srcdir)/Include/cpython/pylifecycle.h \ + $(srcdir)/Include/cpython/pymem.h \ + $(srcdir)/Include/cpython/pystate.h \ + $(srcdir)/Include/cpython/pystats.h \ + $(srcdir)/Include/cpython/pythonrun.h \ + $(srcdir)/Include/cpython/pythread.h \ + $(srcdir)/Include/cpython/setobject.h \ + $(srcdir)/Include/cpython/sysmodule.h \ + $(srcdir)/Include/cpython/traceback.h \ + $(srcdir)/Include/cpython/tracemalloc.h \ + $(srcdir)/Include/cpython/tupleobject.h \ + $(srcdir)/Include/cpython/unicodeobject.h \ + $(srcdir)/Include/cpython/warnings.h \ + $(srcdir)/Include/cpython/weakrefobject.h \ + \ + $(MIMALLOC_HEADERS) \ + \ + $(srcdir)/Include/internal/pycore_abstract.h \ + $(srcdir)/Include/internal/pycore_asdl.h \ + $(srcdir)/Include/internal/pycore_ast.h \ + $(srcdir)/Include/internal/pycore_ast_state.h \ + $(srcdir)/Include/internal/pycore_atexit.h \ + $(srcdir)/Include/internal/pycore_bitutils.h \ + $(srcdir)/Include/internal/pycore_blocks_output_buffer.h \ + $(srcdir)/Include/internal/pycore_brc.h \ + $(srcdir)/Include/internal/pycore_bytes_methods.h \ + $(srcdir)/Include/internal/pycore_bytesobject.h \ + $(srcdir)/Include/internal/pycore_call.h \ + $(srcdir)/Include/internal/pycore_capsule.h \ + $(srcdir)/Include/internal/pycore_ceval.h \ + $(srcdir)/Include/internal/pycore_ceval_state.h \ + $(srcdir)/Include/internal/pycore_code.h \ + $(srcdir)/Include/internal/pycore_codecs.h \ + $(srcdir)/Include/internal/pycore_compile.h \ + $(srcdir)/Include/internal/pycore_complexobject.h \ + $(srcdir)/Include/internal/pycore_condvar.h \ + $(srcdir)/Include/internal/pycore_context.h \ + $(srcdir)/Include/internal/pycore_critical_section.h \ + $(srcdir)/Include/internal/pycore_crossinterp.h \ + $(srcdir)/Include/internal/pycore_descrobject.h \ + $(srcdir)/Include/internal/pycore_dict.h \ + $(srcdir)/Include/internal/pycore_dict_state.h \ + $(srcdir)/Include/internal/pycore_dtoa.h \ + $(srcdir)/Include/internal/pycore_exceptions.h \ + $(srcdir)/Include/internal/pycore_faulthandler.h \ + $(srcdir)/Include/internal/pycore_fileutils.h \ + $(srcdir)/Include/internal/pycore_floatobject.h \ + $(srcdir)/Include/internal/pycore_flowgraph.h \ + $(srcdir)/Include/internal/pycore_format.h \ + $(srcdir)/Include/internal/pycore_frame.h \ + $(srcdir)/Include/internal/pycore_freelist.h \ + $(srcdir)/Include/internal/pycore_function.h \ + $(srcdir)/Include/internal/pycore_gc.h \ + $(srcdir)/Include/internal/pycore_genobject.h \ + $(srcdir)/Include/internal/pycore_getopt.h \ + $(srcdir)/Include/internal/pycore_gil.h \ + $(srcdir)/Include/internal/pycore_global_objects.h \ + $(srcdir)/Include/internal/pycore_global_objects_fini_generated.h \ + $(srcdir)/Include/internal/pycore_global_strings.h \ + $(srcdir)/Include/internal/pycore_hamt.h \ + $(srcdir)/Include/internal/pycore_hashtable.h \ + $(srcdir)/Include/internal/pycore_identifier.h \ + $(srcdir)/Include/internal/pycore_import.h \ + $(srcdir)/Include/internal/pycore_importdl.h \ + $(srcdir)/Include/internal/pycore_initconfig.h \ + $(srcdir)/Include/internal/pycore_instruments.h \ + $(srcdir)/Include/internal/pycore_interp.h \ + $(srcdir)/Include/internal/pycore_intrinsics.h \ + $(srcdir)/Include/internal/pycore_jit.h \ + $(srcdir)/Include/internal/pycore_list.h \ + $(srcdir)/Include/internal/pycore_llist.h \ + $(srcdir)/Include/internal/pycore_lock.h \ + $(srcdir)/Include/internal/pycore_long.h \ + $(srcdir)/Include/internal/pycore_memoryobject.h \ + $(srcdir)/Include/internal/pycore_mimalloc.h \ + $(srcdir)/Include/internal/pycore_modsupport.h \ + $(srcdir)/Include/internal/pycore_moduleobject.h \ + $(srcdir)/Include/internal/pycore_namespace.h \ + $(srcdir)/Include/internal/pycore_object.h \ + $(srcdir)/Include/internal/pycore_object_alloc.h \ + $(srcdir)/Include/internal/pycore_object_stack.h \ + $(srcdir)/Include/internal/pycore_object_state.h \ + $(srcdir)/Include/internal/pycore_obmalloc.h \ + $(srcdir)/Include/internal/pycore_obmalloc_init.h \ + $(srcdir)/Include/internal/pycore_opcode_metadata.h \ + $(srcdir)/Include/internal/pycore_opcode_utils.h \ + $(srcdir)/Include/internal/pycore_optimizer.h \ + $(srcdir)/Include/internal/pycore_parking_lot.h \ + $(srcdir)/Include/internal/pycore_parser.h \ + $(srcdir)/Include/internal/pycore_pathconfig.h \ + $(srcdir)/Include/internal/pycore_pyarena.h \ + $(srcdir)/Include/internal/pycore_pybuffer.h \ + $(srcdir)/Include/internal/pycore_pyerrors.h \ + $(srcdir)/Include/internal/pycore_pyhash.h \ + $(srcdir)/Include/internal/pycore_pylifecycle.h \ + $(srcdir)/Include/internal/pycore_pymath.h \ + $(srcdir)/Include/internal/pycore_pymem.h \ + $(srcdir)/Include/internal/pycore_pymem_init.h \ + $(srcdir)/Include/internal/pycore_pystate.h \ + $(srcdir)/Include/internal/pycore_pystats.h \ + $(srcdir)/Include/internal/pycore_pythonrun.h \ + $(srcdir)/Include/internal/pycore_pythread.h \ + $(srcdir)/Include/internal/pycore_range.h \ + $(srcdir)/Include/internal/pycore_runtime.h \ + $(srcdir)/Include/internal/pycore_runtime_init.h \ + $(srcdir)/Include/internal/pycore_runtime_init_generated.h \ + $(srcdir)/Include/internal/pycore_semaphore.h \ + $(srcdir)/Include/internal/pycore_setobject.h \ + $(srcdir)/Include/internal/pycore_signal.h \ + $(srcdir)/Include/internal/pycore_sliceobject.h \ + $(srcdir)/Include/internal/pycore_strhex.h \ + $(srcdir)/Include/internal/pycore_structseq.h \ + $(srcdir)/Include/internal/pycore_symtable.h \ + $(srcdir)/Include/internal/pycore_sysmodule.h \ + $(srcdir)/Include/internal/pycore_time.h \ + $(srcdir)/Include/internal/pycore_token.h \ + $(srcdir)/Include/internal/pycore_traceback.h \ + $(srcdir)/Include/internal/pycore_tracemalloc.h \ + $(srcdir)/Include/internal/pycore_tstate.h \ + $(srcdir)/Include/internal/pycore_tuple.h \ + $(srcdir)/Include/internal/pycore_typeobject.h \ + $(srcdir)/Include/internal/pycore_typevarobject.h \ + $(srcdir)/Include/internal/pycore_ucnhash.h \ + $(srcdir)/Include/internal/pycore_unicodeobject.h \ + $(srcdir)/Include/internal/pycore_unicodeobject_generated.h \ + $(srcdir)/Include/internal/pycore_unionobject.h \ + $(srcdir)/Include/internal/pycore_uop_ids.h \ + $(srcdir)/Include/internal/pycore_uop_metadata.h \ + $(srcdir)/Include/internal/pycore_warnings.h \ + $(srcdir)/Include/internal/pycore_weakref.h \ + $(DTRACE_HEADERS) \ + @PLATFORM_HEADERS@ \ + \ + $(srcdir)/Python/stdlib_module_names.h + ########################################################################## # Build static libmpdec.a LIBMPDEC_CFLAGS=@LIBMPDEC_CFLAGS@ $(PY_STDMODULE_CFLAGS) $(CCSHARED) @@ -1364,7 +1623,7 @@ regen-unicodedata: regen-all: regen-cases regen-typeslots \ regen-token regen-ast regen-keyword regen-sre regen-frozen \ regen-pegen-metaparser regen-pegen regen-test-frozenmain \ - regen-test-levenshtein regen-global-objects regen-sbom + regen-test-levenshtein regen-global-objects regen-sbom regen-jit @echo @echo "Note: make regen-stdlib-module-names, make regen-limited-abi, " @echo "make regen-configure and make regen-unicodedata should be run manually" @@ -1398,7 +1657,7 @@ Modules/getpath.o: $(srcdir)/Modules/getpath.c Python/frozen_modules/getpath.h M Programs/python.o: $(srcdir)/Programs/python.c $(CC) -c $(PY_CORE_CFLAGS) -o $@ $(srcdir)/Programs/python.c -Programs/_testembed.o: $(srcdir)/Programs/_testembed.c Programs/test_frozenmain.h +Programs/_testembed.o: $(srcdir)/Programs/_testembed.c Programs/test_frozenmain.h $(PYTHON_HEADERS) $(CC) -c $(PY_CORE_CFLAGS) -o $@ $(srcdir)/Programs/_testembed.c Modules/_sre/sre.o: $(srcdir)/Modules/_sre/sre.c $(srcdir)/Modules/_sre/sre.h $(srcdir)/Modules/_sre/sre_constants.h $(srcdir)/Modules/_sre/sre_lib.h @@ -1667,243 +1926,6 @@ regen-typeslots: $(srcdir)/Objects/typeslots.inc.new $(UPDATE_FILE) $(srcdir)/Objects/typeslots.inc $(srcdir)/Objects/typeslots.inc.new -############################################################################ -# Header files - -PYTHON_HEADERS= \ - $(srcdir)/Include/Python.h \ - $(srcdir)/Include/abstract.h \ - $(srcdir)/Include/bltinmodule.h \ - $(srcdir)/Include/boolobject.h \ - $(srcdir)/Include/bytearrayobject.h \ - $(srcdir)/Include/bytesobject.h \ - $(srcdir)/Include/ceval.h \ - $(srcdir)/Include/codecs.h \ - $(srcdir)/Include/compile.h \ - $(srcdir)/Include/complexobject.h \ - $(srcdir)/Include/descrobject.h \ - $(srcdir)/Include/dictobject.h \ - $(srcdir)/Include/dynamic_annotations.h \ - $(srcdir)/Include/enumobject.h \ - $(srcdir)/Include/errcode.h \ - $(srcdir)/Include/fileobject.h \ - $(srcdir)/Include/fileutils.h \ - $(srcdir)/Include/floatobject.h \ - $(srcdir)/Include/frameobject.h \ - $(srcdir)/Include/import.h \ - $(srcdir)/Include/interpreteridobject.h \ - $(srcdir)/Include/intrcheck.h \ - $(srcdir)/Include/iterobject.h \ - $(srcdir)/Include/listobject.h \ - $(srcdir)/Include/longobject.h \ - $(srcdir)/Include/marshal.h \ - $(srcdir)/Include/memoryobject.h \ - $(srcdir)/Include/methodobject.h \ - $(srcdir)/Include/modsupport.h \ - $(srcdir)/Include/moduleobject.h \ - $(srcdir)/Include/object.h \ - $(srcdir)/Include/objimpl.h \ - $(srcdir)/Include/opcode.h \ - $(srcdir)/Include/opcode_ids.h \ - $(srcdir)/Include/osdefs.h \ - $(srcdir)/Include/osmodule.h \ - $(srcdir)/Include/patchlevel.h \ - $(srcdir)/Include/pybuffer.h \ - $(srcdir)/Include/pycapsule.h \ - $(srcdir)/Include/pydtrace.h \ - $(srcdir)/Include/pyerrors.h \ - $(srcdir)/Include/pyframe.h \ - $(srcdir)/Include/pyhash.h \ - $(srcdir)/Include/pylifecycle.h \ - $(srcdir)/Include/pymacconfig.h \ - $(srcdir)/Include/pymacro.h \ - $(srcdir)/Include/pymath.h \ - $(srcdir)/Include/pymem.h \ - $(srcdir)/Include/pyport.h \ - $(srcdir)/Include/pystate.h \ - $(srcdir)/Include/pystats.h \ - $(srcdir)/Include/pystrcmp.h \ - $(srcdir)/Include/pystrtod.h \ - $(srcdir)/Include/pythonrun.h \ - $(srcdir)/Include/pythread.h \ - $(srcdir)/Include/pytypedefs.h \ - $(srcdir)/Include/rangeobject.h \ - $(srcdir)/Include/setobject.h \ - $(srcdir)/Include/sliceobject.h \ - $(srcdir)/Include/structmember.h \ - $(srcdir)/Include/structseq.h \ - $(srcdir)/Include/sysmodule.h \ - $(srcdir)/Include/traceback.h \ - $(srcdir)/Include/tupleobject.h \ - $(srcdir)/Include/unicodeobject.h \ - $(srcdir)/Include/warnings.h \ - $(srcdir)/Include/weakrefobject.h \ - \ - pyconfig.h \ - $(PARSER_HEADERS) \ - \ - $(srcdir)/Include/cpython/abstract.h \ - $(srcdir)/Include/cpython/bytearrayobject.h \ - $(srcdir)/Include/cpython/bytesobject.h \ - $(srcdir)/Include/cpython/cellobject.h \ - $(srcdir)/Include/cpython/ceval.h \ - $(srcdir)/Include/cpython/classobject.h \ - $(srcdir)/Include/cpython/code.h \ - $(srcdir)/Include/cpython/compile.h \ - $(srcdir)/Include/cpython/complexobject.h \ - $(srcdir)/Include/cpython/context.h \ - $(srcdir)/Include/cpython/descrobject.h \ - $(srcdir)/Include/cpython/dictobject.h \ - $(srcdir)/Include/cpython/fileobject.h \ - $(srcdir)/Include/cpython/fileutils.h \ - $(srcdir)/Include/cpython/floatobject.h \ - $(srcdir)/Include/cpython/frameobject.h \ - $(srcdir)/Include/cpython/funcobject.h \ - $(srcdir)/Include/cpython/genobject.h \ - $(srcdir)/Include/cpython/import.h \ - $(srcdir)/Include/cpython/initconfig.h \ - $(srcdir)/Include/cpython/interpreteridobject.h \ - $(srcdir)/Include/cpython/listobject.h \ - $(srcdir)/Include/cpython/longintrepr.h \ - $(srcdir)/Include/cpython/longobject.h \ - $(srcdir)/Include/cpython/memoryobject.h \ - $(srcdir)/Include/cpython/methodobject.h \ - $(srcdir)/Include/cpython/object.h \ - $(srcdir)/Include/cpython/objimpl.h \ - $(srcdir)/Include/cpython/odictobject.h \ - $(srcdir)/Include/cpython/optimizer.h \ - $(srcdir)/Include/cpython/picklebufobject.h \ - $(srcdir)/Include/cpython/pthread_stubs.h \ - $(srcdir)/Include/cpython/pyatomic.h \ - $(srcdir)/Include/cpython/pyatomic_gcc.h \ - $(srcdir)/Include/cpython/pyatomic_std.h \ - $(srcdir)/Include/cpython/pyctype.h \ - $(srcdir)/Include/cpython/pydebug.h \ - $(srcdir)/Include/cpython/pyerrors.h \ - $(srcdir)/Include/cpython/pyfpe.h \ - $(srcdir)/Include/cpython/pyframe.h \ - $(srcdir)/Include/cpython/pyhash.h \ - $(srcdir)/Include/cpython/pylifecycle.h \ - $(srcdir)/Include/cpython/pymem.h \ - $(srcdir)/Include/cpython/pystate.h \ - $(srcdir)/Include/cpython/pystats.h \ - $(srcdir)/Include/cpython/pythonrun.h \ - $(srcdir)/Include/cpython/pythread.h \ - $(srcdir)/Include/cpython/setobject.h \ - $(srcdir)/Include/cpython/sysmodule.h \ - $(srcdir)/Include/cpython/traceback.h \ - $(srcdir)/Include/cpython/tracemalloc.h \ - $(srcdir)/Include/cpython/tupleobject.h \ - $(srcdir)/Include/cpython/unicodeobject.h \ - $(srcdir)/Include/cpython/warnings.h \ - $(srcdir)/Include/cpython/weakrefobject.h \ - \ - $(MIMALLOC_HEADERS) \ - \ - $(srcdir)/Include/internal/pycore_abstract.h \ - $(srcdir)/Include/internal/pycore_asdl.h \ - $(srcdir)/Include/internal/pycore_ast.h \ - $(srcdir)/Include/internal/pycore_ast_state.h \ - $(srcdir)/Include/internal/pycore_atexit.h \ - $(srcdir)/Include/internal/pycore_bitutils.h \ - $(srcdir)/Include/internal/pycore_bytes_methods.h \ - $(srcdir)/Include/internal/pycore_bytesobject.h \ - $(srcdir)/Include/internal/pycore_call.h \ - $(srcdir)/Include/internal/pycore_capsule.h \ - $(srcdir)/Include/internal/pycore_ceval.h \ - $(srcdir)/Include/internal/pycore_ceval_state.h \ - $(srcdir)/Include/internal/pycore_code.h \ - $(srcdir)/Include/internal/pycore_codecs.h \ - $(srcdir)/Include/internal/pycore_compile.h \ - $(srcdir)/Include/internal/pycore_complexobject.h \ - $(srcdir)/Include/internal/pycore_condvar.h \ - $(srcdir)/Include/internal/pycore_context.h \ - $(srcdir)/Include/internal/pycore_critical_section.h \ - $(srcdir)/Include/internal/pycore_crossinterp.h \ - $(srcdir)/Include/internal/pycore_dict.h \ - $(srcdir)/Include/internal/pycore_dict_state.h \ - $(srcdir)/Include/internal/pycore_descrobject.h \ - $(srcdir)/Include/internal/pycore_dtoa.h \ - $(srcdir)/Include/internal/pycore_exceptions.h \ - $(srcdir)/Include/internal/pycore_faulthandler.h \ - $(srcdir)/Include/internal/pycore_fileutils.h \ - $(srcdir)/Include/internal/pycore_floatobject.h \ - $(srcdir)/Include/internal/pycore_format.h \ - $(srcdir)/Include/internal/pycore_frame.h \ - $(srcdir)/Include/internal/pycore_freelist.h \ - $(srcdir)/Include/internal/pycore_function.h \ - $(srcdir)/Include/internal/pycore_genobject.h \ - $(srcdir)/Include/internal/pycore_getopt.h \ - $(srcdir)/Include/internal/pycore_gil.h \ - $(srcdir)/Include/internal/pycore_global_objects.h \ - $(srcdir)/Include/internal/pycore_global_objects_fini_generated.h \ - $(srcdir)/Include/internal/pycore_hamt.h \ - $(srcdir)/Include/internal/pycore_hashtable.h \ - $(srcdir)/Include/internal/pycore_identifier.h \ - $(srcdir)/Include/internal/pycore_import.h \ - $(srcdir)/Include/internal/pycore_initconfig.h \ - $(srcdir)/Include/internal/pycore_interp.h \ - $(srcdir)/Include/internal/pycore_intrinsics.h \ - $(srcdir)/Include/internal/pycore_list.h \ - $(srcdir)/Include/internal/pycore_llist.h \ - $(srcdir)/Include/internal/pycore_lock.h \ - $(srcdir)/Include/internal/pycore_long.h \ - $(srcdir)/Include/internal/pycore_modsupport.h \ - $(srcdir)/Include/internal/pycore_moduleobject.h \ - $(srcdir)/Include/internal/pycore_namespace.h \ - $(srcdir)/Include/internal/pycore_object.h \ - $(srcdir)/Include/internal/pycore_object_alloc.h \ - $(srcdir)/Include/internal/pycore_object_state.h \ - $(srcdir)/Include/internal/pycore_obmalloc.h \ - $(srcdir)/Include/internal/pycore_obmalloc_init.h \ - $(srcdir)/Include/internal/pycore_opcode_metadata.h \ - $(srcdir)/Include/internal/pycore_opcode_utils.h \ - $(srcdir)/Include/internal/pycore_optimizer.h \ - $(srcdir)/Include/internal/pycore_parking_lot.h \ - $(srcdir)/Include/internal/pycore_pathconfig.h \ - $(srcdir)/Include/internal/pycore_pyarena.h \ - $(srcdir)/Include/internal/pycore_pybuffer.h \ - $(srcdir)/Include/internal/pycore_pyerrors.h \ - $(srcdir)/Include/internal/pycore_pyhash.h \ - $(srcdir)/Include/internal/pycore_pylifecycle.h \ - $(srcdir)/Include/internal/pycore_pymem.h \ - $(srcdir)/Include/internal/pycore_pymem_init.h \ - $(srcdir)/Include/internal/pycore_pystate.h \ - $(srcdir)/Include/internal/pycore_pystats.h \ - $(srcdir)/Include/internal/pycore_pythonrun.h \ - $(srcdir)/Include/internal/pycore_pythread.h \ - $(srcdir)/Include/internal/pycore_range.h \ - $(srcdir)/Include/internal/pycore_runtime.h \ - $(srcdir)/Include/internal/pycore_runtime_init_generated.h \ - $(srcdir)/Include/internal/pycore_runtime_init.h \ - $(srcdir)/Include/internal/pycore_semaphore.h \ - $(srcdir)/Include/internal/pycore_setobject.h \ - $(srcdir)/Include/internal/pycore_signal.h \ - $(srcdir)/Include/internal/pycore_sliceobject.h \ - $(srcdir)/Include/internal/pycore_strhex.h \ - $(srcdir)/Include/internal/pycore_structseq.h \ - $(srcdir)/Include/internal/pycore_symtable.h \ - $(srcdir)/Include/internal/pycore_sysmodule.h \ - $(srcdir)/Include/internal/pycore_time.h \ - $(srcdir)/Include/internal/pycore_token.h \ - $(srcdir)/Include/internal/pycore_traceback.h \ - $(srcdir)/Include/internal/pycore_tracemalloc.h \ - $(srcdir)/Include/internal/pycore_tstate.h \ - $(srcdir)/Include/internal/pycore_tuple.h \ - $(srcdir)/Include/internal/pycore_typeobject.h \ - $(srcdir)/Include/internal/pycore_typevarobject.h \ - $(srcdir)/Include/internal/pycore_ucnhash.h \ - $(srcdir)/Include/internal/pycore_unionobject.h \ - $(srcdir)/Include/internal/pycore_unicodeobject.h \ - $(srcdir)/Include/internal/pycore_unicodeobject_generated.h \ - $(srcdir)/Include/internal/pycore_uop_metadata.h \ - $(srcdir)/Include/internal/pycore_warnings.h \ - $(srcdir)/Include/internal/pycore_weakref.h \ - $(DTRACE_HEADERS) \ - @PLATFORM_HEADERS@ \ - \ - $(srcdir)/Python/stdlib_module_names.h - $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) @@ -2639,6 +2661,23 @@ config.status: $(srcdir)/configure Python/asm_trampoline.o: $(srcdir)/Python/asm_trampoline.S $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< + +JIT_DEPS = \ + $(srcdir)/Tools/jit/*.c \ + $(srcdir)/Tools/jit/*.py \ + $(srcdir)/Python/executor_cases.c.h \ + pyconfig.h + +jit_stencils.h: $(JIT_DEPS) + @REGEN_JIT_COMMAND@ + +Python/jit.o: $(srcdir)/Python/jit.c @JIT_STENCILS_H@ + $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< + +.PHONY: regen-jit +regen-jit: + @REGEN_JIT_COMMAND@ + # Some make's put the object file in the current directory .c.o: $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< @@ -2731,6 +2770,7 @@ clean-retain-profile: pycremoval -rm -f Python/deepfreeze/*.[co] -rm -f Python/frozen_modules/*.h -rm -f Python/frozen_modules/MANIFEST + -rm -f jit_stencils.h -find build -type f -a ! -name '*.gc??' -exec rm -f {} ';' -rm -f Include/pydtrace_probes.h -rm -f profile-gen-stamp @@ -2855,6 +2895,9 @@ Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h MODULE_DEPS_STATIC=Modules/config.c MODULE_DEPS_SHARED=$(MODULE_DEPS_STATIC) $(EXPORTSYMS) +MODULE__CURSES_DEPS=$(srcdir)/Include/py_curses.h +MODULE__CURSES_PANEL_DEPS=$(srcdir)/Include/py_curses.h +MODULE__DATETIME_DEPS=$(srcdir)/Include/datetime.h MODULE_CMATH_DEPS=$(srcdir)/Modules/_math.h MODULE_MATH_DEPS=$(srcdir)/Modules/_math.h MODULE_PYEXPAT_DEPS=@LIBEXPAT_INTERNAL@ diff --git a/Misc/ACKS b/Misc/ACKS index 466023f390a421d..8a80e02ecba26a8 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1051,6 +1051,7 @@ Mark Lawrence Chris Laws Michael Layzell Michael Lazar +Peter Lazorchak Brian Leair Mathieu Leduc-Hamel Amandine Lee diff --git a/Misc/NEWS.d/3.11.0a2.rst b/Misc/NEWS.d/3.11.0a2.rst index eb1456f1bcf3536..a6b5fe54b391c53 100644 --- a/Misc/NEWS.d/3.11.0a2.rst +++ b/Misc/NEWS.d/3.11.0a2.rst @@ -1189,7 +1189,7 @@ context objects can now be disabled. .. section: C API Exclude :c:func:`PyWeakref_GET_OBJECT` from the limited C API. It never -worked since the :c:type:`PyWeakReference` structure is opaque in the +worked since the :c:type:`!PyWeakReference` structure is opaque in the limited C API. .. diff --git a/Misc/NEWS.d/3.11.0a4.rst b/Misc/NEWS.d/3.11.0a4.rst index 5abacd8473f3940..78b682f7a22cc61 100644 --- a/Misc/NEWS.d/3.11.0a4.rst +++ b/Misc/NEWS.d/3.11.0a4.rst @@ -161,7 +161,7 @@ faster due to reference-counting optimizations. Patch by Dennis Sweeney. .. nonce: 7oGp-I .. section: Core and Builtins -:opcode:`PREP_RERAISE_STAR` no longer pushes ``lasti`` to the stack. +:opcode:`!PREP_RERAISE_STAR` no longer pushes ``lasti`` to the stack. .. @@ -170,7 +170,7 @@ faster due to reference-counting optimizations. Patch by Dennis Sweeney. .. nonce: IKx4v6 .. section: Core and Builtins -Remove :opcode:`POP_EXCEPT_AND_RERAISE` and replace it by an equivalent +Remove :opcode:`!POP_EXCEPT_AND_RERAISE` and replace it by an equivalent sequence of other opcodes. .. @@ -1171,7 +1171,7 @@ Replaced deprecated usage of :c:func:`PyImport_ImportModuleNoBlock` with .. nonce: sMgDLz .. section: C API -The :c:func:`PyUnicode_CHECK_INTERNED` macro has been excluded from the +The :c:func:`!PyUnicode_CHECK_INTERNED` macro has been excluded from the limited C API. It was never usable there, because it used internal structures which are not available in the limited C API. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/3.11.0a6.rst b/Misc/NEWS.d/3.11.0a6.rst index 2b50b7773492cbd..2fdceef7746d4e9 100644 --- a/Misc/NEWS.d/3.11.0a6.rst +++ b/Misc/NEWS.d/3.11.0a6.rst @@ -248,7 +248,7 @@ Don't un-adapt :opcode:`COMPARE_OP` when collecting specialization stats. .. nonce: RX_AzJ .. section: Core and Builtins -Fix specialization stats gathering for :opcode:`PRECALL` instructions. +Fix specialization stats gathering for :opcode:`!PRECALL` instructions. .. diff --git a/Misc/NEWS.d/3.11.0a7.rst b/Misc/NEWS.d/3.11.0a7.rst index 76699632db223a5..ec99bd0294ceca2 100644 --- a/Misc/NEWS.d/3.11.0a7.rst +++ b/Misc/NEWS.d/3.11.0a7.rst @@ -138,7 +138,7 @@ Replaced :opcode:`JUMP_ABSOLUTE` by the relative jump .. nonce: SwrrFO .. section: Core and Builtins -:c:func:`PyFrame_FastToLocalsWithError` and :c:func:`PyFrame_LocalsToFast` +:c:func:`!PyFrame_FastToLocalsWithError` and :c:func:`!PyFrame_LocalsToFast` are no longer called during profiling nor tracing. C code can access the ``f_locals`` attribute of :c:type:`PyFrameObject` by calling :c:func:`PyFrame_GetLocals`. @@ -295,7 +295,7 @@ oparg) as an adaptive counter. .. nonce: O12Pba .. section: Core and Builtins -Use inline caching for :opcode:`PRECALL` and :opcode:`CALL`, and remove the +Use inline caching for :opcode:`!PRECALL` and :opcode:`CALL`, and remove the internal machinery for managing the (now unused) non-inline caches. .. diff --git a/Misc/NEWS.d/3.11.0b1.rst b/Misc/NEWS.d/3.11.0b1.rst index 2c30dc6e084bfb9..f92966796555733 100644 --- a/Misc/NEWS.d/3.11.0b1.rst +++ b/Misc/NEWS.d/3.11.0b1.rst @@ -403,8 +403,8 @@ so this led to crashes. The problem is now fixed. .. nonce: 6S_uoU .. section: Core and Builtins -Make opcodes :opcode:`JUMP_IF_TRUE_OR_POP` and -:opcode:`JUMP_IF_FALSE_OR_POP` relative rather than absolute. +Make opcodes :opcode:`!JUMP_IF_TRUE_OR_POP` and +:opcode:`!JUMP_IF_FALSE_OR_POP` relative rather than absolute. .. diff --git a/Misc/NEWS.d/3.12.0a4.rst b/Misc/NEWS.d/3.12.0a4.rst index ce2814bbe2e5ab2..82faa5ad0b2031a 100644 --- a/Misc/NEWS.d/3.12.0a4.rst +++ b/Misc/NEWS.d/3.12.0a4.rst @@ -13,8 +13,8 @@ Fix misleading default value in :func:`input`'s ``__text_signature__``. .. nonce: cmGwxv .. section: Core and Builtins -Remove :opcode:`UNARY_POSITIVE`, :opcode:`ASYNC_GEN_WRAP` and -:opcode:`LIST_TO_TUPLE`, replacing them with intrinsics. +Remove :opcode:`!UNARY_POSITIVE`, :opcode:`!ASYNC_GEN_WRAP` and +:opcode:`!LIST_TO_TUPLE`, replacing them with intrinsics. .. diff --git a/Misc/NEWS.d/3.12.0a6.rst b/Misc/NEWS.d/3.12.0a6.rst index 5bd600cd8b6fc0e..cf28bdb92588206 100644 --- a/Misc/NEWS.d/3.12.0a6.rst +++ b/Misc/NEWS.d/3.12.0a6.rst @@ -170,7 +170,7 @@ all as not all platform C libraries generate an error. .. section: Core and Builtins Add :opcode:`CALL_INTRINSIC_2` and use it instead of -:opcode:`PREP_RERAISE_STAR`. +:opcode:`!PREP_RERAISE_STAR`. .. diff --git a/Misc/NEWS.d/3.12.0a7.rst b/Misc/NEWS.d/3.12.0a7.rst index f22050b0dc377b2..a859be8a0474561 100644 --- a/Misc/NEWS.d/3.12.0a7.rst +++ b/Misc/NEWS.d/3.12.0a7.rst @@ -24,7 +24,7 @@ Reduce the number of inline :opcode:`CACHE` entries for .. nonce: PRkGca .. section: Core and Builtins -Removed :opcode:`JUMP_IF_FALSE_OR_POP` and :opcode:`JUMP_IF_TRUE_OR_POP` +Removed :opcode:`!JUMP_IF_FALSE_OR_POP` and :opcode:`!JUMP_IF_TRUE_OR_POP` instructions. .. diff --git a/Misc/NEWS.d/3.12.0b1.rst b/Misc/NEWS.d/3.12.0b1.rst index 007a6ad4ffd4d42..211513d05d00407 100644 --- a/Misc/NEWS.d/3.12.0b1.rst +++ b/Misc/NEWS.d/3.12.0b1.rst @@ -1008,7 +1008,7 @@ Update the bundled copy of pip to version 23.1.2. .. nonce: pst8iT .. section: Library -Make :mod:`dis` display the value of oparg of :opcode:`KW_NAMES`. +Make :mod:`dis` display the value of oparg of :opcode:`!KW_NAMES`. .. diff --git a/Misc/NEWS.d/3.13.0a1.rst b/Misc/NEWS.d/3.13.0a1.rst index 102bddcee5c5c2a..d385b6a4504f97a 100644 --- a/Misc/NEWS.d/3.13.0a1.rst +++ b/Misc/NEWS.d/3.13.0a1.rst @@ -2276,7 +2276,7 @@ creation. .. nonce: m2H5Bk .. section: Library -Remove unnecessary extra ``__slots__`` in :py:class:`datetime`\'s pure +Remove unnecessary extra ``__slots__`` in :class:`~datetime.datetime`\'s pure python implementation to reduce memory size, as they are defined in the superclass. Patch by James Hilton-Balfe diff --git a/Misc/NEWS.d/3.13.0a2.rst b/Misc/NEWS.d/3.13.0a2.rst index d4be4fb8a3d3ab6..e5841e14c02efbf 100644 --- a/Misc/NEWS.d/3.13.0a2.rst +++ b/Misc/NEWS.d/3.13.0a2.rst @@ -565,9 +565,9 @@ part of a :exc:`BaseExceptionGroup`, in addition to the recent support for .. section: Library The :class:`mmap.mmap` class now has an :meth:`~mmap.mmap.seekable` method -that can be used where it requires a file-like object with seekable and the -:meth:`~mmap.mmap.seek` method return the new absolute position. Patch by -Donghee Na. +that can be used when a seekable file-like object is required. +The :meth:`~mmap.mmap.seek` method now returns the new absolute position. +Patch by Donghee Na. .. diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index 11b303e89ad04f7..bd9061601fe1905 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -3395,8 +3395,8 @@ Zackery Spytz. .. nonce: S0Irst .. section: Library -Fix parsing non-ASCII identifiers in :mod:`lib2to3.pgen2.tokenize` (PEP -3131). +Fix parsing non-ASCII identifiers in :mod:`!lib2to3.pgen2.tokenize` +(:pep:`3131`). .. diff --git a/Misc/NEWS.d/3.9.0a1.rst b/Misc/NEWS.d/3.9.0a1.rst index 0772a0fed206527..66d7fc1f32e705d 100644 --- a/Misc/NEWS.d/3.9.0a1.rst +++ b/Misc/NEWS.d/3.9.0a1.rst @@ -5715,8 +5715,8 @@ The :c:macro:`METH_FASTCALL` calling convention has been documented. .. nonce: 4tClQT .. section: C API -The new function :c:func:`PyCode_NewWithPosOnlyArgs` allows to create code -objects like :c:func:`PyCode_New`, but with an extra *posonlyargcount* +The new function :c:func:`!PyCode_NewWithPosOnlyArgs` allows to create code +objects like :c:func:`!PyCode_New`, but with an extra *posonlyargcount* parameter for indicating the number of positonal-only arguments. .. diff --git a/Misc/NEWS.d/3.9.0a5.rst b/Misc/NEWS.d/3.9.0a5.rst index b4594aade3b3ed8..f0015ac54df307a 100644 --- a/Misc/NEWS.d/3.9.0a5.rst +++ b/Misc/NEWS.d/3.9.0a5.rst @@ -1122,7 +1122,7 @@ a different condition than the GIL. .. nonce: Nbl7lF .. section: Tools/Demos -Added support to fix ``getproxies`` in the :mod:`lib2to3.fixes.fix_urllib` +Added support to fix ``getproxies`` in the :mod:`!lib2to3.fixes.fix_urllib` module. Patch by José Roberto Meza Cabrera. .. diff --git a/Misc/NEWS.d/next/Build/2024-02-01-20-08-11.gh-issue-114875.x_2iZ9.rst b/Misc/NEWS.d/next/Build/2024-02-01-20-08-11.gh-issue-114875.x_2iZ9.rst new file mode 100644 index 000000000000000..20e9d6376b973cb --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-02-01-20-08-11.gh-issue-114875.x_2iZ9.rst @@ -0,0 +1 @@ +Add :c:func:`!getgrent` as a prerequisite for building the :mod:`grp` module. diff --git a/Misc/NEWS.d/next/Build/2024-02-08-17-38-56.gh-issue-113632.y9KIGb.rst b/Misc/NEWS.d/next/Build/2024-02-08-17-38-56.gh-issue-113632.y9KIGb.rst new file mode 100644 index 000000000000000..8b02b1b2cd08c90 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-02-08-17-38-56.gh-issue-113632.y9KIGb.rst @@ -0,0 +1,2 @@ +Promote WASI to a tier 2 platform and drop Emscripten from tier 3 in +configure.ac. diff --git a/Misc/NEWS.d/next/Build/2024-02-08-19-36-20.gh-issue-115167.LB9nDK.rst b/Misc/NEWS.d/next/Build/2024-02-08-19-36-20.gh-issue-115167.LB9nDK.rst new file mode 100644 index 000000000000000..c60c4a93fe8906c --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-02-08-19-36-20.gh-issue-115167.LB9nDK.rst @@ -0,0 +1 @@ +Avoid vendoring ``vcruntime140_threads.dll`` when building with Visual Studio 2022 version 17.8. diff --git a/Misc/NEWS.d/next/C API/2023-11-15-13-47-48.gh-issue-112066.22WsqR.rst b/Misc/NEWS.d/next/C API/2023-11-15-13-47-48.gh-issue-112066.22WsqR.rst new file mode 100644 index 000000000000000..ae2b8b2444de972 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-11-15-13-47-48.gh-issue-112066.22WsqR.rst @@ -0,0 +1,5 @@ +Add :c:func:`PyDict_SetDefaultRef`: insert a key and value into a dictionary +if the key is not already present. This is similar to +:meth:`dict.setdefault`, but returns an integer value indicating if the key +was already present. It is also similar to :c:func:`PyDict_SetDefault`, but +returns a strong reference instead of a borrowed reference. diff --git a/Misc/NEWS.d/next/C API/2024-01-23-21-45-02.gh-issue-114329.YRaBoe.rst b/Misc/NEWS.d/next/C API/2024-01-23-21-45-02.gh-issue-114329.YRaBoe.rst new file mode 100644 index 000000000000000..62d4ce0cfb8de54 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-01-23-21-45-02.gh-issue-114329.YRaBoe.rst @@ -0,0 +1,3 @@ +Add :c:func:`PyList_GetItemRef`, which is similar to +:c:func:`PyList_GetItem` but returns a :term:`strong reference` instead of a +:term:`borrowed reference`. diff --git a/Misc/NEWS.d/next/C API/2024-01-29-12-13-24.gh-issue-114685.B07RME.rst b/Misc/NEWS.d/next/C API/2024-01-29-12-13-24.gh-issue-114685.B07RME.rst new file mode 100644 index 000000000000000..55b02d1d8e1e9fc --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-01-29-12-13-24.gh-issue-114685.B07RME.rst @@ -0,0 +1,3 @@ +:c:func:`PyObject_GetBuffer` now raises a :exc:`SystemError` if called with +:c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE` as flags. These flags should +only be used with the ``PyMemoryView_*`` C API. diff --git a/Misc/NEWS.d/next/C API/2024-01-31-15-43-35.gh-issue-114685.n7aRmX.rst b/Misc/NEWS.d/next/C API/2024-01-31-15-43-35.gh-issue-114685.n7aRmX.rst new file mode 100644 index 000000000000000..76ff00645fe57d2 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-01-31-15-43-35.gh-issue-114685.n7aRmX.rst @@ -0,0 +1,3 @@ +:c:func:`PyBuffer_FillInfo` now raises a :exc:`SystemError` if called with +:c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE` as flags. These flags should +only be used with the ``PyMemoryView_*`` C API. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-16-06-52-34.gh-issue-104530.mJnA0W.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-16-06-52-34.gh-issue-104530.mJnA0W.rst new file mode 100644 index 000000000000000..8643a25ae51b13f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-16-06-52-34.gh-issue-104530.mJnA0W.rst @@ -0,0 +1 @@ +Use native Win32 condition variables. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-06-19-09-00.gh-issue-55664.vYYl0V.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-06-19-09-00.gh-issue-55664.vYYl0V.rst new file mode 100644 index 000000000000000..438be9854966501 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-06-06-19-09-00.gh-issue-55664.vYYl0V.rst @@ -0,0 +1 @@ +Add warning when creating :class:`type` using a namespace dictionary with non-string keys. Patched by Daniel Urban and Furkan Onder. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-22-13-21-39.gh-issue-113055.47xBMF.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-22-13-21-39.gh-issue-113055.47xBMF.rst new file mode 100644 index 000000000000000..90f49272218c960 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-22-13-21-39.gh-issue-113055.47xBMF.rst @@ -0,0 +1,5 @@ +Make interp->obmalloc a pointer. For interpreters that share state with the +main interpreter, this points to the same static memory structure. For +interpreters with their own obmalloc state, it is heap allocated. Add +free_obmalloc_arenas() which will free the obmalloc arenas and radix tree +structures for interpreters with their own obmalloc state. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-24-03-25-28.gh-issue-113464.dvjQmA.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-24-03-25-28.gh-issue-113464.dvjQmA.rst new file mode 100644 index 000000000000000..bdee4d645f61c83 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-24-03-25-28.gh-issue-113464.dvjQmA.rst @@ -0,0 +1,4 @@ +Add an option (``--enable-experimental-jit`` for ``configure``-based builds +or ``--experimental-jit`` for ``PCbuild``-based ones) to build an +*experimental* just-in-time compiler, based on `copy-and-patch +<https://fredrikbk.com/publications/copy-and-patch.pdf>`_ diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-11-22-58-45.gh-issue-112050.hDuvDW.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-11-22-58-45.gh-issue-112050.hDuvDW.rst new file mode 100644 index 000000000000000..e5f3d5ea0cea254 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-11-22-58-45.gh-issue-112050.hDuvDW.rst @@ -0,0 +1 @@ +Convert :class:`collections.deque` to use Argument Clinic. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-12-16-40-07.gh-issue-113939.Yi3L-e.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-12-16-40-07.gh-issue-113939.Yi3L-e.rst new file mode 100644 index 000000000000000..28b8e4bdda6be4e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-12-16-40-07.gh-issue-113939.Yi3L-e.rst @@ -0,0 +1,4 @@ +frame.clear(): +Clear frame.f_locals as well, and not only the fast locals. +This is relevant once frame.f_locals was accessed, +which would contain also references to all the locals. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-17-00-52-57.gh-issue-113884.CvEjUE.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-17-00-52-57.gh-issue-113884.CvEjUE.rst new file mode 100644 index 000000000000000..6a39fd2f60ab813 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-17-00-52-57.gh-issue-113884.CvEjUE.rst @@ -0,0 +1 @@ +Make :class:`queue.SimpleQueue` thread safe when the GIL is disabled. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-17-05-09-32.gh-issue-112354.Run9ko.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-17-05-09-32.gh-issue-112354.Run9ko.rst new file mode 100644 index 000000000000000..ed45ba49c3ad421 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-17-05-09-32.gh-issue-112354.Run9ko.rst @@ -0,0 +1,2 @@ +The ``END_FOR`` instruction now pops only one value. This is to better +support side exits in loops. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-18-20-20-37.gh-issue-112529.oVNvDG.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-18-20-20-37.gh-issue-112529.oVNvDG.rst new file mode 100644 index 000000000000000..b3aa43801da488a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-18-20-20-37.gh-issue-112529.oVNvDG.rst @@ -0,0 +1,3 @@ +The free-threaded build now has its own thread-safe GC implementation that +uses mimalloc to find GC tracked objects. It is non-generational, unlike the +existing GC implementation. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-21-17-29-32.gh-issue-114388.UVGO4K.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-21-17-29-32.gh-issue-114388.UVGO4K.rst new file mode 100644 index 000000000000000..52c2742001d9ca6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-21-17-29-32.gh-issue-114388.UVGO4K.rst @@ -0,0 +1,5 @@ +Fix a :exc:`RuntimeWarning` emitted when assign an integer-like value that +is not an instance of :class:`int` to an attribute that corresponds to a C +struct member of :ref:`type <PyMemberDef-types>` T_UINT and T_ULONG. Fix a +double :exc:`RuntimeWarning` emitted when assign a negative integer value to +an attribute that corresponds to a C struct member of type T_UINT. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-22-15-10-01.gh-issue-114456.fBFEJF.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-22-15-10-01.gh-issue-114456.fBFEJF.rst new file mode 100644 index 000000000000000..2b30ad98fb5c79b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-22-15-10-01.gh-issue-114456.fBFEJF.rst @@ -0,0 +1 @@ +Lower the recursion limit under a debug build of WASI. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-25-18-50-49.gh-issue-112529.IbbApA.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-25-18-50-49.gh-issue-112529.IbbApA.rst new file mode 100644 index 000000000000000..2a6d74fb2227023 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-25-18-50-49.gh-issue-112529.IbbApA.rst @@ -0,0 +1,4 @@ +The free-threaded build no longer allocates space for the ``PyGC_Head`` +structure in objects that support cyclic garbage collection. A number of +other fields and data structures are used as replacements, including +``ob_gc_bits``, ``ob_tid``, and mimalloc internal data structures. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-31-09-10-10.gh-issue-107944.XWm1B-.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-31-09-10-10.gh-issue-107944.XWm1B-.rst new file mode 100644 index 000000000000000..8e3fb786c11055b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-31-09-10-10.gh-issue-107944.XWm1B-.rst @@ -0,0 +1 @@ +Improve error message for function calls with bad keyword arguments via getargs diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-01-18-16-52.gh-issue-114806.wrH2J6.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-01-18-16-52.gh-issue-114806.wrH2J6.rst new file mode 100644 index 000000000000000..795f2529df82074 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-01-18-16-52.gh-issue-114806.wrH2J6.rst @@ -0,0 +1,3 @@ +No longer specialize calls to classes, if those classes have metaclasses. +Fixes bug where the ``__call__`` method of the metaclass was not being +called. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-01-23-43-49.gh-issue-76763.o_2J6i.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-01-23-43-49.gh-issue-76763.o_2J6i.rst new file mode 100644 index 000000000000000..d35d3d87073dddd --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-01-23-43-49.gh-issue-76763.o_2J6i.rst @@ -0,0 +1,3 @@ +The :func:`chr` builtin function now always raises :exc:`ValueError` for +values outside the valid range. Previously it raised :exc:`OverflowError` for +very large or small values. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-02-05-27-48.gh-issue-113462.VMml8q.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-02-05-27-48.gh-issue-113462.VMml8q.rst new file mode 100644 index 000000000000000..1a401ecebf019aa --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-02-05-27-48.gh-issue-113462.VMml8q.rst @@ -0,0 +1,2 @@ +Limit the number of versions that a single class can use. Prevents a few +wayward classes using up all the version numbers. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-03-01-48-38.gh-issue-114944.4J5ELD.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-03-01-48-38.gh-issue-114944.4J5ELD.rst new file mode 100644 index 000000000000000..fb41caf7c5f4fae --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-03-01-48-38.gh-issue-114944.4J5ELD.rst @@ -0,0 +1 @@ +Fixes a race between ``PyParkingLot_Park`` and ``_PyParkingLot_UnparkAll``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-03-04-07-18.gh-issue-114887.uLSFmN.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-03-04-07-18.gh-issue-114887.uLSFmN.rst new file mode 100644 index 000000000000000..b4d8cf4089d7238 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-03-04-07-18.gh-issue-114887.uLSFmN.rst @@ -0,0 +1,2 @@ +Changed socket type validation in :meth:`~asyncio.loop.create_datagram_endpoint` to accept all non-stream sockets. +This fixes a regression in compatibility with raw sockets. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-07-00-18-42.gh-issue-112069.jRDRR5.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-07-00-18-42.gh-issue-112069.jRDRR5.rst new file mode 100644 index 000000000000000..51ba6bd1ddaac35 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-07-00-18-42.gh-issue-112069.jRDRR5.rst @@ -0,0 +1 @@ +Adapt :class:`set` and :class:`frozenset` methods to Argument Clinic. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-07-07-50-12.gh-issue-114828.nSXwMi.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-07-07-50-12.gh-issue-114828.nSXwMi.rst new file mode 100644 index 000000000000000..b1c63e0a1518fdc --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-07-07-50-12.gh-issue-114828.nSXwMi.rst @@ -0,0 +1,2 @@ +Fix compilation crashes in uncommon code examples using :func:`super` inside +a comprehension in a class body. diff --git a/Misc/NEWS.d/next/IDLE/2023-04-25-03-01-23.gh-issue-103820.LCSpza.rst b/Misc/NEWS.d/next/IDLE/2023-04-25-03-01-23.gh-issue-103820.LCSpza.rst new file mode 100644 index 000000000000000..b9d7faf047b28e7 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2023-04-25-03-01-23.gh-issue-103820.LCSpza.rst @@ -0,0 +1,2 @@ +Revise IDLE bindings so that events from mouse button 4/5 on non-X11 +windowing systems (i.e. Win32 and Aqua) are not mistaken for scrolling. diff --git a/Misc/NEWS.d/next/Library/2022-07-31-01-24-40.gh-issue-88569.eU0--b.rst b/Misc/NEWS.d/next/Library/2022-07-31-01-24-40.gh-issue-88569.eU0--b.rst new file mode 100644 index 000000000000000..31dd985bb5c3b6c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-07-31-01-24-40.gh-issue-88569.eU0--b.rst @@ -0,0 +1,4 @@ +Add :func:`os.path.isreserved`, which identifies reserved pathnames such +as "NUL", "AUX" and "CON". This function is only available on Windows. + +Deprecate :meth:`pathlib.PurePath.is_reserved`. diff --git a/Misc/NEWS.d/next/Library/2023-03-15-03-21-18.gh-issue-85984.Xaq6ZN.rst b/Misc/NEWS.d/next/Library/2023-03-15-03-21-18.gh-issue-85984.Xaq6ZN.rst new file mode 100644 index 000000000000000..0e54a1fe3c8a1cf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-15-03-21-18.gh-issue-85984.Xaq6ZN.rst @@ -0,0 +1,2 @@ +Add POSIX pseudo-terminal functions :func:`os.posix_openpt`, +:func:`os.grantpt`, :func:`os.unlockpt`, and :func:`os.ptsname`. diff --git a/Misc/NEWS.d/next/Library/2023-04-08-11-41-07.gh-issue-101599.PaWNFh.rst b/Misc/NEWS.d/next/Library/2023-04-08-11-41-07.gh-issue-101599.PaWNFh.rst new file mode 100644 index 000000000000000..a1608a1ae0d2fae --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-08-11-41-07.gh-issue-101599.PaWNFh.rst @@ -0,0 +1 @@ +Changed argparse flag options formatting to remove redundancy. diff --git a/Misc/NEWS.d/next/Library/2023-05-06-04-57-10.gh-issue-96471.C9wAU7.rst b/Misc/NEWS.d/next/Library/2023-05-06-04-57-10.gh-issue-96471.C9wAU7.rst new file mode 100644 index 000000000000000..0bace8d8bd425c7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-06-04-57-10.gh-issue-96471.C9wAU7.rst @@ -0,0 +1 @@ +Add :py:class:`queue.Queue` termination with :py:meth:`~queue.Queue.shutdown`. diff --git a/Misc/NEWS.d/next/Library/2023-06-29-14-26-56.gh-issue-106233.Aqw2HI.rst b/Misc/NEWS.d/next/Library/2023-06-29-14-26-56.gh-issue-106233.Aqw2HI.rst new file mode 100644 index 000000000000000..345c8b20815c951 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-06-29-14-26-56.gh-issue-106233.Aqw2HI.rst @@ -0,0 +1,2 @@ +Fix stacklevel in ``InvalidTZPathWarning`` during :mod:`zoneinfo` module +import. diff --git a/Misc/NEWS.d/next/Library/2023-07-23-12-28-26.gh-issue-75705.aB2-Ww.rst b/Misc/NEWS.d/next/Library/2023-07-23-12-28-26.gh-issue-75705.aB2-Ww.rst new file mode 100644 index 000000000000000..272e31d64cfbd9b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-23-12-28-26.gh-issue-75705.aB2-Ww.rst @@ -0,0 +1 @@ +Set unixfrom envelope in :class:`mailbox.mbox` and :class:`mailbox.MMDF`. diff --git a/Misc/NEWS.d/next/Library/2023-10-19-02-08-12.gh-issue-111051.8h1Dpk.rst b/Misc/NEWS.d/next/Library/2023-10-19-02-08-12.gh-issue-111051.8h1Dpk.rst new file mode 100644 index 000000000000000..adb3241b89ae3e6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-19-02-08-12.gh-issue-111051.8h1Dpk.rst @@ -0,0 +1 @@ +Added check for file modification during debugging with :mod:`pdb` diff --git a/Misc/NEWS.d/next/Library/2023-10-24-19-19-54.gh-issue-82626._hfLRf.rst b/Misc/NEWS.d/next/Library/2023-10-24-19-19-54.gh-issue-82626._hfLRf.rst new file mode 100644 index 000000000000000..92a66b5bf0f635c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-24-19-19-54.gh-issue-82626._hfLRf.rst @@ -0,0 +1,2 @@ +Many functions now emit a warning if a boolean value is passed as a file +descriptor argument. diff --git a/Misc/NEWS.d/next/Library/2023-10-27-19-24-58.gh-issue-43457.84lx9H.rst b/Misc/NEWS.d/next/Library/2023-10-27-19-24-58.gh-issue-43457.84lx9H.rst new file mode 100644 index 000000000000000..401a532ce03e777 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-27-19-24-58.gh-issue-43457.84lx9H.rst @@ -0,0 +1,8 @@ +Fix the :mod:`tkinter` widget method :meth:`!wm_attributes`. It now +accepts the attribute name without the minus prefix to get window attributes +and allows to specify attributes and values to set as keyword arguments. +Add new optional keyword argument *return_python_dict*: calling +``w.wm_attributes(return_python_dict=True)`` returns the attributes as +a dict instead of a tuple. +Calling ``w.wm_attributes()`` now returns a tuple instead of string if +*wantobjects* was set to 0. diff --git a/Misc/NEWS.d/next/Library/2023-11-04-22-32-27.gh-issue-111741.f1ufr8.rst b/Misc/NEWS.d/next/Library/2023-11-04-22-32-27.gh-issue-111741.f1ufr8.rst new file mode 100644 index 000000000000000..e43f93a270ce9c8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-04-22-32-27.gh-issue-111741.f1ufr8.rst @@ -0,0 +1 @@ +Recognise ``image/webp`` as a standard format in the :mod:`mimetypes` module. diff --git a/Misc/NEWS.d/next/Library/2023-11-18-16-30-21.gh-issue-112240.YXS0tj.rst b/Misc/NEWS.d/next/Library/2023-11-18-16-30-21.gh-issue-112240.YXS0tj.rst new file mode 100644 index 000000000000000..686f0311e80dcbf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-18-16-30-21.gh-issue-112240.YXS0tj.rst @@ -0,0 +1,2 @@ +Add option to calendar module CLI to specify the weekday to start each week. +Patch by Steven Ward. diff --git a/Misc/NEWS.d/next/Library/2023-11-27-19-54-43.gh-issue-59013.chpQ0e.rst b/Misc/NEWS.d/next/Library/2023-11-27-19-54-43.gh-issue-59013.chpQ0e.rst new file mode 100644 index 000000000000000..a2be2fb8eacf17c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-27-19-54-43.gh-issue-59013.chpQ0e.rst @@ -0,0 +1 @@ +Set breakpoint on the first executable line of the function, instead of the line of function definition when the user do ``break func`` using :mod:`pdb` diff --git a/Misc/NEWS.d/next/Library/2023-12-09-23-31-17.gh-issue-112919.S5k9QN.rst b/Misc/NEWS.d/next/Library/2023-12-09-23-31-17.gh-issue-112919.S5k9QN.rst new file mode 100644 index 000000000000000..3e99d480139cbea --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-09-23-31-17.gh-issue-112919.S5k9QN.rst @@ -0,0 +1,2 @@ +Speed-up :func:`datetime.datetime.replace`, :func:`datetime.date.replace` and +:func:`datetime.time.replace`. diff --git a/Misc/NEWS.d/next/Library/2024-01-05-16-27-34.gh-issue-113732.fgDRXA.rst b/Misc/NEWS.d/next/Library/2024-01-05-16-27-34.gh-issue-113732.fgDRXA.rst new file mode 100644 index 000000000000000..7582603dcf95f5f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-05-16-27-34.gh-issue-113732.fgDRXA.rst @@ -0,0 +1,2 @@ +Fix support of :data:`~csv.QUOTE_NOTNULL` and :data:`~csv.QUOTE_STRINGS` in +:func:`csv.reader`. diff --git a/Misc/NEWS.d/next/Library/2024-01-11-20-47-49.gh-issue-113951.AzlqFK.rst b/Misc/NEWS.d/next/Library/2024-01-11-20-47-49.gh-issue-113951.AzlqFK.rst new file mode 100644 index 000000000000000..e683472e59b8a49 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-11-20-47-49.gh-issue-113951.AzlqFK.rst @@ -0,0 +1,7 @@ +Fix the behavior of ``tag_unbind()`` methods of :class:`tkinter.Text` and +:class:`tkinter.Canvas` classes with three arguments. Previously, +``widget.tag_unbind(tag, sequence, funcid)`` destroyed the current binding +for *sequence*, leaving *sequence* unbound, and deleted the *funcid* +command. Now it removes only *funcid* from the binding for *sequence*, +keeping other commands, and deletes the *funcid* command. It leaves +*sequence* unbound only if *funcid* was the last bound command. diff --git a/Misc/NEWS.d/next/Library/2024-01-12-09-35-07.gh-issue-112202.t_0V1m.rst b/Misc/NEWS.d/next/Library/2024-01-12-09-35-07.gh-issue-112202.t_0V1m.rst new file mode 100644 index 000000000000000..9abde13bbf8571b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-12-09-35-07.gh-issue-112202.t_0V1m.rst @@ -0,0 +1 @@ +Ensure that a :func:`asyncio.Condition.notify` call does not get lost if the awakened ``Task`` is simultaneously cancelled or encounters any other error. diff --git a/Misc/NEWS.d/next/Library/2024-01-15-18-42-44.gh-issue-109534.wYaLMZ.rst b/Misc/NEWS.d/next/Library/2024-01-15-18-42-44.gh-issue-109534.wYaLMZ.rst new file mode 100644 index 000000000000000..fc9a765a230037d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-15-18-42-44.gh-issue-109534.wYaLMZ.rst @@ -0,0 +1,3 @@ +Fix a reference leak in +:class:`asyncio.selector_events.BaseSelectorEventLoop` when SSL handshakes +fail. Patch contributed by Jamie Phan. diff --git a/Misc/NEWS.d/next/Library/2024-01-15-20-21-33.gh-issue-83648.HzD_fY.rst b/Misc/NEWS.d/next/Library/2024-01-15-20-21-33.gh-issue-83648.HzD_fY.rst new file mode 100644 index 000000000000000..bd3e27b4be0cf58 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-15-20-21-33.gh-issue-83648.HzD_fY.rst @@ -0,0 +1,2 @@ +Support deprecation of options, positional arguments and subcommands in +:mod:`argparse`. diff --git a/Misc/NEWS.d/next/Library/2024-01-18-22-29-28.gh-issue-101438.1-uUi_.rst b/Misc/NEWS.d/next/Library/2024-01-18-22-29-28.gh-issue-101438.1-uUi_.rst new file mode 100644 index 000000000000000..9b69b5deb1b5a06 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-18-22-29-28.gh-issue-101438.1-uUi_.rst @@ -0,0 +1,4 @@ +Avoid reference cycle in ElementTree.iterparse. The iterator returned by +``ElementTree.iterparse`` may hold on to a file descriptor. The reference +cycle prevented prompt clean-up of the file descriptor if the returned +iterator was not exhausted. diff --git a/Misc/NEWS.d/next/Library/2024-01-23-11-04-21.gh-issue-113267.xe_Pxe.rst b/Misc/NEWS.d/next/Library/2024-01-23-11-04-21.gh-issue-113267.xe_Pxe.rst new file mode 100644 index 000000000000000..ad8aaf9250f6d8e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-23-11-04-21.gh-issue-113267.xe_Pxe.rst @@ -0,0 +1,2 @@ +Revert changes in :gh:`106584` which made calls of ``TestResult`` methods +``startTest()`` and ``stopTest()`` unbalanced. diff --git a/Misc/NEWS.d/next/Library/2024-01-23-14-11-49.gh-issue-114315.KeVdzl.rst b/Misc/NEWS.d/next/Library/2024-01-23-14-11-49.gh-issue-114315.KeVdzl.rst new file mode 100644 index 000000000000000..a8a19fc525d0199 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-23-14-11-49.gh-issue-114315.KeVdzl.rst @@ -0,0 +1,2 @@ +Make :class:`threading.Lock` a real class, not a factory function. Add +``__new__`` to ``_thread.lock`` type. diff --git a/Misc/NEWS.d/next/Library/2024-01-23-21-20-40.gh-issue-114492.vKxl5o.rst b/Misc/NEWS.d/next/Library/2024-01-23-21-20-40.gh-issue-114492.vKxl5o.rst new file mode 100644 index 000000000000000..8df8299d0dffcd6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-23-21-20-40.gh-issue-114492.vKxl5o.rst @@ -0,0 +1,2 @@ +Make the result of :func:`termios.tcgetattr` reproducible on Alpine Linux. +Previously it could leave a random garbage in some fields. diff --git a/Misc/NEWS.d/next/Library/2024-01-23-23-13-47.gh-issue-109653.KLBHmT.rst b/Misc/NEWS.d/next/Library/2024-01-23-23-13-47.gh-issue-109653.KLBHmT.rst new file mode 100644 index 000000000000000..76074df9c76fa61 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-23-23-13-47.gh-issue-109653.KLBHmT.rst @@ -0,0 +1 @@ +Reduce the import time of :mod:`threading` module by ~50%. Patch by Daniel Hollas. diff --git a/Misc/NEWS.d/next/Library/2024-01-24-17-25-18.gh-issue-69893.PQq5fR.rst b/Misc/NEWS.d/next/Library/2024-01-24-17-25-18.gh-issue-69893.PQq5fR.rst new file mode 100644 index 000000000000000..1ebf434c33187bf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-24-17-25-18.gh-issue-69893.PQq5fR.rst @@ -0,0 +1,2 @@ +Add the :meth:`!close` method for the iterator returned by +:func:`xml.etree.ElementTree.iterparse`. diff --git a/Misc/NEWS.d/next/Library/2024-01-24-20-11-46.gh-issue-112451.7YrG4p.rst b/Misc/NEWS.d/next/Library/2024-01-24-20-11-46.gh-issue-112451.7YrG4p.rst new file mode 100644 index 000000000000000..126ca36a3b7cb1a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-24-20-11-46.gh-issue-112451.7YrG4p.rst @@ -0,0 +1,2 @@ +Prohibit subclassing pure-Python :class:`datetime.timezone`. This is consistent +with C-extension implementation. Patch by Mariusz Felisiak. diff --git a/Misc/NEWS.d/next/Library/2024-01-24-20-51-49.gh-issue-91602.8fOH8l.rst b/Misc/NEWS.d/next/Library/2024-01-24-20-51-49.gh-issue-91602.8fOH8l.rst new file mode 100644 index 000000000000000..21d39df43e035b6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-24-20-51-49.gh-issue-91602.8fOH8l.rst @@ -0,0 +1,3 @@ +Add *filter* keyword-only parameter to +:meth:`sqlite3.Connection.iterdump` for filtering database objects to dump. +Patch by Mariusz Felisiak. diff --git a/Misc/NEWS.d/next/Library/2024-01-25-19-22-17.gh-issue-83383.3GwO9v.rst b/Misc/NEWS.d/next/Library/2024-01-25-19-22-17.gh-issue-83383.3GwO9v.rst new file mode 100644 index 000000000000000..e6336204dfa2369 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-25-19-22-17.gh-issue-83383.3GwO9v.rst @@ -0,0 +1,5 @@ +Synchronization of the :mod:`dbm.dumb` database is now no-op if there was no +modification since opening or last synchronization. +The directory file for a newly created empty :mod:`dbm.dumb` database is now +created immediately after opening instead of deferring this until +synchronizing or closing. diff --git a/Misc/NEWS.d/next/Library/2024-01-26-16-46-21.gh-issue-77749.NY_7TS.rst b/Misc/NEWS.d/next/Library/2024-01-26-16-46-21.gh-issue-77749.NY_7TS.rst new file mode 100644 index 000000000000000..f1c99c09d2dfe1c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-26-16-46-21.gh-issue-77749.NY_7TS.rst @@ -0,0 +1,2 @@ +:meth:`email.policy.EmailPolicy.fold` now always encodes non-ASCII characters +in headers if :attr:`~email.policy.EmailPolicy.utf8` is false. diff --git a/Misc/NEWS.d/next/Library/2024-01-27-20-11-24.gh-issue-113280.CZPQMf.rst b/Misc/NEWS.d/next/Library/2024-01-27-20-11-24.gh-issue-113280.CZPQMf.rst new file mode 100644 index 000000000000000..3dcdbcf0995616a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-27-20-11-24.gh-issue-113280.CZPQMf.rst @@ -0,0 +1,2 @@ +Fix a leak of open socket in rare cases when error occurred in +:class:`ssl.SSLSocket` creation. diff --git a/Misc/NEWS.d/next/Library/2024-01-28-00-48-12.gh-issue-109653.vF4exe.rst b/Misc/NEWS.d/next/Library/2024-01-28-00-48-12.gh-issue-109653.vF4exe.rst new file mode 100644 index 000000000000000..fb3382098853b30 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-28-00-48-12.gh-issue-109653.vF4exe.rst @@ -0,0 +1 @@ +Improve import time of :mod:`importlib.metadata` and :mod:`email.utils`. diff --git a/Misc/NEWS.d/next/Library/2024-01-28-18-38-18.gh-issue-70303._Lt_pj.rst b/Misc/NEWS.d/next/Library/2024-01-28-18-38-18.gh-issue-70303._Lt_pj.rst new file mode 100644 index 000000000000000..dedda24b481241e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-28-18-38-18.gh-issue-70303._Lt_pj.rst @@ -0,0 +1,2 @@ +Return both files and directories from :meth:`pathlib.Path.glob` if a +pattern ends with "``**``". Previously only directories were returned. diff --git a/Misc/NEWS.d/next/Library/2024-01-28-19-40-40.gh-issue-114678.kYKcJw.rst b/Misc/NEWS.d/next/Library/2024-01-28-19-40-40.gh-issue-114678.kYKcJw.rst new file mode 100644 index 000000000000000..2306af4a39dcf62 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-28-19-40-40.gh-issue-114678.kYKcJw.rst @@ -0,0 +1,3 @@ +Ensure that deprecation warning for 'N' specifier in :class:`~decimal.Decimal` +format is not raised for cases where 'N' appears in other places +in the format specifier. Based on patch by Stefan Krah. diff --git a/Misc/NEWS.d/next/Library/2024-01-30-15-34-08.gh-issue-110190.Z5PQQX.rst b/Misc/NEWS.d/next/Library/2024-01-30-15-34-08.gh-issue-110190.Z5PQQX.rst new file mode 100644 index 000000000000000..af77e409963e04d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-30-15-34-08.gh-issue-110190.Z5PQQX.rst @@ -0,0 +1 @@ +Fix ctypes structs with array on Windows ARM64 platform by setting ``MAX_STRUCT_SIZE`` to 32 in stgdict. Patch by Diego Russo diff --git a/Misc/NEWS.d/next/Library/2024-01-31-20-07-11.gh-issue-109475.lmTb9S.rst b/Misc/NEWS.d/next/Library/2024-01-31-20-07-11.gh-issue-109475.lmTb9S.rst new file mode 100644 index 000000000000000..7582cb2bcd76298 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-31-20-07-11.gh-issue-109475.lmTb9S.rst @@ -0,0 +1,2 @@ +Fix support of explicit option value "--" in :mod:`argparse` (e.g. +``--option=--``). diff --git a/Misc/NEWS.d/next/Library/2024-02-01-10-19-11.gh-issue-114071.vkm2G_.rst b/Misc/NEWS.d/next/Library/2024-02-01-10-19-11.gh-issue-114071.vkm2G_.rst new file mode 100644 index 000000000000000..587ce4d21576371 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-01-10-19-11.gh-issue-114071.vkm2G_.rst @@ -0,0 +1 @@ +Support tuple subclasses using auto() for enum member value. diff --git a/Misc/NEWS.d/next/Library/2024-02-03-16-59-25.gh-issue-114959.dCfAG2.rst b/Misc/NEWS.d/next/Library/2024-02-03-16-59-25.gh-issue-114959.dCfAG2.rst new file mode 100644 index 000000000000000..5c6eaa7525e3b0c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-03-16-59-25.gh-issue-114959.dCfAG2.rst @@ -0,0 +1,2 @@ +:mod:`tarfile` no longer ignores errors when trying to extract a directory on +top of a file. diff --git a/Misc/NEWS.d/next/Library/2024-02-03-17-54-17.gh-issue-114965.gHksCK.rst b/Misc/NEWS.d/next/Library/2024-02-03-17-54-17.gh-issue-114965.gHksCK.rst new file mode 100644 index 000000000000000..d59ff991993792a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-03-17-54-17.gh-issue-114965.gHksCK.rst @@ -0,0 +1 @@ +Update bundled pip to 24.0 diff --git a/Misc/NEWS.d/next/Library/2024-02-04-13-17-33.gh-issue-114628.WJpqqS.rst b/Misc/NEWS.d/next/Library/2024-02-04-13-17-33.gh-issue-114628.WJpqqS.rst new file mode 100644 index 000000000000000..8138adc62c95f32 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-04-13-17-33.gh-issue-114628.WJpqqS.rst @@ -0,0 +1,2 @@ +When csv.Error is raised when handling TypeError, do not print the TypeError +traceback. diff --git a/Misc/NEWS.d/next/Library/2024-02-08-13-26-14.gh-issue-115059.DqP9dr.rst b/Misc/NEWS.d/next/Library/2024-02-08-13-26-14.gh-issue-115059.DqP9dr.rst new file mode 100644 index 000000000000000..331baedd3b24c5c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-08-13-26-14.gh-issue-115059.DqP9dr.rst @@ -0,0 +1 @@ +:meth:`io.BufferedRandom.read1` now flushes the underlying write buffer. diff --git a/Misc/NEWS.d/next/Library/2024-02-08-17-04-58.gh-issue-112903.SN_vUs.rst b/Misc/NEWS.d/next/Library/2024-02-08-17-04-58.gh-issue-112903.SN_vUs.rst new file mode 100644 index 000000000000000..e27f5832553c136 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-08-17-04-58.gh-issue-112903.SN_vUs.rst @@ -0,0 +1,2 @@ +Fix "issubclass() arg 1 must be a class" errors in certain cases of multiple +inheritance with generic aliases (regression in early 3.13 alpha releases). diff --git a/Misc/NEWS.d/next/Library/2024-02-09-07-20-16.gh-issue-115165.yfJLXA.rst b/Misc/NEWS.d/next/Library/2024-02-09-07-20-16.gh-issue-115165.yfJLXA.rst new file mode 100644 index 000000000000000..73d3d001f07f3f4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-09-07-20-16.gh-issue-115165.yfJLXA.rst @@ -0,0 +1,4 @@ +Most exceptions are now ignored when attempting to set the ``__orig_class__`` +attribute on objects returned when calling :mod:`typing` generic aliases +(including generic aliases created using :data:`typing.Annotated`). +Previously only :exc:`AttributeError`` was ignored. Patch by Dave Shawley. diff --git a/Misc/NEWS.d/next/Library/2024-02-10-15-24-20.gh-issue-102840.4mnDq1.rst b/Misc/NEWS.d/next/Library/2024-02-10-15-24-20.gh-issue-102840.4mnDq1.rst new file mode 100644 index 000000000000000..52668a9424a9765 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-10-15-24-20.gh-issue-102840.4mnDq1.rst @@ -0,0 +1,3 @@ +Fix confused traceback when floordiv, mod, or divmod operations happens +between instances of :class:`fractions.Fraction` and :class:`complex`. + diff --git a/Misc/NEWS.d/next/Tests/2023-06-02-05-04-15.gh-issue-105089.KaZFtU.rst b/Misc/NEWS.d/next/Tests/2023-06-02-05-04-15.gh-issue-105089.KaZFtU.rst new file mode 100644 index 000000000000000..d04ef435dd572d8 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-06-02-05-04-15.gh-issue-105089.KaZFtU.rst @@ -0,0 +1,4 @@ +Fix +``test.test_zipfile.test_core.TestWithDirectory.test_create_directory_with_write`` +test in AIX by doing a bitwise AND of 0xFFFF on mode , so that it will be in +sync with ``zinfo.external_attr`` diff --git a/Misc/NEWS.d/next/Tests/2024-02-02-13-18-55.gh-issue-114099.C_ycWg.rst b/Misc/NEWS.d/next/Tests/2024-02-02-13-18-55.gh-issue-114099.C_ycWg.rst new file mode 100644 index 000000000000000..487cd5062fc75b5 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-02-02-13-18-55.gh-issue-114099.C_ycWg.rst @@ -0,0 +1 @@ +Added test exclusions required to run the test suite on iOS. diff --git a/Misc/NEWS.d/next/Tools-Demos/2024-02-05-02-45-51.gh-issue-115015.rgtiDB.rst b/Misc/NEWS.d/next/Tools-Demos/2024-02-05-02-45-51.gh-issue-115015.rgtiDB.rst new file mode 100644 index 000000000000000..d8739d28eb2b73c --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2024-02-05-02-45-51.gh-issue-115015.rgtiDB.rst @@ -0,0 +1,5 @@ +Fix a bug in Argument Clinic that generated incorrect code for methods with +no parameters that use the :ref:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS +<METH_METHOD-METH_FASTCALL-METH_KEYWORDS>` calling convention. Only the +positional parameter count was checked; any keyword argument passed would be +silently accepted. diff --git a/Misc/NEWS.d/next/Tools-Demos/2024-02-05-19-00-32.gh-issue-109991.yJSEkw.rst b/Misc/NEWS.d/next/Tools-Demos/2024-02-05-19-00-32.gh-issue-109991.yJSEkw.rst new file mode 100644 index 000000000000000..4eb4d39629b9bcd --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2024-02-05-19-00-32.gh-issue-109991.yJSEkw.rst @@ -0,0 +1,2 @@ +Update GitHub CI workflows to use OpenSSL 3.0.13 and multissltests to use +1.1.1w, 3.0.13, 3.1.5, and 3.2.1. diff --git a/Misc/NEWS.d/next/Windows/2024-01-23-00-05-05.gh-issue-100107.lkbP_Q.rst b/Misc/NEWS.d/next/Windows/2024-01-23-00-05-05.gh-issue-100107.lkbP_Q.rst new file mode 100644 index 000000000000000..388d61a2b3bd6db --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-01-23-00-05-05.gh-issue-100107.lkbP_Q.rst @@ -0,0 +1 @@ +The ``py.exe`` launcher will no longer attempt to run the Microsoft Store redirector when launching a script containing a ``/usr/bin/env`` shebang diff --git a/Misc/NEWS.d/next/Windows/2024-02-01-14-35-05.gh-issue-111239.SO7SUF.rst b/Misc/NEWS.d/next/Windows/2024-02-01-14-35-05.gh-issue-111239.SO7SUF.rst new file mode 100644 index 000000000000000..ea82c3b941f8026 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-02-01-14-35-05.gh-issue-111239.SO7SUF.rst @@ -0,0 +1 @@ +Update Windows builds to use zlib v1.3.1. diff --git a/Misc/NEWS.d/next/Windows/2024-02-05-16-53-12.gh-issue-109991.YqjnDz.rst b/Misc/NEWS.d/next/Windows/2024-02-05-16-53-12.gh-issue-109991.YqjnDz.rst new file mode 100644 index 000000000000000..d9923c35c2726e6 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-02-05-16-53-12.gh-issue-109991.YqjnDz.rst @@ -0,0 +1 @@ +Update Windows build to use OpenSSL 3.0.13. diff --git a/Misc/NEWS.d/next/Windows/2024-02-06-09-05-13.gh-issue-115009.ShMjZs.rst b/Misc/NEWS.d/next/Windows/2024-02-06-09-05-13.gh-issue-115009.ShMjZs.rst new file mode 100644 index 000000000000000..5bdb6963a243118 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-02-06-09-05-13.gh-issue-115009.ShMjZs.rst @@ -0,0 +1 @@ +Update Windows installer to use SQLite 3.45.1. diff --git a/Misc/NEWS.d/next/macOS/2022-11-18-10-05-35.gh-issue-87804.rhlDmD.rst b/Misc/NEWS.d/next/macOS/2022-11-18-10-05-35.gh-issue-87804.rhlDmD.rst new file mode 100644 index 000000000000000..e6554d5c9f1e1ec --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2022-11-18-10-05-35.gh-issue-87804.rhlDmD.rst @@ -0,0 +1 @@ +On macOS the result of ``os.statvfs`` and ``os.fstatvfs`` now correctly report the size of very large disks, in previous versions the reported number of blocks was wrong for disks with at least 2**32 blocks. diff --git a/Misc/NEWS.d/next/macOS/2024-01-23-11-35-26.gh-issue-114490.FrQOQ0.rst b/Misc/NEWS.d/next/macOS/2024-01-23-11-35-26.gh-issue-114490.FrQOQ0.rst new file mode 100644 index 000000000000000..abd296f86085180 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2024-01-23-11-35-26.gh-issue-114490.FrQOQ0.rst @@ -0,0 +1 @@ +Add Mach-O linkage support for :func:`platform.architecture()`. diff --git a/Misc/NEWS.d/next/macOS/2024-02-05-18-30-27.gh-issue-109991.tun6Yu.rst b/Misc/NEWS.d/next/macOS/2024-02-05-18-30-27.gh-issue-109991.tun6Yu.rst new file mode 100644 index 000000000000000..79b45e7d51da3f5 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2024-02-05-18-30-27.gh-issue-109991.tun6Yu.rst @@ -0,0 +1 @@ +Update macOS installer to use OpenSSL 3.0.13. diff --git a/Misc/NEWS.d/next/macOS/2024-02-06-09-01-10.gh-issue-115009.ysau7e.rst b/Misc/NEWS.d/next/macOS/2024-02-06-09-01-10.gh-issue-115009.ysau7e.rst new file mode 100644 index 000000000000000..47ec488c3cced2d --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2024-02-06-09-01-10.gh-issue-115009.ysau7e.rst @@ -0,0 +1 @@ +Update macOS installer to use SQLite 3.45.1. diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index 5b3cd04ffa7f74b..d783d14255e66f1 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -1570,18 +1570,18 @@ "fileName": "Modules/_decimal/libmpdec/vcdiv64.asm" }, { - "SPDXID": "SPDXRef-FILE-Lib-ensurepip-bundled-pip-23.3.2-py3-none-any.whl", + "SPDXID": "SPDXRef-FILE-Lib-ensurepip-bundled-pip-24.0-py3-none-any.whl", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "8e48f55ab2965ee64bd55cc91a8077d184a33e30" + "checksumValue": "e44313ae1e6af3c2bd3b60ab2fa8c34308d00555" }, { "algorithm": "SHA256", - "checksumValue": "5052d7889c1f9d05224cd41741acb7c5d6fa735ab34e339624a614eaaa7e7d76" + "checksumValue": "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc" } ], - "fileName": "Lib/ensurepip/_bundled/pip-23.3.2-py3-none-any.whl" + "fileName": "Lib/ensurepip/_bundled/pip-24.0-py3-none-any.whl" } ], "packages": [ @@ -1601,7 +1601,7 @@ "referenceType": "cpe23Type" } ], - "licenseConcluded": "MIT", + "licenseConcluded": "NOASSERTION", "name": "expat", "originator": "Organization: Expat development team", "primaryPackagePurpose": "SOURCE", @@ -1623,7 +1623,7 @@ "referenceType": "cpe23Type" } ], - "licenseConcluded": "Apache-2.0", + "licenseConcluded": "NOASSERTION", "name": "hacl-star", "originator": "Organization: HACL* Developers", "primaryPackagePurpose": "SOURCE", @@ -1645,7 +1645,7 @@ "referenceType": "cpe23Type" } ], - "licenseConcluded": "CC0-1.0", + "licenseConcluded": "NOASSERTION", "name": "libb2", "originator": "Organization: BLAKE2 - fast secure hashing", "primaryPackagePurpose": "SOURCE", @@ -1667,7 +1667,7 @@ "referenceType": "purl" } ], - "licenseConcluded": "MIT", + "licenseConcluded": "NOASSERTION", "name": "macholib", "originator": "Person: Ronald Oussoren (ronaldoussoren@mac.com)", "primaryPackagePurpose": "SOURCE", @@ -1689,41 +1689,665 @@ "referenceType": "cpe23Type" } ], - "licenseConcluded": "BSD-2-Clause", + "licenseConcluded": "NOASSERTION", "name": "mpdecimal", "originator": "Organization: bytereef.org", "primaryPackagePurpose": "SOURCE", "versionInfo": "2.5.1" }, + { + "SPDXID": "SPDXRef-PACKAGE-cachecontrol", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "95dedbec849f46dda3137866dc28b9d133fc9af55f5b805ab1291833e4457aa4" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/1d/e3/a22348e6226dcd585d5a4b5f0175b3a16dabfd3912cbeb02f321d00e56c7/cachecontrol-0.13.1-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/cachecontrol@0.13.1", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "cachecontrol", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "0.13.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-colorama", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/colorama@0.4.6", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "colorama", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "0.4.6" + }, + { + "SPDXID": "SPDXRef-PACKAGE-distlib", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/8e/41/9307e4f5f9976bc8b7fea0b66367734e8faf3ec84bc0d412d8cfabbb66cd/distlib-0.3.8-py2.py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/distlib@0.3.8", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "distlib", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "0.3.8" + }, + { + "SPDXID": "SPDXRef-PACKAGE-distro", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "99522ca3e365cac527b44bde033f64c6945d90eb9f769703caaec52b09bbd3ff" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/f4/2c/c90a3adaf0ddb70afe193f5ebfb539612af57cffe677c3126be533df3098/distro-1.8.0-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/distro@1.8.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "distro", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.8.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-msgpack", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "525228efd79bb831cf6830a732e2e80bc1b05436b086d4264814b4b2955b2fa9" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/9f/4a/36d936e54cf71e23ad276564465f6a54fb129e3d61520b76e13e0bb29167/msgpack-1.0.5-cp310-cp310-macosx_10_9_universal2.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/msgpack@1.0.5", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "msgpack", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.0.5" + }, + { + "SPDXID": "SPDXRef-PACKAGE-packaging", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/05/8e/8de486cbd03baba4deef4142bd643a3e7bbe954a784dc1bb17142572d127/packaging-21.3-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/packaging@21.3", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "packaging", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "21.3" + }, + { + "SPDXID": "SPDXRef-PACKAGE-platformdirs", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "cec7b889196b9144d088e4c57d9ceef7374f6c39694ad1577a0aab50d27ea28c" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/9e/d8/563a9fc17153c588c8c2042d2f0f84a89057cdb1c30270f589c88b42d62c/platformdirs-3.8.1-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/platformdirs@3.8.1", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "platformdirs", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "3.8.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-pyparsing", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "d554a96d1a7d3ddaf7183104485bc19fd80543ad6ac5bdb6426719d766fb06c1" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/a4/24/6ae4c9c45cf99d96b06b5d99e25526c060303171fb0aea9da2bfd7dbde93/pyparsing-3.1.0-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/pyparsing@3.1.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "pyparsing", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "3.1.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-pyproject-hooks", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/d5/ea/9ae603de7fbb3df820b23a70f6aff92bf8c7770043254ad8d2dc9d6bcba4/pyproject_hooks-1.0.0-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/pyproject-hooks@1.0.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "pyproject-hooks", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.0.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-requests", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/requests@2.31.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "requests", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "2.31.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-certifi", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/4c/dd/2234eab22353ffc7d94e8d13177aaa050113286e93e7b40eae01fbf7c3d9/certifi-2023.7.22-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/certifi@2023.7.22", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "certifi", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "2023.7.22" + }, + { + "SPDXID": "SPDXRef-PACKAGE-chardet", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "362777fb014af596ad31334fde1e8c327dfdb076e1960d1694662d46a6917ab9" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/74/8f/8fc49109009e8d2169d94d72e6b1f4cd45c13d147ba7d6170fb41f22b08f/chardet-5.1.0-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/chardet@5.1.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "chardet", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "5.1.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-idna", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/fc/34/3030de6f1370931b9dbb4dad48f6ab1015ab1d32447850b9fc94e60097be/idna-3.4-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/idna@3.4", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "idna", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "3.4" + }, + { + "SPDXID": "SPDXRef-PACKAGE-rich", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "8f87bc7ee54675732fa66a05ebfe489e27264caeeff3728c945d25971b6485ec" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/fc/1e/482e5eec0b89b593e81d78f819a9412849814e22225842b598908e7ac560/rich-13.4.2-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/rich@13.4.2", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "rich", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "13.4.2" + }, + { + "SPDXID": "SPDXRef-PACKAGE-pygments", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/34/a7/37c8d68532ba71549db4212cb036dbd6161b40e463aba336770e80c72f84/Pygments-2.15.1-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/pygments@2.15.1", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "pygments", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "2.15.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-typing-extensions", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/ec/6b/63cc3df74987c36fe26157ee12e09e8f9db4de771e0f3404263117e75b95/typing_extensions-4.7.1-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/typing_extensions@4.7.1", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "typing_extensions", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "4.7.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-resolvelib", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "d2da45d1a8dfee81bdd591647783e340ef3bcb104b54c383f70d422ef5cc7dbf" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/d2/fc/e9ccf0521607bcd244aa0b3fbd574f71b65e9ce6a112c83af988bbbe2e23/resolvelib-1.0.1-py2.py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/resolvelib@1.0.1", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "resolvelib", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.0.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-setuptools", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/c7/42/be1c7bbdd83e1bfb160c94b9cafd8e25efc7400346cf7ccdbdb452c467fa/setuptools-68.0.0-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/setuptools@68.0.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "setuptools", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "68.0.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-six", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/six@1.16.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "six", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.16.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-tenacity", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "2f277afb21b851637e8f52e6a613ff08734c347dc19ade928e519d7d2d8569b0" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/e7/b0/c23bd61e1b32c9b96fbca996c87784e196a812da8d621d8d04851f6c8181/tenacity-8.2.2-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/tenacity@8.2.2", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "tenacity", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "8.2.2" + }, + { + "SPDXID": "SPDXRef-PACKAGE-tomli", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/tomli@2.0.1", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "tomli", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "2.0.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-truststore", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "e37a5642ae9fc48caa8f120b6283d77225d600d224965a672c9e8ef49ce4bb4c" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/20/56/7811d5439b6a56374f274a8672d8f18b4deadadeb3a9f0c86424b98b6f96/truststore-0.8.0-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/truststore@0.8.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "truststore", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "0.8.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-webencodings", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/f4/24/2a3e3df732393fed8b3ebf2ec078f05546de641fe1b667ee316ec1dcf3b7/webencodings-0.5.1-py2.py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/webencodings@0.5.1", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "webencodings", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "0.5.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-urllib3", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/48/fe/a5c6cc46e9fe9171d7ecf0f33ee7aae14642f8d74baa7af4d7840f9358be/urllib3-1.26.17-py2.py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/urllib3@1.26.17", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "urllib3", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.26.17" + }, { "SPDXID": "SPDXRef-PACKAGE-pip", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "5052d7889c1f9d05224cd41741acb7c5d6fa735ab34e339624a614eaaa7e7d76" + "checksumValue": "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc" } ], - "downloadLocation": "https://files.pythonhosted.org/packages/15/aa/3f4c7bcee2057a76562a5b33ecbd199be08cdb4443a02e26bd2c3cf6fc39/pip-23.3.2-py3-none-any.whl", + "downloadLocation": "https://files.pythonhosted.org/packages/8a/6a/19e9fe04fca059ccf770861c7d5721ab4c2aebc539889e97c7977528a53b/pip-24.0-py3-none-any.whl", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:pypa:pip:23.3.2:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:pypa:pip:24.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" }, { "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/pip@23.3.2", + "referenceLocator": "pkg:pypi/pip@24.0", "referenceType": "purl" } ], - "licenseConcluded": "MIT", + "licenseConcluded": "NOASSERTION", "name": "pip", "originator": "Organization: Python Packaging Authority", "primaryPackagePurpose": "SOURCE", - "versionInfo": "23.3.2" + "versionInfo": "24.0" } ], "relationships": [ + { + "relatedSpdxElement": "SPDXRef-PACKAGE-cachecontrol", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-certifi", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-chardet", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-colorama", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-distlib", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-distro", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-idna", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-msgpack", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-packaging", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-platformdirs", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-pygments", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-pyparsing", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-pyproject-hooks", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-requests", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-resolvelib", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-rich", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-setuptools", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-six", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-tenacity", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-tomli", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-truststore", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-typing-extensions", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-urllib3", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-webencodings", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, { "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-COPYING", "relationshipType": "CONTAINS", @@ -2285,7 +2909,7 @@ "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" }, { - "relatedSpdxElement": "SPDXRef-FILE-Lib-ensurepip-bundled-pip-23.3.2-py3-none-any.whl", + "relatedSpdxElement": "SPDXRef-FILE-Lib-ensurepip-bundled-pip-24.0-py3-none-any.whl", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-pip" } diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 22b25dd0ec141fd..a9875f6ffd1a56a 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -78,6 +78,10 @@ [feature_macro.Py_REF_DEBUG] doc = 'when Python is compiled in debug mode (with Py_REF_DEBUG)' windows = 'maybe' +[feature_macro.Py_TRACE_REFS] + # nb. This mode is not compatible with Stable ABI/Limited API. + doc = 'when Python is compiled with Py_TRACE_REFS' + windows = 'maybe' # Mentioned in PEP 384: @@ -2481,3 +2485,7 @@ [function._Py_SetRefcnt] added = '3.13' abi_only = true +[data.PyExc_IncompleteInputError] + added = '3.13' +[function.PyList_GetItemRef] + added = '3.13' diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index c8cd53de5e22622..ef77d34b10e47b9 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -44,8 +44,11 @@ find_module_state_by_def(PyTypeObject *type) /*[clinic input] module _collections class _tuplegetter "_tuplegetterobject *" "clinic_state()->tuplegetter_type" +class _collections.deque "dequeobject *" "clinic_state()->deque_type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=7356042a89862e0e]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=a033cc2a8476b3f1]*/ + +typedef struct dequeobject dequeobject; /* We can safely assume type to be the defining class, * since tuplegetter is not a base type */ @@ -53,6 +56,12 @@ class _tuplegetter "_tuplegetterobject *" "clinic_state()->tuplegetter_type" #include "clinic/_collectionsmodule.c.h" #undef clinic_state +/*[python input] +class dequeobject_converter(self_converter): + type = "dequeobject *" +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=b6ae4a3ff852be2f]*/ + /* collections module implementation of a deque() datatype Written and maintained by Raymond D. Hettinger <python@rcn.com> */ @@ -121,7 +130,7 @@ typedef struct BLOCK { struct BLOCK *rightlink; } block; -typedef struct { +struct dequeobject { PyObject_VAR_HEAD block *leftblock; block *rightblock; @@ -132,7 +141,7 @@ typedef struct { Py_ssize_t numfreeblocks; block *freeblocks[MAXFREEBLOCKS]; PyObject *weakreflist; -} dequeobject; +}; /* For debug builds, add error checking to track the endpoints * in the chain of links. The goal is to make sure that link @@ -219,8 +228,17 @@ deque_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *)deque; } +/*[clinic input] +_collections.deque.pop as deque_pop + + deque: dequeobject + +Remove and return the rightmost element. +[clinic start generated code]*/ + static PyObject * -deque_pop(dequeobject *deque, PyObject *unused) +deque_pop_impl(dequeobject *deque) +/*[clinic end generated code: output=2e5f7890c4251f07 input=eb6e6d020f877dec]*/ { PyObject *item; block *prevblock; @@ -254,10 +272,17 @@ deque_pop(dequeobject *deque, PyObject *unused) return item; } -PyDoc_STRVAR(pop_doc, "Remove and return the rightmost element."); +/*[clinic input] +_collections.deque.popleft as deque_popleft + + deque: dequeobject + +Remove and return the leftmost element. +[clinic start generated code]*/ static PyObject * -deque_popleft(dequeobject *deque, PyObject *unused) +deque_popleft_impl(dequeobject *deque) +/*[clinic end generated code: output=62b154897097ff68 input=acb41b9af50a9d9b]*/ { PyObject *item; block *prevblock; @@ -292,8 +317,6 @@ deque_popleft(dequeobject *deque, PyObject *unused) return item; } -PyDoc_STRVAR(popleft_doc, "Remove and return the leftmost element."); - /* The deque's size limit is d.maxlen. The limit can be zero or positive. * If there is no limit, then d.maxlen == -1. * @@ -326,7 +349,7 @@ deque_append_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) deque->rightindex++; deque->rightblock->data[deque->rightindex] = item; if (NEEDS_TRIM(deque, maxlen)) { - PyObject *olditem = deque_popleft(deque, NULL); + PyObject *olditem = deque_popleft_impl(deque); Py_DECREF(olditem); } else { deque->state++; @@ -334,16 +357,25 @@ deque_append_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) return 0; } +/*[clinic input] +_collections.deque.append as deque_append + + deque: dequeobject + item: object + / + +Add an element to the right side of the deque. +[clinic start generated code]*/ + static PyObject * deque_append(dequeobject *deque, PyObject *item) +/*[clinic end generated code: output=507b13efc4853ecc input=f112b83c380528e3]*/ { if (deque_append_internal(deque, Py_NewRef(item), deque->maxlen) < 0) return NULL; Py_RETURN_NONE; } -PyDoc_STRVAR(append_doc, "Add an element to the right side of the deque."); - static inline int deque_appendleft_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) { @@ -362,7 +394,7 @@ deque_appendleft_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) deque->leftindex--; deque->leftblock->data[deque->leftindex] = item; if (NEEDS_TRIM(deque, deque->maxlen)) { - PyObject *olditem = deque_pop(deque, NULL); + PyObject *olditem = deque_pop_impl(deque); Py_DECREF(olditem); } else { deque->state++; @@ -370,16 +402,25 @@ deque_appendleft_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) return 0; } +/*[clinic input] +_collections.deque.appendleft as deque_appendleft + + deque: dequeobject + item: object + / + +Add an element to the left side of the deque. +[clinic start generated code]*/ + static PyObject * deque_appendleft(dequeobject *deque, PyObject *item) +/*[clinic end generated code: output=de0335a64800ffd8 input=bbdaa60a3e956062]*/ { if (deque_appendleft_internal(deque, Py_NewRef(item), deque->maxlen) < 0) return NULL; Py_RETURN_NONE; } -PyDoc_STRVAR(appendleft_doc, "Add an element to the left side of the deque."); - static PyObject* finalize_iterator(PyObject *it) { @@ -410,8 +451,19 @@ consume_iterator(PyObject *it) return finalize_iterator(it); } +/*[clinic input] +_collections.deque.extend as deque_extend + + deque: dequeobject + iterable: object + / + +Extend the right side of the deque with elements from the iterable. +[clinic start generated code]*/ + static PyObject * deque_extend(dequeobject *deque, PyObject *iterable) +/*[clinic end generated code: output=a3a6e74d17063f8d input=cfebfd34d5383339]*/ { PyObject *it, *item; PyObject *(*iternext)(PyObject *); @@ -454,11 +506,19 @@ deque_extend(dequeobject *deque, PyObject *iterable) return finalize_iterator(it); } -PyDoc_STRVAR(extend_doc, -"Extend the right side of the deque with elements from the iterable"); +/*[clinic input] +_collections.deque.extendleft as deque_extendleft + + deque: dequeobject + iterable: object + / + +Extend the left side of the deque with elements from the iterable. +[clinic start generated code]*/ static PyObject * deque_extendleft(dequeobject *deque, PyObject *iterable) +/*[clinic end generated code: output=2dba946c50498c67 input=f4820e695a6f9416]*/ { PyObject *it, *item; PyObject *(*iternext)(PyObject *); @@ -501,9 +561,6 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) return finalize_iterator(it); } -PyDoc_STRVAR(extendleft_doc, -"Extend the left side of the deque with elements from the iterable"); - static PyObject * deque_inplace_concat(dequeobject *deque, PyObject *other) { @@ -517,8 +574,17 @@ deque_inplace_concat(dequeobject *deque, PyObject *other) return (PyObject *)deque; } +/*[clinic input] +_collections.deque.copy as deque_copy + + deque: dequeobject + +Return a shallow copy of a deque. +[clinic start generated code]*/ + static PyObject * -deque_copy(PyObject *deque, PyObject *Py_UNUSED(ignored)) +deque_copy_impl(dequeobject *deque) +/*[clinic end generated code: output=6409b3d1ad2898b5 input=0e22f138bc1fcbee]*/ { PyObject *result; dequeobject *old_deque = (dequeobject *)deque; @@ -537,7 +603,7 @@ deque_copy(PyObject *deque, PyObject *Py_UNUSED(ignored)) PyObject *item = old_deque->leftblock->data[old_deque->leftindex]; rv = deque_append(new_deque, item); } else { - rv = deque_extend(new_deque, deque); + rv = deque_extend(new_deque, (PyObject *)deque); } if (rv != NULL) { Py_DECREF(rv); @@ -547,7 +613,8 @@ deque_copy(PyObject *deque, PyObject *Py_UNUSED(ignored)) return NULL; } if (old_deque->maxlen < 0) - result = PyObject_CallOneArg((PyObject *)(Py_TYPE(deque)), deque); + result = PyObject_CallOneArg((PyObject *)(Py_TYPE(deque)), + (PyObject *)deque); else result = PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "Oi", deque, old_deque->maxlen, NULL); @@ -561,7 +628,18 @@ deque_copy(PyObject *deque, PyObject *Py_UNUSED(ignored)) return result; } -PyDoc_STRVAR(copy_doc, "Return a shallow copy of a deque."); +/*[clinic input] +_collections.deque.__copy__ as deque___copy__ = _collections.deque.copy + +Return a shallow copy of a deque. +[clinic start generated code]*/ + +static PyObject * +deque___copy___impl(dequeobject *deque) +/*[clinic end generated code: output=7c5821504342bf23 input=fce05df783e7912b]*/ +{ + return deque_copy_impl(deque); +} static PyObject * deque_concat(dequeobject *deque, PyObject *other) @@ -580,7 +658,7 @@ deque_concat(dequeobject *deque, PyObject *other) return NULL; } - new_deque = deque_copy((PyObject *)deque, NULL); + new_deque = deque_copy_impl(deque); if (new_deque == NULL) return NULL; result = deque_extend((dequeobject *)new_deque, other); @@ -669,22 +747,29 @@ deque_clear(dequeobject *deque) alternate_method: while (Py_SIZE(deque)) { - item = deque_pop(deque, NULL); + item = deque_pop_impl(deque); assert (item != NULL); Py_DECREF(item); } return 0; } +/*[clinic input] +_collections.deque.clear as deque_clearmethod + + deque: dequeobject + +Remove all elements from the deque. +[clinic start generated code]*/ + static PyObject * -deque_clearmethod(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +deque_clearmethod_impl(dequeobject *deque) +/*[clinic end generated code: output=79b2513e097615c1 input=20488eb932f89f9e]*/ { deque_clear(deque); Py_RETURN_NONE; } -PyDoc_STRVAR(clear_doc, "Remove all elements from the deque."); - static PyObject * deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) { @@ -768,7 +853,7 @@ deque_repeat(dequeobject *deque, Py_ssize_t n) dequeobject *new_deque; PyObject *rv; - new_deque = (dequeobject *)deque_copy((PyObject *) deque, NULL); + new_deque = (dequeobject *)deque_copy_impl(deque); if (new_deque == NULL) return NULL; rv = deque_inplace_repeat(new_deque, n); @@ -925,36 +1010,36 @@ _deque_rotate(dequeobject *deque, Py_ssize_t n) return rv; } -static PyObject * -deque_rotate(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) -{ - Py_ssize_t n=1; +/*[clinic input] +_collections.deque.rotate as deque_rotate - if (!_PyArg_CheckPositional("deque.rotate", nargs, 0, 1)) { - return NULL; - } - if (nargs) { - PyObject *index = _PyNumber_Index(args[0]); - if (index == NULL) { - return NULL; - } - n = PyLong_AsSsize_t(index); - Py_DECREF(index); - if (n == -1 && PyErr_Occurred()) { - return NULL; - } - } + deque: dequeobject + n: Py_ssize_t = 1 + / +Rotate the deque n steps to the right. If n is negative, rotates left. +[clinic start generated code]*/ + +static PyObject * +deque_rotate_impl(dequeobject *deque, Py_ssize_t n) +/*[clinic end generated code: output=96c2402a371eb15d input=d22070f49cc06c76]*/ +{ if (!_deque_rotate(deque, n)) Py_RETURN_NONE; return NULL; } -PyDoc_STRVAR(rotate_doc, -"Rotate the deque n steps to the right (default n=1). If n is negative, rotates left."); +/*[clinic input] +_collections.deque.reverse as deque_reverse + + deque: dequeobject + +Reverse *IN PLACE*. +[clinic start generated code]*/ static PyObject * -deque_reverse(dequeobject *deque, PyObject *unused) +deque_reverse_impl(dequeobject *deque) +/*[clinic end generated code: output=bdeebc2cf8c1f064 input=f139787f406101c9]*/ { block *leftblock = deque->leftblock; block *rightblock = deque->rightblock; @@ -991,11 +1076,19 @@ deque_reverse(dequeobject *deque, PyObject *unused) Py_RETURN_NONE; } -PyDoc_STRVAR(reverse_doc, -"D.reverse() -- reverse *IN PLACE*"); +/*[clinic input] +_collections.deque.count as deque_count + + deque: dequeobject + value as v: object + / + +Return number of occurrences of value. +[clinic start generated code]*/ static PyObject * deque_count(dequeobject *deque, PyObject *v) +/*[clinic end generated code: output=7405d289d94d7b9b input=1892925260ff5d78]*/ { block *b = deque->leftblock; Py_ssize_t index = deque->leftindex; @@ -1030,9 +1123,6 @@ deque_count(dequeobject *deque, PyObject *v) return PyLong_FromSsize_t(count); } -PyDoc_STRVAR(count_doc, -"D.count(value) -- return number of occurrences of value"); - static int deque_contains(dequeobject *deque, PyObject *v) { @@ -1071,22 +1161,33 @@ deque_len(dequeobject *deque) return Py_SIZE(deque); } +/*[clinic input] +@text_signature "($self, value, [start, [stop]])" +_collections.deque.index as deque_index + + deque: dequeobject + value as v: object + start: object(converter='_PyEval_SliceIndexNotNone', type='Py_ssize_t', c_default='0') = NULL + stop: object(converter='_PyEval_SliceIndexNotNone', type='Py_ssize_t', c_default='Py_SIZE(deque)') = NULL + / + +Return first index of value. + +Raises ValueError if the value is not present. +[clinic start generated code]*/ + static PyObject * -deque_index(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +deque_index_impl(dequeobject *deque, PyObject *v, Py_ssize_t start, + Py_ssize_t stop) +/*[clinic end generated code: output=df45132753175ef9 input=140210c099830f64]*/ { - Py_ssize_t i, n, start=0, stop=Py_SIZE(deque); - PyObject *v, *item; + Py_ssize_t i, n; + PyObject *item; block *b = deque->leftblock; Py_ssize_t index = deque->leftindex; size_t start_state = deque->state; int cmp; - if (!_PyArg_ParseStack(args, nargs, "O|O&O&:index", &v, - _PyEval_SliceIndexNotNone, &start, - _PyEval_SliceIndexNotNone, &stop)) { - return NULL; - } - if (start < 0) { start += Py_SIZE(deque); if (start < 0) @@ -1138,10 +1239,6 @@ deque_index(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) return NULL; } -PyDoc_STRVAR(index_doc, -"D.index(value, [start, [stop]]) -- return first index of value.\n" -"Raises ValueError if the value is not present."); - /* insert(), remove(), and delitem() are implemented in terms of rotate() for simplicity and reasonable performance near the end points. If for some reason these methods become popular, it is not @@ -1150,18 +1247,24 @@ PyDoc_STRVAR(index_doc, boost (by moving each pointer only once instead of twice). */ +/*[clinic input] +_collections.deque.insert as deque_insert + + deque: dequeobject + index: Py_ssize_t + value: object + / + +Insert value before index. +[clinic start generated code]*/ + static PyObject * -deque_insert(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +deque_insert_impl(dequeobject *deque, Py_ssize_t index, PyObject *value) +/*[clinic end generated code: output=ef4d2c15d5532b80 input=3e5c1c120d70c0e6]*/ { - Py_ssize_t index; Py_ssize_t n = Py_SIZE(deque); - PyObject *value; PyObject *rv; - if (!_PyArg_ParseStack(args, nargs, "nO:insert", &index, &value)) { - return NULL; - } - if (deque->maxlen == Py_SIZE(deque)) { PyErr_SetString(PyExc_IndexError, "deque already at its maximum size"); return NULL; @@ -1184,12 +1287,6 @@ deque_insert(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) Py_RETURN_NONE; } -PyDoc_STRVAR(insert_doc, -"D.insert(index, object) -- insert object before index"); - -PyDoc_STRVAR(remove_doc, -"D.remove(value) -- remove first occurrence of value."); - static int valid_index(Py_ssize_t i, Py_ssize_t limit) { @@ -1246,15 +1343,26 @@ deque_del_item(dequeobject *deque, Py_ssize_t i) assert (i >= 0 && i < Py_SIZE(deque)); if (_deque_rotate(deque, -i)) return -1; - item = deque_popleft(deque, NULL); + item = deque_popleft_impl(deque); rv = _deque_rotate(deque, i); assert (item != NULL); Py_DECREF(item); return rv; } +/*[clinic input] +_collections.deque.remove as deque_remove + + deque: dequeobject + value: object + / + +Remove first occurrence of value. +[clinic start generated code]*/ + static PyObject * deque_remove(dequeobject *deque, PyObject *value) +/*[clinic end generated code: output=49e1666d612fe911 input=d972f32d15990880]*/ { PyObject *item; block *b = deque->leftblock; @@ -1375,8 +1483,17 @@ deque_traverse(dequeobject *deque, visitproc visit, void *arg) return 0; } +/*[clinic input] +_collections.deque.__reduce__ as deque___reduce__ + + deque: dequeobject + +Return state information for pickling. +[clinic start generated code]*/ + static PyObject * -deque_reduce(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +deque___reduce___impl(dequeobject *deque) +/*[clinic end generated code: output=cb85d9e0b7d2c5ad input=991a933a5bc7a526]*/ { PyObject *state, *it; @@ -1510,26 +1627,23 @@ deque_richcompare(PyObject *v, PyObject *w, int op) return NULL; } +/*[clinic input] +@text_signature "([iterable[, maxlen]])" +_collections.deque.__init__ as deque_init + + deque: dequeobject + iterable: object = NULL + maxlen as maxlenobj: object = NULL + +A list-like sequence optimized for data accesses near its endpoints. +[clinic start generated code]*/ + static int -deque_init(dequeobject *deque, PyObject *args, PyObject *kwdargs) +deque_init_impl(dequeobject *deque, PyObject *iterable, PyObject *maxlenobj) +/*[clinic end generated code: output=7084a39d71218dcd input=5ebdffc48a2d27ae]*/ + { - PyObject *iterable = NULL; - PyObject *maxlenobj = NULL; Py_ssize_t maxlen = -1; - char *kwlist[] = {"iterable", "maxlen", 0}; - - if (kwdargs == NULL && PyTuple_GET_SIZE(args) <= 2) { - if (PyTuple_GET_SIZE(args) > 0) { - iterable = PyTuple_GET_ITEM(args, 0); - } - if (PyTuple_GET_SIZE(args) > 1) { - maxlenobj = PyTuple_GET_ITEM(args, 1); - } - } else { - if (!PyArg_ParseTupleAndKeywords(args, kwdargs, "|OO:deque", kwlist, - &iterable, &maxlenobj)) - return -1; - } if (maxlenobj != NULL && maxlenobj != Py_None) { maxlen = PyLong_AsSsize_t(maxlenobj); if (maxlen == -1 && PyErr_Occurred()) @@ -1551,8 +1665,17 @@ deque_init(dequeobject *deque, PyObject *args, PyObject *kwdargs) return 0; } +/*[clinic input] +_collections.deque.__sizeof__ as deque___sizeof__ + + deque: dequeobject + +Return the size of the deque in memory, in bytes. +[clinic start generated code]*/ + static PyObject * -deque_sizeof(dequeobject *deque, void *unused) +deque___sizeof___impl(dequeobject *deque) +/*[clinic end generated code: output=4d36e9fb4f30bbaf input=4e7c9a00c03c3290]*/ { size_t res = _PyObject_SIZE(Py_TYPE(deque)); size_t blocks; @@ -1563,9 +1686,6 @@ deque_sizeof(dequeobject *deque, void *unused) return PyLong_FromSize_t(res); } -PyDoc_STRVAR(sizeof_doc, -"D.__sizeof__() -- size of D in memory, in bytes"); - static PyObject * deque_get_maxlen(dequeobject *deque, void *Py_UNUSED(ignored)) { @@ -1574,6 +1694,22 @@ deque_get_maxlen(dequeobject *deque, void *Py_UNUSED(ignored)) return PyLong_FromSsize_t(deque->maxlen); } +static PyObject *deque_reviter(dequeobject *deque); + +/*[clinic input] +_collections.deque.__reversed__ as deque___reversed__ + + deque: dequeobject + +Return a reverse iterator over the deque. +[clinic start generated code]*/ + +static PyObject * +deque___reversed___impl(dequeobject *deque) +/*[clinic end generated code: output=3e7e7e715883cf2e input=3d494c25a6fe5c7e]*/ +{ + return deque_reviter(deque); +} /* deque object ********************************************************/ @@ -1584,47 +1720,26 @@ static PyGetSetDef deque_getset[] = { }; static PyObject *deque_iter(dequeobject *deque); -static PyObject *deque_reviter(dequeobject *deque, PyObject *Py_UNUSED(ignored)); -PyDoc_STRVAR(reversed_doc, - "D.__reversed__() -- return a reverse iterator over the deque"); static PyMethodDef deque_methods[] = { - {"append", (PyCFunction)deque_append, - METH_O, append_doc}, - {"appendleft", (PyCFunction)deque_appendleft, - METH_O, appendleft_doc}, - {"clear", (PyCFunction)deque_clearmethod, - METH_NOARGS, clear_doc}, - {"__copy__", deque_copy, - METH_NOARGS, copy_doc}, - {"copy", deque_copy, - METH_NOARGS, copy_doc}, - {"count", (PyCFunction)deque_count, - METH_O, count_doc}, - {"extend", (PyCFunction)deque_extend, - METH_O, extend_doc}, - {"extendleft", (PyCFunction)deque_extendleft, - METH_O, extendleft_doc}, - {"index", _PyCFunction_CAST(deque_index), - METH_FASTCALL, index_doc}, - {"insert", _PyCFunction_CAST(deque_insert), - METH_FASTCALL, insert_doc}, - {"pop", (PyCFunction)deque_pop, - METH_NOARGS, pop_doc}, - {"popleft", (PyCFunction)deque_popleft, - METH_NOARGS, popleft_doc}, - {"__reduce__", (PyCFunction)deque_reduce, - METH_NOARGS, reduce_doc}, - {"remove", (PyCFunction)deque_remove, - METH_O, remove_doc}, - {"__reversed__", (PyCFunction)deque_reviter, - METH_NOARGS, reversed_doc}, - {"reverse", (PyCFunction)deque_reverse, - METH_NOARGS, reverse_doc}, - {"rotate", _PyCFunction_CAST(deque_rotate), - METH_FASTCALL, rotate_doc}, - {"__sizeof__", (PyCFunction)deque_sizeof, - METH_NOARGS, sizeof_doc}, + DEQUE_APPEND_METHODDEF + DEQUE_APPENDLEFT_METHODDEF + DEQUE_CLEARMETHOD_METHODDEF + DEQUE___COPY___METHODDEF + DEQUE_COPY_METHODDEF + DEQUE_COUNT_METHODDEF + DEQUE_EXTEND_METHODDEF + DEQUE_EXTENDLEFT_METHODDEF + DEQUE_INDEX_METHODDEF + DEQUE_INSERT_METHODDEF + DEQUE_POP_METHODDEF + DEQUE_POPLEFT_METHODDEF + DEQUE___REDUCE___METHODDEF + DEQUE_REMOVE_METHODDEF + DEQUE___REVERSED___METHODDEF + DEQUE_REVERSE_METHODDEF + DEQUE_ROTATE_METHODDEF + DEQUE___SIZEOF___METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ @@ -1635,17 +1750,12 @@ static PyMemberDef deque_members[] = { {NULL}, }; -PyDoc_STRVAR(deque_doc, -"deque([iterable[, maxlen]]) --> deque object\n\ -\n\ -A list-like sequence optimized for data accesses near its endpoints."); - static PyType_Slot deque_slots[] = { {Py_tp_dealloc, deque_dealloc}, {Py_tp_repr, deque_repr}, {Py_tp_hash, PyObject_HashNotImplemented}, {Py_tp_getattro, PyObject_GenericGetAttr}, - {Py_tp_doc, (void *)deque_doc}, + {Py_tp_doc, (void *)deque_init__doc__}, {Py_tp_traverse, deque_traverse}, {Py_tp_clear, deque_clear}, {Py_tp_richcompare, deque_richcompare}, @@ -1834,7 +1944,7 @@ static PyType_Spec dequeiter_spec = { /*********************** Deque Reverse Iterator **************************/ static PyObject * -deque_reviter(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +deque_reviter(dequeobject *deque) { dequeiterobject *it; collections_state *state = find_module_state_by_def(Py_TYPE(deque)); @@ -1889,7 +1999,7 @@ dequereviter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; assert(type == state->dequereviter_type); - it = (dequeiterobject*)deque_reviter((dequeobject *)deque, NULL); + it = (dequeiterobject *)deque_reviter((dequeobject *)deque); if (!it) return NULL; /* consume items from the queue */ diff --git a/Modules/_csv.c b/Modules/_csv.c index 929c21584ac2ef0..3aa648b8e9cec44 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -131,7 +131,7 @@ typedef struct { Py_UCS4 *field; /* temporary buffer */ Py_ssize_t field_size; /* size of allocated buffer */ Py_ssize_t field_len; /* length of current field */ - int numeric_field; /* treat field as numeric */ + bool unquoted_field; /* true if no quotes around the current field */ unsigned long line_num; /* Source-file line number */ } ReaderObj; @@ -644,22 +644,33 @@ _call_dialect(_csvstate *module_state, PyObject *dialect_inst, PyObject *kwargs) static int parse_save_field(ReaderObj *self) { + int quoting = self->dialect->quoting; PyObject *field; - field = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - (void *) self->field, self->field_len); - if (field == NULL) - return -1; - self->field_len = 0; - if (self->numeric_field) { - PyObject *tmp; - - self->numeric_field = 0; - tmp = PyNumber_Float(field); - Py_DECREF(field); - if (tmp == NULL) + if (self->unquoted_field && + self->field_len == 0 && + (quoting == QUOTE_NOTNULL || quoting == QUOTE_STRINGS)) + { + field = Py_NewRef(Py_None); + } + else { + field = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + (void *) self->field, self->field_len); + if (field == NULL) { return -1; - field = tmp; + } + if (self->unquoted_field && + self->field_len != 0 && + (quoting == QUOTE_NONNUMERIC || quoting == QUOTE_STRINGS)) + { + PyObject *tmp = PyNumber_Float(field); + Py_DECREF(field); + if (tmp == NULL) { + return -1; + } + field = tmp; + } + self->field_len = 0; } if (PyList_Append(self->fields, field) < 0) { Py_DECREF(field); @@ -721,6 +732,7 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) /* fallthru */ case START_FIELD: /* expecting field */ + self->unquoted_field = true; if (c == '\n' || c == '\r' || c == EOL) { /* save empty field - return [fields] */ if (parse_save_field(self) < 0) @@ -730,10 +742,12 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) else if (c == dialect->quotechar && dialect->quoting != QUOTE_NONE) { /* start quoted field */ + self->unquoted_field = false; self->state = IN_QUOTED_FIELD; } else if (c == dialect->escapechar) { /* possible escaped character */ + self->unquoted_field = false; self->state = ESCAPED_CHAR; } else if (c == ' ' && dialect->skipinitialspace) @@ -746,8 +760,6 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) } else { /* begin new unquoted field */ - if (dialect->quoting == QUOTE_NONNUMERIC) - self->numeric_field = 1; if (parse_add_char(self, module_state, c) < 0) return -1; self->state = IN_FIELD; @@ -892,7 +904,7 @@ parse_reset(ReaderObj *self) return -1; self->field_len = 0; self->state = START_RECORD; - self->numeric_field = 0; + self->unquoted_field = false; return 0; } diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 2397015ba658895..deafa696fdd0d03 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -707,7 +707,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct /* * The value of MAX_STRUCT_SIZE depends on the platform Python is running on. */ -#if defined(__aarch64__) || defined(__arm__) +#if defined(__aarch64__) || defined(__arm__) || defined(_M_ARM64) # define MAX_STRUCT_SIZE 32 #elif defined(__powerpc64__) # define MAX_STRUCT_SIZE 64 diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index cb5403e8461ff0e..9b8e0a719d9048c 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -61,16 +61,6 @@ static datetime_state _datetime_global_state; #define STATIC_STATE() (&_datetime_global_state) -/*[clinic input] -module datetime -class datetime.datetime "PyDateTime_DateTime *" "&PyDateTime_DateTimeType" -class datetime.date "PyDateTime_Date *" "&PyDateTime_DateType" -class datetime.IsoCalendarDate "PyDateTime_IsoCalendarDate *" "&PyDateTime_IsoCalendarDateType" -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=81bec0fa19837f63]*/ - -#include "clinic/_datetimemodule.c.h" - /* We require that C int be at least 32 bits, and use int virtually * everywhere. In just a few cases we use a temp long, where a Python * API returns a C long. In such cases, we have to ensure that the @@ -161,6 +151,17 @@ static PyTypeObject PyDateTime_TimeZoneType; static int check_tzinfo_subclass(PyObject *p); +/*[clinic input] +module datetime +class datetime.datetime "PyDateTime_DateTime *" "&PyDateTime_DateTimeType" +class datetime.date "PyDateTime_Date *" "&PyDateTime_DateType" +class datetime.time "PyDateTime_Time *" "&PyDateTime_TimeType" +class datetime.IsoCalendarDate "PyDateTime_IsoCalendarDate *" "&PyDateTime_IsoCalendarDateType" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6f65a48dd22fa40f]*/ + +#include "clinic/_datetimemodule.c.h" + /* --------------------------------------------------------------------------- * Math utilities. @@ -3466,24 +3467,22 @@ date_timetuple(PyDateTime_Date *self, PyObject *Py_UNUSED(ignored)) 0, 0, 0, -1); } +/*[clinic input] +datetime.date.replace + + year: int(c_default="GET_YEAR(self)") = unchanged + month: int(c_default="GET_MONTH(self)") = unchanged + day: int(c_default="GET_DAY(self)") = unchanged + +Return date with new specified fields. +[clinic start generated code]*/ + static PyObject * -date_replace(PyDateTime_Date *self, PyObject *args, PyObject *kw) +datetime_date_replace_impl(PyDateTime_Date *self, int year, int month, + int day) +/*[clinic end generated code: output=2a9430d1e6318aeb input=0d1f02685b3e90f6]*/ { - PyObject *clone; - PyObject *tuple; - int year = GET_YEAR(self); - int month = GET_MONTH(self); - int day = GET_DAY(self); - - if (! PyArg_ParseTupleAndKeywords(args, kw, "|iii:replace", date_kws, - &year, &month, &day)) - return NULL; - tuple = Py_BuildValue("iii", year, month, day); - if (tuple == NULL) - return NULL; - clone = date_new(Py_TYPE(self), tuple, NULL); - Py_DECREF(tuple); - return clone; + return new_date_ex(year, month, day, Py_TYPE(self)); } static Py_hash_t @@ -3596,10 +3595,9 @@ static PyMethodDef date_methods[] = { PyDoc_STR("Return the day of the week represented by the date.\n" "Monday == 0 ... Sunday == 6")}, - {"replace", _PyCFunction_CAST(date_replace), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("Return date with new specified fields.")}, + DATETIME_DATE_REPLACE_METHODDEF - {"__replace__", _PyCFunction_CAST(date_replace), METH_VARARGS | METH_KEYWORDS}, + {"__replace__", _PyCFunction_CAST(datetime_date_replace), METH_FASTCALL | METH_KEYWORDS}, {"__reduce__", (PyCFunction)date_reduce, METH_NOARGS, PyDoc_STR("__reduce__() -> (cls, state)")}, @@ -4573,36 +4571,28 @@ time_hash(PyDateTime_Time *self) return self->hashcode; } +/*[clinic input] +datetime.time.replace + + hour: int(c_default="TIME_GET_HOUR(self)") = unchanged + minute: int(c_default="TIME_GET_MINUTE(self)") = unchanged + second: int(c_default="TIME_GET_SECOND(self)") = unchanged + microsecond: int(c_default="TIME_GET_MICROSECOND(self)") = unchanged + tzinfo: object(c_default="HASTZINFO(self) ? self->tzinfo : Py_None") = unchanged + * + fold: int(c_default="TIME_GET_FOLD(self)") = unchanged + +Return time with new specified fields. +[clinic start generated code]*/ + static PyObject * -time_replace(PyDateTime_Time *self, PyObject *args, PyObject *kw) +datetime_time_replace_impl(PyDateTime_Time *self, int hour, int minute, + int second, int microsecond, PyObject *tzinfo, + int fold) +/*[clinic end generated code: output=0b89a44c299e4f80 input=9b6a35b1e704b0ca]*/ { - PyObject *clone; - PyObject *tuple; - int hh = TIME_GET_HOUR(self); - int mm = TIME_GET_MINUTE(self); - int ss = TIME_GET_SECOND(self); - int us = TIME_GET_MICROSECOND(self); - PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None; - int fold = TIME_GET_FOLD(self); - - if (! PyArg_ParseTupleAndKeywords(args, kw, "|iiiiO$i:replace", - time_kws, - &hh, &mm, &ss, &us, &tzinfo, &fold)) - return NULL; - if (fold != 0 && fold != 1) { - PyErr_SetString(PyExc_ValueError, - "fold must be either 0 or 1"); - return NULL; - } - tuple = Py_BuildValue("iiiiO", hh, mm, ss, us, tzinfo); - if (tuple == NULL) - return NULL; - clone = time_new(Py_TYPE(self), tuple, NULL); - if (clone != NULL) { - TIME_SET_FOLD(clone, fold); - } - Py_DECREF(tuple); - return clone; + return new_time_ex2(hour, minute, second, microsecond, tzinfo, fold, + Py_TYPE(self)); } static PyObject * @@ -4732,10 +4722,9 @@ static PyMethodDef time_methods[] = { {"dst", (PyCFunction)time_dst, METH_NOARGS, PyDoc_STR("Return self.tzinfo.dst(self).")}, - {"replace", _PyCFunction_CAST(time_replace), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("Return time with new specified fields.")}, + DATETIME_TIME_REPLACE_METHODDEF - {"__replace__", _PyCFunction_CAST(time_replace), METH_VARARGS | METH_KEYWORDS}, + {"__replace__", _PyCFunction_CAST(datetime_time_replace), METH_FASTCALL | METH_KEYWORDS}, {"fromisoformat", (PyCFunction)time_fromisoformat, METH_O | METH_CLASS, PyDoc_STR("string -> time from a string in ISO 8601 format")}, @@ -6042,40 +6031,32 @@ datetime_hash(PyDateTime_DateTime *self) return self->hashcode; } +/*[clinic input] +datetime.datetime.replace + + year: int(c_default="GET_YEAR(self)") = unchanged + month: int(c_default="GET_MONTH(self)") = unchanged + day: int(c_default="GET_DAY(self)") = unchanged + hour: int(c_default="DATE_GET_HOUR(self)") = unchanged + minute: int(c_default="DATE_GET_MINUTE(self)") = unchanged + second: int(c_default="DATE_GET_SECOND(self)") = unchanged + microsecond: int(c_default="DATE_GET_MICROSECOND(self)") = unchanged + tzinfo: object(c_default="HASTZINFO(self) ? self->tzinfo : Py_None") = unchanged + * + fold: int(c_default="DATE_GET_FOLD(self)") = unchanged + +Return datetime with new specified fields. +[clinic start generated code]*/ + static PyObject * -datetime_replace(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) +datetime_datetime_replace_impl(PyDateTime_DateTime *self, int year, + int month, int day, int hour, int minute, + int second, int microsecond, PyObject *tzinfo, + int fold) +/*[clinic end generated code: output=00bc96536833fddb input=9b38253d56d9bcad]*/ { - PyObject *clone; - PyObject *tuple; - int y = GET_YEAR(self); - int m = GET_MONTH(self); - int d = GET_DAY(self); - int hh = DATE_GET_HOUR(self); - int mm = DATE_GET_MINUTE(self); - int ss = DATE_GET_SECOND(self); - int us = DATE_GET_MICROSECOND(self); - PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None; - int fold = DATE_GET_FOLD(self); - - if (! PyArg_ParseTupleAndKeywords(args, kw, "|iiiiiiiO$i:replace", - datetime_kws, - &y, &m, &d, &hh, &mm, &ss, &us, - &tzinfo, &fold)) - return NULL; - if (fold != 0 && fold != 1) { - PyErr_SetString(PyExc_ValueError, - "fold must be either 0 or 1"); - return NULL; - } - tuple = Py_BuildValue("iiiiiiiO", y, m, d, hh, mm, ss, us, tzinfo); - if (tuple == NULL) - return NULL; - clone = datetime_new(Py_TYPE(self), tuple, NULL); - if (clone != NULL) { - DATE_SET_FOLD(clone, fold); - } - Py_DECREF(tuple); - return clone; + return new_datetime_ex2(year, month, day, hour, minute, second, + microsecond, tzinfo, fold, Py_TYPE(self)); } static PyObject * @@ -6597,10 +6578,9 @@ static PyMethodDef datetime_methods[] = { {"dst", (PyCFunction)datetime_dst, METH_NOARGS, PyDoc_STR("Return self.tzinfo.dst(self).")}, - {"replace", _PyCFunction_CAST(datetime_replace), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("Return datetime with new specified fields.")}, + DATETIME_DATETIME_REPLACE_METHODDEF - {"__replace__", _PyCFunction_CAST(datetime_replace), METH_VARARGS | METH_KEYWORDS}, + {"__replace__", _PyCFunction_CAST(datetime_datetime_replace), METH_FASTCALL | METH_KEYWORDS}, {"astimezone", _PyCFunction_CAST(datetime_astimezone), METH_VARARGS | METH_KEYWORDS, PyDoc_STR("tz -> convert to local time in new timezone tz\n")}, diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 8b93f8e2cbcf0bb..127f5f2887d4cd1 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -3446,6 +3446,14 @@ dec_format(PyObject *dec, PyObject *args) if (fmt == NULL) { return NULL; } + + if (size > 0 && fmt[size-1] == 'N') { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Format specifier 'N' is deprecated", 1) < 0) { + return NULL; + } + } + /* NOTE: If https://github.com/python/cpython/pull/29438 lands, the * format string manipulation below can be eliminated by enhancing * the forked mpd_parse_fmt_str(). */ @@ -3593,12 +3601,6 @@ dec_format(PyObject *dec, PyObject *args) if (replace_fillchar) { dec_replace_fillchar(decstring); } - if (strchr(fmt, 'N') != NULL) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Format specifier 'N' is deprecated", 1) < 0) { - goto finish; - } - } result = PyUnicode_DecodeUTF8(decstring, size, NULL); diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index b574c96d3f96255..544510812116541 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -267,7 +267,7 @@ typedef struct { LOCAL(int) create_extra(ElementObject* self, PyObject* attrib) { - self->extra = PyObject_Malloc(sizeof(ElementObjectExtra)); + self->extra = PyMem_Malloc(sizeof(ElementObjectExtra)); if (!self->extra) { PyErr_NoMemory(); return -1; @@ -295,10 +295,11 @@ dealloc_extra(ElementObjectExtra *extra) for (i = 0; i < extra->length; i++) Py_DECREF(extra->children[i]); - if (extra->children != extra->_children) - PyObject_Free(extra->children); + if (extra->children != extra->_children) { + PyMem_Free(extra->children); + } - PyObject_Free(extra); + PyMem_Free(extra); } LOCAL(void) @@ -495,14 +496,16 @@ element_resize(ElementObject* self, Py_ssize_t extra) * "children", which needs at least 4 bytes. Although it's a * false alarm always assume at least one child to be safe. */ - children = PyObject_Realloc(self->extra->children, - size * sizeof(PyObject*)); - if (!children) + children = PyMem_Realloc(self->extra->children, + size * sizeof(PyObject*)); + if (!children) { goto nomemory; + } } else { - children = PyObject_Malloc(size * sizeof(PyObject*)); - if (!children) + children = PyMem_Malloc(size * sizeof(PyObject*)); + if (!children) { goto nomemory; + } /* copy existing children from static area to malloc buffer */ memcpy(children, self->extra->children, self->extra->length * sizeof(PyObject*)); @@ -3044,7 +3047,7 @@ _elementtree_TreeBuilder_start_impl(TreeBuilderObject *self, PyObject *tag, #define EXPAT(st, func) ((st)->expat_capi->func) static XML_Memory_Handling_Suite ExpatMemoryHandler = { - PyObject_Malloc, PyObject_Realloc, PyObject_Free}; + PyMem_Malloc, PyMem_Realloc, PyMem_Free}; typedef struct { PyObject_HEAD diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index f02207ace9f3d26..8ebe9ec7095586f 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1050,6 +1050,16 @@ _io__Buffered_read1_impl(buffered *self, Py_ssize_t n) Py_DECREF(res); return NULL; } + /* Flush the write buffer if necessary */ + if (self->writable) { + PyObject *r = buffered_flush_and_rewind_unlocked(self); + if (r == NULL) { + LEAVE_BUFFERED(self) + Py_DECREF(res); + return NULL; + } + Py_DECREF(r); + } _bufferedreader_reset_buf(self); r = _bufferedreader_raw_read(self, PyBytes_AS_STRING(res), n); LEAVE_BUFFERED(self) diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index d5bec5f71f5be88..64eddcd314a803e 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -96,7 +96,7 @@ _io__BufferedIOBase_detach_impl(PyObject *self, PyTypeObject *cls); static PyObject * _io__BufferedIOBase_detach(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "detach() takes no arguments"); return NULL; } @@ -1245,4 +1245,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=442b05b9a117df6c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4249187a725a3b3e input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index 37023e49087647e..620e9e3b84ea192 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -96,7 +96,7 @@ _io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls); static PyObject * _io_BytesIO_getbuffer(bytesio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "getbuffer() takes no arguments"); return NULL; } @@ -534,4 +534,4 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=2be0e05a8871b7e2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ef116925b8b9e535 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index cf3ba28b066cf7f..5b5487d63eba900 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -27,7 +27,7 @@ _io_FileIO_close_impl(fileio *self, PyTypeObject *cls); static PyObject * _io_FileIO_close(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "close() takes no arguments"); return NULL; } @@ -528,4 +528,4 @@ _io_FileIO_isatty(fileio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=1c0f4a36f76b0c6a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e3d9446b4087020e input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/iobase.c.h b/Modules/_io/clinic/iobase.c.h index 6bdfa1444015acc..bae80a265fab075 100644 --- a/Modules/_io/clinic/iobase.c.h +++ b/Modules/_io/clinic/iobase.c.h @@ -262,7 +262,7 @@ _io__IOBase_fileno_impl(PyObject *self, PyTypeObject *cls); static PyObject * _io__IOBase_fileno(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "fileno() takes no arguments"); return NULL; } @@ -438,4 +438,4 @@ _io__RawIOBase_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _io__RawIOBase_readall_impl(self); } -/*[clinic end generated code: output=5a22bc5db0ecaacb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e7326fbefc52bfba input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 23b3cc8d71e098c..f04ee729abc9edc 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -27,7 +27,7 @@ _io__TextIOBase_detach_impl(PyObject *self, PyTypeObject *cls); static PyObject * _io__TextIOBase_detach(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "detach() takes no arguments"); return NULL; } @@ -1292,4 +1292,4 @@ _io_TextIOWrapper__CHUNK_SIZE_set(textio *self, PyObject *value, void *Py_UNUSED return return_value; } -/*[clinic end generated code: output=d01aa598647c1385 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=93a5a91a22100a28 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index 6cab295c44611d4..4696ecc5c843e6e 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -29,7 +29,7 @@ _io__WindowsConsoleIO_close_impl(winconsoleio *self, PyTypeObject *cls); static PyObject * _io__WindowsConsoleIO_close(winconsoleio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "close() takes no arguments"); return NULL; } @@ -457,4 +457,4 @@ _io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ -/*[clinic end generated code: output=04108fc26b187386 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2c2bc86713b21dd6 input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 9cf268ca0b26c81..6bb156e41fe43cb 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -269,6 +269,13 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, self->fd = -1; } + if (PyBool_Check(nameobj)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "bool is used as a file descriptor", 1)) + { + return -1; + } + } fd = PyLong_AsInt(nameobj); if (fd < 0) { if (!PyErr_Occurred()) { diff --git a/Modules/_json.c b/Modules/_json.c index 24b292ce70e5ebd..c55299899e77fe5 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -691,11 +691,10 @@ _parse_object_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ss key = scanstring_unicode(pystr, idx + 1, s->strict, &next_idx); if (key == NULL) goto bail; - memokey = PyDict_SetDefault(memo, key, key); - if (memokey == NULL) { + if (PyDict_SetDefaultRef(memo, key, key, &memokey) < 0) { goto bail; } - Py_SETREF(key, Py_NewRef(memokey)); + Py_SETREF(key, memokey); idx = next_idx; /* skip whitespace between key and : delimiter, read :, skip whitespace */ diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index 8fca3cdd0deb188..18b24855c52ad61 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -3,8 +3,9 @@ #endif #include "Python.h" -#include "pycore_ceval.h" // _PyEval_MakePendingCalls() +#include "pycore_ceval.h" // Py_MakePendingCalls() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_parking_lot.h" #include "pycore_time.h" // _PyTime_t #include <stdbool.h> @@ -151,7 +152,9 @@ RingBuf_Get(RingBuf *buf) return item; } -// Returns 0 on success or -1 if the buffer failed to grow +// Returns 0 on success or -1 if the buffer failed to grow. +// +// Steals a reference to item. static int RingBuf_Put(RingBuf *buf, PyObject *item) { @@ -164,7 +167,7 @@ RingBuf_Put(RingBuf *buf, PyObject *item) return -1; } } - buf->items[buf->put_idx] = Py_NewRef(item); + buf->items[buf->put_idx] = item; buf->put_idx = (buf->put_idx + 1) % buf->items_cap; buf->num_items++; return 0; @@ -184,9 +187,13 @@ RingBuf_IsEmpty(RingBuf *buf) typedef struct { PyObject_HEAD - PyThread_type_lock lock; - int locked; + + // Are there threads waiting for items + bool has_threads_waiting; + + // Items in the queue RingBuf buf; + PyObject *weakreflist; } simplequeueobject; @@ -209,12 +216,6 @@ simplequeue_dealloc(simplequeueobject *self) PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - if (self->lock != NULL) { - /* Unlock the lock so it's safe to free it */ - if (self->locked > 0) - PyThread_release_lock(self->lock); - PyThread_free_lock(self->lock); - } (void)simplequeue_clear(self); if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) self); @@ -249,12 +250,6 @@ simplequeue_new_impl(PyTypeObject *type) self = (simplequeueobject *) type->tp_alloc(type, 0); if (self != NULL) { self->weakreflist = NULL; - self->lock = PyThread_allocate_lock(); - if (self->lock == NULL) { - Py_DECREF(self); - PyErr_SetString(PyExc_MemoryError, "can't allocate lock"); - return NULL; - } if (RingBuf_Init(&self->buf) < 0) { Py_DECREF(self); return NULL; @@ -264,7 +259,29 @@ simplequeue_new_impl(PyTypeObject *type) return (PyObject *) self; } +typedef struct { + bool handed_off; + simplequeueobject *queue; + PyObject *item; +} HandoffData; + +static void +maybe_handoff_item(HandoffData *data, PyObject **item, int has_more_waiters) +{ + if (item == NULL) { + // No threads were waiting + data->handed_off = false; + } + else { + // There was at least one waiting thread, hand off the item + *item = data->item; + data->handed_off = true; + } + data->queue->has_threads_waiting = has_more_waiters; +} + /*[clinic input] +@critical_section _queue.SimpleQueue.put item: object block: bool = True @@ -280,21 +297,28 @@ never blocks. They are provided for compatibility with the Queue class. static PyObject * _queue_SimpleQueue_put_impl(simplequeueobject *self, PyObject *item, int block, PyObject *timeout) -/*[clinic end generated code: output=4333136e88f90d8b input=6e601fa707a782d5]*/ +/*[clinic end generated code: output=4333136e88f90d8b input=a16dbb33363c0fa8]*/ { - /* BEGIN GIL-protected critical section */ - if (RingBuf_Put(&self->buf, item) < 0) - return NULL; - if (self->locked) { - /* A get() may be waiting, wake it up */ - self->locked = 0; - PyThread_release_lock(self->lock); + HandoffData data = { + .handed_off = 0, + .item = Py_NewRef(item), + .queue = self, + }; + if (self->has_threads_waiting) { + // Try to hand the item off directly if there are threads waiting + _PyParkingLot_Unpark(&self->has_threads_waiting, + (_Py_unpark_fn_t *)maybe_handoff_item, &data); + } + if (!data.handed_off) { + if (RingBuf_Put(&self->buf, item) < 0) { + return NULL; + } } - /* END GIL-protected critical section */ Py_RETURN_NONE; } /*[clinic input] +@critical_section _queue.SimpleQueue.put_nowait item: object @@ -307,12 +331,23 @@ for compatibility with the Queue class. static PyObject * _queue_SimpleQueue_put_nowait_impl(simplequeueobject *self, PyObject *item) -/*[clinic end generated code: output=0990536715efb1f1 input=36b1ea96756b2ece]*/ +/*[clinic end generated code: output=0990536715efb1f1 input=ce949cc2cd8a4119]*/ { return _queue_SimpleQueue_put_impl(self, item, 0, Py_None); } +static PyObject * +empty_error(PyTypeObject *cls) +{ + PyObject *module = PyType_GetModule(cls); + assert(module != NULL); + simplequeue_state *state = simplequeue_get_state(module); + PyErr_SetNone(state->EmptyError); + return NULL; +} + /*[clinic input] +@critical_section _queue.SimpleQueue.get cls: defining_class @@ -335,23 +370,15 @@ in that case). static PyObject * _queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, int block, PyObject *timeout_obj) -/*[clinic end generated code: output=5c2cca914cd1e55b input=5b4047bfbc645ec1]*/ +/*[clinic end generated code: output=5c2cca914cd1e55b input=f7836c65e5839c51]*/ { _PyTime_t endtime = 0; - _PyTime_t timeout; - PyObject *item; - PyLockStatus r; - PY_TIMEOUT_T microseconds; - PyThreadState *tstate = PyThreadState_Get(); // XXX Use PyThread_ParseTimeoutArg(). - if (block == 0) { - /* Non-blocking */ - microseconds = 0; - } - else if (timeout_obj != Py_None) { + if (block != 0 && !Py_IsNone(timeout_obj)) { /* With timeout */ + _PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_CEILING) < 0) { return NULL; @@ -361,65 +388,64 @@ _queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, "'timeout' must be a non-negative number"); return NULL; } - microseconds = _PyTime_AsMicroseconds(timeout, - _PyTime_ROUND_CEILING); - if (microseconds > PY_TIMEOUT_MAX) { - PyErr_SetString(PyExc_OverflowError, - "timeout value is too large"); - return NULL; - } endtime = _PyDeadline_Init(timeout); } - else { - /* Infinitely blocking */ - microseconds = -1; - } - /* put() signals the queue to be non-empty by releasing the lock. - * So we simply try to acquire the lock in a loop, until the condition - * (queue non-empty) becomes true. - */ - while (RingBuf_IsEmpty(&self->buf)) { - /* First a simple non-blocking try without releasing the GIL */ - r = PyThread_acquire_lock_timed(self->lock, 0, 0); - if (r == PY_LOCK_FAILURE && microseconds != 0) { - Py_BEGIN_ALLOW_THREADS - r = PyThread_acquire_lock_timed(self->lock, microseconds, 1); - Py_END_ALLOW_THREADS + for (;;) { + if (!RingBuf_IsEmpty(&self->buf)) { + return RingBuf_Get(&self->buf); } - if (r == PY_LOCK_INTR && _PyEval_MakePendingCalls(tstate) < 0) { - return NULL; - } - if (r == PY_LOCK_FAILURE) { - PyObject *module = PyType_GetModule(cls); - simplequeue_state *state = simplequeue_get_state(module); - /* Timed out */ - PyErr_SetNone(state->EmptyError); - return NULL; + if (!block) { + return empty_error(cls); } - self->locked = 1; - /* Adjust timeout for next iteration (if any) */ - if (microseconds > 0) { - timeout = _PyDeadline_Get(endtime); - microseconds = _PyTime_AsMicroseconds(timeout, - _PyTime_ROUND_CEILING); + int64_t timeout_ns = -1; + if (endtime != 0) { + timeout_ns = _PyDeadline_Get(endtime); + if (timeout_ns < 0) { + return empty_error(cls); + } } - } - /* BEGIN GIL-protected critical section */ - item = RingBuf_Get(&self->buf); - if (self->locked) { - PyThread_release_lock(self->lock); - self->locked = 0; + bool waiting = 1; + self->has_threads_waiting = waiting; + + PyObject *item = NULL; + int st = _PyParkingLot_Park(&self->has_threads_waiting, &waiting, + sizeof(bool), timeout_ns, &item, + /* detach */ 1); + switch (st) { + case Py_PARK_OK: { + assert(item != NULL); + return item; + } + case Py_PARK_TIMEOUT: { + return empty_error(cls); + } + case Py_PARK_INTR: { + // Interrupted + if (Py_MakePendingCalls() < 0) { + return NULL; + } + break; + } + case Py_PARK_AGAIN: { + // This should be impossible with the current implementation of + // PyParkingLot, but would be possible if critical sections / + // the GIL were released before the thread was added to the + // internal thread queue in the parking lot. + break; + } + default: { + Py_UNREACHABLE(); + } + } } - /* END GIL-protected critical section */ - - return item; } /*[clinic input] +@critical_section _queue.SimpleQueue.get_nowait cls: defining_class @@ -434,12 +460,13 @@ raise the Empty exception. static PyObject * _queue_SimpleQueue_get_nowait_impl(simplequeueobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=620c58e2750f8b8a input=842f732bf04216d3]*/ +/*[clinic end generated code: output=620c58e2750f8b8a input=d48be63633fefae9]*/ { return _queue_SimpleQueue_get_impl(self, cls, 0, Py_None); } /*[clinic input] +@critical_section _queue.SimpleQueue.empty -> bool Return True if the queue is empty, False otherwise (not reliable!). @@ -447,12 +474,13 @@ Return True if the queue is empty, False otherwise (not reliable!). static int _queue_SimpleQueue_empty_impl(simplequeueobject *self) -/*[clinic end generated code: output=1a02a1b87c0ef838 input=1a98431c45fd66f9]*/ +/*[clinic end generated code: output=1a02a1b87c0ef838 input=96cb22df5a67d831]*/ { return RingBuf_IsEmpty(&self->buf); } /*[clinic input] +@critical_section _queue.SimpleQueue.qsize -> Py_ssize_t Return the approximate size of the queue (not reliable!). @@ -460,7 +488,7 @@ Return the approximate size of the queue (not reliable!). static Py_ssize_t _queue_SimpleQueue_qsize_impl(simplequeueobject *self) -/*[clinic end generated code: output=f9dcd9d0a90e121e input=7a74852b407868a1]*/ +/*[clinic end generated code: output=f9dcd9d0a90e121e input=e218623cb8c16a79]*/ { return RingBuf_Len(&self->buf); } diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index db5eb77891e52e5..811314b5cd8aed8 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -1204,21 +1204,67 @@ pysqlite_connection_interrupt(pysqlite_Connection *self, PyObject *Py_UNUSED(ign } PyDoc_STRVAR(pysqlite_connection_iterdump__doc__, -"iterdump($self, /)\n" +"iterdump($self, /, *, filter=None)\n" "--\n" "\n" -"Returns iterator to the dump of the database in an SQL text format."); +"Returns iterator to the dump of the database in an SQL text format.\n" +"\n" +" filter\n" +" An optional LIKE pattern for database objects to dump"); #define PYSQLITE_CONNECTION_ITERDUMP_METHODDEF \ - {"iterdump", (PyCFunction)pysqlite_connection_iterdump, METH_NOARGS, pysqlite_connection_iterdump__doc__}, + {"iterdump", _PyCFunction_CAST(pysqlite_connection_iterdump), METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_iterdump__doc__}, static PyObject * -pysqlite_connection_iterdump_impl(pysqlite_Connection *self); +pysqlite_connection_iterdump_impl(pysqlite_Connection *self, + PyObject *filter); static PyObject * -pysqlite_connection_iterdump(pysqlite_Connection *self, PyObject *Py_UNUSED(ignored)) +pysqlite_connection_iterdump(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return pysqlite_connection_iterdump_impl(self); + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(filter), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"filter", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "iterdump", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *filter = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + filter = args[0]; +skip_optional_kwonly: + return_value = pysqlite_connection_iterdump_impl(self, filter); + +exit: + return return_value; } PyDoc_STRVAR(pysqlite_connection_backup__doc__, @@ -1551,7 +1597,9 @@ deserialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { @@ -1818,4 +1866,4 @@ getconfig(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=90b5b9c14261b8d7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3c6d0b748fac016f input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 0a6633972cc5ef7..f97afcf5fcf16ef 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -1979,12 +1979,17 @@ pysqlite_connection_interrupt_impl(pysqlite_Connection *self) /*[clinic input] _sqlite3.Connection.iterdump as pysqlite_connection_iterdump + * + filter: object = None + An optional LIKE pattern for database objects to dump + Returns iterator to the dump of the database in an SQL text format. [clinic start generated code]*/ static PyObject * -pysqlite_connection_iterdump_impl(pysqlite_Connection *self) -/*[clinic end generated code: output=586997aaf9808768 input=1911ca756066da89]*/ +pysqlite_connection_iterdump_impl(pysqlite_Connection *self, + PyObject *filter) +/*[clinic end generated code: output=fd81069c4bdeb6b0 input=4ae6d9a898f108df]*/ { if (!pysqlite_check_connection(self)) { return NULL; @@ -1998,9 +2003,16 @@ pysqlite_connection_iterdump_impl(pysqlite_Connection *self) } return NULL; } - - PyObject *retval = PyObject_CallOneArg(iterdump, (PyObject *)self); + PyObject *args[3] = {NULL, (PyObject *)self, filter}; + PyObject *kwnames = Py_BuildValue("(s)", "filter"); + if (!kwnames) { + Py_DECREF(iterdump); + return NULL; + } + Py_ssize_t nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; + PyObject *retval = PyObject_Vectorcall(iterdump, args + 1, nargsf, kwnames); Py_DECREF(iterdump); + Py_DECREF(kwnames); return retval; } diff --git a/Modules/_sre/clinic/sre.c.h b/Modules/_sre/clinic/sre.c.h index cd3fbbc720bdf1e..48336c7a2fca262 100644 --- a/Modules/_sre/clinic/sre.c.h +++ b/Modules/_sre/clinic/sre.c.h @@ -1434,7 +1434,7 @@ _sre_SRE_Scanner_match_impl(ScannerObject *self, PyTypeObject *cls); static PyObject * _sre_SRE_Scanner_match(ScannerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "match() takes no arguments"); return NULL; } @@ -1455,10 +1455,10 @@ _sre_SRE_Scanner_search_impl(ScannerObject *self, PyTypeObject *cls); static PyObject * _sre_SRE_Scanner_search(ScannerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "search() takes no arguments"); return NULL; } return _sre_SRE_Scanner_search_impl(self, cls); } -/*[clinic end generated code: output=ad513f31b99505fa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c3e711f0b2f43d66 input=a9049054013a1b77]*/ diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index d451974b9cf81e6..00fbd9674b8cdda 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -39,13 +39,14 @@ static const char copyright[] = " SRE 2.2.2 Copyright (c) 1997-2002 by Secret Labs AB "; #include "Python.h" -#include "pycore_dict.h" // _PyDict_Next() -#include "pycore_long.h" // _PyLong_GetZero() -#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION +#include "pycore_dict.h" // _PyDict_Next() +#include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_moduleobject.h" // _PyModule_GetState() -#include "sre.h" // SRE_CODE +#include "sre.h" // SRE_CODE -#include <ctype.h> // tolower(), toupper(), isalnum() +#include <ctype.h> // tolower(), toupper(), isalnum() #define SRE_CODE_BITS (8 * sizeof(SRE_CODE)) @@ -2349,26 +2350,28 @@ _sre_SRE_Match_groupdict_impl(MatchObject *self, PyObject *default_value) if (!result || !self->pattern->groupindex) return result; + Py_BEGIN_CRITICAL_SECTION(self->pattern->groupindex); while (_PyDict_Next(self->pattern->groupindex, &pos, &key, &value, &hash)) { int status; Py_INCREF(key); value = match_getslice(self, key, default_value); if (!value) { Py_DECREF(key); - goto failed; + Py_CLEAR(result); + goto exit; } status = _PyDict_SetItem_KnownHash(result, key, value, hash); Py_DECREF(value); Py_DECREF(key); - if (status < 0) - goto failed; + if (status < 0) { + Py_CLEAR(result); + goto exit; + } } +exit: + Py_END_CRITICAL_SECTION(); return result; - -failed: - Py_DECREF(result); - return NULL; } /*[clinic input] diff --git a/Modules/_sre/sre_lib.h b/Modules/_sre/sre_lib.h index f5497d9ff2b93fd..97fbb0a75e54b65 100644 --- a/Modules/_sre/sre_lib.h +++ b/Modules/_sre/sre_lib.h @@ -1122,7 +1122,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) /* install new repeat context */ /* TODO(https://github.com/python/cpython/issues/67877): Fix this * potential memory leak. */ - ctx->u.rep = (SRE_REPEAT*) PyObject_Malloc(sizeof(*ctx->u.rep)); + ctx->u.rep = (SRE_REPEAT*) PyMem_Malloc(sizeof(*ctx->u.rep)); if (!ctx->u.rep) { PyErr_NoMemory(); RETURN_FAILURE; @@ -1136,7 +1136,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) state->ptr = ptr; DO_JUMP(JUMP_REPEAT, jump_repeat, pattern+pattern[0]); state->repeat = ctx->u.rep->prev; - PyObject_Free(ctx->u.rep); + PyMem_Free(ctx->u.rep); if (ret) { RETURN_ON_ERROR(ret); diff --git a/Modules/_testcapi/buffer.c b/Modules/_testcapi/buffer.c index 942774156c6c47f..7e2f6e5e29482c2 100644 --- a/Modules/_testcapi/buffer.c +++ b/Modules/_testcapi/buffer.c @@ -54,8 +54,10 @@ static int testbuf_getbuf(testBufObject *self, Py_buffer *view, int flags) { int buf = PyObject_GetBuffer(self->obj, view, flags); - Py_SETREF(view->obj, Py_NewRef(self)); - self->references++; + if (buf == 0) { + Py_SETREF(view->obj, Py_NewRef(self)); + self->references++; + } return buf; } diff --git a/Modules/_testcapi/dict.c b/Modules/_testcapi/dict.c index 42e056b7d07a31f..fe03c24f75e196b 100644 --- a/Modules/_testcapi/dict.c +++ b/Modules/_testcapi/dict.c @@ -225,6 +225,31 @@ dict_setdefault(PyObject *self, PyObject *args) return PyDict_SetDefault(mapping, key, defaultobj); } +static PyObject * +dict_setdefaultref(PyObject *self, PyObject *args) +{ + PyObject *obj, *key, *default_value, *result = UNINITIALIZED_PTR; + if (!PyArg_ParseTuple(args, "OOO", &obj, &key, &default_value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(key); + NULLABLE(default_value); + switch (PyDict_SetDefaultRef(obj, key, default_value, &result)) { + case -1: + assert(result == NULL); + return NULL; + case 0: + assert(result == default_value); + return result; + case 1: + return result; + default: + Py_FatalError("PyDict_SetDefaultRef() returned invalid code"); + Py_UNREACHABLE(); + } +} + static PyObject * dict_delitem(PyObject *self, PyObject *args) { @@ -433,6 +458,7 @@ static PyMethodDef test_methods[] = { {"dict_delitem", dict_delitem, METH_VARARGS}, {"dict_delitemstring", dict_delitemstring, METH_VARARGS}, {"dict_setdefault", dict_setdefault, METH_VARARGS}, + {"dict_setdefaultref", dict_setdefaultref, METH_VARARGS}, {"dict_keys", dict_keys, METH_O}, {"dict_values", dict_values, METH_O}, {"dict_items", dict_items, METH_O}, diff --git a/Modules/_testcapi/list.c b/Modules/_testcapi/list.c index 10e18699f01bc1f..2cb6499e28336df 100644 --- a/Modules/_testcapi/list.c +++ b/Modules/_testcapi/list.c @@ -59,6 +59,18 @@ list_get_item(PyObject *Py_UNUSED(module), PyObject *args) return Py_XNewRef(PyList_GET_ITEM(obj, i)); } +static PyObject * +list_get_item_ref(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &obj, &i)) { + return NULL; + } + NULLABLE(obj); + return PyList_GetItemRef(obj, i); +} + static PyObject * list_setitem(PyObject *Py_UNUSED(module), PyObject *args) { @@ -191,6 +203,7 @@ static PyMethodDef test_methods[] = { {"list_get_size", list_get_size, METH_O}, {"list_getitem", list_getitem, METH_VARARGS}, {"list_get_item", list_get_item, METH_VARARGS}, + {"list_get_item_ref", list_get_item_ref, METH_VARARGS}, {"list_setitem", list_setitem, METH_VARARGS}, {"list_set_item", list_set_item, METH_VARARGS}, {"list_insert", list_insert, METH_VARARGS}, diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index a763ff46a3c2901..1eb0db2c2e65761 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -15,8 +15,8 @@ module _testcapi /*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/ // Test dict watching -static PyObject *g_dict_watch_events; -static int g_dict_watchers_installed; +static PyObject *g_dict_watch_events = NULL; +static int g_dict_watchers_installed = 0; static int dict_watch_callback(PyDict_WatchEvent event, diff --git a/Modules/_testcapi_feature_macros.inc b/Modules/_testcapi_feature_macros.inc index a076e7149800743..f5f3524f2c0177f 100644 --- a/Modules/_testcapi_feature_macros.inc +++ b/Modules/_testcapi_feature_macros.inc @@ -38,6 +38,15 @@ if (res) { Py_DECREF(result); return NULL; } +#ifdef Py_TRACE_REFS + res = PyDict_SetItemString(result, "Py_TRACE_REFS", Py_True); +#else + res = PyDict_SetItemString(result, "Py_TRACE_REFS", Py_False); +#endif +if (res) { + Py_DECREF(result); return NULL; +} + #ifdef USE_STACKCHECK res = PyDict_SetItemString(result, "USE_STACKCHECK", Py_True); #else diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 6def680190b1a63..e67de3eeb6e17eb 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1261,6 +1261,26 @@ make_memoryview_from_NULL_pointer(PyObject *self, PyObject *Py_UNUSED(ignored)) return PyMemoryView_FromBuffer(&info); } +static PyObject * +buffer_fill_info(PyObject *self, PyObject *args) +{ + Py_buffer info; + const char *data; + Py_ssize_t size; + int readonly; + int flags; + + if (!PyArg_ParseTuple(args, "s#ii:buffer_fill_info", + &data, &size, &readonly, &flags)) { + return NULL; + } + + if (PyBuffer_FillInfo(&info, NULL, (void *)data, size, readonly, flags) < 0) { + return NULL; + } + return PyMemoryView_FromBuffer(&info); +} + static PyObject * test_from_contiguous(PyObject* self, PyObject *Py_UNUSED(ignored)) { @@ -3314,6 +3334,7 @@ static PyMethodDef TestMethods[] = { {"eval_code_ex", eval_eval_code_ex, METH_VARARGS}, {"make_memoryview_from_NULL_pointer", make_memoryview_from_NULL_pointer, METH_NOARGS}, + {"buffer_fill_info", buffer_fill_info, METH_VARARGS}, {"crash_no_current_thread", crash_no_current_thread, METH_NOARGS}, {"test_current_tstate_matches", test_current_tstate_matches, METH_NOARGS}, {"run_in_subinterp", run_in_subinterp, METH_VARARGS}, diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 15e0093f15ba1e6..fb0936bbccd3183 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -1213,6 +1213,40 @@ clone_with_conv_f2_impl(PyObject *module, custom_t path) } +/*[clinic input] +class _testclinic.TestClass "PyObject *" "PyObject" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=668a591c65bec947]*/ + +/*[clinic input] +_testclinic.TestClass.meth_method_no_params + cls: defining_class + / +[clinic start generated code]*/ + +static PyObject * +_testclinic_TestClass_meth_method_no_params_impl(PyObject *self, + PyTypeObject *cls) +/*[clinic end generated code: output=c140f100080c2fc8 input=6bd34503d11c63c1]*/ +{ + Py_RETURN_NONE; +} + +static struct PyMethodDef test_class_methods[] = { + _TESTCLINIC_TESTCLASS_METH_METHOD_NO_PARAMS_METHODDEF + {NULL, NULL} +}; + +static PyTypeObject TestClass = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_testclinic.TestClass", + .tp_basicsize = sizeof(PyObject), + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = PyType_GenericNew, + .tp_methods = test_class_methods, +}; + + /*[clinic input] output push destination deprstar new file '{dirname}/clinic/_testclinic_depr.c.h' @@ -1906,6 +1940,9 @@ PyInit__testclinic(void) if (m == NULL) { return NULL; } + if (PyModule_AddType(m, &TestClass) < 0) { + goto error; + } if (PyModule_AddType(m, &DeprStarNew) < 0) { goto error; } diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 7d277df164d3ec6..0bb739b5398b113 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1635,6 +1635,21 @@ get_type_module_name(PyObject *self, PyObject *type) return _PyType_GetModuleName((PyTypeObject *)type); } +static PyObject * +get_rare_event_counters(PyObject *self, PyObject *type) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + + return Py_BuildValue( + "{sksksksksk}", + "set_class", (unsigned long)interp->rare_events.set_class, + "set_bases", (unsigned long)interp->rare_events.set_bases, + "set_eval_frame_func", (unsigned long)interp->rare_events.set_eval_frame_func, + "builtin_dict", (unsigned long)interp->rare_events.builtin_dict, + "func_modification", (unsigned long)interp->rare_events.func_modification + ); +} + #ifdef Py_GIL_DISABLED static PyObject * @@ -1711,6 +1726,7 @@ static PyMethodDef module_functions[] = { {"restore_crossinterp_data", restore_crossinterp_data, METH_VARARGS}, _TESTINTERNALCAPI_TEST_LONG_NUMBITS_METHODDEF {"get_type_module_name", get_type_module_name, METH_O}, + {"get_rare_event_counters", get_rare_event_counters, METH_NOARGS}, #ifdef Py_GIL_DISABLED {"py_thread_id", get_py_thread_id, METH_NOARGS}, #endif @@ -1736,8 +1752,18 @@ module_exec(PyObject *module) return 1; } + Py_ssize_t sizeof_gc_head = 0; +#ifndef Py_GIL_DISABLED + sizeof_gc_head = sizeof(PyGC_Head); +#endif + if (PyModule_Add(module, "SIZEOF_PYGC_HEAD", - PyLong_FromSsize_t(sizeof(PyGC_Head))) < 0) { + PyLong_FromSsize_t(sizeof_gc_head)) < 0) { + return 1; + } + + if (PyModule_Add(module, "SIZEOF_MANAGED_PRE_HEADER", + PyLong_FromSsize_t(2 * sizeof(PyObject*))) < 0) { return 1; } diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 99f97eb6d0adcce..df02b023012fbde 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -5,6 +5,7 @@ #include "Python.h" #include "pycore_interp.h" // _PyInterpreterState.threads.count #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_pylifecycle.h" #include "pycore_pystate.h" // _PyThreadState_SetCurrent() #include "pycore_sysmodule.h" // _PySys_GetAttr() @@ -43,6 +44,7 @@ get_thread_state(PyObject *module) typedef struct { PyObject_HEAD + struct llist_node node; // linked list node (see _pythread_runtime_state) PyThread_ident_t ident; PyThread_handle_t handle; char joinable; @@ -58,6 +60,11 @@ new_thread_handle(thread_module_state* state) self->ident = 0; self->handle = 0; self->joinable = 0; + + HEAD_LOCK(&_PyRuntime); + llist_insert_tail(&_PyRuntime.threads.handles, &self->node); + HEAD_UNLOCK(&_PyRuntime); + return self; } @@ -65,6 +72,14 @@ static void ThreadHandle_dealloc(ThreadHandleObject *self) { PyObject *tp = (PyObject *) Py_TYPE(self); + + // Remove ourself from the global list of handles + HEAD_LOCK(&_PyRuntime); + if (self->node.next != NULL) { + llist_remove(&self->node); + } + HEAD_UNLOCK(&_PyRuntime); + if (self->joinable) { int ret = PyThread_detach_thread(self->handle); if (ret) { @@ -76,6 +91,28 @@ ThreadHandle_dealloc(ThreadHandleObject *self) Py_DECREF(tp); } +void +_PyThread_AfterFork(struct _pythread_runtime_state *state) +{ + // gh-115035: We mark ThreadHandles as not joinable early in the child's + // after-fork handler. We do this before calling any Python code to ensure + // that it happens before any ThreadHandles are deallocated, such as by a + // GC cycle. + PyThread_ident_t current = PyThread_get_thread_ident_ex(); + + struct llist_node *node; + llist_for_each_safe(node, &state->handles) { + ThreadHandleObject *hobj = llist_data(node, ThreadHandleObject, node); + if (hobj->ident == current) { + continue; + } + + // Disallow calls to detach() and join() as they could crash. + hobj->joinable = 0; + llist_remove(node); + } +} + static PyObject * ThreadHandle_repr(ThreadHandleObject *self) { @@ -90,21 +127,6 @@ ThreadHandle_get_ident(ThreadHandleObject *self, void *ignored) } -static PyObject * -ThreadHandle_after_fork_alive(ThreadHandleObject *self, void* ignored) -{ - PyThread_update_thread_after_fork(&self->ident, &self->handle); - Py_RETURN_NONE; -} - -static PyObject * -ThreadHandle_after_fork_dead(ThreadHandleObject *self, void* ignored) -{ - // Disallow calls to detach() and join() as they could crash. - self->joinable = 0; - Py_RETURN_NONE; -} - static PyObject * ThreadHandle_detach(ThreadHandleObject *self, void* ignored) { @@ -156,8 +178,6 @@ static PyGetSetDef ThreadHandle_getsetlist[] = { static PyMethodDef ThreadHandle_methods[] = { - {"after_fork_alive", (PyCFunction)ThreadHandle_after_fork_alive, METH_NOARGS}, - {"after_fork_dead", (PyCFunction)ThreadHandle_after_fork_dead, METH_NOARGS}, {"detach", (PyCFunction)ThreadHandle_detach, METH_NOARGS}, {"join", (PyCFunction)ThreadHandle_join, METH_NOARGS}, {0, 0} @@ -349,6 +369,27 @@ lock__at_fork_reinit(lockobject *self, PyObject *Py_UNUSED(args)) } #endif /* HAVE_FORK */ +static lockobject *newlockobject(PyObject *module); + +static PyObject * +lock_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + // convert to AC? + if (!_PyArg_NoKeywords("lock", kwargs)) { + goto error; + } + if (!_PyArg_CheckPositional("lock", PyTuple_GET_SIZE(args), 0, 0)) { + goto error; + } + + PyObject *module = PyType_GetModuleByDef(type, &thread_module); + assert(module != NULL); + return (PyObject *)newlockobject(module); + +error: + return NULL; +} + static PyMethodDef lock_methods[] = { {"acquire_lock", _PyCFunction_CAST(lock_PyThread_acquire_lock), @@ -398,6 +439,7 @@ static PyType_Slot lock_type_slots[] = { {Py_tp_methods, lock_methods}, {Py_tp_traverse, lock_traverse}, {Py_tp_members, lock_type_members}, + {Py_tp_new, lock_new}, {0, 0} }; @@ -405,7 +447,7 @@ static PyType_Spec lock_type_spec = { .name = "_thread.lock", .basicsize = sizeof(lockobject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_IMMUTABLETYPE), .slots = lock_type_slots, }; @@ -1442,8 +1484,6 @@ A subthread can use this function to interrupt the main thread.\n\ Note: the default signal handler for SIGINT raises ``KeyboardInterrupt``." ); -static lockobject *newlockobject(PyObject *module); - static PyObject * thread_PyThread_allocate_lock(PyObject *module, PyObject *Py_UNUSED(ignored)) { @@ -1841,10 +1881,14 @@ thread_module_exec(PyObject *module) } // Lock - state->lock_type = (PyTypeObject *)PyType_FromSpec(&lock_type_spec); + state->lock_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, &lock_type_spec, NULL); if (state->lock_type == NULL) { return -1; } + if (PyModule_AddType(module, state->lock_type) < 0) { + return -1; + } + // Old alias: lock -> LockType if (PyDict_SetItemString(d, "LockType", (PyObject *)state->lock_type) < 0) { return -1; } diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 26302b559817b39..5e5eb123c4ccfff 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -441,7 +441,7 @@ _winapi_ConnectNamedPipe_impl(PyObject *module, HANDLE handle, /*[clinic input] _winapi.CreateFile -> HANDLE - file_name: LPCTSTR + file_name: LPCWSTR desired_access: DWORD share_mode: DWORD security_attributes: LPSECURITY_ATTRIBUTES @@ -452,12 +452,12 @@ _winapi.CreateFile -> HANDLE [clinic start generated code]*/ static HANDLE -_winapi_CreateFile_impl(PyObject *module, LPCTSTR file_name, +_winapi_CreateFile_impl(PyObject *module, LPCWSTR file_name, DWORD desired_access, DWORD share_mode, LPSECURITY_ATTRIBUTES security_attributes, DWORD creation_disposition, DWORD flags_and_attributes, HANDLE template_file) -/*[clinic end generated code: output=417ddcebfc5a3d53 input=6423c3e40372dbd5]*/ +/*[clinic end generated code: output=818c811e5e04d550 input=1fa870ed1c2e3d69]*/ { HANDLE handle; @@ -468,14 +468,15 @@ _winapi_CreateFile_impl(PyObject *module, LPCTSTR file_name, } Py_BEGIN_ALLOW_THREADS - handle = CreateFile(file_name, desired_access, - share_mode, security_attributes, - creation_disposition, - flags_and_attributes, template_file); + handle = CreateFileW(file_name, desired_access, + share_mode, security_attributes, + creation_disposition, + flags_and_attributes, template_file); Py_END_ALLOW_THREADS - if (handle == INVALID_HANDLE_VALUE) + if (handle == INVALID_HANDLE_VALUE) { PyErr_SetFromWindowsErr(0); + } return handle; } diff --git a/Modules/_xxtestfuzz/fuzzer.c b/Modules/_xxtestfuzz/fuzzer.c index e133b4d3c444809..6ea9f64d6285304 100644 --- a/Modules/_xxtestfuzz/fuzzer.c +++ b/Modules/_xxtestfuzz/fuzzer.c @@ -502,7 +502,6 @@ static int fuzz_elementtree_parsewhole(const char* data, size_t size) { } #define MAX_PYCOMPILE_TEST_SIZE 16384 -static char pycompile_scratch[MAX_PYCOMPILE_TEST_SIZE]; static const int start_vals[] = {Py_eval_input, Py_single_input, Py_file_input}; const size_t NUM_START_VALS = sizeof(start_vals) / sizeof(start_vals[0]); @@ -531,6 +530,8 @@ static int fuzz_pycompile(const char* data, size_t size) { unsigned char optimize_idx = (unsigned char) data[1]; int optimize = optimize_vals[optimize_idx % NUM_OPTIMIZE_VALS]; + char pycompile_scratch[MAX_PYCOMPILE_TEST_SIZE]; + // Create a NUL-terminated C string from the remaining input memcpy(pycompile_scratch, data + 2, size - 2); // Put a NUL terminator just after the copied data. (Space was reserved already.) @@ -549,7 +550,13 @@ static int fuzz_pycompile(const char* data, size_t size) { PyObject *result = Py_CompileStringExFlags(pycompile_scratch, "<fuzz input>", start, flags, optimize); if (result == NULL) { - /* compilation failed, most likely from a syntax error */ + /* Compilation failed, most likely from a syntax error. If it was a + SystemError we abort. There's no non-bug reason to raise a + SystemError. */ + if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_SystemError)) { + PyErr_Print(); + abort(); + } PyErr_Clear(); } else { Py_DECREF(result); diff --git a/Modules/cjkcodecs/clinic/multibytecodec.c.h b/Modules/cjkcodecs/clinic/multibytecodec.c.h index 305ade17b1f1aa8..b5639d5cf10a22a 100644 --- a/Modules/cjkcodecs/clinic/multibytecodec.c.h +++ b/Modules/cjkcodecs/clinic/multibytecodec.c.h @@ -668,7 +668,7 @@ _multibytecodec_MultibyteStreamWriter_reset_impl(MultibyteStreamWriterObject *se static PyObject * _multibytecodec_MultibyteStreamWriter_reset(MultibyteStreamWriterObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "reset() takes no arguments"); return NULL; } @@ -682,4 +682,4 @@ PyDoc_STRVAR(_multibytecodec___create_codec__doc__, #define _MULTIBYTECODEC___CREATE_CODEC_METHODDEF \ {"__create_codec", (PyCFunction)_multibytecodec___create_codec, METH_O, _multibytecodec___create_codec__doc__}, -/*[clinic end generated code: output=219a363662d2fbff input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ee767a6d93c7108a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index d941c280a4300ba..6a9c8ff6d8fdd9e 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -120,7 +120,7 @@ _asyncio_Future_exception_impl(FutureObj *self, PyTypeObject *cls); static PyObject * _asyncio_Future_exception(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "exception() takes no arguments"); return NULL; } @@ -453,7 +453,7 @@ _asyncio_Future_get_loop_impl(FutureObj *self, PyTypeObject *cls); static PyObject * _asyncio_Future_get_loop(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "get_loop() takes no arguments"); return NULL; } @@ -1487,4 +1487,4 @@ _asyncio_current_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=f3864d8e2af7635f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b26155080c82c472 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index 12fea806ab52093..1c0f37442ab3509 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -297,7 +297,9 @@ _codecs_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { @@ -1099,7 +1101,9 @@ _codecs_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_ if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { @@ -1175,7 +1179,9 @@ _codecs_raw_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ss if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { @@ -1644,7 +1650,9 @@ _codecs_readbuffer_encode(PyObject *module, PyObject *const *args, Py_ssize_t na if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { @@ -2738,4 +2746,4 @@ _codecs_lookup_error(PyObject *module, PyObject *arg) #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=d8d9e372f7ccba35 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e50d5fdf65bd45fa input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_collectionsmodule.c.h b/Modules/clinic/_collectionsmodule.c.h index 591ab50c76a8e8f..60fb12a22316195 100644 --- a/Modules/clinic/_collectionsmodule.c.h +++ b/Modules/clinic/_collectionsmodule.c.h @@ -2,9 +2,425 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_modsupport.h" // _PyArg_CheckPositional() +PyDoc_STRVAR(deque_pop__doc__, +"pop($self, /)\n" +"--\n" +"\n" +"Remove and return the rightmost element."); + +#define DEQUE_POP_METHODDEF \ + {"pop", (PyCFunction)deque_pop, METH_NOARGS, deque_pop__doc__}, + +static PyObject * +deque_pop_impl(dequeobject *deque); + +static PyObject * +deque_pop(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + return deque_pop_impl(deque); +} + +PyDoc_STRVAR(deque_popleft__doc__, +"popleft($self, /)\n" +"--\n" +"\n" +"Remove and return the leftmost element."); + +#define DEQUE_POPLEFT_METHODDEF \ + {"popleft", (PyCFunction)deque_popleft, METH_NOARGS, deque_popleft__doc__}, + +static PyObject * +deque_popleft_impl(dequeobject *deque); + +static PyObject * +deque_popleft(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + return deque_popleft_impl(deque); +} + +PyDoc_STRVAR(deque_append__doc__, +"append($self, item, /)\n" +"--\n" +"\n" +"Add an element to the right side of the deque."); + +#define DEQUE_APPEND_METHODDEF \ + {"append", (PyCFunction)deque_append, METH_O, deque_append__doc__}, + +PyDoc_STRVAR(deque_appendleft__doc__, +"appendleft($self, item, /)\n" +"--\n" +"\n" +"Add an element to the left side of the deque."); + +#define DEQUE_APPENDLEFT_METHODDEF \ + {"appendleft", (PyCFunction)deque_appendleft, METH_O, deque_appendleft__doc__}, + +PyDoc_STRVAR(deque_extend__doc__, +"extend($self, iterable, /)\n" +"--\n" +"\n" +"Extend the right side of the deque with elements from the iterable."); + +#define DEQUE_EXTEND_METHODDEF \ + {"extend", (PyCFunction)deque_extend, METH_O, deque_extend__doc__}, + +PyDoc_STRVAR(deque_extendleft__doc__, +"extendleft($self, iterable, /)\n" +"--\n" +"\n" +"Extend the left side of the deque with elements from the iterable."); + +#define DEQUE_EXTENDLEFT_METHODDEF \ + {"extendleft", (PyCFunction)deque_extendleft, METH_O, deque_extendleft__doc__}, + +PyDoc_STRVAR(deque_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"Return a shallow copy of a deque."); + +#define DEQUE_COPY_METHODDEF \ + {"copy", (PyCFunction)deque_copy, METH_NOARGS, deque_copy__doc__}, + +static PyObject * +deque_copy_impl(dequeobject *deque); + +static PyObject * +deque_copy(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + return deque_copy_impl(deque); +} + +PyDoc_STRVAR(deque___copy____doc__, +"__copy__($self, /)\n" +"--\n" +"\n" +"Return a shallow copy of a deque."); + +#define DEQUE___COPY___METHODDEF \ + {"__copy__", (PyCFunction)deque___copy__, METH_NOARGS, deque___copy____doc__}, + +static PyObject * +deque___copy___impl(dequeobject *deque); + +static PyObject * +deque___copy__(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + return deque___copy___impl(deque); +} + +PyDoc_STRVAR(deque_clearmethod__doc__, +"clear($self, /)\n" +"--\n" +"\n" +"Remove all elements from the deque."); + +#define DEQUE_CLEARMETHOD_METHODDEF \ + {"clear", (PyCFunction)deque_clearmethod, METH_NOARGS, deque_clearmethod__doc__}, + +static PyObject * +deque_clearmethod_impl(dequeobject *deque); + +static PyObject * +deque_clearmethod(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + return deque_clearmethod_impl(deque); +} + +PyDoc_STRVAR(deque_rotate__doc__, +"rotate($self, n=1, /)\n" +"--\n" +"\n" +"Rotate the deque n steps to the right. If n is negative, rotates left."); + +#define DEQUE_ROTATE_METHODDEF \ + {"rotate", _PyCFunction_CAST(deque_rotate), METH_FASTCALL, deque_rotate__doc__}, + +static PyObject * +deque_rotate_impl(dequeobject *deque, Py_ssize_t n); + +static PyObject * +deque_rotate(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t n = 1; + + if (!_PyArg_CheckPositional("rotate", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + n = ival; + } +skip_optional: + return_value = deque_rotate_impl(deque, n); + +exit: + return return_value; +} + +PyDoc_STRVAR(deque_reverse__doc__, +"reverse($self, /)\n" +"--\n" +"\n" +"Reverse *IN PLACE*."); + +#define DEQUE_REVERSE_METHODDEF \ + {"reverse", (PyCFunction)deque_reverse, METH_NOARGS, deque_reverse__doc__}, + +static PyObject * +deque_reverse_impl(dequeobject *deque); + +static PyObject * +deque_reverse(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + return deque_reverse_impl(deque); +} + +PyDoc_STRVAR(deque_count__doc__, +"count($self, value, /)\n" +"--\n" +"\n" +"Return number of occurrences of value."); + +#define DEQUE_COUNT_METHODDEF \ + {"count", (PyCFunction)deque_count, METH_O, deque_count__doc__}, + +PyDoc_STRVAR(deque_index__doc__, +"index($self, value, [start, [stop]])\n" +"--\n" +"\n" +"Return first index of value.\n" +"\n" +"Raises ValueError if the value is not present."); + +#define DEQUE_INDEX_METHODDEF \ + {"index", _PyCFunction_CAST(deque_index), METH_FASTCALL, deque_index__doc__}, + +static PyObject * +deque_index_impl(dequeobject *deque, PyObject *v, Py_ssize_t start, + Py_ssize_t stop); + +static PyObject * +deque_index(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *v; + Py_ssize_t start = 0; + Py_ssize_t stop = Py_SIZE(deque); + + if (!_PyArg_CheckPositional("index", nargs, 1, 3)) { + goto exit; + } + v = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndexNotNone(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndexNotNone(args[2], &stop)) { + goto exit; + } +skip_optional: + return_value = deque_index_impl(deque, v, start, stop); + +exit: + return return_value; +} + +PyDoc_STRVAR(deque_insert__doc__, +"insert($self, index, value, /)\n" +"--\n" +"\n" +"Insert value before index."); + +#define DEQUE_INSERT_METHODDEF \ + {"insert", _PyCFunction_CAST(deque_insert), METH_FASTCALL, deque_insert__doc__}, + +static PyObject * +deque_insert_impl(dequeobject *deque, Py_ssize_t index, PyObject *value); + +static PyObject * +deque_insert(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t index; + PyObject *value; + + if (!_PyArg_CheckPositional("insert", nargs, 2, 2)) { + goto exit; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + index = ival; + } + value = args[1]; + return_value = deque_insert_impl(deque, index, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(deque_remove__doc__, +"remove($self, value, /)\n" +"--\n" +"\n" +"Remove first occurrence of value."); + +#define DEQUE_REMOVE_METHODDEF \ + {"remove", (PyCFunction)deque_remove, METH_O, deque_remove__doc__}, + +PyDoc_STRVAR(deque___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n" +"Return state information for pickling."); + +#define DEQUE___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)deque___reduce__, METH_NOARGS, deque___reduce____doc__}, + +static PyObject * +deque___reduce___impl(dequeobject *deque); + +static PyObject * +deque___reduce__(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + return deque___reduce___impl(deque); +} + +PyDoc_STRVAR(deque_init__doc__, +"deque([iterable[, maxlen]])\n" +"--\n" +"\n" +"A list-like sequence optimized for data accesses near its endpoints."); + +static int +deque_init_impl(dequeobject *deque, PyObject *iterable, PyObject *maxlenobj); + +static int +deque_init(PyObject *deque, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(iterable), &_Py_ID(maxlen), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"iterable", "maxlen", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "deque", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + PyObject *iterable = NULL; + PyObject *maxlenobj = NULL; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 2, 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (fastargs[0]) { + iterable = fastargs[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + maxlenobj = fastargs[1]; +skip_optional_pos: + return_value = deque_init_impl((dequeobject *)deque, iterable, maxlenobj); + +exit: + return return_value; +} + +PyDoc_STRVAR(deque___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n" +"Return the size of the deque in memory, in bytes."); + +#define DEQUE___SIZEOF___METHODDEF \ + {"__sizeof__", (PyCFunction)deque___sizeof__, METH_NOARGS, deque___sizeof____doc__}, + +static PyObject * +deque___sizeof___impl(dequeobject *deque); + +static PyObject * +deque___sizeof__(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + return deque___sizeof___impl(deque); +} + +PyDoc_STRVAR(deque___reversed____doc__, +"__reversed__($self, /)\n" +"--\n" +"\n" +"Return a reverse iterator over the deque."); + +#define DEQUE___REVERSED___METHODDEF \ + {"__reversed__", (PyCFunction)deque___reversed__, METH_NOARGS, deque___reversed____doc__}, + +static PyObject * +deque___reversed___impl(dequeobject *deque); + +static PyObject * +deque___reversed__(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + return deque___reversed___impl(deque); +} + PyDoc_STRVAR(_collections__count_elements__doc__, "_count_elements($module, mapping, iterable, /)\n" "--\n" @@ -72,4 +488,4 @@ tuplegetter_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=c896a72f8c45930d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3633a5cbc23e8440 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_curses_panel.c.h b/Modules/clinic/_curses_panel.c.h index 7945d93b5433f7d..457f71370afda9f 100644 --- a/Modules/clinic/_curses_panel.c.h +++ b/Modules/clinic/_curses_panel.c.h @@ -19,7 +19,7 @@ _curses_panel_panel_bottom_impl(PyCursesPanelObject *self, PyTypeObject *cls); static PyObject * _curses_panel_panel_bottom(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "bottom() takes no arguments"); return NULL; } @@ -43,7 +43,7 @@ _curses_panel_panel_hide_impl(PyCursesPanelObject *self, PyTypeObject *cls); static PyObject * _curses_panel_panel_hide(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "hide() takes no arguments"); return NULL; } @@ -65,7 +65,7 @@ _curses_panel_panel_show_impl(PyCursesPanelObject *self, PyTypeObject *cls); static PyObject * _curses_panel_panel_show(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "show() takes no arguments"); return NULL; } @@ -87,7 +87,7 @@ _curses_panel_panel_top_impl(PyCursesPanelObject *self, PyTypeObject *cls); static PyObject * _curses_panel_panel_top(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "top() takes no arguments"); return NULL; } @@ -327,7 +327,7 @@ _curses_panel_panel_userptr_impl(PyCursesPanelObject *self, static PyObject * _curses_panel_panel_userptr(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "userptr() takes no arguments"); return NULL; } @@ -418,4 +418,4 @@ _curses_panel_update_panels(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _curses_panel_update_panels_impl(module); } -/*[clinic end generated code: output=636beecf71d96ff1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7bac14e9a1194c87 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_datetimemodule.c.h b/Modules/clinic/_datetimemodule.c.h index 1ee50fc2a137621..48499e0aaf7783d 100644 --- a/Modules/clinic/_datetimemodule.c.h +++ b/Modules/clinic/_datetimemodule.c.h @@ -82,6 +82,207 @@ iso_calendar_date_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return return_value; } +PyDoc_STRVAR(datetime_date_replace__doc__, +"replace($self, /, year=unchanged, month=unchanged, day=unchanged)\n" +"--\n" +"\n" +"Return date with new specified fields."); + +#define DATETIME_DATE_REPLACE_METHODDEF \ + {"replace", _PyCFunction_CAST(datetime_date_replace), METH_FASTCALL|METH_KEYWORDS, datetime_date_replace__doc__}, + +static PyObject * +datetime_date_replace_impl(PyDateTime_Date *self, int year, int month, + int day); + +static PyObject * +datetime_date_replace(PyDateTime_Date *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(year), &_Py_ID(month), &_Py_ID(day), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"year", "month", "day", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "replace", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int year = GET_YEAR(self); + int month = GET_MONTH(self); + int day = GET_DAY(self); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 3, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + year = PyLong_AsInt(args[0]); + if (year == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[1]) { + month = PyLong_AsInt(args[1]); + if (month == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + day = PyLong_AsInt(args[2]); + if (day == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_pos: + return_value = datetime_date_replace_impl(self, year, month, day); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time_replace__doc__, +"replace($self, /, hour=unchanged, minute=unchanged, second=unchanged,\n" +" microsecond=unchanged, tzinfo=unchanged, *, fold=unchanged)\n" +"--\n" +"\n" +"Return time with new specified fields."); + +#define DATETIME_TIME_REPLACE_METHODDEF \ + {"replace", _PyCFunction_CAST(datetime_time_replace), METH_FASTCALL|METH_KEYWORDS, datetime_time_replace__doc__}, + +static PyObject * +datetime_time_replace_impl(PyDateTime_Time *self, int hour, int minute, + int second, int microsecond, PyObject *tzinfo, + int fold); + +static PyObject * +datetime_time_replace(PyDateTime_Time *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 6 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(hour), &_Py_ID(minute), &_Py_ID(second), &_Py_ID(microsecond), &_Py_ID(tzinfo), &_Py_ID(fold), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"hour", "minute", "second", "microsecond", "tzinfo", "fold", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "replace", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[6]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int hour = TIME_GET_HOUR(self); + int minute = TIME_GET_MINUTE(self); + int second = TIME_GET_SECOND(self); + int microsecond = TIME_GET_MICROSECOND(self); + PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None; + int fold = TIME_GET_FOLD(self); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 5, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + hour = PyLong_AsInt(args[0]); + if (hour == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[1]) { + minute = PyLong_AsInt(args[1]); + if (minute == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[2]) { + second = PyLong_AsInt(args[2]); + if (second == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[3]) { + microsecond = PyLong_AsInt(args[3]); + if (microsecond == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[4]) { + tzinfo = args[4]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + fold = PyLong_AsInt(args[5]); + if (fold == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_kwonly: + return_value = datetime_time_replace_impl(self, hour, minute, second, microsecond, tzinfo, fold); + +exit: + return return_value; +} + PyDoc_STRVAR(datetime_datetime_now__doc__, "now($type, /, tz=None)\n" "--\n" @@ -146,4 +347,153 @@ datetime_datetime_now(PyTypeObject *type, PyObject *const *args, Py_ssize_t narg exit: return return_value; } -/*[clinic end generated code: output=562813dd3e164794 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(datetime_datetime_replace__doc__, +"replace($self, /, year=unchanged, month=unchanged, day=unchanged,\n" +" hour=unchanged, minute=unchanged, second=unchanged,\n" +" microsecond=unchanged, tzinfo=unchanged, *, fold=unchanged)\n" +"--\n" +"\n" +"Return datetime with new specified fields."); + +#define DATETIME_DATETIME_REPLACE_METHODDEF \ + {"replace", _PyCFunction_CAST(datetime_datetime_replace), METH_FASTCALL|METH_KEYWORDS, datetime_datetime_replace__doc__}, + +static PyObject * +datetime_datetime_replace_impl(PyDateTime_DateTime *self, int year, + int month, int day, int hour, int minute, + int second, int microsecond, PyObject *tzinfo, + int fold); + +static PyObject * +datetime_datetime_replace(PyDateTime_DateTime *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 9 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(year), &_Py_ID(month), &_Py_ID(day), &_Py_ID(hour), &_Py_ID(minute), &_Py_ID(second), &_Py_ID(microsecond), &_Py_ID(tzinfo), &_Py_ID(fold), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"year", "month", "day", "hour", "minute", "second", "microsecond", "tzinfo", "fold", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "replace", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[9]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int year = GET_YEAR(self); + int month = GET_MONTH(self); + int day = GET_DAY(self); + int hour = DATE_GET_HOUR(self); + int minute = DATE_GET_MINUTE(self); + int second = DATE_GET_SECOND(self); + int microsecond = DATE_GET_MICROSECOND(self); + PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None; + int fold = DATE_GET_FOLD(self); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 8, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + year = PyLong_AsInt(args[0]); + if (year == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[1]) { + month = PyLong_AsInt(args[1]); + if (month == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[2]) { + day = PyLong_AsInt(args[2]); + if (day == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[3]) { + hour = PyLong_AsInt(args[3]); + if (hour == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[4]) { + minute = PyLong_AsInt(args[4]); + if (minute == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[5]) { + second = PyLong_AsInt(args[5]); + if (second == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[6]) { + microsecond = PyLong_AsInt(args[6]); + if (microsecond == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[7]) { + tzinfo = args[7]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + fold = PyLong_AsInt(args[8]); + if (fold == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_kwonly: + return_value = datetime_datetime_replace_impl(self, year, month, day, hour, minute, second, microsecond, tzinfo, fold); + +exit: + return return_value; +} +/*[clinic end generated code: output=c7a04b865b1e0890 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_dbmmodule.c.h b/Modules/clinic/_dbmmodule.c.h index 5a4aba2825e03a6..d06271e18a49b25 100644 --- a/Modules/clinic/_dbmmodule.c.h +++ b/Modules/clinic/_dbmmodule.c.h @@ -37,7 +37,7 @@ _dbm_dbm_keys_impl(dbmobject *self, PyTypeObject *cls); static PyObject * _dbm_dbm_keys(dbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "keys() takes no arguments"); return NULL; } @@ -149,7 +149,7 @@ _dbm_dbm_clear_impl(dbmobject *self, PyTypeObject *cls); static PyObject * _dbm_dbm_clear(dbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "clear() takes no arguments"); return NULL; } @@ -218,4 +218,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=96fdd4bd7bd256c5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=743ce0cea116747e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h index 02375c8a61e73e4..9622591a1aa8552 100644 --- a/Modules/clinic/_elementtree.c.h +++ b/Modules/clinic/_elementtree.c.h @@ -87,7 +87,7 @@ _elementtree_Element___copy___impl(ElementObject *self, PyTypeObject *cls); static PyObject * _elementtree_Element___copy__(ElementObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "__copy__() takes no arguments"); return NULL; } @@ -644,7 +644,7 @@ _elementtree_Element_itertext_impl(ElementObject *self, PyTypeObject *cls); static PyObject * _elementtree_Element_itertext(ElementObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "itertext() takes no arguments"); return NULL; } @@ -1219,4 +1219,4 @@ _elementtree_XMLParser__setevents(XMLParserObject *self, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=8fdaa17d3262800a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=218ec9e6a889f796 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_gdbmmodule.c.h b/Modules/clinic/_gdbmmodule.c.h index c7164e519d0e7de..626e4678809d4f5 100644 --- a/Modules/clinic/_gdbmmodule.c.h +++ b/Modules/clinic/_gdbmmodule.c.h @@ -106,7 +106,7 @@ _gdbm_gdbm_keys_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_keys(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "keys() takes no arguments"); return NULL; } @@ -132,7 +132,7 @@ _gdbm_gdbm_firstkey_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_firstkey(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "firstkey() takes no arguments"); return NULL; } @@ -211,7 +211,7 @@ _gdbm_gdbm_reorganize_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_reorganize(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "reorganize() takes no arguments"); return NULL; } @@ -236,7 +236,7 @@ _gdbm_gdbm_sync_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_sync(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "sync() takes no arguments"); return NULL; } @@ -258,7 +258,7 @@ _gdbm_gdbm_clear_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_clear(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "clear() takes no arguments"); return NULL; } @@ -340,4 +340,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=c5ee922363d5a81f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6b4c19905ac9967d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lsprof.c.h b/Modules/clinic/_lsprof.c.h index dfc003eb54774ca..b3b7fda5660bfd5 100644 --- a/Modules/clinic/_lsprof.c.h +++ b/Modules/clinic/_lsprof.c.h @@ -39,10 +39,10 @@ _lsprof_Profiler_getstats_impl(ProfilerObject *self, PyTypeObject *cls); static PyObject * _lsprof_Profiler_getstats(ProfilerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "getstats() takes no arguments"); return NULL; } return _lsprof_Profiler_getstats_impl(self, cls); } -/*[clinic end generated code: output=0615a53cce828f06 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5c9d87d89863dc83 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_pickle.c.h b/Modules/clinic/_pickle.c.h index fb086925e3941d6..5a6ae7be6b6ea77 100644 --- a/Modules/clinic/_pickle.c.h +++ b/Modules/clinic/_pickle.c.h @@ -328,7 +328,7 @@ _pickle_Unpickler_load_impl(UnpicklerObject *self, PyTypeObject *cls); static PyObject * _pickle_Unpickler_load(UnpicklerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "load() takes no arguments"); return NULL; } @@ -1077,4 +1077,4 @@ _pickle_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=ebe78653233827a6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bd63c85a8737b0aa input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_queuemodule.c.h b/Modules/clinic/_queuemodule.c.h index 8e2a430835e35f7..6f4c715c7229651 100644 --- a/Modules/clinic/_queuemodule.c.h +++ b/Modules/clinic/_queuemodule.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_NoKeywords() PyDoc_STRVAR(simplequeue_new__doc__, @@ -107,7 +108,9 @@ _queue_SimpleQueue_put(simplequeueobject *self, PyObject *const *args, Py_ssize_ } timeout = args[2]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _queue_SimpleQueue_put_impl(self, item, block, timeout); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -165,7 +168,9 @@ _queue_SimpleQueue_put_nowait(simplequeueobject *self, PyObject *const *args, Py goto exit; } item = args[0]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _queue_SimpleQueue_put_nowait_impl(self, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -244,7 +249,9 @@ _queue_SimpleQueue_get(simplequeueobject *self, PyTypeObject *cls, PyObject *con } timeout_obj = args[1]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _queue_SimpleQueue_get_impl(self, cls, block, timeout_obj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -269,11 +276,18 @@ _queue_SimpleQueue_get_nowait_impl(simplequeueobject *self, static PyObject * _queue_SimpleQueue_get_nowait(simplequeueobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + PyObject *return_value = NULL; + + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "get_nowait() takes no arguments"); - return NULL; + goto exit; } - return _queue_SimpleQueue_get_nowait_impl(self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _queue_SimpleQueue_get_nowait_impl(self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_queue_SimpleQueue_empty__doc__, @@ -294,7 +308,9 @@ _queue_SimpleQueue_empty(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) PyObject *return_value = NULL; int _return_value; + Py_BEGIN_CRITICAL_SECTION(self); _return_value = _queue_SimpleQueue_empty_impl(self); + Py_END_CRITICAL_SECTION(); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -322,7 +338,9 @@ _queue_SimpleQueue_qsize(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) PyObject *return_value = NULL; Py_ssize_t _return_value; + Py_BEGIN_CRITICAL_SECTION(self); _return_value = _queue_SimpleQueue_qsize_impl(self); + Py_END_CRITICAL_SECTION(); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -331,4 +349,4 @@ _queue_SimpleQueue_qsize(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=457310b20cb61cf8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=44a718f40072018a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 19c0f619b92f45c..2940f16a2cb7f6a 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -1297,7 +1297,9 @@ _ssl_RAND_add(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&view, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&view, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &view, PyBUF_SIMPLE) != 0) { @@ -1662,4 +1664,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=6342ea0062ab16c7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fd1c3378fbba5240 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index fea30e778381dea..bb516be37ec3f09 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -3141,4 +3141,26 @@ clone_with_conv_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py exit: return return_value; } -/*[clinic end generated code: output=90743ac900d60f9f input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_testclinic_TestClass_meth_method_no_params__doc__, +"meth_method_no_params($self, /)\n" +"--\n" +"\n"); + +#define _TESTCLINIC_TESTCLASS_METH_METHOD_NO_PARAMS_METHODDEF \ + {"meth_method_no_params", _PyCFunction_CAST(_testclinic_TestClass_meth_method_no_params), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _testclinic_TestClass_meth_method_no_params__doc__}, + +static PyObject * +_testclinic_TestClass_meth_method_no_params_impl(PyObject *self, + PyTypeObject *cls); + +static PyObject * +_testclinic_TestClass_meth_method_no_params(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "meth_method_no_params() takes no arguments"); + return NULL; + } + return _testclinic_TestClass_meth_method_no_params_impl(self, cls); +} +/*[clinic end generated code: output=6520c1ca5392a3f0 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testmultiphase.c.h b/Modules/clinic/_testmultiphase.c.h index c0a00954c16cbe2..7ac361ece1acc3e 100644 --- a/Modules/clinic/_testmultiphase.c.h +++ b/Modules/clinic/_testmultiphase.c.h @@ -27,7 +27,7 @@ _testmultiphase_StateAccessType_get_defining_module_impl(StateAccessTypeObject * static PyObject * _testmultiphase_StateAccessType_get_defining_module(StateAccessTypeObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "get_defining_module() takes no arguments"); return NULL; } @@ -50,7 +50,7 @@ _testmultiphase_StateAccessType_getmodulebydef_bad_def_impl(StateAccessTypeObjec static PyObject * _testmultiphase_StateAccessType_getmodulebydef_bad_def(StateAccessTypeObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "getmodulebydef_bad_def() takes no arguments"); return NULL; } @@ -156,10 +156,10 @@ _testmultiphase_StateAccessType_get_count_impl(StateAccessTypeObject *self, static PyObject * _testmultiphase_StateAccessType_get_count(StateAccessTypeObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "get_count() takes no arguments"); return NULL; } return _testmultiphase_StateAccessType_get_count_impl(self, cls); } -/*[clinic end generated code: output=d8c262af27b3b98d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2c199bad52e9cda7 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index 3a3231c051ef713..d1052f38919ddef 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -162,7 +162,7 @@ PyDoc_STRVAR(_winapi_CreateFile__doc__, {"CreateFile", _PyCFunction_CAST(_winapi_CreateFile), METH_FASTCALL, _winapi_CreateFile__doc__}, static HANDLE -_winapi_CreateFile_impl(PyObject *module, LPCTSTR file_name, +_winapi_CreateFile_impl(PyObject *module, LPCWSTR file_name, DWORD desired_access, DWORD share_mode, LPSECURITY_ATTRIBUTES security_attributes, DWORD creation_disposition, @@ -172,7 +172,7 @@ static PyObject * _winapi_CreateFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - LPCTSTR file_name; + LPCWSTR file_name = NULL; DWORD desired_access; DWORD share_mode; LPSECURITY_ATTRIBUTES security_attributes; @@ -181,8 +181,8 @@ _winapi_CreateFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs) HANDLE template_file; HANDLE _return_value; - if (!_PyArg_ParseStack(args, nargs, "skk" F_POINTER "kk" F_HANDLE ":CreateFile", - &file_name, &desired_access, &share_mode, &security_attributes, &creation_disposition, &flags_and_attributes, &template_file)) { + if (!_PyArg_ParseStack(args, nargs, "O&kk" F_POINTER "kk" F_HANDLE ":CreateFile", + _PyUnicode_WideCharString_Converter, &file_name, &desired_access, &share_mode, &security_attributes, &creation_disposition, &flags_and_attributes, &template_file)) { goto exit; } _return_value = _winapi_CreateFile_impl(module, file_name, desired_access, share_mode, security_attributes, creation_disposition, flags_and_attributes, template_file); @@ -195,6 +195,9 @@ _winapi_CreateFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return_value = HANDLE_TO_PYNUM(_return_value); exit: + /* Cleanup for file_name */ + PyMem_Free((void *)file_name); + return return_value; } @@ -1479,4 +1482,4 @@ _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } -/*[clinic end generated code: output=e1a9908bb82a6379 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2350d4f2275d3a6f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index dbce03135416499..0b764e43e194375 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -652,7 +652,7 @@ array_arrayiterator___reduce___impl(arrayiterobject *self, PyTypeObject *cls); static PyObject * array_arrayiterator___reduce__(arrayiterobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "__reduce__() takes no arguments"); return NULL; } @@ -667,4 +667,4 @@ PyDoc_STRVAR(array_arrayiterator___setstate____doc__, #define ARRAY_ARRAYITERATOR___SETSTATE___METHODDEF \ {"__setstate__", (PyCFunction)array_arrayiterator___setstate__, METH_O, array_arrayiterator___setstate____doc__}, -/*[clinic end generated code: output=bf086c01e7e482bf input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3be987238a4bb431 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/md5module.c.h b/Modules/clinic/md5module.c.h index 7d4d3108dab9b6c..ee7fb3d7c613f22 100644 --- a/Modules/clinic/md5module.c.h +++ b/Modules/clinic/md5module.c.h @@ -23,7 +23,7 @@ MD5Type_copy_impl(MD5object *self, PyTypeObject *cls); static PyObject * MD5Type_copy(MD5object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -148,4 +148,4 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw exit: return return_value; } -/*[clinic end generated code: output=bfadda44914804a8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4dbca39332d3f52f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index ba3e1cfa8dbc21b..b49d64d4281889e 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -4465,6 +4465,156 @@ os_sched_getaffinity(PyObject *module, PyObject *arg) #endif /* defined(HAVE_SCHED_H) && defined(HAVE_SCHED_SETAFFINITY) */ +#if defined(HAVE_POSIX_OPENPT) + +PyDoc_STRVAR(os_posix_openpt__doc__, +"posix_openpt($module, oflag, /)\n" +"--\n" +"\n" +"Open and return a file descriptor for a master pseudo-terminal device.\n" +"\n" +"Performs a posix_openpt() C function call. The oflag argument is used to\n" +"set file status flags and file access modes as specified in the manual page\n" +"of posix_openpt() of your system."); + +#define OS_POSIX_OPENPT_METHODDEF \ + {"posix_openpt", (PyCFunction)os_posix_openpt, METH_O, os_posix_openpt__doc__}, + +static int +os_posix_openpt_impl(PyObject *module, int oflag); + +static PyObject * +os_posix_openpt(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int oflag; + int _return_value; + + oflag = PyLong_AsInt(arg); + if (oflag == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = os_posix_openpt_impl(module, oflag); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); + +exit: + return return_value; +} + +#endif /* defined(HAVE_POSIX_OPENPT) */ + +#if defined(HAVE_GRANTPT) + +PyDoc_STRVAR(os_grantpt__doc__, +"grantpt($module, fd, /)\n" +"--\n" +"\n" +"Grant access to the slave pseudo-terminal device.\n" +"\n" +" fd\n" +" File descriptor of a master pseudo-terminal device.\n" +"\n" +"Performs a grantpt() C function call."); + +#define OS_GRANTPT_METHODDEF \ + {"grantpt", (PyCFunction)os_grantpt, METH_O, os_grantpt__doc__}, + +static PyObject * +os_grantpt_impl(PyObject *module, int fd); + +static PyObject * +os_grantpt(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int fd; + + if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + goto exit; + } + return_value = os_grantpt_impl(module, fd); + +exit: + return return_value; +} + +#endif /* defined(HAVE_GRANTPT) */ + +#if defined(HAVE_UNLOCKPT) + +PyDoc_STRVAR(os_unlockpt__doc__, +"unlockpt($module, fd, /)\n" +"--\n" +"\n" +"Unlock a pseudo-terminal master/slave pair.\n" +"\n" +" fd\n" +" File descriptor of a master pseudo-terminal device.\n" +"\n" +"Performs an unlockpt() C function call."); + +#define OS_UNLOCKPT_METHODDEF \ + {"unlockpt", (PyCFunction)os_unlockpt, METH_O, os_unlockpt__doc__}, + +static PyObject * +os_unlockpt_impl(PyObject *module, int fd); + +static PyObject * +os_unlockpt(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int fd; + + if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + goto exit; + } + return_value = os_unlockpt_impl(module, fd); + +exit: + return return_value; +} + +#endif /* defined(HAVE_UNLOCKPT) */ + +#if (defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R)) + +PyDoc_STRVAR(os_ptsname__doc__, +"ptsname($module, fd, /)\n" +"--\n" +"\n" +"Return the name of the slave pseudo-terminal device.\n" +"\n" +" fd\n" +" File descriptor of a master pseudo-terminal device.\n" +"\n" +"If the ptsname_r() C function is available, it is called;\n" +"otherwise, performs a ptsname() C function call."); + +#define OS_PTSNAME_METHODDEF \ + {"ptsname", (PyCFunction)os_ptsname, METH_O, os_ptsname__doc__}, + +static PyObject * +os_ptsname_impl(PyObject *module, int fd); + +static PyObject * +os_ptsname(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int fd; + + if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + goto exit; + } + return_value = os_ptsname_impl(module, fd); + +exit: + return return_value; +} + +#endif /* (defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R)) */ + #if (defined(HAVE_OPENPTY) || defined(HAVE__GETPTY) || defined(HAVE_DEV_PTMX)) PyDoc_STRVAR(os_openpty__doc__, @@ -11062,7 +11212,7 @@ os_DirEntry_is_symlink(DirEntry *self, PyTypeObject *defining_class, PyObject *c PyObject *return_value = NULL; int _return_value; - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "is_symlink() takes no arguments"); goto exit; } @@ -11991,6 +12141,22 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #define OS_SCHED_GETAFFINITY_METHODDEF #endif /* !defined(OS_SCHED_GETAFFINITY_METHODDEF) */ +#ifndef OS_POSIX_OPENPT_METHODDEF + #define OS_POSIX_OPENPT_METHODDEF +#endif /* !defined(OS_POSIX_OPENPT_METHODDEF) */ + +#ifndef OS_GRANTPT_METHODDEF + #define OS_GRANTPT_METHODDEF +#endif /* !defined(OS_GRANTPT_METHODDEF) */ + +#ifndef OS_UNLOCKPT_METHODDEF + #define OS_UNLOCKPT_METHODDEF +#endif /* !defined(OS_UNLOCKPT_METHODDEF) */ + +#ifndef OS_PTSNAME_METHODDEF + #define OS_PTSNAME_METHODDEF +#endif /* !defined(OS_PTSNAME_METHODDEF) */ + #ifndef OS_OPENPTY_METHODDEF #define OS_OPENPTY_METHODDEF #endif /* !defined(OS_OPENPTY_METHODDEF) */ @@ -12422,4 +12588,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=18c128534c355d84 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=268af5cbc8baa9d4 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha1module.c.h b/Modules/clinic/sha1module.c.h index ee391656fb67c31..b89c7e505c788e9 100644 --- a/Modules/clinic/sha1module.c.h +++ b/Modules/clinic/sha1module.c.h @@ -23,7 +23,7 @@ SHA1Type_copy_impl(SHA1object *self, PyTypeObject *cls); static PyObject * SHA1Type_copy(SHA1object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -148,4 +148,4 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * exit: return return_value; } -/*[clinic end generated code: output=41fc7579213b57b4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=af5a640df662066f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha2module.c.h b/Modules/clinic/sha2module.c.h index ec31d5545be4c15..cf4b88d52856b86 100644 --- a/Modules/clinic/sha2module.c.h +++ b/Modules/clinic/sha2module.c.h @@ -23,7 +23,7 @@ SHA256Type_copy_impl(SHA256object *self, PyTypeObject *cls); static PyObject * SHA256Type_copy(SHA256object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -45,7 +45,7 @@ SHA512Type_copy_impl(SHA512object *self, PyTypeObject *cls); static PyObject * SHA512Type_copy(SHA512object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -437,4 +437,4 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject exit: return return_value; } -/*[clinic end generated code: output=1482d9de086e45c4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b46da764024b1764 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index 6b09abe309bf486..7ff3edf5a557f80 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -637,7 +637,7 @@ zlib_Compress_copy_impl(compobject *self, PyTypeObject *cls); static PyObject * zlib_Compress_copy(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -662,7 +662,7 @@ zlib_Compress___copy___impl(compobject *self, PyTypeObject *cls); static PyObject * zlib_Compress___copy__(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "__copy__() takes no arguments"); return NULL; } @@ -735,7 +735,7 @@ zlib_Decompress_copy_impl(compobject *self, PyTypeObject *cls); static PyObject * zlib_Decompress_copy(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -760,7 +760,7 @@ zlib_Decompress___copy___impl(compobject *self, PyTypeObject *cls); static PyObject * zlib_Decompress___copy__(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "__copy__() takes no arguments"); return NULL; } @@ -1098,4 +1098,4 @@ zlib_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #ifndef ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #define ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #endif /* !defined(ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF) */ -/*[clinic end generated code: output=6dd97dc851c39031 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8bb840fb6af43dd4 input=a9049054013a1b77]*/ diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index a2e3c2300b3ce85..95d646c9c65b3c4 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -119,6 +119,13 @@ faulthandler_get_fileno(PyObject **file_ptr) } } else if (PyLong_Check(file)) { + if (PyBool_Check(file)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "bool is used as a file descriptor", 1)) + { + return -1; + } + } fd = PyLong_AsInt(file); if (fd == -1 && PyErr_Occurred()) return -1; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index ffddef34ecce7a5..a2b66b9b78c169d 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -235,6 +235,26 @@ referentsvisit(PyObject *obj, void *arg) return PyList_Append(list, obj) < 0; } +static int +append_referrents(PyObject *result, PyObject *args) +{ + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(args); i++) { + PyObject *obj = PyTuple_GET_ITEM(args, i); + if (!_PyObject_IS_GC(obj)) { + continue; + } + + traverseproc traverse = Py_TYPE(obj)->tp_traverse; + if (!traverse) { + continue; + } + if (traverse(obj, referentsvisit, result)) { + return -1; + } + } + return 0; +} + /*[clinic input] gc.get_referents @@ -247,29 +267,24 @@ static PyObject * gc_get_referents_impl(PyObject *module, PyObject *args) /*[clinic end generated code: output=d47dc02cefd06fe8 input=b3ceab0c34038cbf]*/ { - Py_ssize_t i; if (PySys_Audit("gc.get_referents", "(O)", args) < 0) { return NULL; } + PyInterpreterState *interp = _PyInterpreterState_GET(); PyObject *result = PyList_New(0); if (result == NULL) return NULL; - for (i = 0; i < PyTuple_GET_SIZE(args); i++) { - traverseproc traverse; - PyObject *obj = PyTuple_GET_ITEM(args, i); + // NOTE: stop the world is a no-op in default build + _PyEval_StopTheWorld(interp); + int err = append_referrents(result, args); + _PyEval_StartTheWorld(interp); - if (!_PyObject_IS_GC(obj)) - continue; - traverse = Py_TYPE(obj)->tp_traverse; - if (! traverse) - continue; - if (traverse(obj, referentsvisit, result)) { - Py_DECREF(result); - return NULL; - } + if (err < 0) { + Py_CLEAR(result); } + return result; } diff --git a/Modules/getpath.c b/Modules/getpath.c index a3c8fc269d1c3cf..abed139028244ad 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -262,6 +262,10 @@ getpath_joinpath(PyObject *Py_UNUSED(self), PyObject *args) } /* Convert all parts to wchar and accumulate max final length */ wchar_t **parts = (wchar_t **)PyMem_Malloc(n * sizeof(wchar_t *)); + if (parts == NULL) { + PyErr_NoMemory(); + return NULL; + } memset(parts, 0, n * sizeof(wchar_t *)); Py_ssize_t cchFinal = 0; Py_ssize_t first = 0; diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 2a796c1c55d2f0a..a877bfcd6afb687 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -2074,11 +2074,6 @@ math_trunc(PyObject *module, PyObject *x) return PyFloat_Type.tp_as_number->nb_int(x); } - if (!_PyType_IsReady(Py_TYPE(x))) { - if (PyType_Ready(Py_TYPE(x)) < 0) - return NULL; - } - math_module_state *state = get_math_module_state(module); trunc = _PyObject_LookupSpecial(x, state->str___trunc__); if (trunc == NULL) { @@ -2570,7 +2565,7 @@ math_dist_impl(PyObject *module, PyObject *p, PyObject *q) goto error_exit; } if (n > NUM_STACK_ELEMS) { - diffs = (double *) PyObject_Malloc(n * sizeof(double)); + diffs = (double *) PyMem_Malloc(n * sizeof(double)); if (diffs == NULL) { PyErr_NoMemory(); goto error_exit; @@ -2590,7 +2585,7 @@ math_dist_impl(PyObject *module, PyObject *p, PyObject *q) } result = vector_norm(n, diffs, max, found_nan); if (diffs != diffs_on_stack) { - PyObject_Free(diffs); + PyMem_Free(diffs); } if (p_allocated) { Py_DECREF(p); @@ -2602,7 +2597,7 @@ math_dist_impl(PyObject *module, PyObject *p, PyObject *q) error_exit: if (diffs != diffs_on_stack) { - PyObject_Free(diffs); + PyMem_Free(diffs); } if (p_allocated) { Py_DECREF(p); @@ -2626,7 +2621,7 @@ math_hypot(PyObject *self, PyObject *const *args, Py_ssize_t nargs) double *coordinates = coord_on_stack; if (nargs > NUM_STACK_ELEMS) { - coordinates = (double *) PyObject_Malloc(nargs * sizeof(double)); + coordinates = (double *) PyMem_Malloc(nargs * sizeof(double)); if (coordinates == NULL) { return PyErr_NoMemory(); } @@ -2643,13 +2638,13 @@ math_hypot(PyObject *self, PyObject *const *args, Py_ssize_t nargs) } result = vector_norm(nargs, coordinates, max, found_nan); if (coordinates != coord_on_stack) { - PyObject_Free(coordinates); + PyMem_Free(coordinates); } return PyFloat_FromDouble(result); error_exit: if (coordinates != coord_on_stack) { - PyObject_Free(coordinates); + PyMem_Free(coordinates); } return NULL; } diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 007fc1cb116f84c..d05b4ba723ce8c8 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -52,6 +52,12 @@ # define EX_OK EXIT_SUCCESS #endif +#ifdef __APPLE__ + /* Needed for the implementation of os.statvfs */ +# include <sys/param.h> +# include <sys/mount.h> +#endif + /* On android API level 21, 'AT_EACCESS' is not declared although * HAVE_FACCESSAT is defined. */ #ifdef __ANDROID__ @@ -637,6 +643,10 @@ PyOS_AfterFork_Child(void) tstate->native_thread_id = PyThread_get_thread_native_id(); #endif +#ifdef Py_GIL_DISABLED + _Py_brc_after_fork(tstate->interp); +#endif + status = _PyEval_ReInitThreads(tstate); if (_PyStatus_EXCEPTION(status)) { goto fatal_error; @@ -969,6 +979,13 @@ _fd_converter(PyObject *o, int *p) int overflow; long long_value; + if (PyBool_Check(o)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "bool is used as a file descriptor", 1)) + { + return 0; + } + } PyObject *index = _PyNumber_Index(o); if (index == NULL) { return 0; @@ -1620,7 +1637,7 @@ convertenviron(void) Py_DECREF(d); return NULL; } - if (PyDict_SetDefault(d, k, v) == NULL) { + if (PyDict_SetDefaultRef(d, k, v, NULL) < 0) { Py_DECREF(v); Py_DECREF(k); Py_DECREF(d); @@ -8358,6 +8375,149 @@ os_sched_getaffinity_impl(PyObject *module, pid_t pid) #endif /* HAVE_SCHED_H */ +#ifdef HAVE_POSIX_OPENPT +/*[clinic input] +os.posix_openpt -> int + + oflag: int + / + +Open and return a file descriptor for a master pseudo-terminal device. + +Performs a posix_openpt() C function call. The oflag argument is used to +set file status flags and file access modes as specified in the manual page +of posix_openpt() of your system. +[clinic start generated code]*/ + +static int +os_posix_openpt_impl(PyObject *module, int oflag) +/*[clinic end generated code: output=ee0bc2624305fc79 input=0de33d0e29693caa]*/ +{ + int fd; + +#if defined(O_CLOEXEC) + oflag |= O_CLOEXEC; +#endif + + fd = posix_openpt(oflag); + if (fd == -1) { + posix_error(); + return -1; + } + + // Just in case, likely a no-op given O_CLOEXEC above. + if (_Py_set_inheritable(fd, 0, NULL) < 0) { + close(fd); + return -1; + } + + return fd; +} +#endif /* HAVE_POSIX_OPENPT */ + +#ifdef HAVE_GRANTPT +/*[clinic input] +os.grantpt + + fd: fildes + File descriptor of a master pseudo-terminal device. + / + +Grant access to the slave pseudo-terminal device. + +Performs a grantpt() C function call. +[clinic start generated code]*/ + +static PyObject * +os_grantpt_impl(PyObject *module, int fd) +/*[clinic end generated code: output=dfd580015cf548ab input=0668e3b96760e849]*/ +{ + int ret; + int saved_errno; + PyOS_sighandler_t sig_saved; + + sig_saved = PyOS_setsig(SIGCHLD, SIG_DFL); + + ret = grantpt(fd); + if (ret == -1) + saved_errno = errno; + + PyOS_setsig(SIGCHLD, sig_saved); + + if (ret == -1) { + errno = saved_errno; + return posix_error(); + } + + Py_RETURN_NONE; +} +#endif /* HAVE_GRANTPT */ + +#ifdef HAVE_UNLOCKPT +/*[clinic input] +os.unlockpt + + fd: fildes + File descriptor of a master pseudo-terminal device. + / + +Unlock a pseudo-terminal master/slave pair. + +Performs an unlockpt() C function call. +[clinic start generated code]*/ + +static PyObject * +os_unlockpt_impl(PyObject *module, int fd) +/*[clinic end generated code: output=e08d354dec12d30c input=de7ab1f59f69a2b4]*/ +{ + if (unlockpt(fd) == -1) + return posix_error(); + + Py_RETURN_NONE; +} +#endif /* HAVE_UNLOCKPT */ + +#if defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R) +/*[clinic input] +os.ptsname + + fd: fildes + File descriptor of a master pseudo-terminal device. + / + +Return the name of the slave pseudo-terminal device. + +If the ptsname_r() C function is available, it is called; +otherwise, performs a ptsname() C function call. +[clinic start generated code]*/ + +static PyObject * +os_ptsname_impl(PyObject *module, int fd) +/*[clinic end generated code: output=ef300fadc5675872 input=1369ccc0546f3130]*/ +{ +#ifdef HAVE_PTSNAME_R + int ret; + char name[MAXPATHLEN+1]; + + ret = ptsname_r(fd, name, sizeof(name)); + if (ret != 0) { + errno = ret; + return posix_error(); + } +#else + char *name; + + name = ptsname(fd); + /* POSIX manpage: Upon failure, ptsname() shall return a null pointer and may set errno. + *MAY* set errno? Hmm... */ + if (name == NULL) + return posix_error(); +#endif /* HAVE_PTSNAME_R */ + + return PyUnicode_DecodeFSDefault(name); +} +#endif /* defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R) */ + /* AIX uses /dev/ptc but is otherwise the same as /dev/ptmx */ #if defined(HAVE_DEV_PTC) && !defined(HAVE_DEV_PTMX) # define DEV_PTY_FILE "/dev/ptc" @@ -12732,6 +12892,59 @@ os_WSTOPSIG_impl(PyObject *module, int status) #endif #include <sys/statvfs.h> +#ifdef __APPLE__ +/* On macOS struct statvfs uses 32-bit integers for block counts, + * resulting in overflow when filesystems are larger tan 4TB. Therefore + * os.statvfs is implemented in terms of statfs(2). + */ + +static PyObject* +_pystatvfs_fromstructstatfs(PyObject *module, struct statfs st) { + PyObject *StatVFSResultType = get_posix_state(module)->StatVFSResultType; + PyObject *v = PyStructSequence_New((PyTypeObject *)StatVFSResultType); + if (v == NULL) + return NULL; + + long flags = 0; + if (st.f_flags & MNT_RDONLY) { + flags |= ST_RDONLY; + } + if (st.f_flags & MNT_NOSUID) { + flags |= ST_NOSUID; + } + + _Static_assert(sizeof(st.f_blocks) == sizeof(long long), "assuming large file"); + + PyStructSequence_SET_ITEM(v, 0, PyLong_FromLong((long) st.f_iosize)); + PyStructSequence_SET_ITEM(v, 1, PyLong_FromLong((long) st.f_bsize)); + PyStructSequence_SET_ITEM(v, 2, + PyLong_FromLongLong((long long) st.f_blocks)); + PyStructSequence_SET_ITEM(v, 3, + PyLong_FromLongLong((long long) st.f_bfree)); + PyStructSequence_SET_ITEM(v, 4, + PyLong_FromLongLong((long long) st.f_bavail)); + PyStructSequence_SET_ITEM(v, 5, + PyLong_FromLongLong((long long) st.f_files)); + PyStructSequence_SET_ITEM(v, 6, + PyLong_FromLongLong((long long) st.f_ffree)); + PyStructSequence_SET_ITEM(v, 7, + PyLong_FromLongLong((long long) st.f_ffree)); + PyStructSequence_SET_ITEM(v, 8, PyLong_FromLong((long) flags)); + + PyStructSequence_SET_ITEM(v, 9, PyLong_FromLong((long) NAME_MAX)); + PyStructSequence_SET_ITEM(v, 10, PyLong_FromUnsignedLong(st.f_fsid.val[0])); + if (PyErr_Occurred()) { + Py_DECREF(v); + return NULL; + } + + return v; +} + +#else + + + static PyObject* _pystatvfs_fromstructstatvfs(PyObject *module, struct statvfs st) { PyObject *StatVFSResultType = get_posix_state(module)->StatVFSResultType; @@ -12783,6 +12996,8 @@ _pystatvfs_fromstructstatvfs(PyObject *module, struct statvfs st) { return v; } +#endif + /*[clinic input] os.fstatvfs @@ -12800,6 +13015,22 @@ os_fstatvfs_impl(PyObject *module, int fd) { int result; int async_err = 0; +#ifdef __APPLE__ + struct statfs st; + /* On macOS os.fstatvfs is implemented using fstatfs(2) because + * the former uses 32-bit values for block counts. + */ + do { + Py_BEGIN_ALLOW_THREADS + result = fstatfs(fd, &st); + Py_END_ALLOW_THREADS + } while (result != 0 && errno == EINTR && + !(async_err = PyErr_CheckSignals())); + if (result != 0) + return (!async_err) ? posix_error() : NULL; + + return _pystatvfs_fromstructstatfs(module, st); +#else struct statvfs st; do { @@ -12812,6 +13043,7 @@ os_fstatvfs_impl(PyObject *module, int fd) return (!async_err) ? posix_error() : NULL; return _pystatvfs_fromstructstatvfs(module, st); +#endif } #endif /* defined(HAVE_FSTATVFS) && defined(HAVE_SYS_STATVFS_H) */ @@ -12835,6 +13067,28 @@ os_statvfs_impl(PyObject *module, path_t *path) /*[clinic end generated code: output=87106dd1beb8556e input=3f5c35791c669bd9]*/ { int result; + +#ifdef __APPLE__ + /* On macOS os.statvfs is implemented using statfs(2)/fstatfs(2) because + * the former uses 32-bit values for block counts. + */ + struct statfs st; + + Py_BEGIN_ALLOW_THREADS + if (path->fd != -1) { + result = fstatfs(path->fd, &st); + } + else + result = statfs(path->narrow, &st); + Py_END_ALLOW_THREADS + + if (result) { + return path_error(path); + } + + return _pystatvfs_fromstructstatfs(module, st); + +#else struct statvfs st; Py_BEGIN_ALLOW_THREADS @@ -12852,6 +13106,7 @@ os_statvfs_impl(PyObject *module, path_t *path) } return _pystatvfs_fromstructstatvfs(module, st); +#endif } #endif /* defined(HAVE_STATVFS) && defined(HAVE_SYS_STATVFS_H) */ @@ -16275,6 +16530,10 @@ static PyMethodDef posix_methods[] = { OS_SCHED_YIELD_METHODDEF OS_SCHED_SETAFFINITY_METHODDEF OS_SCHED_GETAFFINITY_METHODDEF + OS_POSIX_OPENPT_METHODDEF + OS_GRANTPT_METHODDEF + OS_UNLOCKPT_METHODDEF + OS_PTSNAME_METHODDEF OS_OPENPTY_METHODDEF OS_LOGIN_TTY_METHODDEF OS_FORKPTY_METHODDEF diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index ec44892d101e44e..62cd262a7885e93 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -21,7 +21,7 @@ module pyexpat #define XML_COMBINED_VERSION (10000*XML_MAJOR_VERSION+100*XML_MINOR_VERSION+XML_MICRO_VERSION) static XML_Memory_Handling_Suite ExpatMemoryHandler = { - PyObject_Malloc, PyObject_Realloc, PyObject_Free}; + PyMem_Malloc, PyMem_Realloc, PyMem_Free}; enum HandlerTypes { StartElement, @@ -1615,7 +1615,8 @@ static int init_handler_descrs(pyexpat_state *state) if (descr == NULL) return -1; - if (PyDict_SetDefault(state->xml_parse_type->tp_dict, PyDescr_NAME(descr), descr) == NULL) { + if (PyDict_SetDefaultRef(state->xml_parse_type->tp_dict, + PyDescr_NAME(descr), descr, NULL) < 0) { Py_DECREF(descr); return -1; } diff --git a/Modules/termios.c b/Modules/termios.c index c4f0fd9d50044aa..69dbd88be5fcc20 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -98,6 +98,8 @@ termios_tcgetattr_impl(PyObject *module, int fd) struct termios mode; int r; + /* Alpine Linux can leave some fields uninitialized. */ + memset(&mode, 0, sizeof(mode)); Py_BEGIN_ALLOW_THREADS r = tcgetattr(fd, &mode); Py_END_ALLOW_THREADS diff --git a/Objects/abstract.c b/Objects/abstract.c index 1ec5c5b8c3dc2f5..07d4b89fe188c84 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -425,6 +425,12 @@ PyObject_AsWriteBuffer(PyObject *obj, int PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) { + if (flags != PyBUF_SIMPLE) { /* fast path */ + if (flags == PyBUF_READ || flags == PyBUF_WRITE) { + PyErr_BadInternalCall(); + return -1; + } + } PyBufferProcs *pb = Py_TYPE(obj)->tp_as_buffer; if (pb == NULL || pb->bf_getbuffer == NULL) { @@ -761,11 +767,17 @@ PyBuffer_FillInfo(Py_buffer *view, PyObject *obj, void *buf, Py_ssize_t len, return -1; } - if (((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE) && - (readonly == 1)) { - PyErr_SetString(PyExc_BufferError, - "Object is not writable."); - return -1; + if (flags != PyBUF_SIMPLE) { /* fast path */ + if (flags == PyBUF_READ || flags == PyBUF_WRITE) { + PyErr_BadInternalCall(); + return -1; + } + if (((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE) && + (readonly == 1)) { + PyErr_SetString(PyExc_BufferError, + "Object is not writable."); + return -1; + } } view->obj = Py_XNewRef(obj); diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 659de7d3dd5a994..acc59b926448ca4 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -132,7 +132,7 @@ PyByteArray_FromStringAndSize(const char *bytes, Py_ssize_t size) } else { alloc = size + 1; - new->ob_bytes = PyObject_Malloc(alloc); + new->ob_bytes = PyMem_Malloc(alloc); if (new->ob_bytes == NULL) { Py_DECREF(new); return PyErr_NoMemory(); @@ -221,17 +221,17 @@ PyByteArray_Resize(PyObject *self, Py_ssize_t requested_size) } if (logical_offset > 0) { - sval = PyObject_Malloc(alloc); + sval = PyMem_Malloc(alloc); if (sval == NULL) { PyErr_NoMemory(); return -1; } memcpy(sval, PyByteArray_AS_STRING(self), Py_MIN((size_t)requested_size, (size_t)Py_SIZE(self))); - PyObject_Free(obj->ob_bytes); + PyMem_Free(obj->ob_bytes); } else { - sval = PyObject_Realloc(obj->ob_bytes, alloc); + sval = PyMem_Realloc(obj->ob_bytes, alloc); if (sval == NULL) { PyErr_NoMemory(); return -1; @@ -951,7 +951,7 @@ bytearray_repr(PyByteArrayObject *self) } newsize += 6 + length * 4; - buffer = PyObject_Malloc(newsize); + buffer = PyMem_Malloc(newsize); if (buffer == NULL) { PyErr_NoMemory(); return NULL; @@ -1008,7 +1008,7 @@ bytearray_repr(PyByteArrayObject *self) } v = PyUnicode_FromStringAndSize(buffer, p - buffer); - PyObject_Free(buffer); + PyMem_Free(buffer); return v; } @@ -1088,7 +1088,7 @@ bytearray_dealloc(PyByteArrayObject *self) PyErr_Print(); } if (self->ob_bytes != 0) { - PyObject_Free(self->ob_bytes); + PyMem_Free(self->ob_bytes); } Py_TYPE(self)->tp_free((PyObject *)self); } diff --git a/Objects/clinic/dictobject.c.h b/Objects/clinic/dictobject.c.h index 641514235c23414..daaef211b1db494 100644 --- a/Objects/clinic/dictobject.c.h +++ b/Objects/clinic/dictobject.c.h @@ -2,6 +2,7 @@ preserve [clinic start generated code]*/ +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(dict_fromkeys__doc__, @@ -38,6 +39,24 @@ dict_fromkeys(PyTypeObject *type, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(dict_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"Return a shallow copy of the dict."); + +#define DICT_COPY_METHODDEF \ + {"copy", (PyCFunction)dict_copy, METH_NOARGS, dict_copy__doc__}, + +static PyObject * +dict_copy_impl(PyDictObject *self); + +static PyObject * +dict_copy(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +{ + return dict_copy_impl(self); +} + PyDoc_STRVAR(dict___contains____doc__, "__contains__($self, key, /)\n" "--\n" @@ -47,6 +66,21 @@ PyDoc_STRVAR(dict___contains____doc__, #define DICT___CONTAINS___METHODDEF \ {"__contains__", (PyCFunction)dict___contains__, METH_O|METH_COEXIST, dict___contains____doc__}, +static PyObject * +dict___contains___impl(PyDictObject *self, PyObject *key); + +static PyObject * +dict___contains__(PyDictObject *self, PyObject *key) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = dict___contains___impl(self, key); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(dict_get__doc__, "get($self, key, default=None, /)\n" "--\n" @@ -75,7 +109,9 @@ dict_get(PyDictObject *self, PyObject *const *args, Py_ssize_t nargs) } default_value = args[1]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = dict_get_impl(self, key, default_value); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -112,12 +148,32 @@ dict_setdefault(PyDictObject *self, PyObject *const *args, Py_ssize_t nargs) } default_value = args[1]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = dict_setdefault_impl(self, key, default_value); + Py_END_CRITICAL_SECTION(); exit: return return_value; } +PyDoc_STRVAR(dict_clear__doc__, +"clear($self, /)\n" +"--\n" +"\n" +"Remove all items from the dict."); + +#define DICT_CLEAR_METHODDEF \ + {"clear", (PyCFunction)dict_clear, METH_NOARGS, dict_clear__doc__}, + +static PyObject * +dict_clear_impl(PyDictObject *self); + +static PyObject * +dict_clear(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +{ + return dict_clear_impl(self); +} + PyDoc_STRVAR(dict_pop__doc__, "pop($self, key, default=<unrepresentable>, /)\n" "--\n" @@ -173,7 +229,31 @@ dict_popitem_impl(PyDictObject *self); static PyObject * dict_popitem(PyDictObject *self, PyObject *Py_UNUSED(ignored)) { - return dict_popitem_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = dict_popitem_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(dict___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n" +"Return the size of the dict in memory, in bytes."); + +#define DICT___SIZEOF___METHODDEF \ + {"__sizeof__", (PyCFunction)dict___sizeof__, METH_NOARGS, dict___sizeof____doc__}, + +static PyObject * +dict___sizeof___impl(PyDictObject *self); + +static PyObject * +dict___sizeof__(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +{ + return dict___sizeof___impl(self); } PyDoc_STRVAR(dict___reversed____doc__, @@ -193,4 +273,58 @@ dict___reversed__(PyDictObject *self, PyObject *Py_UNUSED(ignored)) { return dict___reversed___impl(self); } -/*[clinic end generated code: output=17c3c4cf9a9b95a7 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(dict_keys__doc__, +"keys($self, /)\n" +"--\n" +"\n" +"Return a set-like object providing a view on the dict\'s keys."); + +#define DICT_KEYS_METHODDEF \ + {"keys", (PyCFunction)dict_keys, METH_NOARGS, dict_keys__doc__}, + +static PyObject * +dict_keys_impl(PyDictObject *self); + +static PyObject * +dict_keys(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +{ + return dict_keys_impl(self); +} + +PyDoc_STRVAR(dict_items__doc__, +"items($self, /)\n" +"--\n" +"\n" +"Return a set-like object providing a view on the dict\'s items."); + +#define DICT_ITEMS_METHODDEF \ + {"items", (PyCFunction)dict_items, METH_NOARGS, dict_items__doc__}, + +static PyObject * +dict_items_impl(PyDictObject *self); + +static PyObject * +dict_items(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +{ + return dict_items_impl(self); +} + +PyDoc_STRVAR(dict_values__doc__, +"values($self, /)\n" +"--\n" +"\n" +"Return an object providing a view on the dict\'s values."); + +#define DICT_VALUES_METHODDEF \ + {"values", (PyCFunction)dict_values, METH_NOARGS, dict_values__doc__}, + +static PyObject * +dict_values_impl(PyDictObject *self); + +static PyObject * +dict_values(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +{ + return dict_values_impl(self); +} +/*[clinic end generated code: output=c8fda06bac5b05f3 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/setobject.c.h b/Objects/clinic/setobject.c.h new file mode 100644 index 000000000000000..f3c96995ede60dd --- /dev/null +++ b/Objects/clinic/setobject.c.h @@ -0,0 +1,414 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + +PyDoc_STRVAR(set_pop__doc__, +"pop($self, /)\n" +"--\n" +"\n" +"Remove and return an arbitrary set element.\n" +"\n" +"Raises KeyError if the set is empty."); + +#define SET_POP_METHODDEF \ + {"pop", (PyCFunction)set_pop, METH_NOARGS, set_pop__doc__}, + +static PyObject * +set_pop_impl(PySetObject *so); + +static PyObject * +set_pop(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + return set_pop_impl(so); +} + +PyDoc_STRVAR(set_update__doc__, +"update($self, /, *others)\n" +"--\n" +"\n" +"Update the set, adding elements from all others."); + +#define SET_UPDATE_METHODDEF \ + {"update", _PyCFunction_CAST(set_update), METH_FASTCALL, set_update__doc__}, + +static PyObject * +set_update_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_update(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("update", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_update_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"Return a shallow copy of a set."); + +#define SET_COPY_METHODDEF \ + {"copy", (PyCFunction)set_copy, METH_NOARGS, set_copy__doc__}, + +static PyObject * +set_copy_impl(PySetObject *so); + +static PyObject * +set_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + return set_copy_impl(so); +} + +PyDoc_STRVAR(frozenset_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"Return a shallow copy of a set."); + +#define FROZENSET_COPY_METHODDEF \ + {"copy", (PyCFunction)frozenset_copy, METH_NOARGS, frozenset_copy__doc__}, + +static PyObject * +frozenset_copy_impl(PySetObject *so); + +static PyObject * +frozenset_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + return frozenset_copy_impl(so); +} + +PyDoc_STRVAR(set_clear__doc__, +"clear($self, /)\n" +"--\n" +"\n" +"Remove all elements from this set."); + +#define SET_CLEAR_METHODDEF \ + {"clear", (PyCFunction)set_clear, METH_NOARGS, set_clear__doc__}, + +static PyObject * +set_clear_impl(PySetObject *so); + +static PyObject * +set_clear(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + return set_clear_impl(so); +} + +PyDoc_STRVAR(set_union__doc__, +"union($self, /, *others)\n" +"--\n" +"\n" +"Return a new set with elements from the set and all others."); + +#define SET_UNION_METHODDEF \ + {"union", _PyCFunction_CAST(set_union), METH_FASTCALL, set_union__doc__}, + +static PyObject * +set_union_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_union(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("union", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_union_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_intersection_multi__doc__, +"intersection($self, /, *others)\n" +"--\n" +"\n" +"Return a new set with elements common to the set and all others."); + +#define SET_INTERSECTION_MULTI_METHODDEF \ + {"intersection", _PyCFunction_CAST(set_intersection_multi), METH_FASTCALL, set_intersection_multi__doc__}, + +static PyObject * +set_intersection_multi_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_intersection_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("intersection", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_intersection_multi_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_intersection_update_multi__doc__, +"intersection_update($self, /, *others)\n" +"--\n" +"\n" +"Update the set, keeping only elements found in it and all others."); + +#define SET_INTERSECTION_UPDATE_MULTI_METHODDEF \ + {"intersection_update", _PyCFunction_CAST(set_intersection_update_multi), METH_FASTCALL, set_intersection_update_multi__doc__}, + +static PyObject * +set_intersection_update_multi_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_intersection_update_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("intersection_update", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_intersection_update_multi_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_isdisjoint__doc__, +"isdisjoint($self, other, /)\n" +"--\n" +"\n" +"Return True if two sets have a null intersection."); + +#define SET_ISDISJOINT_METHODDEF \ + {"isdisjoint", (PyCFunction)set_isdisjoint, METH_O, set_isdisjoint__doc__}, + +PyDoc_STRVAR(set_difference_update__doc__, +"difference_update($self, /, *others)\n" +"--\n" +"\n" +"Update the set, removing elements found in others."); + +#define SET_DIFFERENCE_UPDATE_METHODDEF \ + {"difference_update", _PyCFunction_CAST(set_difference_update), METH_FASTCALL, set_difference_update__doc__}, + +static PyObject * +set_difference_update_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_difference_update(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("difference_update", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_difference_update_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_difference_multi__doc__, +"difference($self, /, *others)\n" +"--\n" +"\n" +"Return a new set with elements in the set that are not in the others."); + +#define SET_DIFFERENCE_MULTI_METHODDEF \ + {"difference", _PyCFunction_CAST(set_difference_multi), METH_FASTCALL, set_difference_multi__doc__}, + +static PyObject * +set_difference_multi_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_difference_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("difference", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_difference_multi_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_symmetric_difference_update__doc__, +"symmetric_difference_update($self, other, /)\n" +"--\n" +"\n" +"Update the set, keeping only elements found in either set, but not in both."); + +#define SET_SYMMETRIC_DIFFERENCE_UPDATE_METHODDEF \ + {"symmetric_difference_update", (PyCFunction)set_symmetric_difference_update, METH_O, set_symmetric_difference_update__doc__}, + +PyDoc_STRVAR(set_symmetric_difference__doc__, +"symmetric_difference($self, other, /)\n" +"--\n" +"\n" +"Return a new set with elements in either the set or other but not both."); + +#define SET_SYMMETRIC_DIFFERENCE_METHODDEF \ + {"symmetric_difference", (PyCFunction)set_symmetric_difference, METH_O, set_symmetric_difference__doc__}, + +PyDoc_STRVAR(set_issubset__doc__, +"issubset($self, other, /)\n" +"--\n" +"\n" +"Report whether another set contains this set."); + +#define SET_ISSUBSET_METHODDEF \ + {"issubset", (PyCFunction)set_issubset, METH_O, set_issubset__doc__}, + +PyDoc_STRVAR(set_issuperset__doc__, +"issuperset($self, other, /)\n" +"--\n" +"\n" +"Report whether this set contains another set."); + +#define SET_ISSUPERSET_METHODDEF \ + {"issuperset", (PyCFunction)set_issuperset, METH_O, set_issuperset__doc__}, + +PyDoc_STRVAR(set_add__doc__, +"add($self, object, /)\n" +"--\n" +"\n" +"Add an element to a set.\n" +"\n" +"This has no effect if the element is already present."); + +#define SET_ADD_METHODDEF \ + {"add", (PyCFunction)set_add, METH_O, set_add__doc__}, + +PyDoc_STRVAR(set___contains____doc__, +"__contains__($self, object, /)\n" +"--\n" +"\n" +"x.__contains__(y) <==> y in x."); + +#define SET___CONTAINS___METHODDEF \ + {"__contains__", (PyCFunction)set___contains__, METH_O|METH_COEXIST, set___contains____doc__}, + +PyDoc_STRVAR(set_remove__doc__, +"remove($self, object, /)\n" +"--\n" +"\n" +"Remove an element from a set; it must be a member.\n" +"\n" +"If the element is not a member, raise a KeyError."); + +#define SET_REMOVE_METHODDEF \ + {"remove", (PyCFunction)set_remove, METH_O, set_remove__doc__}, + +PyDoc_STRVAR(set_discard__doc__, +"discard($self, object, /)\n" +"--\n" +"\n" +"Remove an element from a set if it is a member.\n" +"\n" +"Unlike set.remove(), the discard() method does not raise\n" +"an exception when an element is missing from the set."); + +#define SET_DISCARD_METHODDEF \ + {"discard", (PyCFunction)set_discard, METH_O, set_discard__doc__}, + +PyDoc_STRVAR(set___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n" +"Return state information for pickling."); + +#define SET___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)set___reduce__, METH_NOARGS, set___reduce____doc__}, + +static PyObject * +set___reduce___impl(PySetObject *so); + +static PyObject * +set___reduce__(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + return set___reduce___impl(so); +} + +PyDoc_STRVAR(set___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n" +"S.__sizeof__() -> size of S in memory, in bytes."); + +#define SET___SIZEOF___METHODDEF \ + {"__sizeof__", (PyCFunction)set___sizeof__, METH_NOARGS, set___sizeof____doc__}, + +static PyObject * +set___sizeof___impl(PySetObject *so); + +static PyObject * +set___sizeof__(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + return set___sizeof___impl(so); +} +/*[clinic end generated code: output=34a30591148da884 input=a9049054013a1b77]*/ diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 2482a918ba983bd..9b1defa5cbc609f 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -113,17 +113,19 @@ As a consequence of this, split keys have a maximum size of 16. #define PyDict_MINSIZE 8 #include "Python.h" -#include "pycore_bitutils.h" // _Py_bit_length -#include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_ceval.h" // _PyEval_GetBuiltin() -#include "pycore_code.h" // stats -#include "pycore_dict.h" // export _PyDict_SizeOf() -#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() -#include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() -#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() -#include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_setobject.h" // _PySet_NextEntry() -#include "stringlib/eq.h" // unicode_eq() +#include "pycore_bitutils.h" // _Py_bit_length +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_code.h" // stats +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION, Py_END_CRITICAL_SECTION +#include "pycore_dict.h" // export _PyDict_SizeOf() +#include "pycore_freelist.h" // _PyFreeListState_GET() +#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() +#include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_setobject.h" // _PySet_NextEntry() +#include "stringlib/eq.h" // unicode_eq() #include <stdbool.h> @@ -140,6 +142,21 @@ To avoid slowing down lookups on a near-full table, we resize the table when it's USABLE_FRACTION (currently two-thirds) full. */ +#ifdef Py_GIL_DISABLED + +static inline void +ASSERT_DICT_LOCKED(PyObject *op) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); +} +#define ASSERT_DICT_LOCKED(op) ASSERT_DICT_LOCKED(_Py_CAST(PyObject*, op)) + +#else + +#define ASSERT_DICT_LOCKED(op) + +#endif + #define PERTURB_SHIFT 5 /* @@ -239,43 +256,57 @@ static int dictresize(PyInterpreterState *interp, PyDictObject *mp, static PyObject* dict_iter(PyObject *dict); +static int +contains_lock_held(PyDictObject *mp, PyObject *key); +static int +contains_known_hash_lock_held(PyDictObject *mp, PyObject *key, Py_ssize_t hash); +static int +setitem_lock_held(PyDictObject *mp, PyObject *key, PyObject *value); +static int +dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_value, + PyObject **result, int incref_result); + #include "clinic/dictobject.c.h" -#if PyDict_MAXFREELIST > 0 -static struct _Py_dict_state * -get_dict_state(PyInterpreterState *interp) +#ifdef WITH_FREELISTS +static struct _Py_dict_freelist * +get_dict_state(void) { - return &interp->dict_state; + _PyFreeListState *state = _PyFreeListState_GET(); + return &state->dicts; } #endif void -_PyDict_ClearFreeList(PyInterpreterState *interp) +_PyDict_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization) { -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = &interp->dict_state; - while (state->numfree) { +#ifdef WITH_FREELISTS + struct _Py_dict_freelist *state = &freelist_state->dicts; + while (state->numfree > 0) { PyDictObject *op = state->free_list[--state->numfree]; assert(PyDict_CheckExact(op)); PyObject_GC_Del(op); } - while (state->keys_numfree) { - PyObject_Free(state->keys_free_list[--state->keys_numfree]); + while (state->keys_numfree > 0) { + PyMem_Free(state->keys_free_list[--state->keys_numfree]); + } + if (is_finalization) { + state->numfree = -1; + state->keys_numfree = -1; } #endif } - void -_PyDict_Fini(PyInterpreterState *interp) +_PyDict_Fini(PyInterpreterState *Py_UNUSED(interp)) { - _PyDict_ClearFreeList(interp); -#if defined(Py_DEBUG) && PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = &interp->dict_state; - state->numfree = -1; - state->keys_numfree = -1; + // With Py_GIL_DISABLED: + // the freelists for the current thread state have already been cleared. +#ifndef Py_GIL_DISABLED + _PyFreeListState *state = _PyFreeListState_GET(); + _PyDict_ClearFreeList(state, 1); #endif } @@ -290,9 +321,8 @@ unicode_get_hash(PyObject *o) void _PyDict_DebugMallocStats(FILE *out) { -#if PyDict_MAXFREELIST > 0 - PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _Py_dict_state *state = get_dict_state(interp); +#ifdef WITH_FREELISTS + struct _Py_dict_freelist *state = get_dict_state(); _PyDebugAllocatorStats(out, "free PyDictObject", state->numfree, sizeof(PyDictObject)); #endif @@ -300,7 +330,7 @@ _PyDict_DebugMallocStats(FILE *out) #define DK_MASK(dk) (DK_SIZE(dk)-1) -static void free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys); +static void free_keys_object(PyDictKeysObject *keys); /* PyDictKeysObject has refcounts like PyObject does, so we have the following two functions to mirror what Py_INCREF() and Py_DECREF() do. @@ -332,7 +362,23 @@ dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk) _Py_DecRefTotal(_PyInterpreterState_GET()); #endif if (--dk->dk_refcnt == 0) { - free_keys_object(interp, dk); + if (DK_IS_UNICODE(dk)) { + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dk); + Py_ssize_t i, n; + for (i = 0, n = dk->dk_nentries; i < n; i++) { + Py_XDECREF(entries[i].me_key); + Py_XDECREF(entries[i].me_value); + } + } + else { + PyDictKeyEntry *entries = DK_ENTRIES(dk); + Py_ssize_t i, n; + for (i = 0, n = dk->dk_nentries; i < n; i++) { + Py_XDECREF(entries[i].me_key); + Py_XDECREF(entries[i].me_value); + } + } + free_keys_object(dk); } } @@ -627,12 +673,8 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) log2_bytes = log2_size + 2; } -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(interp); -#ifdef Py_DEBUG - // new_keys_object() must not be called after _PyDict_Fini() - assert(state->keys_numfree != -1); -#endif +#ifdef WITH_FREELISTS + struct _Py_dict_freelist *state = get_dict_state(); if (log2_size == PyDict_LOG_MINSIZE && unicode && state->keys_numfree > 0) { dk = state->keys_free_list[--state->keys_numfree]; OBJECT_STAT_INC(from_freelist); @@ -640,9 +682,9 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) else #endif { - dk = PyObject_Malloc(sizeof(PyDictKeysObject) - + ((size_t)1 << log2_bytes) - + entry_size * usable); + dk = PyMem_Malloc(sizeof(PyDictKeysObject) + + ((size_t)1 << log2_bytes) + + entry_size * usable); if (dk == NULL) { PyErr_NoMemory(); return NULL; @@ -664,40 +706,20 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) } static void -free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys) +free_keys_object(PyDictKeysObject *keys) { - assert(keys != Py_EMPTY_KEYS); - if (DK_IS_UNICODE(keys)) { - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); - Py_ssize_t i, n; - for (i = 0, n = keys->dk_nentries; i < n; i++) { - Py_XDECREF(entries[i].me_key); - Py_XDECREF(entries[i].me_value); - } - } - else { - PyDictKeyEntry *entries = DK_ENTRIES(keys); - Py_ssize_t i, n; - for (i = 0, n = keys->dk_nentries; i < n; i++) { - Py_XDECREF(entries[i].me_key); - Py_XDECREF(entries[i].me_value); - } - } -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(interp); -#ifdef Py_DEBUG - // free_keys_object() must not be called after _PyDict_Fini() - assert(state->keys_numfree != -1); -#endif +#ifdef WITH_FREELISTS + struct _Py_dict_freelist *state = get_dict_state(); if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE && state->keys_numfree < PyDict_MAXFREELIST + && state->keys_numfree >= 0 && DK_IS_UNICODE(keys)) { state->keys_free_list[state->keys_numfree++] = keys; OBJECT_STAT_INC(to_freelist); return; } #endif - PyObject_Free(keys); + PyMem_Free(keys); } static inline PyDictValues* @@ -731,13 +753,9 @@ new_dict(PyInterpreterState *interp, { PyDictObject *mp; assert(keys != NULL); -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(interp); -#ifdef Py_DEBUG - // new_dict() must not be called after _PyDict_Fini() - assert(state->numfree != -1); -#endif - if (state->numfree) { +#ifdef WITH_FREELISTS + struct _Py_dict_freelist *state = get_dict_state(); + if (state->numfree > 0) { mp = state->free_list[--state->numfree]; assert (mp != NULL); assert (Py_IS_TYPE(mp, &PyDict_Type)); @@ -797,8 +815,10 @@ clone_combined_dict_keys(PyDictObject *orig) assert(orig->ma_keys != Py_EMPTY_KEYS); assert(orig->ma_keys->dk_refcnt == 1); + ASSERT_DICT_LOCKED(orig); + size_t keys_size = _PyDict_KeysSize(orig->ma_keys); - PyDictKeysObject *keys = PyObject_Malloc(keys_size); + PyDictKeysObject *keys = PyMem_Malloc(keys_size); if (keys == NULL) { PyErr_NoMemory(); return NULL; @@ -875,11 +895,11 @@ lookdict_index(PyDictKeysObject *k, Py_hash_t hash, Py_ssize_t index) Py_UNREACHABLE(); } -// Search non-Unicode key from Unicode table -static Py_ssize_t -unicodekeys_lookup_generic(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +static inline Py_ALWAYS_INLINE Py_ssize_t +do_lookup(PyDictObject *mp, PyDictKeysObject *dk, PyObject *key, Py_hash_t hash, + Py_ssize_t (*check_lookup)(PyDictObject *, PyDictKeysObject *, void *, Py_ssize_t ix, PyObject *key, Py_hash_t)) { - PyDictUnicodeEntry *ep0 = DK_UNICODE_ENTRIES(dk); + void *ep0 = _DK_ENTRIES(dk); size_t mask = DK_MASK(dk); size_t perturb = hash; size_t i = (size_t)hash & mask; @@ -887,73 +907,26 @@ unicodekeys_lookup_generic(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key for (;;) { ix = dictkeys_get_index(dk, i); if (ix >= 0) { - PyDictUnicodeEntry *ep = &ep0[ix]; - assert(ep->me_key != NULL); - assert(PyUnicode_CheckExact(ep->me_key)); - if (ep->me_key == key) { + Py_ssize_t cmp = check_lookup(mp, dk, ep0, ix, key, hash); + if (cmp < 0) { + return cmp; + } else if (cmp) { return ix; } - if (unicode_get_hash(ep->me_key) == hash) { - PyObject *startkey = ep->me_key; - Py_INCREF(startkey); - int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); - Py_DECREF(startkey); - if (cmp < 0) { - return DKIX_ERROR; - } - if (dk == mp->ma_keys && ep->me_key == startkey) { - if (cmp > 0) { - return ix; - } - } - else { - /* The dict was mutated, restart */ - return DKIX_KEY_CHANGED; - } - } } else if (ix == DKIX_EMPTY) { return DKIX_EMPTY; } perturb >>= PERTURB_SHIFT; i = mask & (i*5 + perturb + 1); - } - Py_UNREACHABLE(); -} -// Search Unicode key from Unicode table. -static Py_ssize_t _Py_HOT_FUNCTION -unicodekeys_lookup_unicode(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) -{ - PyDictUnicodeEntry *ep0 = DK_UNICODE_ENTRIES(dk); - size_t mask = DK_MASK(dk); - size_t perturb = hash; - size_t i = (size_t)hash & mask; - Py_ssize_t ix; - for (;;) { - ix = dictkeys_get_index(dk, i); - if (ix >= 0) { - PyDictUnicodeEntry *ep = &ep0[ix]; - assert(ep->me_key != NULL); - assert(PyUnicode_CheckExact(ep->me_key)); - if (ep->me_key == key || - (unicode_get_hash(ep->me_key) == hash && unicode_eq(ep->me_key, key))) { - return ix; - } - } - else if (ix == DKIX_EMPTY) { - return DKIX_EMPTY; - } - perturb >>= PERTURB_SHIFT; - i = mask & (i*5 + perturb + 1); // Manual loop unrolling ix = dictkeys_get_index(dk, i); if (ix >= 0) { - PyDictUnicodeEntry *ep = &ep0[ix]; - assert(ep->me_key != NULL); - assert(PyUnicode_CheckExact(ep->me_key)); - if (ep->me_key == key || - (unicode_get_hash(ep->me_key) == hash && unicode_eq(ep->me_key, key))) { + Py_ssize_t cmp = check_lookup(mp, dk, ep0, ix, key, hash); + if (cmp < 0) { + return cmp; + } else if (cmp) { return ix; } } @@ -966,49 +939,94 @@ unicodekeys_lookup_unicode(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) Py_UNREACHABLE(); } -// Search key from Generic table. +static inline Py_ALWAYS_INLINE Py_ssize_t +compare_unicode_generic(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; + assert(ep->me_key != NULL); + assert(PyUnicode_CheckExact(ep->me_key)); + assert(!PyUnicode_CheckExact(key)); + // TODO: Thread safety + + if (unicode_get_hash(ep->me_key) == hash) { + PyObject *startkey = ep->me_key; + Py_INCREF(startkey); + int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + Py_DECREF(startkey); + if (cmp < 0) { + return DKIX_ERROR; + } + if (dk == mp->ma_keys && ep->me_key == startkey) { + return cmp; + } + else { + /* The dict was mutated, restart */ + return DKIX_KEY_CHANGED; + } + } + return 0; +} + +// Search non-Unicode key from Unicode table static Py_ssize_t -dictkeys_generic_lookup(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +unicodekeys_lookup_generic(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) { - PyDictKeyEntry *ep0 = DK_ENTRIES(dk); - size_t mask = DK_MASK(dk); - size_t perturb = hash; - size_t i = (size_t)hash & mask; - Py_ssize_t ix; - for (;;) { - ix = dictkeys_get_index(dk, i); - if (ix >= 0) { - PyDictKeyEntry *ep = &ep0[ix]; - assert(ep->me_key != NULL); - if (ep->me_key == key) { - return ix; - } - if (ep->me_hash == hash) { - PyObject *startkey = ep->me_key; - Py_INCREF(startkey); - int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); - Py_DECREF(startkey); - if (cmp < 0) { - return DKIX_ERROR; - } - if (dk == mp->ma_keys && ep->me_key == startkey) { - if (cmp > 0) { - return ix; - } - } - else { - /* The dict was mutated, restart */ - return DKIX_KEY_CHANGED; - } - } + return do_lookup(mp, dk, key, hash, compare_unicode_generic); +} + +static inline Py_ALWAYS_INLINE Py_ssize_t +compare_unicode_unicode(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; + assert(ep->me_key != NULL); + assert(PyUnicode_CheckExact(ep->me_key)); + if (ep->me_key == key || + (unicode_get_hash(ep->me_key) == hash && unicode_eq(ep->me_key, key))) { + return 1; + } + return 0; +} + +static Py_ssize_t _Py_HOT_FUNCTION +unicodekeys_lookup_unicode(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + return do_lookup(NULL, dk, key, hash, compare_unicode_unicode); +} + +static inline Py_ALWAYS_INLINE Py_ssize_t +compare_generic(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictKeyEntry *ep = &((PyDictKeyEntry *)ep0)[ix]; + assert(ep->me_key != NULL); + if (ep->me_key == key) { + return 1; + } + if (ep->me_hash == hash) { + PyObject *startkey = ep->me_key; + Py_INCREF(startkey); + int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + Py_DECREF(startkey); + if (cmp < 0) { + return DKIX_ERROR; } - else if (ix == DKIX_EMPTY) { - return DKIX_EMPTY; + if (dk == mp->ma_keys && ep->me_key == startkey) { + return cmp; + } + else { + /* The dict was mutated, restart */ + return DKIX_KEY_CHANGED; } - perturb >>= PERTURB_SHIFT; - i = mask & (i*5 + perturb + 1); } - Py_UNREACHABLE(); + return 0; +} + +static Py_ssize_t +dictkeys_generic_lookup(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + return do_lookup(mp, dk, key, hash, compare_generic); } /* Lookup a string in a (all unicode) dict keys. @@ -1240,6 +1258,8 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, { PyObject *old_value; + ASSERT_DICT_LOCKED(mp); + if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) { if (insertion_resize(interp, mp, 0) < 0) goto Fail; @@ -1336,6 +1356,7 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) { assert(mp->ma_keys == Py_EMPTY_KEYS); + ASSERT_DICT_LOCKED(mp); uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_ADDED, mp, key, value); @@ -1429,6 +1450,8 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, PyDictKeysObject *oldkeys; PyDictValues *oldvalues; + ASSERT_DICT_LOCKED(mp); + if (log2_newsize >= SIZEOF_SIZE_T*8) { PyErr_NoMemory(); return -1; @@ -1544,32 +1567,13 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, } } - // We can not use free_keys_object here because key's reference - // are moved already. if (oldkeys != Py_EMPTY_KEYS) { #ifdef Py_REF_DEBUG _Py_DecRefTotal(_PyInterpreterState_GET()); #endif assert(oldkeys->dk_kind != DICT_KEYS_SPLIT); assert(oldkeys->dk_refcnt == 1); -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(interp); -#ifdef Py_DEBUG - // dictresize() must not be called after _PyDict_Fini() - assert(state->keys_numfree != -1); -#endif - if (DK_LOG_SIZE(oldkeys) == PyDict_LOG_MINSIZE && - DK_IS_UNICODE(oldkeys) && - state->keys_numfree < PyDict_MAXFREELIST) - { - state->keys_free_list[state->keys_numfree++] = oldkeys; - OBJECT_STAT_INC(to_freelist); - } - else -#endif - { - PyObject_Free(oldkeys); - } + free_keys_object(oldkeys); } } @@ -1642,7 +1646,7 @@ _PyDict_FromItems(PyObject *const *keys, Py_ssize_t keys_offset, for (Py_ssize_t i = 0; i < length; i++) { PyObject *key = *ks; PyObject *value = *vs; - if (PyDict_SetItem(dict, key, value) < 0) { + if (setitem_lock_held((PyDictObject *)dict, key, value) < 0) { Py_DECREF(dict); return NULL; } @@ -1717,6 +1721,7 @@ PyDict_GetItem(PyObject *op, PyObject *key) Py_ssize_t _PyDict_LookupIndex(PyDictObject *mp, PyObject *key) { + // TODO: Thread safety PyObject *value; assert(PyDict_CheckExact((PyObject*)mp)); assert(PyUnicode_CheckExact(key)); @@ -1893,9 +1898,11 @@ _PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) } /* Consumes references to key and value */ -int -_PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) +static int +setitem_take2_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) { + ASSERT_DICT_LOCKED(mp); + assert(key); assert(value); assert(PyDict_Check(mp)); @@ -1908,7 +1915,9 @@ _PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) return -1; } } + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (mp->ma_keys == Py_EMPTY_KEYS) { return insert_to_emptydict(interp, mp, key, hash, value); } @@ -1916,6 +1925,16 @@ _PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) return insertdict(interp, mp, key, hash, value); } +int +_PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(mp); + res = setitem_take2_lock_held(mp, key, value); + Py_END_CRITICAL_SECTION(); + return res; +} + /* CAUTION: PyDict_SetItem() must guarantee that it won't resize the * dictionary if it's merely replacing the value for an existing key. * This means that it's safe to loop over a dictionary with PyDict_Next() @@ -1935,6 +1954,16 @@ PyDict_SetItem(PyObject *op, PyObject *key, PyObject *value) Py_NewRef(key), Py_NewRef(value)); } +static int +setitem_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) +{ + assert(key); + assert(value); + return setitem_take2_lock_held(mp, + Py_NewRef(key), Py_NewRef(value)); +} + + int _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, Py_hash_t hash) @@ -1950,12 +1979,21 @@ _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, assert(hash != -1); mp = (PyDictObject *)op; + int res; PyInterpreterState *interp = _PyInterpreterState_GET(); + + Py_BEGIN_CRITICAL_SECTION(mp); + if (mp->ma_keys == Py_EMPTY_KEYS) { - return insert_to_emptydict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); + res = insert_to_emptydict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); } - /* insertdict() handles any resizing that might be necessary */ - return insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); + else { + /* insertdict() handles any resizing that might be necessary */ + res = insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); + } + + Py_END_CRITICAL_SECTION(); + return res; } static void @@ -1980,6 +2018,8 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, { PyObject *old_key; + ASSERT_DICT_LOCKED(mp); + Py_ssize_t hashpos = lookdict_index(mp->ma_keys, hash, ix); assert(hashpos >= 0); @@ -2031,8 +2071,8 @@ PyDict_DelItem(PyObject *op, PyObject *key) return _PyDict_DelItem_KnownHash(op, key, hash); } -int -_PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) +static int +delitem_knownhash_lock_held(PyObject *op, PyObject *key, Py_hash_t hash) { Py_ssize_t ix; PyDictObject *mp; @@ -2042,6 +2082,9 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) PyErr_BadInternalCall(); return -1; } + + ASSERT_DICT_LOCKED(op); + assert(key); assert(hash != -1); mp = (PyDictObject *)op; @@ -2059,13 +2102,19 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) return delitem_common(mp, hash, ix, old_value, new_version); } -/* This function promises that the predicate -> deletion sequence is atomic - * (i.e. protected by the GIL), assuming the predicate itself doesn't - * release the GIL. - */ int -_PyDict_DelItemIf(PyObject *op, PyObject *key, - int (*predicate)(PyObject *value)) +_PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(op); + res = delitem_knownhash_lock_held(op, key, hash); + Py_END_CRITICAL_SECTION(); + return res; +} + +static int +delitemif_lock_held(PyObject *op, PyObject *key, + int (*predicate)(PyObject *value)) { Py_ssize_t hashpos, ix; PyDictObject *mp; @@ -2073,6 +2122,8 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, PyObject *old_value; int res; + ASSERT_DICT_LOCKED(op); + if (!PyDict_Check(op)) { PyErr_BadInternalCall(); return -1; @@ -2106,16 +2157,32 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, return 0; } } +/* This function promises that the predicate -> deletion sequence is atomic + * (i.e. protected by the GIL or the per-dict mutex in free threaded builds), + * assuming the predicate itself doesn't release the GIL (or cause re-entrancy + * which would release the per-dict mutex) + */ +int +_PyDict_DelItemIf(PyObject *op, PyObject *key, + int (*predicate)(PyObject *value)) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(op); + res = delitemif_lock_held(op, key, predicate); + Py_END_CRITICAL_SECTION(); + return res; +} - -void -PyDict_Clear(PyObject *op) +static void +clear_lock_held(PyObject *op) { PyDictObject *mp; PyDictKeysObject *oldkeys; PyDictValues *oldvalues; Py_ssize_t i, n; + ASSERT_DICT_LOCKED(op); + if (!PyDict_Check(op)) return; mp = ((PyDictObject *)op); @@ -2148,6 +2215,14 @@ PyDict_Clear(PyObject *op) ASSERT_CONSISTENT(mp); } +void +PyDict_Clear(PyObject *op) +{ + Py_BEGIN_CRITICAL_SECTION(op); + clear_lock_held(op); + Py_END_CRITICAL_SECTION(); +} + /* Internal version of PyDict_Next that returns a hash value in addition * to the key and value. * Return 1 on success, return 0 when the reached the end of the dictionary @@ -2164,6 +2239,9 @@ _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, if (!PyDict_Check(op)) return 0; + + ASSERT_DICT_LOCKED(op); + mp = (PyDictObject *)op; i = *ppos; if (mp->ma_values) { @@ -2237,7 +2315,11 @@ _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, int PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) { - return _PyDict_Next(op, ppos, pkey, pvalue, NULL); + int res; + Py_BEGIN_CRITICAL_SECTION(op); + res = _PyDict_Next(op, ppos, pkey, pvalue, NULL); + Py_END_CRITICAL_SECTION(); + return res; } @@ -2248,6 +2330,8 @@ _PyDict_Pop_KnownHash(PyDictObject *mp, PyObject *key, Py_hash_t hash, { assert(PyDict_Check(mp)); + ASSERT_DICT_LOCKED(mp); + if (mp->ma_used == 0) { if (result) { *result = NULL; @@ -2287,10 +2371,11 @@ _PyDict_Pop_KnownHash(PyDictObject *mp, PyObject *key, Py_hash_t hash, return 1; } - -int -PyDict_Pop(PyObject *op, PyObject *key, PyObject **result) +static int +pop_lock_held(PyObject *op, PyObject *key, PyObject **result) { + ASSERT_DICT_LOCKED(op); + if (!PyDict_Check(op)) { if (result) { *result = NULL; @@ -2320,6 +2405,17 @@ PyDict_Pop(PyObject *op, PyObject *key, PyObject **result) return _PyDict_Pop_KnownHash(dict, key, hash, result); } +int +PyDict_Pop(PyObject *op, PyObject *key, PyObject **result) +{ + int err; + Py_BEGIN_CRITICAL_SECTION(op); + err = pop_lock_held(op, key, result); + Py_END_CRITICAL_SECTION(); + + return err; +} + int PyDict_PopString(PyObject *op, const char *key, PyObject **result) @@ -2352,6 +2448,55 @@ _PyDict_Pop(PyObject *dict, PyObject *key, PyObject *default_value) return result; } +static PyDictObject * +dict_dict_fromkeys(PyInterpreterState *interp, PyDictObject *mp, + PyObject *iterable, PyObject *value) +{ + PyObject *oldvalue; + Py_ssize_t pos = 0; + PyObject *key; + Py_hash_t hash; + int unicode = DK_IS_UNICODE(((PyDictObject*)iterable)->ma_keys); + uint8_t new_size = Py_MAX( + estimate_log2_keysize(PyDict_GET_SIZE(iterable)), + DK_LOG_SIZE(mp->ma_keys)); + if (dictresize(interp, mp, new_size, unicode)) { + Py_DECREF(mp); + return NULL; + } + + while (_PyDict_Next(iterable, &pos, &key, &oldvalue, &hash)) { + if (insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value))) { + Py_DECREF(mp); + return NULL; + } + } + return mp; +} + +static PyDictObject * +dict_set_fromkeys(PyInterpreterState *interp, PyDictObject *mp, + PyObject *iterable, PyObject *value) +{ + Py_ssize_t pos = 0; + PyObject *key; + Py_hash_t hash; + + if (dictresize(interp, mp, + estimate_log2_keysize(PySet_GET_SIZE(iterable)), 0)) { + Py_DECREF(mp); + return NULL; + } + + while (_PySet_NextEntry(iterable, &pos, &key, &hash)) { + if (insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value))) { + Py_DECREF(mp); + return NULL; + } + } + return mp; +} /* Internal version of dict.from_keys(). It is subclass-friendly. */ PyObject * @@ -2367,49 +2512,22 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) if (d == NULL) return NULL; - if (PyDict_CheckExact(d) && ((PyDictObject *)d)->ma_used == 0) { + + if (PyDict_CheckExact(d)) { if (PyDict_CheckExact(iterable)) { PyDictObject *mp = (PyDictObject *)d; - PyObject *oldvalue; - Py_ssize_t pos = 0; - PyObject *key; - Py_hash_t hash; - - int unicode = DK_IS_UNICODE(((PyDictObject*)iterable)->ma_keys); - if (dictresize(interp, mp, - estimate_log2_keysize(PyDict_GET_SIZE(iterable)), - unicode)) { - Py_DECREF(d); - return NULL; - } - while (_PyDict_Next(iterable, &pos, &key, &oldvalue, &hash)) { - if (insertdict(interp, mp, - Py_NewRef(key), hash, Py_NewRef(value))) { - Py_DECREF(d); - return NULL; - } - } + Py_BEGIN_CRITICAL_SECTION2(d, iterable); + d = (PyObject *)dict_dict_fromkeys(interp, mp, iterable, value); + Py_END_CRITICAL_SECTION2(); return d; } - if (PyAnySet_CheckExact(iterable)) { + else if (PyAnySet_CheckExact(iterable)) { PyDictObject *mp = (PyDictObject *)d; - Py_ssize_t pos = 0; - PyObject *key; - Py_hash_t hash; - if (dictresize(interp, mp, - estimate_log2_keysize(PySet_GET_SIZE(iterable)), 0)) { - Py_DECREF(d); - return NULL; - } - - while (_PySet_NextEntry(iterable, &pos, &key, &hash)) { - if (insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value))) { - Py_DECREF(d); - return NULL; - } - } + Py_BEGIN_CRITICAL_SECTION2(d, iterable); + d = (PyObject *)dict_set_fromkeys(interp, mp, iterable, value); + Py_END_CRITICAL_SECTION2(); return d; } } @@ -2421,12 +2539,17 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) } if (PyDict_CheckExact(d)) { + Py_BEGIN_CRITICAL_SECTION(d); while ((key = PyIter_Next(it)) != NULL) { - status = PyDict_SetItem(d, key, value); + status = setitem_lock_held((PyDictObject *)d, key, value); Py_DECREF(key); - if (status < 0) - goto Fail; + if (status < 0) { + assert(PyErr_Occurred()); + goto dict_iter_exit; + } } +dict_iter_exit: + Py_END_CRITICAL_SECTION(); } else { while ((key = PyIter_Next(it)) != NULL) { status = PyObject_SetItem(d, key, value); @@ -2480,13 +2603,10 @@ dict_dealloc(PyObject *self) assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS); dictkeys_decref(interp, keys); } -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(interp); -#ifdef Py_DEBUG - // new_dict() must not be called after _PyDict_Fini() - assert(state->numfree != -1); -#endif - if (state->numfree < PyDict_MAXFREELIST && Py_IS_TYPE(mp, &PyDict_Type)) { +#ifdef WITH_FREELISTS + struct _Py_dict_freelist *state = get_dict_state(); + if (state->numfree < PyDict_MAXFREELIST && state->numfree >=0 && + Py_IS_TYPE(mp, &PyDict_Type)) { state->free_list[state->numfree++] = mp; OBJECT_STAT_INC(to_freelist); } @@ -2500,7 +2620,7 @@ dict_dealloc(PyObject *self) static PyObject * -dict_repr(PyObject *self) +dict_repr_lock_held(PyObject *self) { PyDictObject *mp = (PyDictObject *)self; Py_ssize_t i; @@ -2530,7 +2650,7 @@ dict_repr(PyObject *self) Note that repr may mutate the dict. */ i = 0; first = 1; - while (PyDict_Next((PyObject *)mp, &i, &key, &value)) { + while (_PyDict_Next((PyObject *)mp, &i, &key, &value, NULL)) { PyObject *s; int res; @@ -2583,15 +2703,25 @@ dict_repr(PyObject *self) return NULL; } +static PyObject * +dict_repr(PyObject *self) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(self); + res = dict_repr_lock_held(self); + Py_END_CRITICAL_SECTION(); + return res; +} + static Py_ssize_t dict_length(PyObject *self) { PyDictObject *mp = (PyDictObject *)self; - return mp->ma_used; + return _Py_atomic_load_ssize_relaxed(&mp->ma_used); } static PyObject * -dict_subscript(PyObject *self, PyObject *key) +dict_subscript_lock_held(PyObject *self, PyObject *key) { PyDictObject *mp = (PyDictObject *)self; Py_ssize_t ix; @@ -2626,6 +2756,16 @@ dict_subscript(PyObject *self, PyObject *key) return Py_NewRef(value); } +static PyObject * +dict_subscript(PyObject *self, PyObject *key) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(self); + res = dict_subscript_lock_held(self, key); + Py_END_CRITICAL_SECTION(); + return res; +} + static int dict_ass_sub(PyObject *mp, PyObject *v, PyObject *w) { @@ -2642,8 +2782,15 @@ static PyMappingMethods dict_as_mapping = { }; static PyObject * -dict_keys(PyDictObject *mp) +keys_lock_held(PyObject *dict) { + ASSERT_DICT_LOCKED(dict); + + if (dict == NULL || !PyDict_Check(dict)) { + PyErr_BadInternalCall(); + return NULL; + } + PyDictObject *mp = (PyDictObject *)dict; PyObject *v; Py_ssize_t n; @@ -2672,9 +2819,27 @@ dict_keys(PyDictObject *mp) return v; } +PyObject * +PyDict_Keys(PyObject *dict) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(dict); + res = keys_lock_held(dict); + Py_END_CRITICAL_SECTION(); + + return res; +} + static PyObject * -dict_values(PyDictObject *mp) +values_lock_held(PyObject *dict) { + ASSERT_DICT_LOCKED(dict); + + if (dict == NULL || !PyDict_Check(dict)) { + PyErr_BadInternalCall(); + return NULL; + } + PyDictObject *mp = (PyDictObject *)dict; PyObject *v; Py_ssize_t n; @@ -2703,9 +2868,26 @@ dict_values(PyDictObject *mp) return v; } +PyObject * +PyDict_Values(PyObject *dict) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(dict); + res = values_lock_held(dict); + Py_END_CRITICAL_SECTION(); + return res; +} + static PyObject * -dict_items(PyDictObject *mp) +items_lock_held(PyObject *dict) { + ASSERT_DICT_LOCKED(dict); + + if (dict == NULL || !PyDict_Check(dict)) { + PyErr_BadInternalCall(); + return NULL; + } + PyDictObject *mp = (PyDictObject *)dict; PyObject *v; Py_ssize_t i, n; PyObject *item; @@ -2749,6 +2931,17 @@ dict_items(PyDictObject *mp) return v; } +PyObject * +PyDict_Items(PyObject *dict) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(dict); + res = items_lock_held(dict); + Py_END_CRITICAL_SECTION(); + + return res; +} + /*[clinic input] @classmethod dict.fromkeys @@ -2827,8 +3020,8 @@ dict_update(PyObject *self, PyObject *args, PyObject *kwds) producing iterable objects of length 2. */ -int -PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) +static int +merge_from_seq2_lock_held(PyObject *d, PyObject *seq2, int override) { PyObject *it; /* iter(seq2) */ Py_ssize_t i; /* index into seq2 of current element */ @@ -2880,14 +3073,14 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) Py_INCREF(key); Py_INCREF(value); if (override) { - if (PyDict_SetItem(d, key, value) < 0) { + if (setitem_lock_held((PyDictObject *)d, key, value) < 0) { Py_DECREF(key); Py_DECREF(value); goto Fail; } } else { - if (PyDict_SetDefault(d, key, value) == NULL) { + if (dict_setdefault_ref_lock_held(d, key, value, NULL, 0) < 0) { Py_DECREF(key); Py_DECREF(value); goto Fail; @@ -2912,6 +3105,117 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) return Py_SAFE_DOWNCAST(i, Py_ssize_t, int); } +int +PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(d); + res = merge_from_seq2_lock_held(d, seq2, override); + Py_END_CRITICAL_SECTION(); + + return res; +} + +static int +dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *other, int override) +{ + if (other == mp || other->ma_used == 0) + /* a.update(a) or a.update({}); nothing to do */ + return 0; + if (mp->ma_used == 0) { + /* Since the target dict is empty, PyDict_GetItem() + * always returns NULL. Setting override to 1 + * skips the unnecessary test. + */ + override = 1; + PyDictKeysObject *okeys = other->ma_keys; + + // If other is clean, combined, and just allocated, just clone it. + if (other->ma_values == NULL && + other->ma_used == okeys->dk_nentries && + (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || + USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) { + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL); + PyDictKeysObject *keys = clone_combined_dict_keys(other); + if (keys == NULL) + return -1; + + dictkeys_decref(interp, mp->ma_keys); + mp->ma_keys = keys; + if (mp->ma_values != NULL) { + free_values(mp->ma_values); + mp->ma_values = NULL; + } + + mp->ma_used = other->ma_used; + mp->ma_version_tag = new_version; + ASSERT_CONSISTENT(mp); + + if (_PyObject_GC_IS_TRACKED(other) && !_PyObject_GC_IS_TRACKED(mp)) { + /* Maintain tracking. */ + _PyObject_GC_TRACK(mp); + } + + return 0; + } + } + /* Do one big resize at the start, rather than + * incrementally resizing as we insert new items. Expect + * that there will be no (or few) overlapping keys. + */ + if (USABLE_FRACTION(DK_SIZE(mp->ma_keys)) < other->ma_used) { + int unicode = DK_IS_UNICODE(other->ma_keys); + if (dictresize(interp, mp, + estimate_log2_keysize(mp->ma_used + other->ma_used), + unicode)) { + return -1; + } + } + + Py_ssize_t orig_size = other->ma_keys->dk_nentries; + Py_ssize_t pos = 0; + Py_hash_t hash; + PyObject *key, *value; + + while (_PyDict_Next((PyObject*)other, &pos, &key, &value, &hash)) { + int err = 0; + Py_INCREF(key); + Py_INCREF(value); + if (override == 1) { + err = insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value)); + } + else { + err = contains_known_hash_lock_held(mp, key, hash); + if (err == 0) { + err = insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value)); + } + else if (err > 0) { + if (override != 0) { + _PyErr_SetKeyError(key); + Py_DECREF(value); + Py_DECREF(key); + return -1; + } + err = 0; + } + } + Py_DECREF(value); + Py_DECREF(key); + if (err != 0) + return -1; + + if (orig_size != other->ma_keys->dk_nentries) { + PyErr_SetString(PyExc_RuntimeError, + "dict mutated during update"); + return -1; + } + } + return 0; +} + static int dict_merge(PyInterpreterState *interp, PyObject *a, PyObject *b, int override) { @@ -2929,127 +3233,44 @@ dict_merge(PyInterpreterState *interp, PyObject *a, PyObject *b, int override) return -1; } mp = (PyDictObject*)a; + int res = 0; if (PyDict_Check(b) && (Py_TYPE(b)->tp_iter == dict_iter)) { other = (PyDictObject*)b; - if (other == mp || other->ma_used == 0) - /* a.update(a) or a.update({}); nothing to do */ - return 0; - if (mp->ma_used == 0) { - /* Since the target dict is empty, PyDict_GetItem() - * always returns NULL. Setting override to 1 - * skips the unnecessary test. - */ - override = 1; - PyDictKeysObject *okeys = other->ma_keys; - - // If other is clean, combined, and just allocated, just clone it. - if (other->ma_values == NULL && - other->ma_used == okeys->dk_nentries && - (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || - USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) { - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_CLONED, mp, b, NULL); - PyDictKeysObject *keys = clone_combined_dict_keys(other); - if (keys == NULL) { - return -1; - } - - dictkeys_decref(interp, mp->ma_keys); - mp->ma_keys = keys; - if (mp->ma_values != NULL) { - free_values(mp->ma_values); - mp->ma_values = NULL; - } - - mp->ma_used = other->ma_used; - mp->ma_version_tag = new_version; - ASSERT_CONSISTENT(mp); - - if (_PyObject_GC_IS_TRACKED(other) && !_PyObject_GC_IS_TRACKED(mp)) { - /* Maintain tracking. */ - _PyObject_GC_TRACK(mp); - } - - return 0; - } - } - /* Do one big resize at the start, rather than - * incrementally resizing as we insert new items. Expect - * that there will be no (or few) overlapping keys. - */ - if (USABLE_FRACTION(DK_SIZE(mp->ma_keys)) < other->ma_used) { - int unicode = DK_IS_UNICODE(other->ma_keys); - if (dictresize(interp, mp, - estimate_log2_keysize(mp->ma_used + other->ma_used), - unicode)) { - return -1; - } - } - - Py_ssize_t orig_size = other->ma_keys->dk_nentries; - Py_ssize_t pos = 0; - Py_hash_t hash; - PyObject *key, *value; - - while (_PyDict_Next((PyObject*)other, &pos, &key, &value, &hash)) { - int err = 0; - Py_INCREF(key); - Py_INCREF(value); - if (override == 1) { - err = insertdict(interp, mp, - Py_NewRef(key), hash, Py_NewRef(value)); - } - else { - err = _PyDict_Contains_KnownHash(a, key, hash); - if (err == 0) { - err = insertdict(interp, mp, - Py_NewRef(key), hash, Py_NewRef(value)); - } - else if (err > 0) { - if (override != 0) { - _PyErr_SetKeyError(key); - Py_DECREF(value); - Py_DECREF(key); - return -1; - } - err = 0; - } - } - Py_DECREF(value); - Py_DECREF(key); - if (err != 0) - return -1; - - if (orig_size != other->ma_keys->dk_nentries) { - PyErr_SetString(PyExc_RuntimeError, - "dict mutated during update"); - return -1; - } - } + int res; + Py_BEGIN_CRITICAL_SECTION2(a, b); + res = dict_dict_merge(interp, (PyDictObject *)a, other, override); + ASSERT_CONSISTENT(a); + Py_END_CRITICAL_SECTION2(); + return res; } else { /* Do it the generic, slower way */ + Py_BEGIN_CRITICAL_SECTION(a); PyObject *keys = PyMapping_Keys(b); PyObject *iter; PyObject *key, *value; int status; - if (keys == NULL) + if (keys == NULL) { /* Docstring says this is equivalent to E.keys() so * if E doesn't have a .keys() method we want * AttributeError to percolate up. Might as well * do the same for any other error. */ - return -1; + res = -1; + goto slow_exit; + } iter = PyObject_GetIter(keys); Py_DECREF(keys); - if (iter == NULL) - return -1; + if (iter == NULL) { + res = -1; + goto slow_exit; + } for (key = PyIter_Next(iter); key; key = PyIter_Next(iter)) { if (override != 1) { - status = PyDict_Contains(a, key); + status = contains_lock_held(mp, key); if (status != 0) { if (status > 0) { if (override == 0) { @@ -3060,30 +3281,39 @@ dict_merge(PyInterpreterState *interp, PyObject *a, PyObject *b, int override) } Py_DECREF(key); Py_DECREF(iter); - return -1; + res = -1; + goto slow_exit; } } value = PyObject_GetItem(b, key); if (value == NULL) { Py_DECREF(iter); Py_DECREF(key); - return -1; + res = -1; + goto slow_exit; } - status = PyDict_SetItem(a, key, value); + status = setitem_lock_held(mp, key, value); Py_DECREF(key); Py_DECREF(value); if (status < 0) { Py_DECREF(iter); + res = -1; + goto slow_exit; return -1; } } Py_DECREF(iter); - if (PyErr_Occurred()) + if (PyErr_Occurred()) { /* Iterator completed, via error */ - return -1; + res = -1; + goto slow_exit; + } + +slow_exit: + ASSERT_CONSISTENT(a); + Py_END_CRITICAL_SECTION(); + return res; } - ASSERT_CONSISTENT(a); - return 0; } int @@ -3108,23 +3338,27 @@ _PyDict_MergeEx(PyObject *a, PyObject *b, int override) return dict_merge(interp, a, b, override); } +/*[clinic input] +dict.copy + +Return a shallow copy of the dict. +[clinic start generated code]*/ + static PyObject * -dict_copy(PyObject *mp, PyObject *Py_UNUSED(ignored)) +dict_copy_impl(PyDictObject *self) +/*[clinic end generated code: output=ffb782cf970a5c39 input=73935f042b639de4]*/ { - return PyDict_Copy(mp); + return PyDict_Copy((PyObject *)self); } -PyObject * -PyDict_Copy(PyObject *o) +static PyObject * +copy_lock_held(PyObject *o) { PyObject *copy; PyDictObject *mp; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (o == NULL || !PyDict_Check(o)) { - PyErr_BadInternalCall(); - return NULL; - } + ASSERT_DICT_LOCKED(o); mp = (PyDictObject *)o; if (mp->ma_used == 0) { @@ -3197,54 +3431,41 @@ PyDict_Copy(PyObject *o) return (PyObject *)new; } - - copy = PyDict_New(); - if (copy == NULL) - return NULL; - if (dict_merge(interp, copy, o, 1) == 0) - return copy; - Py_DECREF(copy); - return NULL; -} - -Py_ssize_t -PyDict_Size(PyObject *mp) -{ - if (mp == NULL || !PyDict_Check(mp)) { - PyErr_BadInternalCall(); - return -1; - } - return ((PyDictObject *)mp)->ma_used; -} - -PyObject * -PyDict_Keys(PyObject *mp) -{ - if (mp == NULL || !PyDict_Check(mp)) { - PyErr_BadInternalCall(); - return NULL; - } - return dict_keys((PyDictObject *)mp); + + copy = PyDict_New(); + if (copy == NULL) + return NULL; + if (dict_merge(interp, copy, o, 1) == 0) + return copy; + Py_DECREF(copy); + return NULL; } PyObject * -PyDict_Values(PyObject *mp) +PyDict_Copy(PyObject *o) { - if (mp == NULL || !PyDict_Check(mp)) { + if (o == NULL || !PyDict_Check(o)) { PyErr_BadInternalCall(); return NULL; } - return dict_values((PyDictObject *)mp); + + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(o); + + res = copy_lock_held(o); + + Py_END_CRITICAL_SECTION(); + return res; } -PyObject * -PyDict_Items(PyObject *mp) +Py_ssize_t +PyDict_Size(PyObject *mp) { if (mp == NULL || !PyDict_Check(mp)) { PyErr_BadInternalCall(); - return NULL; + return -1; } - return dict_items((PyDictObject *)mp); + return ((PyDictObject *)mp)->ma_used; } /* Return 1 if dicts equal, 0 if not, -1 if error. @@ -3252,10 +3473,13 @@ PyDict_Items(PyObject *mp) * Uses only Py_EQ comparison. */ static int -dict_equal(PyDictObject *a, PyDictObject *b) +dict_equal_lock_held(PyDictObject *a, PyDictObject *b) { Py_ssize_t i; + ASSERT_DICT_LOCKED(a); + ASSERT_DICT_LOCKED(b); + if (a->ma_used != b->ma_used) /* can't be equal if # of entries differ */ return 0; @@ -3310,6 +3534,17 @@ dict_equal(PyDictObject *a, PyDictObject *b) return 1; } +static int +dict_equal(PyDictObject *a, PyDictObject *b) +{ + int res; + Py_BEGIN_CRITICAL_SECTION2(a, b); + res = dict_equal_lock_held(a, b); + Py_END_CRITICAL_SECTION2(); + + return res; +} + static PyObject * dict_richcompare(PyObject *v, PyObject *w, int op) { @@ -3333,6 +3568,7 @@ dict_richcompare(PyObject *v, PyObject *w, int op) /*[clinic input] @coexist +@critical_section dict.__contains__ key: object @@ -3342,8 +3578,8 @@ True if the dictionary has the specified key, else False. [clinic start generated code]*/ static PyObject * -dict___contains__(PyDictObject *self, PyObject *key) -/*[clinic end generated code: output=a3d03db709ed6e6b input=fe1cb42ad831e820]*/ +dict___contains___impl(PyDictObject *self, PyObject *key) +/*[clinic end generated code: output=1b314e6da7687dae input=bc76ec9c157cb81b]*/ { register PyDictObject *mp = self; Py_hash_t hash; @@ -3364,6 +3600,7 @@ dict___contains__(PyDictObject *self, PyObject *key) } /*[clinic input] +@critical_section dict.get key: object @@ -3375,7 +3612,7 @@ Return the value for key if key is in the dictionary, else default. static PyObject * dict_get_impl(PyDictObject *self, PyObject *key, PyObject *default_value) -/*[clinic end generated code: output=bba707729dee05bf input=279ddb5790b6b107]*/ +/*[clinic end generated code: output=bba707729dee05bf input=a631d3f18f584c60]*/ { PyObject *val = NULL; Py_hash_t hash; @@ -3395,51 +3632,77 @@ dict_get_impl(PyDictObject *self, PyObject *key, PyObject *default_value) return Py_NewRef(val); } -PyObject * -PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) +static int +dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_value, + PyObject **result, int incref_result) { PyDictObject *mp = (PyDictObject *)d; PyObject *value; Py_hash_t hash; PyInterpreterState *interp = _PyInterpreterState_GET(); + ASSERT_DICT_LOCKED(d); + if (!PyDict_Check(d)) { PyErr_BadInternalCall(); - return NULL; + if (result) { + *result = NULL; + } + return -1; } if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { hash = PyObject_Hash(key); - if (hash == -1) - return NULL; + if (hash == -1) { + if (result) { + *result = NULL; + } + return -1; + } } if (mp->ma_keys == Py_EMPTY_KEYS) { if (insert_to_emptydict(interp, mp, Py_NewRef(key), hash, - Py_NewRef(defaultobj)) < 0) { - return NULL; + Py_NewRef(default_value)) < 0) { + if (result) { + *result = NULL; + } + return -1; + } + if (result) { + *result = incref_result ? Py_NewRef(default_value) : default_value; } - return defaultobj; + return 0; } if (!PyUnicode_CheckExact(key) && DK_IS_UNICODE(mp->ma_keys)) { if (insertion_resize(interp, mp, 0) < 0) { - return NULL; + if (result) { + *result = NULL; + } + return -1; } } Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); - if (ix == DKIX_ERROR) - return NULL; + if (ix == DKIX_ERROR) { + if (result) { + *result = NULL; + } + return -1; + } if (ix == DKIX_EMPTY) { uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, defaultobj); + interp, PyDict_EVENT_ADDED, mp, key, default_value); mp->ma_keys->dk_version = 0; - value = defaultobj; + value = default_value; if (mp->ma_keys->dk_usable <= 0) { if (insertion_resize(interp, mp, 1) < 0) { - return NULL; + if (result) { + *result = NULL; + } + return -1; } } Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash); @@ -3471,11 +3734,16 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) mp->ma_keys->dk_usable--; mp->ma_keys->dk_nentries++; assert(mp->ma_keys->dk_usable >= 0); + ASSERT_CONSISTENT(mp); + if (result) { + *result = incref_result ? Py_NewRef(value) : value; + } + return 0; } else if (value == NULL) { uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, defaultobj); - value = defaultobj; + interp, PyDict_EVENT_ADDED, mp, key, default_value); + value = default_value; assert(_PyDict_HasSplitTable(mp)); assert(mp->ma_values->values[ix] == NULL); MAINTAIN_TRACKING(mp, key, value); @@ -3483,13 +3751,43 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) _PyDictValues_AddToInsertionOrder(mp->ma_values, ix); mp->ma_used++; mp->ma_version_tag = new_version; + ASSERT_CONSISTENT(mp); + if (result) { + *result = incref_result ? Py_NewRef(value) : value; + } + return 0; } ASSERT_CONSISTENT(mp); - return value; + if (result) { + *result = incref_result ? Py_NewRef(value) : value; + } + return 1; +} + +int +PyDict_SetDefaultRef(PyObject *d, PyObject *key, PyObject *default_value, + PyObject **result) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(d); + res = dict_setdefault_ref_lock_held(d, key, default_value, result, 1); + Py_END_CRITICAL_SECTION(); + return res; +} + +PyObject * +PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(d); + dict_setdefault_ref_lock_held(d, key, defaultobj, &result, 0); + Py_END_CRITICAL_SECTION(); + return result; } /*[clinic input] +@critical_section dict.setdefault key: object @@ -3504,18 +3802,25 @@ Return the value for key if key is in the dictionary, else default. static PyObject * dict_setdefault_impl(PyDictObject *self, PyObject *key, PyObject *default_value) -/*[clinic end generated code: output=f8c1101ebf69e220 input=0f063756e815fd9d]*/ +/*[clinic end generated code: output=f8c1101ebf69e220 input=9237af9a0a224302]*/ { PyObject *val; - - val = PyDict_SetDefault((PyObject *)self, key, default_value); - return Py_XNewRef(val); + dict_setdefault_ref_lock_held((PyObject *)self, key, default_value, &val, 1); + return val; } + +/*[clinic input] +dict.clear + +Remove all items from the dict. +[clinic start generated code]*/ + static PyObject * -dict_clear(PyObject *mp, PyObject *Py_UNUSED(ignored)) +dict_clear_impl(PyDictObject *self) +/*[clinic end generated code: output=5139a830df00830a input=0bf729baba97a4c2]*/ { - PyDict_Clear(mp); + PyDict_Clear((PyObject *)self); Py_RETURN_NONE; } @@ -3540,6 +3845,7 @@ dict_pop_impl(PyDictObject *self, PyObject *key, PyObject *default_value) } /*[clinic input] +@critical_section dict.popitem Remove and return a (key, value) pair as a 2-tuple. @@ -3550,7 +3856,7 @@ Raises KeyError if the dict is empty. static PyObject * dict_popitem_impl(PyDictObject *self) -/*[clinic end generated code: output=e65fcb04420d230d input=1c38a49f21f64941]*/ +/*[clinic end generated code: output=e65fcb04420d230d input=ef28b4da5f0f762e]*/ { Py_ssize_t i, j; PyObject *res; @@ -3676,8 +3982,8 @@ dict_tp_clear(PyObject *op) static PyObject *dictiter_new(PyDictObject *, PyTypeObject *); -Py_ssize_t -_PyDict_SizeOf(PyDictObject *mp) +static Py_ssize_t +sizeof_lock_held(PyDictObject *mp) { size_t res = _PyObject_SIZE(Py_TYPE(mp)); if (mp->ma_values) { @@ -3692,6 +3998,17 @@ _PyDict_SizeOf(PyDictObject *mp) return (Py_ssize_t)res; } +Py_ssize_t +_PyDict_SizeOf(PyDictObject *mp) +{ + Py_ssize_t res; + Py_BEGIN_CRITICAL_SECTION(mp); + res = sizeof_lock_held(mp); + Py_END_CRITICAL_SECTION(); + + return res; +} + size_t _PyDict_KeysSize(PyDictKeysObject *keys) { @@ -3703,11 +4020,17 @@ _PyDict_KeysSize(PyDictKeysObject *keys) return size; } +/*[clinic input] +dict.__sizeof__ + +Return the size of the dict in memory, in bytes. +[clinic start generated code]*/ + static PyObject * -dict_sizeof(PyObject *self, PyObject *Py_UNUSED(ignored)) +dict___sizeof___impl(PyDictObject *self) +/*[clinic end generated code: output=44279379b3824bda input=4fec4ddfc44a4d1a]*/ { - PyDictObject *mp = (PyDictObject *)self; - return PyLong_FromSsize_t(_PyDict_SizeOf(mp)); + return PyLong_FromSsize_t(_PyDict_SizeOf(self)); } static PyObject * @@ -3739,70 +4062,59 @@ dict_ior(PyObject *self, PyObject *other) PyDoc_STRVAR(getitem__doc__, "__getitem__($self, key, /)\n--\n\nReturn self[key]."); -PyDoc_STRVAR(sizeof__doc__, -"D.__sizeof__() -> size of D in memory, in bytes"); - PyDoc_STRVAR(update__doc__, "D.update([E, ]**F) -> None. Update D from dict/iterable E and F.\n\ If E is present and has a .keys() method, then does: for k in E: D[k] = E[k]\n\ If E is present and lacks a .keys() method, then does: for k, v in E: D[k] = v\n\ In either case, this is followed by: for k in F: D[k] = F[k]"); -PyDoc_STRVAR(clear__doc__, -"D.clear() -> None. Remove all items from D."); - -PyDoc_STRVAR(copy__doc__, -"D.copy() -> a shallow copy of D"); - /* Forward */ -static PyObject *dictkeys_new(PyObject *, PyObject *); -static PyObject *dictitems_new(PyObject *, PyObject *); -static PyObject *dictvalues_new(PyObject *, PyObject *); - -PyDoc_STRVAR(keys__doc__, - "D.keys() -> a set-like object providing a view on D's keys"); -PyDoc_STRVAR(items__doc__, - "D.items() -> a set-like object providing a view on D's items"); -PyDoc_STRVAR(values__doc__, - "D.values() -> an object providing a view on D's values"); static PyMethodDef mapp_methods[] = { DICT___CONTAINS___METHODDEF {"__getitem__", dict_subscript, METH_O | METH_COEXIST, getitem__doc__}, - {"__sizeof__", dict_sizeof, METH_NOARGS, - sizeof__doc__}, + DICT___SIZEOF___METHODDEF DICT_GET_METHODDEF DICT_SETDEFAULT_METHODDEF DICT_POP_METHODDEF DICT_POPITEM_METHODDEF - {"keys", dictkeys_new, METH_NOARGS, - keys__doc__}, - {"items", dictitems_new, METH_NOARGS, - items__doc__}, - {"values", dictvalues_new, METH_NOARGS, - values__doc__}, + DICT_KEYS_METHODDEF + DICT_ITEMS_METHODDEF + DICT_VALUES_METHODDEF {"update", _PyCFunction_CAST(dict_update), METH_VARARGS | METH_KEYWORDS, update__doc__}, DICT_FROMKEYS_METHODDEF - {"clear", dict_clear, METH_NOARGS, - clear__doc__}, - {"copy", dict_copy, METH_NOARGS, - copy__doc__}, + DICT_CLEAR_METHODDEF + DICT_COPY_METHODDEF DICT___REVERSED___METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ }; -/* Return 1 if `key` is in dict `op`, 0 if not, and -1 on error. */ -int -PyDict_Contains(PyObject *op, PyObject *key) +static int +contains_known_hash_lock_held(PyDictObject *mp, PyObject *key, Py_ssize_t hash) +{ + Py_ssize_t ix; + PyObject *value; + + ASSERT_DICT_LOCKED(mp); + + ix = _Py_dict_lookup(mp, key, hash, &value); + if (ix == DKIX_ERROR) + return -1; + return (ix != DKIX_EMPTY && value != NULL); +} + +static int +contains_lock_held(PyDictObject *mp, PyObject *key) { Py_hash_t hash; Py_ssize_t ix; - PyDictObject *mp = (PyDictObject *)op; PyObject *value; + ASSERT_DICT_LOCKED(mp); + if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { hash = PyObject_Hash(key); if (hash == -1) @@ -3814,6 +4126,17 @@ PyDict_Contains(PyObject *op, PyObject *key) return (ix != DKIX_EMPTY && value != NULL); } +/* Return 1 if `key` is in dict `op`, 0 if not, and -1 on error. */ +int +PyDict_Contains(PyObject *op, PyObject *key) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(op); + res = contains_lock_held((PyDictObject *)op, key); + Py_END_CRITICAL_SECTION(); + return res; +} + int PyDict_ContainsString(PyObject *op, const char *key) { @@ -4180,17 +4503,15 @@ static PyMethodDef dictiter_methods[] = { }; static PyObject* -dictiter_iternextkey(PyObject *self) +dictiter_iternextkey_lock_held(PyDictObject *d, PyObject *self) { dictiterobject *di = (dictiterobject *)self; PyObject *key; Py_ssize_t i; PyDictKeysObject *k; - PyDictObject *d = di->di_dict; - if (d == NULL) - return NULL; assert (PyDict_Check(d)); + ASSERT_DICT_LOCKED(d); if (di->di_used != d->ma_used) { PyErr_SetString(PyExc_RuntimeError, @@ -4248,6 +4569,23 @@ dictiter_iternextkey(PyObject *self) return NULL; } +static PyObject* +dictiter_iternextkey(PyObject *self) +{ + dictiterobject *di = (dictiterobject *)self; + PyDictObject *d = di->di_dict; + + if (d == NULL) + return NULL; + + PyObject *value; + Py_BEGIN_CRITICAL_SECTION(d); + value = dictiter_iternextkey_lock_held(d, self); + Py_END_CRITICAL_SECTION(); + + return value; +} + PyTypeObject PyDictIterKey_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_keyiterator", /* tp_name */ @@ -4282,16 +4620,14 @@ PyTypeObject PyDictIterKey_Type = { }; static PyObject * -dictiter_iternextvalue(PyObject *self) +dictiter_iternextvalue_lock_held(PyDictObject *d, PyObject *self) { dictiterobject *di = (dictiterobject *)self; PyObject *value; Py_ssize_t i; - PyDictObject *d = di->di_dict; - if (d == NULL) - return NULL; assert (PyDict_Check(d)); + ASSERT_DICT_LOCKED(d); if (di->di_used != d->ma_used) { PyErr_SetString(PyExc_RuntimeError, @@ -4348,6 +4684,23 @@ dictiter_iternextvalue(PyObject *self) return NULL; } +static PyObject * +dictiter_iternextvalue(PyObject *self) +{ + dictiterobject *di = (dictiterobject *)self; + PyDictObject *d = di->di_dict; + + if (d == NULL) + return NULL; + + PyObject *value; + Py_BEGIN_CRITICAL_SECTION(d); + value = dictiter_iternextvalue_lock_held(d, self); + Py_END_CRITICAL_SECTION(); + + return value; +} + PyTypeObject PyDictIterValue_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_valueiterator", /* tp_name */ @@ -4382,15 +4735,12 @@ PyTypeObject PyDictIterValue_Type = { }; static PyObject * -dictiter_iternextitem(PyObject *self) +dictiter_iternextitem_lock_held(PyDictObject *d, PyObject *self) { dictiterobject *di = (dictiterobject *)self; PyObject *key, *value, *result; Py_ssize_t i; - PyDictObject *d = di->di_dict; - if (d == NULL) - return NULL; assert (PyDict_Check(d)); if (di->di_used != d->ma_used) { @@ -4473,6 +4823,22 @@ dictiter_iternextitem(PyObject *self) return NULL; } +static PyObject * +dictiter_iternextitem(PyObject *self) +{ + dictiterobject *di = (dictiterobject *)self; + PyDictObject *d = di->di_dict; + + if (d == NULL) + return NULL; + + PyObject *item; + Py_BEGIN_CRITICAL_SECTION(d); + item = dictiter_iternextitem_lock_held(d, self); + Py_END_CRITICAL_SECTION(); + return item; +} + PyTypeObject PyDictIterItem_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_itemiterator", /* tp_name */ @@ -4510,15 +4876,12 @@ PyTypeObject PyDictIterItem_Type = { /* dictreviter */ static PyObject * -dictreviter_iternext(PyObject *self) +dictreviter_iter_PyDict_Next(PyDictObject *d, PyObject *self) { dictiterobject *di = (dictiterobject *)self; - PyDictObject *d = di->di_dict; - if (d == NULL) { - return NULL; - } assert (PyDict_Check(d)); + ASSERT_DICT_LOCKED(d); if (di->di_used != d->ma_used) { PyErr_SetString(PyExc_RuntimeError, @@ -4609,6 +4972,23 @@ dictreviter_iternext(PyObject *self) return NULL; } +static PyObject * +dictreviter_iternext(PyObject *self) +{ + dictiterobject *di = (dictiterobject *)self; + PyDictObject *d = di->di_dict; + + if (d == NULL) + return NULL; + + PyObject *value; + Py_BEGIN_CRITICAL_SECTION(d); + value = dictreviter_iter_PyDict_Next(d, self); + Py_END_CRITICAL_SECTION(); + + return value; +} + PyTypeObject PyDictRevIterKey_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_reversekeyiterator", @@ -5037,14 +5417,12 @@ dictviews_or(PyObject* self, PyObject *other) } static PyObject * -dictitems_xor(PyObject *self, PyObject *other) +dictitems_xor_lock_held(PyObject *d1, PyObject *d2) { - assert(PyDictItems_Check(self)); - assert(PyDictItems_Check(other)); - PyObject *d1 = (PyObject *)((_PyDictViewObject *)self)->dv_dict; - PyObject *d2 = (PyObject *)((_PyDictViewObject *)other)->dv_dict; + ASSERT_DICT_LOCKED(d1); + ASSERT_DICT_LOCKED(d2); - PyObject *temp_dict = PyDict_Copy(d1); + PyObject *temp_dict = copy_lock_held(d1); if (temp_dict == NULL) { return NULL; } @@ -5122,6 +5500,22 @@ dictitems_xor(PyObject *self, PyObject *other) return NULL; } +static PyObject * +dictitems_xor(PyObject *self, PyObject *other) +{ + assert(PyDictItems_Check(self)); + assert(PyDictItems_Check(other)); + PyObject *d1 = (PyObject *)((_PyDictViewObject *)self)->dv_dict; + PyObject *d2 = (PyObject *)((_PyDictViewObject *)other)->dv_dict; + + PyObject *res; + Py_BEGIN_CRITICAL_SECTION2(d1, d2); + res = dictitems_xor_lock_held(d1, d2); + Py_END_CRITICAL_SECTION2(); + + return res; +} + static PyObject* dictviews_xor(PyObject* self, PyObject *other) { @@ -5263,10 +5657,17 @@ PyTypeObject PyDictKeys_Type = { .tp_getset = dictview_getset, }; +/*[clinic input] +dict.keys + +Return a set-like object providing a view on the dict's keys. +[clinic start generated code]*/ + static PyObject * -dictkeys_new(PyObject *dict, PyObject *Py_UNUSED(ignored)) +dict_keys_impl(PyDictObject *self) +/*[clinic end generated code: output=aac2830c62990358 input=42f48a7a771212a7]*/ { - return _PyDictView_New(dict, &PyDictKeys_Type); + return _PyDictView_New((PyObject *)self, &PyDictKeys_Type); } static PyObject * @@ -5368,10 +5769,17 @@ PyTypeObject PyDictItems_Type = { .tp_getset = dictview_getset, }; +/*[clinic input] +dict.items + +Return a set-like object providing a view on the dict's items. +[clinic start generated code]*/ + static PyObject * -dictitems_new(PyObject *dict, PyObject *Py_UNUSED(ignored)) +dict_items_impl(PyDictObject *self) +/*[clinic end generated code: output=88c7db7150c7909a input=87c822872eb71f5a]*/ { - return _PyDictView_New(dict, &PyDictItems_Type); + return _PyDictView_New((PyObject *)self, &PyDictItems_Type); } static PyObject * @@ -5451,10 +5859,17 @@ PyTypeObject PyDictValues_Type = { .tp_getset = dictview_getset, }; +/*[clinic input] +dict.values + +Return an object providing a view on the dict's values. +[clinic start generated code]*/ + static PyObject * -dictvalues_new(PyObject *dict, PyObject *Py_UNUSED(ignored)) +dict_values_impl(PyDictObject *self) +/*[clinic end generated code: output=ce9f2e9e8a959dd4 input=b46944f85493b230]*/ { - return _PyDictView_New(dict, &PyDictValues_Type); + return _PyDictView_New((PyObject *)self, &PyDictValues_Type); } static PyObject * @@ -5574,6 +5989,18 @@ _PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values) return make_dict_from_instance_attributes(interp, keys, values); } +static bool +has_unique_reference(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + return (_Py_IsOwnedByCurrentThread(op) && + op->ob_ref_local == 1 && + _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared) == 0); +#else + return Py_REFCNT(op) == 1; +#endif +} + // Return true if the dict was dematerialized, false otherwise. bool _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv) @@ -5590,7 +6017,9 @@ _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv) return false; } assert(_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_HEAPTYPE)); - if (dict->ma_keys != CACHED_KEYS(Py_TYPE(obj)) || Py_REFCNT(dict) != 1) { + if (dict->ma_keys != CACHED_KEYS(Py_TYPE(obj)) || + !has_unique_reference((PyObject *)dict)) + { return false; } assert(dict->ma_values); @@ -5973,7 +6402,8 @@ PyDict_AddWatcher(PyDict_WatchCallback callback) { PyInterpreterState *interp = _PyInterpreterState_GET(); - for (int i = 0; i < DICT_MAX_WATCHERS; i++) { + /* Start at 2, as 0 and 1 are reserved for CPython */ + for (int i = 2; i < DICT_MAX_WATCHERS; i++) { if (!interp->dict_state.watchers[i]) { interp->dict_state.watchers[i] = callback; return i; diff --git a/Objects/exceptions.c b/Objects/exceptions.c index a685ed803cd02db..cff55d05163b6ba 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -2566,6 +2566,11 @@ MiddlingExtendsException(PyExc_SyntaxError, IndentationError, SyntaxError, MiddlingExtendsException(PyExc_IndentationError, TabError, SyntaxError, "Improper mixture of spaces and tabs."); +/* + * IncompleteInputError extends SyntaxError + */ +MiddlingExtendsException(PyExc_SyntaxError, IncompleteInputError, SyntaxError, + "incomplete input."); /* * LookupError extends Exception @@ -3635,6 +3640,7 @@ static struct static_exception static_exceptions[] = { // Level 4: Other subclasses ITEM(IndentationError), // base: SyntaxError(Exception) + ITEM(IncompleteInputError), // base: SyntaxError(Exception) ITEM(IndexError), // base: LookupError(Exception) ITEM(KeyError), // base: LookupError(Exception) ITEM(ModuleNotFoundError), // base: ImportError(Exception) diff --git a/Objects/fileobject.c b/Objects/fileobject.c index 5522eba34eace9b..e30ab952dff571b 100644 --- a/Objects/fileobject.c +++ b/Objects/fileobject.c @@ -174,6 +174,13 @@ PyObject_AsFileDescriptor(PyObject *o) PyObject *meth; if (PyLong_Check(o)) { + if (PyBool_Check(o)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "bool is used as a file descriptor", 1)) + { + return -1; + } + } fd = PyLong_AsInt(o); } else if (PyObject_GetOptionalAttr(o, &_Py_ID(fileno), &meth) < 0) { diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 912c450a5e1055c..9b322c52d4daea6 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -32,7 +32,7 @@ get_float_state(void) { _PyFreeListState *state = _PyFreeListState_GET(); assert(state != NULL); - return &state->float_state; + return &state->floats; } #endif @@ -1993,7 +1993,7 @@ void _PyFloat_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization) { #ifdef WITH_FREELISTS - struct _Py_float_state *state = &freelist_state->float_state; + struct _Py_float_state *state = &freelist_state->floats; PyFloatObject *f = state->free_list; while (f != NULL) { PyFloatObject *next = (PyFloatObject*) Py_TYPE(f); @@ -2010,12 +2010,6 @@ _PyFloat_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization) #endif } -void -_PyFloat_Fini(_PyFreeListState *state) -{ - _PyFloat_ClearFreeList(state, 1); -} - void _PyFloat_FiniType(PyInterpreterState *interp) { diff --git a/Objects/frameobject.c b/Objects/frameobject.c index cafe4ef6141d9af..a914c61aac2fd51 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -926,6 +926,7 @@ frame_tp_clear(PyFrameObject *f) Py_CLEAR(locals[i]); } f->f_frame->stacktop = 0; + Py_CLEAR(f->f_frame->f_locals); return 0; } diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 2620dc69bfd79bd..08b2823d8cf024e 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -53,6 +53,15 @@ handle_func_event(PyFunction_WatchEvent event, PyFunctionObject *func, if (interp->active_func_watchers) { notify_func_watchers(interp, event, func, new_value); } + switch (event) { + case PyFunction_EVENT_MODIFY_CODE: + case PyFunction_EVENT_MODIFY_DEFAULTS: + case PyFunction_EVENT_MODIFY_KWDEFAULTS: + RARE_EVENT_INTERP_INC(interp, func_modification); + break; + default: + break; + } } int diff --git a/Objects/genobject.c b/Objects/genobject.c index e9aeb7ab9a9fa83..59ab7abf6180bd1 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1633,7 +1633,7 @@ static struct _Py_async_gen_state * get_async_gen_state(void) { _PyFreeListState *state = _PyFreeListState_GET(); - return &state->async_gen_state; + return &state->async_gens; } #endif @@ -1659,7 +1659,7 @@ void _PyAsyncGen_ClearFreeLists(_PyFreeListState *freelist_state, int is_finalization) { #ifdef WITH_FREELISTS - struct _Py_async_gen_state *state = &freelist_state->async_gen_state; + struct _Py_async_gen_state *state = &freelist_state->async_gens; while (state->value_numfree > 0) { _PyAsyncGenWrappedValue *o; @@ -1682,13 +1682,6 @@ _PyAsyncGen_ClearFreeLists(_PyFreeListState *freelist_state, int is_finalization #endif } -void -_PyAsyncGen_Fini(_PyFreeListState *state) -{ - _PyAsyncGen_ClearFreeLists(state, 1); -} - - static PyObject * async_gen_unwrap_value(PyAsyncGenObject *gen, PyObject *result) { diff --git a/Objects/listobject.c b/Objects/listobject.c index 401d1026133f4e1..7fdb91eab890b5f 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -26,11 +26,10 @@ get_list_state(void) { _PyFreeListState *state = _PyFreeListState_GET(); assert(state != NULL); - return &state->list_state; + return &state->lists; } #endif - /* Ensure ob_item has room for at least newsize elements, and set * ob_size to newsize. If newsize > ob_size on entry, the content * of the new slots at exit is undefined heap trash; it's the caller's @@ -124,7 +123,7 @@ void _PyList_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization) { #ifdef WITH_FREELISTS - struct _Py_list_state *state = &freelist_state->list_state; + struct _Py_list_state *state = &freelist_state->lists; while (state->numfree > 0) { PyListObject *op = state->free_list[--state->numfree]; assert(PyList_CheckExact(op)); @@ -136,12 +135,6 @@ _PyList_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization) #endif } -void -_PyList_Fini(_PyFreeListState *state) -{ - _PyList_ClearFreeList(state, 1); -} - /* Print summary info about the state of the optimized allocator */ void _PyList_DebugMallocStats(FILE *out) @@ -221,8 +214,9 @@ PyList_Size(PyObject *op) PyErr_BadInternalCall(); return -1; } - else - return Py_SIZE(op); + else { + return PyList_GET_SIZE(op); + } } static inline int @@ -253,6 +247,30 @@ PyList_GetItem(PyObject *op, Py_ssize_t i) return ((PyListObject *)op) -> ob_item[i]; } +PyObject * +PyList_GetItemRef(PyObject *op, Py_ssize_t i) +{ + if (!PyList_Check(op)) { + PyErr_SetString(PyExc_TypeError, "expected a list"); + return NULL; + } + if (!valid_index(i, Py_SIZE(op))) { + _Py_DECLARE_STR(list_err, "list index out of range"); + PyErr_SetObject(PyExc_IndexError, &_Py_STR(list_err)); + return NULL; + } + return Py_NewRef(PyList_GET_ITEM(op, i)); +} + +static inline PyObject* +list_get_item_ref(PyListObject *op, Py_ssize_t i) +{ + if (!valid_index(i, Py_SIZE(op))) { + return NULL; + } + return Py_NewRef(PyList_GET_ITEM(op, i)); +} + int PyList_SetItem(PyObject *op, Py_ssize_t i, PyObject *newitem) @@ -328,7 +346,7 @@ PyList_Insert(PyObject *op, Py_ssize_t where, PyObject *newitem) int _PyList_AppendTakeRefListResize(PyListObject *self, PyObject *newitem) { - Py_ssize_t len = PyList_GET_SIZE(self); + Py_ssize_t len = Py_SIZE(self); assert(self->allocated == -1 || self->allocated == len); if (list_resize(self, len + 1) < 0) { Py_DECREF(newitem); @@ -342,7 +360,11 @@ int PyList_Append(PyObject *op, PyObject *newitem) { if (PyList_Check(op) && (newitem != NULL)) { - return _PyList_AppendTakeRef((PyListObject *)op, Py_NewRef(newitem)); + int ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = _PyList_AppendTakeRef((PyListObject *)op, Py_NewRef(newitem)); + Py_END_CRITICAL_SECTION(); + return ret; } PyErr_BadInternalCall(); return -1; @@ -383,18 +405,11 @@ list_dealloc(PyObject *self) } static PyObject * -list_repr(PyObject *self) +list_repr_impl(PyListObject *v) { - PyListObject *v = (PyListObject *)self; - Py_ssize_t i; PyObject *s; _PyUnicodeWriter writer; - - if (Py_SIZE(v) == 0) { - return PyUnicode_FromString("[]"); - } - - i = Py_ReprEnter((PyObject*)v); + Py_ssize_t i = Py_ReprEnter((PyObject*)v); if (i != 0) { return i > 0 ? PyUnicode_FromString("[...]") : NULL; } @@ -439,34 +454,50 @@ list_repr(PyObject *self) return NULL; } +static PyObject * +list_repr(PyObject *self) +{ + if (PyList_GET_SIZE(self) == 0) { + return PyUnicode_FromString("[]"); + } + PyListObject *v = (PyListObject *)self; + PyObject *ret = NULL; + Py_BEGIN_CRITICAL_SECTION(v); + ret = list_repr_impl(v); + Py_END_CRITICAL_SECTION(); + return ret; +} + static Py_ssize_t list_length(PyObject *a) { - return Py_SIZE(a); + return PyList_GET_SIZE(a); } static int list_contains(PyObject *aa, PyObject *el) { - PyListObject *a = (PyListObject *)aa; - PyObject *item; - Py_ssize_t i; - int cmp; - for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(a); ++i) { - item = PyList_GET_ITEM(a, i); - Py_INCREF(item); - cmp = PyObject_RichCompareBool(item, el, Py_EQ); + for (Py_ssize_t i = 0; ; i++) { + PyObject *item = list_get_item_ref((PyListObject *)aa, i); + if (item == NULL) { + // out-of-bounds + return 0; + } + int cmp = PyObject_RichCompareBool(item, el, Py_EQ); Py_DECREF(item); + if (cmp != 0) { + return cmp; + } } - return cmp; + return 0; } static PyObject * list_item(PyObject *aa, Py_ssize_t i) { PyListObject *a = (PyListObject *)aa; - if (!valid_index(i, Py_SIZE(a))) { + if (!valid_index(i, PyList_GET_SIZE(a))) { PyErr_SetObject(PyExc_IndexError, &_Py_STR(list_err)); return NULL; } @@ -504,6 +535,8 @@ PyList_GetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) PyErr_BadInternalCall(); return NULL; } + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(a); if (ilow < 0) { ilow = 0; } @@ -516,7 +549,9 @@ PyList_GetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) else if (ihigh > Py_SIZE(a)) { ihigh = Py_SIZE(a); } - return list_slice((PyListObject *)a, ilow, ihigh); + ret = list_slice((PyListObject *)a, ilow, ihigh); + Py_END_CRITICAL_SECTION(); + return ret; } static PyObject * @@ -2690,8 +2725,6 @@ list_index_impl(PyListObject *self, PyObject *value, Py_ssize_t start, Py_ssize_t stop) /*[clinic end generated code: output=ec51b88787e4e481 input=40ec5826303a0eb1]*/ { - Py_ssize_t i; - if (start < 0) { start += Py_SIZE(self); if (start < 0) @@ -2702,9 +2735,12 @@ list_index_impl(PyListObject *self, PyObject *value, Py_ssize_t start, if (stop < 0) stop = 0; } - for (i = start; i < stop && i < Py_SIZE(self); i++) { - PyObject *obj = self->ob_item[i]; - Py_INCREF(obj); + for (Py_ssize_t i = start; i < stop; i++) { + PyObject *obj = list_get_item_ref(self, i); + if (obj == NULL) { + // out-of-bounds + break; + } int cmp = PyObject_RichCompareBool(obj, value, Py_EQ); Py_DECREF(obj); if (cmp > 0) @@ -2730,15 +2766,17 @@ list_count(PyListObject *self, PyObject *value) /*[clinic end generated code: output=b1f5d284205ae714 input=3bdc3a5e6f749565]*/ { Py_ssize_t count = 0; - Py_ssize_t i; - - for (i = 0; i < Py_SIZE(self); i++) { - PyObject *obj = self->ob_item[i]; + for (Py_ssize_t i = 0; ; i++) { + PyObject *obj = list_get_item_ref(self, i); + if (obj == NULL) { + // out-of-bounds + break; + } if (obj == value) { count++; + Py_DECREF(obj); continue; } - Py_INCREF(obj); int cmp = PyObject_RichCompareBool(obj, value, Py_EQ); Py_DECREF(obj); if (cmp > 0) diff --git a/Objects/mimalloc/heap.c b/Objects/mimalloc/heap.c index 164b28f0fab2402..154dad0b1284805 100644 --- a/Objects/mimalloc/heap.c +++ b/Objects/mimalloc/heap.c @@ -538,7 +538,6 @@ bool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t *page, mi_ mi_assert(page != NULL); if (page == NULL) return true; - _mi_page_free_collect(page,true); mi_assert_internal(page->local_free == NULL); if (page->used == 0) return true; @@ -635,6 +634,7 @@ bool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t *page, mi_ typedef bool (mi_heap_area_visit_fun)(const mi_heap_t* heap, const mi_heap_area_ex_t* area, void* arg); void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) { + _mi_page_free_collect(page,true); const size_t bsize = mi_page_block_size(page); const size_t ubsize = mi_page_usable_block_size(page); area->reserved = page->reserved * bsize; diff --git a/Objects/object.c b/Objects/object.c index 587c5528c01345b..275aa6713c8c217 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2,6 +2,7 @@ /* Generic object operations; and implementation of None */ #include "Python.h" +#include "pycore_brc.h" // _Py_brc_queue_object() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _Py_EnterRecursiveCallTstate() #include "pycore_context.h" // _PyContextTokenMissing_Type @@ -344,12 +345,10 @@ _Py_DecRefSharedDebug(PyObject *o, const char *filename, int lineno) &shared, new_shared)); if (should_queue) { - // TODO: the inter-thread queue is not yet implemented. For now, - // we just merge the refcount here. - Py_ssize_t refcount = _Py_ExplicitMergeRefcount(o, -1); - if (refcount == 0) { - _Py_Dealloc(o); - } +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyInterpreterState_GET()); +#endif + _Py_brc_queue_object(o); } else if (new_shared == _Py_REF_MERGED) { // refcount is zero AND merged @@ -399,10 +398,6 @@ _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra) Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); do { refcnt = Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, shared, _Py_REF_SHARED_SHIFT); - if (_Py_REF_IS_MERGED(shared)) { - return refcnt; - } - refcnt += (Py_ssize_t)op->ob_ref_local; refcnt += extra; @@ -410,6 +405,10 @@ _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra) } while (!_Py_atomic_compare_exchange_ssize(&op->ob_ref_shared, &shared, new_shared)); +#ifdef Py_REF_DEBUG + _Py_AddRefTotal(_PyInterpreterState_GET(), extra); +#endif + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 0); _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); return refcnt; @@ -794,6 +793,21 @@ PyObject_Bytes(PyObject *v) return PyBytes_FromObject(v); } +void +_PyObject_ClearFreeLists(_PyFreeListState *state, int is_finalization) +{ + // In the free-threaded build, freelists are per-PyThreadState and cleared in PyThreadState_Clear() + // In the default build, freelists are per-interpreter and cleared in finalize_interp_types() + _PyFloat_ClearFreeList(state, is_finalization); + _PyTuple_ClearFreeList(state, is_finalization); + _PyList_ClearFreeList(state, is_finalization); + _PyDict_ClearFreeList(state, is_finalization); + _PyContext_ClearFreeList(state, is_finalization); + _PyAsyncGen_ClearFreeLists(state, is_finalization); + // Only be cleared if is_finalization is true. + _PyObjectStackChunk_ClearFreeList(state, is_finalization); + _PySlice_ClearFreeList(state, is_finalization); +} /* def _PyObject_FunctionStr(x): @@ -2671,7 +2685,12 @@ _PyTrash_thread_deposit_object(struct _py_trashcan *trash, PyObject *op) _PyObject_ASSERT(op, _PyObject_IS_GC(op)); _PyObject_ASSERT(op, !_PyObject_GC_IS_TRACKED(op)); _PyObject_ASSERT(op, Py_REFCNT(op) == 0); +#ifdef Py_GIL_DISABLED + _PyObject_ASSERT(op, op->ob_tid == 0); + op->ob_tid = (uintptr_t)trash->delete_later; +#else _PyGCHead_SET_PREV(_Py_AS_GC(op), (PyGC_Head*)trash->delete_later); +#endif trash->delete_later = op; } @@ -2697,8 +2716,12 @@ _PyTrash_thread_destroy_chain(struct _py_trashcan *trash) PyObject *op = trash->delete_later; destructor dealloc = Py_TYPE(op)->tp_dealloc; - trash->delete_later = - (PyObject*) _PyGCHead_PREV(_Py_AS_GC(op)); +#ifdef Py_GIL_DISABLED + trash->delete_later = (PyObject*) op->ob_tid; + op->ob_tid = 0; +#else + trash->delete_later = (PyObject*) _PyGCHead_PREV(_Py_AS_GC(op)); +#endif /* Call the deallocator directly. This used to try to * fool Py_DECREF into calling it indirectly, but diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 16d5bcb53e7eb75..6a12c3dca38b36d 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -7,6 +7,7 @@ #include "pycore_pyerrors.h" // _Py_FatalErrorFormat() #include "pycore_pymem.h" #include "pycore_pystate.h" // _PyInterpreterState_GET +#include "pycore_obmalloc_init.h" #include <stdlib.h> // malloc() #include <stdbool.h> @@ -1016,6 +1017,13 @@ static int running_on_valgrind = -1; typedef struct _obmalloc_state OMState; +/* obmalloc state for main interpreter and shared by all interpreters without + * their own obmalloc state. By not explicitly initalizing this structure, it + * will be allocated in the BSS which is a small performance win. The radix + * tree arrays are fairly large but are sparsely used. */ +static struct _obmalloc_state obmalloc_state_main; +static bool obmalloc_state_initialized; + static inline int has_own_state(PyInterpreterState *interp) { @@ -1028,10 +1036,8 @@ static inline OMState * get_state(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (!has_own_state(interp)) { - interp = _PyInterpreterState_Main(); - } - return &interp->obmalloc; + assert(interp->obmalloc != NULL); // otherwise not initialized or freed + return interp->obmalloc; } // These macros all rely on a local "state" variable. @@ -1067,7 +1073,12 @@ get_mimalloc_allocated_blocks(PyInterpreterState *interp) mi_heap_visit_blocks(heap, false, &count_blocks, &allocated_blocks); } } - // TODO(sgross): count blocks in abandoned segments. + + mi_abandoned_pool_t *pool = &interp->mimalloc.abandoned_pool; + for (uint8_t tag = 0; tag < _Py_MIMALLOC_HEAP_COUNT; tag++) { + _mi_abandoned_pool_visit_blocks(pool, tag, false, &count_blocks, + &allocated_blocks); + } #else // TODO(sgross): this only counts the current thread's blocks. mi_heap_t *heap = mi_heap_get_default(); @@ -1094,7 +1105,11 @@ _PyInterpreterState_GetAllocatedBlocks(PyInterpreterState *interp) "the interpreter doesn't have its own allocator"); } #endif - OMState *state = &interp->obmalloc; + OMState *state = interp->obmalloc; + + if (state == NULL) { + return 0; + } Py_ssize_t n = raw_allocated_blocks; /* add up allocated blocks for used pools */ @@ -1116,6 +1131,8 @@ _PyInterpreterState_GetAllocatedBlocks(PyInterpreterState *interp) return n; } +static void free_obmalloc_arenas(PyInterpreterState *interp); + void _PyInterpreterState_FinalizeAllocatedBlocks(PyInterpreterState *interp) { @@ -1124,10 +1141,20 @@ _PyInterpreterState_FinalizeAllocatedBlocks(PyInterpreterState *interp) return; } #endif - if (has_own_state(interp)) { + if (has_own_state(interp) && interp->obmalloc != NULL) { Py_ssize_t leaked = _PyInterpreterState_GetAllocatedBlocks(interp); assert(has_own_state(interp) || leaked == 0); interp->runtime->obmalloc.interpreter_leaks += leaked; + if (_PyMem_obmalloc_state_on_heap(interp) && leaked == 0) { + // free the obmalloc arenas and radix tree nodes. If leaked > 0 + // then some of the memory allocated by obmalloc has not been + // freed. It might be safe to free the arenas in that case but + // it's possible that extension modules are still using that + // memory. So, it is safer to not free and to leak. Perhaps there + // should be warning when this happens. It should be possible to + // use a tool like "-fsanitize=address" to track down these leaks. + free_obmalloc_arenas(interp); + } } } @@ -1167,6 +1194,7 @@ get_num_global_allocated_blocks(_PyRuntimeState *runtime) } } else { + _PyEval_StopTheWorldAll(&_PyRuntime); HEAD_LOCK(runtime); PyInterpreterState *interp = PyInterpreterState_Head(); assert(interp != NULL); @@ -1186,6 +1214,7 @@ get_num_global_allocated_blocks(_PyRuntimeState *runtime) } } HEAD_UNLOCK(runtime); + _PyEval_StartTheWorldAll(&_PyRuntime); #ifdef Py_DEBUG assert(got_main); #endif @@ -2717,9 +2746,96 @@ _PyDebugAllocatorStats(FILE *out, (void)printone(out, buf2, num_blocks * sizeof_block); } +// Return true if the obmalloc state structure is heap allocated, +// by PyMem_RawCalloc(). For the main interpreter, this structure +// allocated in the BSS. Allocating that way gives some memory savings +// and a small performance win (at least on a demand paged OS). On +// 64-bit platforms, the obmalloc structure is 256 kB. Most of that +// memory is for the arena_map_top array. Since normally only one entry +// of that array is used, only one page of resident memory is actually +// used, rather than the full 256 kB. +bool _PyMem_obmalloc_state_on_heap(PyInterpreterState *interp) +{ +#if WITH_PYMALLOC + return interp->obmalloc && interp->obmalloc != &obmalloc_state_main; +#else + return false; +#endif +} + +#ifdef WITH_PYMALLOC +static void +init_obmalloc_pools(PyInterpreterState *interp) +{ + // initialize the obmalloc->pools structure. This must be done + // before the obmalloc alloc/free functions can be called. + poolp temp[OBMALLOC_USED_POOLS_SIZE] = + _obmalloc_pools_INIT(interp->obmalloc->pools); + memcpy(&interp->obmalloc->pools.used, temp, sizeof(temp)); +} +#endif /* WITH_PYMALLOC */ + +int _PyMem_init_obmalloc(PyInterpreterState *interp) +{ +#ifdef WITH_PYMALLOC + /* Initialize obmalloc, but only for subinterpreters, + since the main interpreter is initialized statically. */ + if (_Py_IsMainInterpreter(interp) + || _PyInterpreterState_HasFeature(interp, + Py_RTFLAGS_USE_MAIN_OBMALLOC)) { + interp->obmalloc = &obmalloc_state_main; + if (!obmalloc_state_initialized) { + init_obmalloc_pools(interp); + obmalloc_state_initialized = true; + } + } else { + interp->obmalloc = PyMem_RawCalloc(1, sizeof(struct _obmalloc_state)); + if (interp->obmalloc == NULL) { + return -1; + } + init_obmalloc_pools(interp); + } +#endif /* WITH_PYMALLOC */ + return 0; // success +} + #ifdef WITH_PYMALLOC +static void +free_obmalloc_arenas(PyInterpreterState *interp) +{ + OMState *state = interp->obmalloc; + for (uint i = 0; i < maxarenas; ++i) { + // free each obmalloc memory arena + struct arena_object *ao = &allarenas[i]; + _PyObject_Arena.free(_PyObject_Arena.ctx, + (void *)ao->address, ARENA_SIZE); + } + // free the array containing pointers to all arenas + PyMem_RawFree(allarenas); +#if WITH_PYMALLOC_RADIX_TREE +#ifdef USE_INTERIOR_NODES + // Free the middle and bottom nodes of the radix tree. These are allocated + // by arena_map_mark_used() but not freed when arenas are freed. + for (int i1 = 0; i1 < MAP_TOP_LENGTH; i1++) { + arena_map_mid_t *mid = arena_map_root.ptrs[i1]; + if (mid == NULL) { + continue; + } + for (int i2 = 0; i2 < MAP_MID_LENGTH; i2++) { + arena_map_bot_t *bot = arena_map_root.ptrs[i1]->ptrs[i2]; + if (bot == NULL) { + continue; + } + PyMem_RawFree(bot); + } + PyMem_RawFree(mid); + } +#endif +#endif +} + #ifdef Py_DEBUG /* Is target in the list? The list is traversed via the nextpool pointers. * The list may be NULL-terminated, or circular. Return 1 if target is in diff --git a/Objects/odictobject.c b/Objects/odictobject.c index b5280c39e1be542..421bc52992d7354 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -465,12 +465,13 @@ Potential Optimizations */ #include "Python.h" -#include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_ceval.h" // _PyEval_GetBuiltin() -#include "pycore_dict.h" // _Py_dict_lookup() -#include "pycore_object.h" // _PyObject_GC_UNTRACK() -#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() -#include <stddef.h> // offsetof() +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_critical_section.h" //_Py_BEGIN_CRITICAL_SECTION +#include "pycore_dict.h" // _Py_dict_lookup() +#include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include <stddef.h> // offsetof() #include "clinic/odictobject.c.h" @@ -1039,6 +1040,8 @@ _odict_popkey_hash(PyObject *od, PyObject *key, PyObject *failobj, { PyObject *value = NULL; + Py_BEGIN_CRITICAL_SECTION(od); + _ODictNode *node = _odict_find_node_hash((PyODictObject *)od, key, hash); if (node != NULL) { /* Pop the node first to avoid a possible dict resize (due to @@ -1046,7 +1049,7 @@ _odict_popkey_hash(PyObject *od, PyObject *key, PyObject *failobj, resolution. */ int res = _odict_clear_node((PyODictObject *)od, node, key, hash); if (res < 0) { - return NULL; + goto done; } /* Now delete the value from the dict. */ if (_PyDict_Pop_KnownHash((PyDictObject *)od, key, hash, @@ -1063,6 +1066,8 @@ _odict_popkey_hash(PyObject *od, PyObject *key, PyObject *failobj, PyErr_SetObject(PyExc_KeyError, key); } } + Py_END_CRITICAL_SECTION(); +done: return value; } diff --git a/Objects/setobject.c b/Objects/setobject.c index 88d20019bfb4a7f..6a4c8c45f0836d1 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -32,13 +32,27 @@ */ #include "Python.h" -#include "pycore_ceval.h" // _PyEval_GetBuiltin() -#include "pycore_dict.h" // _PyDict_Contains_KnownHash() -#include "pycore_modsupport.h" // _PyArg_NoKwnames() -#include "pycore_object.h" // _PyObject_GC_UNTRACK() -#include "pycore_pyerrors.h" // _PyErr_SetKeyError() -#include "pycore_setobject.h" // _PySet_NextEntry() definition -#include <stddef.h> // offsetof() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION, Py_END_CRITICAL_SECTION +#include "pycore_dict.h" // _PyDict_Contains_KnownHash() +#include "pycore_modsupport.h" // _PyArg_NoKwnames() +#include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_pyerrors.h" // _PyErr_SetKeyError() +#include "pycore_setobject.h" // _PySet_NextEntry() definition +#include <stddef.h> // offsetof() +#include "clinic/setobject.c.h" + +/*[clinic input] +class set "PySetObject *" "&PySet_Type" +class frozenset "PySetObject *" "&PyFrozenSet_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=97ad1d3e9f117079]*/ + +/*[python input] +class setobject_converter(self_converter): + type = "PySetObject *" +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=33a44506d4d57793]*/ /* Object used as dummy key to fill deleted entries */ static PyObject _dummy_struct; @@ -630,8 +644,18 @@ set_merge(PySetObject *so, PyObject *otherset) return 0; } +/*[clinic input] +set.pop + so: setobject + +Remove and return an arbitrary set element. + +Raises KeyError if the set is empty. +[clinic start generated code]*/ + static PyObject * -set_pop(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set_pop_impl(PySetObject *so) +/*[clinic end generated code: output=4d65180f1271871b input=4a3f5552e660a260]*/ { /* Make sure the search finger is in bounds */ setentry *entry = so->table + (so->finger & so->mask); @@ -655,9 +679,6 @@ set_pop(PySetObject *so, PyObject *Py_UNUSED(ignored)) return key; } -PyDoc_STRVAR(pop_doc, "Remove and return an arbitrary set element.\n\ -Raises KeyError if the set is empty."); - static int set_traverse(PySetObject *so, visitproc visit, void *arg) { @@ -903,11 +924,17 @@ set_update_internal(PySetObject *so, PyObject *other) if (set_table_resize(so, (so->used + dictsize)*2) != 0) return -1; } + int err = 0; + Py_BEGIN_CRITICAL_SECTION(other); while (_PyDict_Next(other, &pos, &key, &value, &hash)) { - if (set_add_entry(so, key, hash)) - return -1; + if (set_add_entry(so, key, hash)) { + err = -1; + goto exit; + } } - return 0; +exit: + Py_END_CRITICAL_SECTION(); + return err; } it = PyObject_GetIter(other); @@ -928,8 +955,18 @@ set_update_internal(PySetObject *so, PyObject *other) return 0; } +/*[clinic input] +set.update + so: setobject + *others as args: object + / + +Update the set, adding elements from all others. +[clinic start generated code]*/ + static PyObject * -set_update(PySetObject *so, PyObject *args) +set_update_impl(PySetObject *so, PyObject *args) +/*[clinic end generated code: output=34f6371704974c8a input=eb47c4fbaeb3286e]*/ { Py_ssize_t i; @@ -941,12 +978,6 @@ set_update(PySetObject *so, PyObject *args) Py_RETURN_NONE; } -PyDoc_STRVAR(update_doc, -"update($self, /, *others)\n\ ---\n\ -\n\ -Update the set, adding elements from all others."); - /* XXX Todo: If aligned memory allocations become available, make the set object 64 byte aligned so that most of the fields @@ -1094,14 +1125,30 @@ set_swap_bodies(PySetObject *a, PySetObject *b) } } +/*[clinic input] +set.copy + so: setobject + +Return a shallow copy of a set. +[clinic start generated code]*/ + static PyObject * -set_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set_copy_impl(PySetObject *so) +/*[clinic end generated code: output=c9223a1e1cc6b041 input=2b80b288d47b8cf1]*/ { return make_new_set_basetype(Py_TYPE(so), (PyObject *)so); } +/*[clinic input] +frozenset.copy + so: setobject + +Return a shallow copy of a set. +[clinic start generated code]*/ + static PyObject * -frozenset_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) +frozenset_copy_impl(PySetObject *so) +/*[clinic end generated code: output=b356263526af9e70 input=3dc65577d344eff7]*/ { if (PyFrozenSet_CheckExact(so)) { return Py_NewRef(so); @@ -1109,19 +1156,33 @@ frozenset_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) return set_copy(so, NULL); } -PyDoc_STRVAR(copy_doc, "Return a shallow copy of a set."); +/*[clinic input] +set.clear + so: setobject + +Remove all elements from this set. +[clinic start generated code]*/ static PyObject * -set_clear(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set_clear_impl(PySetObject *so) +/*[clinic end generated code: output=4e71d5a83904161a input=74ac19794da81a39]*/ { set_clear_internal(so); Py_RETURN_NONE; } -PyDoc_STRVAR(clear_doc, "Remove all elements from this set."); +/*[clinic input] +set.union + so: setobject + *others as args: object + / + +Return a new set with elements from the set and all others. +[clinic start generated code]*/ static PyObject * -set_union(PySetObject *so, PyObject *args) +set_union_impl(PySetObject *so, PyObject *args) +/*[clinic end generated code: output=2c83d05a446a1477 input=2e2024fa1e40ac84]*/ { PySetObject *result; PyObject *other; @@ -1143,12 +1204,6 @@ set_union(PySetObject *so, PyObject *args) return (PyObject *)result; } -PyDoc_STRVAR(union_doc, -"union($self, /, *others)\n\ ---\n\ -\n\ -Return a new set with elements from the set and all others."); - static PyObject * set_or(PySetObject *so, PyObject *other) { @@ -1263,8 +1318,18 @@ set_intersection(PySetObject *so, PyObject *other) return NULL; } +/*[clinic input] +set.intersection as set_intersection_multi + so: setobject + *others as args: object + / + +Return a new set with elements common to the set and all others. +[clinic start generated code]*/ + static PyObject * -set_intersection_multi(PySetObject *so, PyObject *args) +set_intersection_multi_impl(PySetObject *so, PyObject *args) +/*[clinic end generated code: output=2406ef3387adbe2f input=04108ea6d7f0532b]*/ { Py_ssize_t i; @@ -1284,12 +1349,6 @@ set_intersection_multi(PySetObject *so, PyObject *args) return result; } -PyDoc_STRVAR(intersection_doc, -"intersection($self, /, *others)\n\ ---\n\ -\n\ -Return a new set with elements common to the set and all others."); - static PyObject * set_intersection_update(PySetObject *so, PyObject *other) { @@ -1303,12 +1362,22 @@ set_intersection_update(PySetObject *so, PyObject *other) Py_RETURN_NONE; } +/*[clinic input] +set.intersection_update as set_intersection_update_multi + so: setobject + *others as args: object + / + +Update the set, keeping only elements found in it and all others. +[clinic start generated code]*/ + static PyObject * -set_intersection_update_multi(PySetObject *so, PyObject *args) +set_intersection_update_multi_impl(PySetObject *so, PyObject *args) +/*[clinic end generated code: output=251c1f729063609d input=ff8f119f97458d16]*/ { PyObject *tmp; - tmp = set_intersection_multi(so, args); + tmp = set_intersection_multi_impl(so, args); if (tmp == NULL) return NULL; set_swap_bodies(so, (PySetObject *)tmp); @@ -1316,12 +1385,6 @@ set_intersection_update_multi(PySetObject *so, PyObject *args) Py_RETURN_NONE; } -PyDoc_STRVAR(intersection_update_doc, -"intersection_update($self, /, *others)\n\ ---\n\ -\n\ -Update the set, keeping only elements found in it and all others."); - static PyObject * set_and(PySetObject *so, PyObject *other) { @@ -1344,8 +1407,18 @@ set_iand(PySetObject *so, PyObject *other) return Py_NewRef(so); } +/*[clinic input] +set.isdisjoint + so: setobject + other: object + / + +Return True if two sets have a null intersection. +[clinic start generated code]*/ + static PyObject * set_isdisjoint(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=a92bbf9a2db6a3da input=c254ddec8a2326e3]*/ { PyObject *key, *it, *tmp; int rv; @@ -1403,9 +1476,6 @@ set_isdisjoint(PySetObject *so, PyObject *other) Py_RETURN_TRUE; } -PyDoc_STRVAR(isdisjoint_doc, -"Return True if two sets have a null intersection."); - static int set_difference_update_internal(PySetObject *so, PyObject *other) { @@ -1464,8 +1534,18 @@ set_difference_update_internal(PySetObject *so, PyObject *other) return set_table_resize(so, so->used>50000 ? so->used*2 : so->used*4); } +/*[clinic input] +set.difference_update + so: setobject + *others as args: object + / + +Update the set, removing elements found in others. +[clinic start generated code]*/ + static PyObject * -set_difference_update(PySetObject *so, PyObject *args) +set_difference_update_impl(PySetObject *so, PyObject *args) +/*[clinic end generated code: output=28685b2fc63e41c4 input=e7abb43c9f2c5a73]*/ { Py_ssize_t i; @@ -1477,12 +1557,6 @@ set_difference_update(PySetObject *so, PyObject *args) Py_RETURN_NONE; } -PyDoc_STRVAR(difference_update_doc, -"difference_update($self, /, *others)\n\ ---\n\ -\n\ -Update the set, removing elements found in others."); - static PyObject * set_copy_and_difference(PySetObject *so, PyObject *other) { @@ -1573,8 +1647,18 @@ set_difference(PySetObject *so, PyObject *other) return result; } +/*[clinic input] +set.difference as set_difference_multi + so: setobject + *others as args: object + / + +Return a new set with elements in the set that are not in the others. +[clinic start generated code]*/ + static PyObject * -set_difference_multi(PySetObject *so, PyObject *args) +set_difference_multi_impl(PySetObject *so, PyObject *args) +/*[clinic end generated code: output=3130c3bb3cac873d input=d8ae9bb6d518ab95]*/ { Py_ssize_t i; PyObject *result, *other; @@ -1597,11 +1681,6 @@ set_difference_multi(PySetObject *so, PyObject *args) return result; } -PyDoc_STRVAR(difference_doc, -"difference($self, /, *others)\n\ ---\n\ -\n\ -Return a new set with elements in the set that are not in the others."); static PyObject * set_sub(PySetObject *so, PyObject *other) { @@ -1620,8 +1699,45 @@ set_isub(PySetObject *so, PyObject *other) return Py_NewRef(so); } +static PyObject * +set_symmetric_difference_update_dict(PySetObject *so, PyObject *other) +{ + PyObject *key; + Py_ssize_t pos = 0; + Py_hash_t hash; + PyObject *value; + int rv; + + while (_PyDict_Next(other, &pos, &key, &value, &hash)) { + Py_INCREF(key); + rv = set_discard_entry(so, key, hash); + if (rv < 0) { + Py_DECREF(key); + return NULL; + } + if (rv == DISCARD_NOTFOUND) { + if (set_add_entry(so, key, hash)) { + Py_DECREF(key); + return NULL; + } + } + Py_DECREF(key); + } + Py_RETURN_NONE; +} + +/*[clinic input] +set.symmetric_difference_update + so: setobject + other: object + / + +Update the set, keeping only elements found in either set, but not in both. +[clinic start generated code]*/ + static PyObject * set_symmetric_difference_update(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=fbb049c0806028de input=a50acf0365e1f0a5]*/ { PySetObject *otherset; PyObject *key; @@ -1634,23 +1750,13 @@ set_symmetric_difference_update(PySetObject *so, PyObject *other) return set_clear(so, NULL); if (PyDict_CheckExact(other)) { - PyObject *value; - while (_PyDict_Next(other, &pos, &key, &value, &hash)) { - Py_INCREF(key); - rv = set_discard_entry(so, key, hash); - if (rv < 0) { - Py_DECREF(key); - return NULL; - } - if (rv == DISCARD_NOTFOUND) { - if (set_add_entry(so, key, hash)) { - Py_DECREF(key); - return NULL; - } - } - Py_DECREF(key); - } - Py_RETURN_NONE; + PyObject *res; + + Py_BEGIN_CRITICAL_SECTION(other); + res = set_symmetric_difference_update_dict(so, other); + Py_END_CRITICAL_SECTION(); + + return res; } if (PyAnySet_Check(other)) { @@ -1684,14 +1790,18 @@ set_symmetric_difference_update(PySetObject *so, PyObject *other) Py_RETURN_NONE; } -PyDoc_STRVAR(symmetric_difference_update_doc, -"symmetric_difference_update($self, other, /)\n\ ---\n\ -\n\ -Update the set, keeping only elements found in either set, but not in both."); +/*[clinic input] +set.symmetric_difference + so: setobject + other: object + / + +Return a new set with elements in either the set or other but not both. +[clinic start generated code]*/ static PyObject * set_symmetric_difference(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=f95364211b88775a input=f18af370ad72ebac]*/ { PyObject *rv; PySetObject *otherset; @@ -1708,12 +1818,6 @@ set_symmetric_difference(PySetObject *so, PyObject *other) return (PyObject *)otherset; } -PyDoc_STRVAR(symmetric_difference_doc, -"symmetric_difference($self, other, /)\n\ ---\n\ -\n\ -Return a new set with elements in either the set or other but not both."); - static PyObject * set_xor(PySetObject *so, PyObject *other) { @@ -1736,8 +1840,18 @@ set_ixor(PySetObject *so, PyObject *other) return Py_NewRef(so); } +/*[clinic input] +set.issubset + so: setobject + other: object + / + +Report whether another set contains this set. +[clinic start generated code]*/ + static PyObject * set_issubset(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=78aef1f377aedef1 input=37fbc579b609db0c]*/ { setentry *entry; Py_ssize_t pos = 0; @@ -1770,10 +1884,18 @@ set_issubset(PySetObject *so, PyObject *other) Py_RETURN_TRUE; } -PyDoc_STRVAR(issubset_doc, "Report whether another set contains this set."); +/*[clinic input] +set.issuperset + so: setobject + other: object + / + +Report whether this set contains another set. +[clinic start generated code]*/ static PyObject * set_issuperset(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=7d2b71dd714a7ec7 input=fd5dab052f2e9bb3]*/ { if (PyAnySet_Check(other)) { return set_issubset((PySetObject *)other, (PyObject *)so); @@ -1802,8 +1924,6 @@ set_issuperset(PySetObject *so, PyObject *other) Py_RETURN_TRUE; } -PyDoc_STRVAR(issuperset_doc, "Report whether this set contains another set."); - static PyObject * set_richcompare(PySetObject *v, PyObject *w, int op) { @@ -1847,19 +1967,26 @@ set_richcompare(PySetObject *v, PyObject *w, int op) Py_RETURN_NOTIMPLEMENTED; } +/*[clinic input] +set.add + so: setobject + object as key: object + / + +Add an element to a set. + +This has no effect if the element is already present. +[clinic start generated code]*/ + static PyObject * set_add(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=cd9c2d5c2069c2ba input=96f1efe029e47972]*/ { if (set_add_key(so, key)) return NULL; Py_RETURN_NONE; } -PyDoc_STRVAR(add_doc, -"Add an element to a set.\n\ -\n\ -This has no effect if the element is already present."); - static int set_contains(PySetObject *so, PyObject *key) { @@ -1880,8 +2007,19 @@ set_contains(PySetObject *so, PyObject *key) return rv; } +/*[clinic input] +@coexist +set.__contains__ + so: setobject + object as key: object + / + +x.__contains__(y) <==> y in x. +[clinic start generated code]*/ + static PyObject * -set_direct_contains(PySetObject *so, PyObject *key) +set___contains__(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=b5948bc5c590d3ca input=cf4c72db704e4cf0]*/ { long result; @@ -1891,10 +2029,20 @@ set_direct_contains(PySetObject *so, PyObject *key) return PyBool_FromLong(result); } -PyDoc_STRVAR(contains_doc, "x.__contains__(y) <==> y in x."); +/*[clinic input] +set.remove + so: setobject + object as key: object + / + +Remove an element from a set; it must be a member. + +If the element is not a member, raise a KeyError. +[clinic start generated code]*/ static PyObject * set_remove(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=08ae496d0cd2b8c1 input=10132515dfe8ebd7]*/ { PyObject *tmpkey; int rv; @@ -1920,13 +2068,21 @@ set_remove(PySetObject *so, PyObject *key) Py_RETURN_NONE; } -PyDoc_STRVAR(remove_doc, -"Remove an element from a set; it must be a member.\n\ -\n\ -If the element is not a member, raise a KeyError."); +/*[clinic input] +set.discard + so: setobject + object as key: object + / + +Remove an element from a set if it is a member. + +Unlike set.remove(), the discard() method does not raise +an exception when an element is missing from the set. +[clinic start generated code]*/ static PyObject * set_discard(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=9181b60d7bb7d480 input=82a689eba94d5ad9]*/ { PyObject *tmpkey; int rv; @@ -1947,14 +2103,16 @@ set_discard(PySetObject *so, PyObject *key) Py_RETURN_NONE; } -PyDoc_STRVAR(discard_doc, -"Remove an element from a set if it is a member.\n\ -\n\ -Unlike set.remove(), the discard() method does not raise\n\ -an exception when an element is missing from the set."); +/*[clinic input] +set.__reduce__ + so: setobject + +Return state information for pickling. +[clinic start generated code]*/ static PyObject * -set_reduce(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set___reduce___impl(PySetObject *so) +/*[clinic end generated code: output=9af7d0e029df87ee input=531375e87a24a449]*/ { PyObject *keys=NULL, *args=NULL, *result=NULL, *state=NULL; @@ -1975,8 +2133,16 @@ set_reduce(PySetObject *so, PyObject *Py_UNUSED(ignored)) return result; } +/*[clinic input] +set.__sizeof__ + so: setobject + +S.__sizeof__() -> size of S in memory, in bytes. +[clinic start generated code]*/ + static PyObject * -set_sizeof(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set___sizeof___impl(PySetObject *so) +/*[clinic end generated code: output=4bfa3df7bd38ed88 input=0f214fc2225319fc]*/ { size_t res = _PyObject_SIZE(Py_TYPE(so)); if (so->table != so->smalltable) { @@ -1985,7 +2151,6 @@ set_sizeof(PySetObject *so, PyObject *Py_UNUSED(ignored)) return PyLong_FromSize_t(res); } -PyDoc_STRVAR(sizeof_doc, "S.__sizeof__() -> size of S in memory, in bytes"); static int set_init(PySetObject *self, PyObject *args, PyObject *kwds) { @@ -2039,46 +2204,26 @@ static PySequenceMethods set_as_sequence = { /* set object ********************************************************/ static PyMethodDef set_methods[] = { - {"add", (PyCFunction)set_add, METH_O, - add_doc}, - {"clear", (PyCFunction)set_clear, METH_NOARGS, - clear_doc}, - {"__contains__",(PyCFunction)set_direct_contains, METH_O | METH_COEXIST, - contains_doc}, - {"copy", (PyCFunction)set_copy, METH_NOARGS, - copy_doc}, - {"discard", (PyCFunction)set_discard, METH_O, - discard_doc}, - {"difference", (PyCFunction)set_difference_multi, METH_VARARGS, - difference_doc}, - {"difference_update", (PyCFunction)set_difference_update, METH_VARARGS, - difference_update_doc}, - {"intersection",(PyCFunction)set_intersection_multi, METH_VARARGS, - intersection_doc}, - {"intersection_update",(PyCFunction)set_intersection_update_multi, METH_VARARGS, - intersection_update_doc}, - {"isdisjoint", (PyCFunction)set_isdisjoint, METH_O, - isdisjoint_doc}, - {"issubset", (PyCFunction)set_issubset, METH_O, - issubset_doc}, - {"issuperset", (PyCFunction)set_issuperset, METH_O, - issuperset_doc}, - {"pop", (PyCFunction)set_pop, METH_NOARGS, - pop_doc}, - {"__reduce__", (PyCFunction)set_reduce, METH_NOARGS, - reduce_doc}, - {"remove", (PyCFunction)set_remove, METH_O, - remove_doc}, - {"__sizeof__", (PyCFunction)set_sizeof, METH_NOARGS, - sizeof_doc}, - {"symmetric_difference",(PyCFunction)set_symmetric_difference, METH_O, - symmetric_difference_doc}, - {"symmetric_difference_update",(PyCFunction)set_symmetric_difference_update, METH_O, - symmetric_difference_update_doc}, - {"union", (PyCFunction)set_union, METH_VARARGS, - union_doc}, - {"update", (PyCFunction)set_update, METH_VARARGS, - update_doc}, + SET_ADD_METHODDEF + SET_CLEAR_METHODDEF + SET___CONTAINS___METHODDEF + SET_COPY_METHODDEF + SET_DISCARD_METHODDEF + SET_DIFFERENCE_MULTI_METHODDEF + SET_DIFFERENCE_UPDATE_METHODDEF + SET_INTERSECTION_MULTI_METHODDEF + SET_INTERSECTION_UPDATE_MULTI_METHODDEF + SET_ISDISJOINT_METHODDEF + SET_ISSUBSET_METHODDEF + SET_ISSUPERSET_METHODDEF + SET_POP_METHODDEF + SET___REDUCE___METHODDEF + SET_REMOVE_METHODDEF + SET___SIZEOF___METHODDEF + SET_SYMMETRIC_DIFFERENCE_METHODDEF + SET_SYMMETRIC_DIFFERENCE_UPDATE_METHODDEF + SET_UNION_METHODDEF + SET_UPDATE_METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ }; @@ -2171,28 +2316,17 @@ PyTypeObject PySet_Type = { static PyMethodDef frozenset_methods[] = { - {"__contains__",(PyCFunction)set_direct_contains, METH_O | METH_COEXIST, - contains_doc}, - {"copy", (PyCFunction)frozenset_copy, METH_NOARGS, - copy_doc}, - {"difference", (PyCFunction)set_difference_multi, METH_VARARGS, - difference_doc}, - {"intersection", (PyCFunction)set_intersection_multi, METH_VARARGS, - intersection_doc}, - {"isdisjoint", (PyCFunction)set_isdisjoint, METH_O, - isdisjoint_doc}, - {"issubset", (PyCFunction)set_issubset, METH_O, - issubset_doc}, - {"issuperset", (PyCFunction)set_issuperset, METH_O, - issuperset_doc}, - {"__reduce__", (PyCFunction)set_reduce, METH_NOARGS, - reduce_doc}, - {"__sizeof__", (PyCFunction)set_sizeof, METH_NOARGS, - sizeof_doc}, - {"symmetric_difference",(PyCFunction)set_symmetric_difference, METH_O, - symmetric_difference_doc}, - {"union", (PyCFunction)set_union, METH_VARARGS, - union_doc}, + SET___CONTAINS___METHODDEF + FROZENSET_COPY_METHODDEF + SET_DIFFERENCE_MULTI_METHODDEF + SET_INTERSECTION_MULTI_METHODDEF + SET_ISDISJOINT_METHODDEF + SET_ISSUBSET_METHODDEF + SET_ISSUPERSET_METHODDEF + SET___REDUCE___METHODDEF + SET___SIZEOF___METHODDEF + SET_SYMMETRIC_DIFFERENCE_METHODDEF + SET_UNION_METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index 440c1da30620c38..9880c123c80f95e 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -103,24 +103,20 @@ PyObject _Py_EllipsisObject = _PyObject_HEAD_INIT(&PyEllipsis_Type); /* Slice object implementation */ -void _PySlice_ClearCache(_PyFreeListState *state) +void _PySlice_ClearFreeList(_PyFreeListState *state, int is_finalization) { + if (!is_finalization) { + return; + } #ifdef WITH_FREELISTS - PySliceObject *obj = state->slice_state.slice_cache; + PySliceObject *obj = state->slices.slice_cache; if (obj != NULL) { - state->slice_state.slice_cache = NULL; + state->slices.slice_cache = NULL; PyObject_GC_Del(obj); } #endif } -void _PySlice_Fini(_PyFreeListState *state) -{ -#ifdef WITH_FREELISTS - _PySlice_ClearCache(state); -#endif -} - /* start, stop, and step are python objects with None indicating no index is present. */ @@ -132,9 +128,9 @@ _PyBuildSlice_Consume2(PyObject *start, PyObject *stop, PyObject *step) PySliceObject *obj; #ifdef WITH_FREELISTS _PyFreeListState *state = _PyFreeListState_GET(); - if (state->slice_state.slice_cache != NULL) { - obj = state->slice_state.slice_cache; - state->slice_state.slice_cache = NULL; + if (state->slices.slice_cache != NULL) { + obj = state->slices.slice_cache; + state->slices.slice_cache = NULL; _Py_NewReference((PyObject *)obj); } else @@ -370,8 +366,8 @@ slice_dealloc(PySliceObject *r) Py_DECREF(r->stop); #ifdef WITH_FREELISTS _PyFreeListState *state = _PyFreeListState_GET(); - if (state->slice_state.slice_cache == NULL) { - state->slice_state.slice_cache = r; + if (state->slices.slice_cache == NULL) { + state->slices.slice_cache = r; } else #endif diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index e1b8e4004c61638..7d73c3fb0f7f2cc 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -964,11 +964,6 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) static void maybe_freelist_clear(_PyFreeListState *, int); -void -_PyTuple_Fini(_PyFreeListState *state) -{ - maybe_freelist_clear(state, 1); -} void _PyTuple_ClearFreeList(_PyFreeListState *state, int is_finalization) @@ -1125,7 +1120,7 @@ tuple_iter(PyObject *seq) * freelists * *************/ -#define STATE (state->tuple_state) +#define STATE (state->tuples) #define FREELIST_FINALIZED (STATE.numfree[0] < 0) static inline PyTupleObject * diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 3a35a5b5975898e..c65d0ec2acae526 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -908,6 +908,8 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { } } +#define MAX_VERSIONS_PER_CLASS 1000 + static int assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) { @@ -922,7 +924,10 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) if (!_PyType_HasFeature(type, Py_TPFLAGS_READY)) { return 0; } - + if (type->tp_versions_used >= MAX_VERSIONS_PER_CLASS) { + return 0; + } + type->tp_versions_used++; if (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) { /* static types */ if (NEXT_GLOBAL_VERSION_TAG > _Py_MAX_GLOBAL_TYPE_VERSION_TAG) { @@ -1371,6 +1376,7 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) res = 0; } + RARE_EVENT_INC(set_bases); Py_DECREF(old_bases); Py_DECREF(old_base); @@ -3492,7 +3498,7 @@ type_new_set_doc(PyTypeObject *type) // Silently truncate the docstring if it contains a null byte Py_ssize_t size = strlen(doc_str) + 1; - char *tp_doc = (char *)PyObject_Malloc(size); + char *tp_doc = (char *)PyMem_Malloc(size); if (tp_doc == NULL) { PyErr_NoMemory(); return -1; @@ -3827,6 +3833,17 @@ type_new_impl(type_new_ctx *ctx) // Put the proper slots in place fixup_slot_dispatchers(type); + if (!_PyDict_HasOnlyStringKeys(type->tp_dict)) { + if (PyErr_WarnFormat( + PyExc_RuntimeWarning, + 1, + "non-string key in the __dict__ of class %.200s", + type->tp_name) == -1) + { + goto error; + } + } + if (type_new_set_names(type) < 0) { goto error; } @@ -4165,12 +4182,12 @@ _PyType_FromMetaclass_impl( goto finally; } if (slot->pfunc == NULL) { - PyObject_Free(tp_doc); + PyMem_Free(tp_doc); tp_doc = NULL; } else { size_t len = strlen(slot->pfunc)+1; - tp_doc = PyObject_Malloc(len); + tp_doc = PyMem_Malloc(len); if (tp_doc == NULL) { PyErr_NoMemory(); goto finally; @@ -4500,7 +4517,7 @@ _PyType_FromMetaclass_impl( Py_CLEAR(res); } Py_XDECREF(bases); - PyObject_Free(tp_doc); + PyMem_Free(tp_doc); Py_XDECREF(ht_name); PyMem_Free(_ht_tpname); return (PyObject*)res; @@ -5098,7 +5115,7 @@ type_dealloc(PyObject *self) /* A type's tp_doc is heap allocated, unlike the tp_doc slots * of most other objects. It's okay to cast it to char *. */ - PyObject_Free((char *)type->tp_doc); + PyMem_Free((char *)type->tp_doc); PyHeapTypeObject *et = (PyHeapTypeObject *)type; Py_XDECREF(et->ht_name); @@ -5842,6 +5859,8 @@ object_set_class(PyObject *self, PyObject *value, void *closure) Py_SET_TYPE(self, newto); if (oldto->tp_flags & Py_TPFLAGS_HEAPTYPE) Py_DECREF(oldto); + + RARE_EVENT_INC(set_class); return 0; } else { @@ -6664,7 +6683,7 @@ type_add_method(PyTypeObject *type, PyMethodDef *meth) int err; PyObject *dict = lookup_tp_dict(type); if (!(meth->ml_flags & METH_COEXIST)) { - err = PyDict_SetDefault(dict, name, descr) == NULL; + err = PyDict_SetDefaultRef(dict, name, descr, NULL) < 0; } else { err = PyDict_SetItem(dict, name, descr) < 0; @@ -6712,7 +6731,7 @@ type_add_members(PyTypeObject *type) if (descr == NULL) return -1; - if (PyDict_SetDefault(dict, PyDescr_NAME(descr), descr) == NULL) { + if (PyDict_SetDefaultRef(dict, PyDescr_NAME(descr), descr, NULL) < 0) { Py_DECREF(descr); return -1; } @@ -6737,7 +6756,7 @@ type_add_getset(PyTypeObject *type) return -1; } - if (PyDict_SetDefault(dict, PyDescr_NAME(descr), descr) == NULL) { + if (PyDict_SetDefaultRef(dict, PyDescr_NAME(descr), descr, NULL) < 0) { Py_DECREF(descr); return -1; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 4b03cc3f4da5fab..0a569a950e88e29 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -996,7 +996,7 @@ resize_compact(PyObject *unicode, Py_ssize_t length) new_size = (struct_size + (length + 1) * char_size); if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) { - PyObject_Free(_PyUnicode_UTF8(unicode)); + PyMem_Free(_PyUnicode_UTF8(unicode)); _PyUnicode_UTF8(unicode) = NULL; _PyUnicode_UTF8_LENGTH(unicode) = 0; } @@ -1049,7 +1049,7 @@ resize_inplace(PyObject *unicode, Py_ssize_t length) if (!share_utf8 && _PyUnicode_HAS_UTF8_MEMORY(unicode)) { - PyObject_Free(_PyUnicode_UTF8(unicode)); + PyMem_Free(_PyUnicode_UTF8(unicode)); _PyUnicode_UTF8(unicode) = NULL; _PyUnicode_UTF8_LENGTH(unicode) = 0; } @@ -1590,10 +1590,10 @@ unicode_dealloc(PyObject *unicode) return; } if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) { - PyObject_Free(_PyUnicode_UTF8(unicode)); + PyMem_Free(_PyUnicode_UTF8(unicode)); } if (!PyUnicode_IS_COMPACT(unicode) && _PyUnicode_DATA_ANY(unicode)) { - PyObject_Free(_PyUnicode_DATA_ANY(unicode)); + PyMem_Free(_PyUnicode_DATA_ANY(unicode)); } Py_TYPE(unicode)->tp_free(unicode); @@ -5203,7 +5203,7 @@ unicode_fill_utf8(PyObject *unicode) PyBytes_AS_STRING(writer.buffer); Py_ssize_t len = end - start; - char *cache = PyObject_Malloc(len + 1); + char *cache = PyMem_Malloc(len + 1); if (cache == NULL) { _PyBytesWriter_Dealloc(&writer); PyErr_NoMemory(); @@ -14674,7 +14674,7 @@ unicode_subtype_new(PyTypeObject *type, PyObject *unicode) PyErr_NoMemory(); goto onError; } - data = PyObject_Malloc((length + 1) * char_size); + data = PyMem_Malloc((length + 1) * char_size); if (data == NULL) { PyErr_NoMemory(); goto onError; @@ -14894,16 +14894,18 @@ _PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p) PyObject *interned = get_interned_dict(interp); assert(interned != NULL); - PyObject *t = PyDict_SetDefault(interned, s, s); - if (t == NULL) { + PyObject *t; + int res = PyDict_SetDefaultRef(interned, s, s, &t); + if (res < 0) { PyErr_Clear(); return; } - - if (t != s) { - Py_SETREF(*p, Py_NewRef(t)); + else if (res == 1) { + // value was already present (not inserted) + Py_SETREF(*p, t); return; } + Py_DECREF(t); if (_Py_IsImmortal(s)) { // XXX Restrict this to the main interpreter? diff --git a/PC/launcher2.c b/PC/launcher2.c index 2a8f8a101fc8a61..e426eccd7000447 100644 --- a/PC/launcher2.c +++ b/PC/launcher2.c @@ -572,6 +572,21 @@ findArgv0End(const wchar_t *buffer, int bufferLength) *** COMMAND-LINE PARSING *** \******************************************************************************/ +// Adapted from https://stackoverflow.com/a/65583702 +typedef struct AppExecLinkFile { // For tag IO_REPARSE_TAG_APPEXECLINK + DWORD reparseTag; + WORD reparseDataLength; + WORD reserved; + ULONG version; + wchar_t stringList[MAX_PATH * 4]; // Multistring (Consecutive UTF-16 strings each ending with a NUL) + /* There are normally 4 strings here. Ex: + Package ID: L"Microsoft.DesktopAppInstaller_8wekyb3d8bbwe" + Entry Point: L"Microsoft.DesktopAppInstaller_8wekyb3d8bbwe!PythonRedirector" + Executable: L"C:\Program Files\WindowsApps\Microsoft.DesktopAppInstaller_1.17.106910_x64__8wekyb3d8bbwe\AppInstallerPythonRedirector.exe" + Applic. Type: L"0" // Integer as ASCII. "0" = Desktop bridge application; Else sandboxed UWP application + */ +} AppExecLinkFile; + int parseCommandLine(SearchInfo *search) @@ -763,6 +778,55 @@ _shebangStartsWith(const wchar_t *buffer, int bufferLength, const wchar_t *prefi } +int +ensure_no_redirector_stub(wchar_t* filename, wchar_t* buffer) +{ + // Make sure we didn't find a reparse point that will open the Microsoft Store + // If we did, pretend there was no shebang and let normal handling take over + WIN32_FIND_DATAW findData; + HANDLE hFind = FindFirstFileW(buffer, &findData); + if (!hFind) { + // Let normal handling take over + debug(L"# Did not find %s on PATH\n", filename); + return RC_NO_SHEBANG; + } + + FindClose(hFind); + + if (!(findData.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT && + findData.dwReserved0 & IO_REPARSE_TAG_APPEXECLINK)) { + return 0; + } + + HANDLE hReparsePoint = CreateFileW(buffer, 0, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT, NULL); + if (!hReparsePoint) { + // Let normal handling take over + debug(L"# Did not find %s on PATH\n", filename); + return RC_NO_SHEBANG; + } + + AppExecLinkFile appExecLink; + + if (!DeviceIoControl(hReparsePoint, FSCTL_GET_REPARSE_POINT, NULL, 0, &appExecLink, sizeof(appExecLink), NULL, NULL)) { + // Let normal handling take over + debug(L"# Did not find %s on PATH\n", filename); + CloseHandle(hReparsePoint); + return RC_NO_SHEBANG; + } + + CloseHandle(hReparsePoint); + + const wchar_t* redirectorPackageId = L"Microsoft.DesktopAppInstaller_8wekyb3d8bbwe"; + + if (0 == wcscmp(appExecLink.stringList, redirectorPackageId)) { + debug(L"# ignoring redirector that would launch store\n"); + return RC_NO_SHEBANG; + } + + return 0; +} + + int searchPath(SearchInfo *search, const wchar_t *shebang, int shebangLength) { @@ -826,6 +890,11 @@ searchPath(SearchInfo *search, const wchar_t *shebang, int shebangLength) return RC_BAD_VIRTUAL_PATH; } + int result = ensure_no_redirector_stub(filename, buffer); + if (result) { + return result; + } + // Check that we aren't going to call ourselves again // If we are, pretend there was no shebang and let normal handling take over if (GetModuleFileNameW(NULL, filename, MAXLEN) && diff --git a/PC/python3dll.c b/PC/python3dll.c index 07aa84c91f9fc7f..aa6bfe2c4022db0 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -324,6 +324,7 @@ EXPORT_FUNC(PyIter_Send) EXPORT_FUNC(PyList_Append) EXPORT_FUNC(PyList_AsTuple) EXPORT_FUNC(PyList_GetItem) +EXPORT_FUNC(PyList_GetItemRef) EXPORT_FUNC(PyList_GetSlice) EXPORT_FUNC(PyList_Insert) EXPORT_FUNC(PyList_New) @@ -830,6 +831,7 @@ EXPORT_DATA(PyExc_FutureWarning) EXPORT_DATA(PyExc_GeneratorExit) EXPORT_DATA(PyExc_ImportError) EXPORT_DATA(PyExc_ImportWarning) +EXPORT_DATA(PyExc_IncompleteInputError) EXPORT_DATA(PyExc_IndentationError) EXPORT_DATA(PyExc_IndexError) EXPORT_DATA(PyExc_InterruptedError) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 610581bc96cb1a2..49f529ebbc2f9b0 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -191,6 +191,7 @@ <ClCompile Include="..\Python\ast_opt.c" /> <ClCompile Include="..\Python\ast_unparse.c" /> <ClCompile Include="..\Python\bltinmodule.c" /> + <ClCompile Include="..\Python\brc.c" /> <ClCompile Include="..\Python\bootstrap_hash.c" /> <ClCompile Include="..\Python\ceval.c" /> <ClCompile Include="..\Python\codecs.c" /> @@ -224,12 +225,14 @@ <ClCompile Include="..\Python\initconfig.c" /> <ClCompile Include="..\Python\intrinsics.c" /> <ClCompile Include="..\Python\instrumentation.c" /> + <ClCompile Include="..\Python\jit.c" /> <ClCompile Include="..\Python\legacy_tracing.c" /> <ClCompile Include="..\Python\lock.c" /> <ClCompile Include="..\Python\marshal.c" /> <ClCompile Include="..\Python\modsupport.c" /> <ClCompile Include="..\Python\mysnprintf.c" /> <ClCompile Include="..\Python\mystrtoul.c" /> + <ClCompile Include="..\Python\object_stack.c" /> <ClCompile Include="..\Python\optimizer.c" /> <ClCompile Include="..\Python\optimizer_analysis.c" /> <ClCompile Include="..\Python\parking_lot.c" /> diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 3141913c0438695..5b1bd7552b4cd98 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -46,6 +46,9 @@ <ClCompile Include="..\Python\bltinmodule.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Python\brc.c"> + <Filter>Python</Filter> + </ClCompile> <ClCompile Include="..\Objects\boolobject.c"> <Filter>Source Files</Filter> </ClCompile> @@ -250,6 +253,9 @@ <ClCompile Include="..\Objects\iterobject.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Python\jit.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\Objects\listobject.c"> <Filter>Source Files</Filter> </ClCompile> @@ -289,6 +295,9 @@ <ClCompile Include="..\Objects\object.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Python\object_stack.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\Objects\obmalloc.c"> <Filter>Source Files</Filter> </ClCompile> diff --git a/PCbuild/build.bat b/PCbuild/build.bat index e61267b5852a8f2..83b50db44670337 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -36,6 +36,7 @@ echo. overrides -c and -d echo. --disable-gil Enable experimental support for running without the GIL. echo. --test-marker Enable the test marker within the build. echo. --regen Regenerate all opcodes, grammar and tokens. +echo. --experimental-jit Enable the experimental just-in-time compiler. echo. echo.Available flags to avoid building certain modules. echo.These flags have no effect if '-e' is not given: @@ -85,6 +86,7 @@ if "%~1"=="--disable-gil" (set UseDisableGil=true) & shift & goto CheckOpts if "%~1"=="--test-marker" (set UseTestMarker=true) & shift & goto CheckOpts if "%~1"=="-V" shift & goto Version if "%~1"=="--regen" (set Regen=true) & shift & goto CheckOpts +if "%~1"=="--experimental-jit" (set UseJIT=true) & shift & goto CheckOpts rem These use the actual property names used by MSBuild. We could just let rem them in through the environment, but we specify them on the command line rem anyway for visibility so set defaults after this @@ -176,6 +178,7 @@ echo on /p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^ /p:DisableGil=%UseDisableGil%^ /p:UseTestMarker=%UseTestMarker% %GITProperty%^ + /p:UseJIT=%UseJIT%^ %1 %2 %3 %4 %5 %6 %7 %8 %9 @echo off diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 3919c0592ec00d0..60ce12b725e2331 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -53,12 +53,12 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 -if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.11 -set libraries=%libraries% sqlite-3.44.2.0 +if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.13 +set libraries=%libraries% sqlite-3.45.1.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.1 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.1 set libraries=%libraries% xz-5.2.5 -set libraries=%libraries% zlib-1.2.13 +set libraries=%libraries% zlib-1.3.1 for %%e in (%libraries%) do ( if exist "%EXTERNALS_DIR%\%%e" ( @@ -76,7 +76,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 -if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.11 +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.13 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.13.1 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index fd5fbc9e910eee4..9c85e5efa4af4ad 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -250,7 +250,7 @@ public override bool Execute() { <VCRuntimeDLL Include="$(VCRuntimeDLL)" /> </ItemGroup> <ItemGroup Condition="$(VCInstallDir) != '' and $(VCRuntimeDLL) == ''"> - <VCRuntimeDLL Include="$(VCRedistDir)\Microsoft.VC*.CRT\vcruntime*.dll" /> + <VCRuntimeDLL Include="$(VCRedistDir)\Microsoft.VC*.CRT\vcruntime*.dll" Exclude="$(VCRedistDir)\Microsoft.VC*.CRT\vcruntime*_threads.dll" /> </ItemGroup> <Warning Text="vcruntime*.dll not found under $(VCRedistDir)." Condition="@(VCRuntimeDLL) == ''" /> diff --git a/PCbuild/python.props b/PCbuild/python.props index e8796081c4eaf37..e21f1f60464bc8c 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -68,17 +68,17 @@ <Import Project="$(ExternalProps)" Condition="$(ExternalProps) != '' and Exists('$(ExternalProps)')" /> <PropertyGroup> - <sqlite3Dir Condition="$(sqlite3Dir) == ''">$(ExternalsDir)sqlite-3.44.2.0\</sqlite3Dir> + <sqlite3Dir Condition="$(sqlite3Dir) == ''">$(ExternalsDir)sqlite-3.45.1.0\</sqlite3Dir> <bz2Dir Condition="$(bz2Dir) == ''">$(ExternalsDir)bzip2-1.0.8\</bz2Dir> <lzmaDir Condition="$(lzmaDir) == ''">$(ExternalsDir)xz-5.2.5\</lzmaDir> <libffiDir Condition="$(libffiDir) == ''">$(ExternalsDir)libffi-3.4.4\</libffiDir> <libffiOutDir Condition="$(libffiOutDir) == ''">$(libffiDir)$(ArchName)\</libffiOutDir> <libffiIncludeDir Condition="$(libffiIncludeDir) == ''">$(libffiOutDir)include</libffiIncludeDir> - <opensslDir Condition="$(opensslDir) == ''">$(ExternalsDir)openssl-3.0.11\</opensslDir> - <opensslOutDir Condition="$(opensslOutDir) == ''">$(ExternalsDir)openssl-bin-3.0.11\$(ArchName)\</opensslOutDir> + <opensslDir Condition="$(opensslDir) == ''">$(ExternalsDir)openssl-3.0.13\</opensslDir> + <opensslOutDir Condition="$(opensslOutDir) == ''">$(ExternalsDir)openssl-bin-3.0.13\$(ArchName)\</opensslOutDir> <opensslIncludeDir Condition="$(opensslIncludeDir) == ''">$(opensslOutDir)include</opensslIncludeDir> <nasmDir Condition="$(nasmDir) == ''">$(ExternalsDir)\nasm-2.11.06\</nasmDir> - <zlibDir Condition="$(zlibDir) == ''">$(ExternalsDir)\zlib-1.2.13\</zlibDir> + <zlibDir Condition="$(zlibDir) == ''">$(ExternalsDir)\zlib-1.3.1\</zlibDir> </PropertyGroup> <PropertyGroup> diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 57275fb2039ee0b..4cc0ca4b9af8deb 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -104,6 +104,7 @@ <AdditionalIncludeDirectories Condition="$(IncludeExternals)">$(zlibDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <PreprocessorDefinitions>_USRDLL;Py_BUILD_CORE;Py_BUILD_CORE_BUILTIN;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions Condition="$(IncludeExternals)">_Py_HAVE_ZLIB;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions Condition="'$(UseJIT)' == 'true'">_Py_JIT;%(PreprocessorDefinitions)</PreprocessorDefinitions> </ClCompile> <Link> <AdditionalDependencies>version.lib;ws2_32.lib;pathcch.lib;bcrypt.lib;%(AdditionalDependencies)</AdditionalDependencies> @@ -205,6 +206,7 @@ <ClInclude Include="..\Include\internal\pycore_ast_state.h" /> <ClInclude Include="..\Include\internal\pycore_atexit.h" /> <ClInclude Include="..\Include\internal\pycore_bitutils.h" /> + <ClInclude Include="..\Include\internal\pycore_brc.h" /> <ClInclude Include="..\Include\internal\pycore_bytes_methods.h" /> <ClInclude Include="..\Include\internal\pycore_bytesobject.h" /> <ClInclude Include="..\Include\internal\pycore_call.h" /> @@ -247,6 +249,7 @@ <ClInclude Include="..\Include\internal\pycore_initconfig.h" /> <ClInclude Include="..\Include\internal\pycore_interp.h" /> <ClInclude Include="..\Include\internal\pycore_intrinsics.h" /> + <ClInclude Include="..\Include\internal\pycore_jit.h" /> <ClInclude Include="..\Include\internal\pycore_list.h" /> <ClInclude Include="..\Include\internal\pycore_llist.h" /> <ClInclude Include="..\Include\internal\pycore_lock.h" /> @@ -551,6 +554,7 @@ <ClCompile Include="..\Python\ast_unparse.c" /> <ClCompile Include="..\Python\bltinmodule.c" /> <ClCompile Include="..\Python\bootstrap_hash.c" /> + <ClCompile Include="..\Python\brc.c" /> <ClCompile Include="..\Python\ceval.c" /> <ClCompile Include="..\Python\codecs.c" /> <ClCompile Include="..\Python\compile.c" /> @@ -585,12 +589,14 @@ <ClCompile Include="..\Python\initconfig.c" /> <ClCompile Include="..\Python\intrinsics.c" /> <ClCompile Include="..\Python\instrumentation.c" /> + <ClCompile Include="..\Python\jit.c" /> <ClCompile Include="..\Python\legacy_tracing.c" /> <ClCompile Include="..\Python\lock.c" /> <ClCompile Include="..\Python\marshal.c" /> <ClCompile Include="..\Python\modsupport.c" /> <ClCompile Include="..\Python\mysnprintf.c" /> <ClCompile Include="..\Python\mystrtoul.c" /> + <ClCompile Include="..\Python\object_stack.c" /> <ClCompile Include="..\Python\optimizer.c" /> <ClCompile Include="..\Python\optimizer_analysis.c" /> <ClCompile Include="..\Python\parking_lot.c" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 51cbb079b5b550b..ceaa21217267cf1 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -546,6 +546,9 @@ <ClInclude Include="..\Include\internal\pycore_bitutils.h"> <Filter>Include\internal</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_brc.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_bytes_methods.h"> <Filter>Include\internal</Filter> </ClInclude> @@ -669,6 +672,9 @@ <ClInclude Include="..\Include\internal\pycore_intrinsics.h"> <Filter>Include\cpython</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_jit.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_list.h"> <Filter>Include\internal</Filter> </ClInclude> @@ -1250,6 +1256,9 @@ <ClCompile Include="..\Python\bltinmodule.c"> <Filter>Python</Filter> </ClCompile> + <ClCompile Include="..\Python\brc.c"> + <Filter>Python</Filter> + </ClCompile> <ClCompile Include="..\Python\ceval.c"> <Filter>Python</Filter> </ClCompile> @@ -1337,6 +1346,9 @@ <ClCompile Include="..\Python\instrumentation.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Python\jit.c"> + <Filter>Python</Filter> + </ClCompile> <ClCompile Include="..\Python\legacy_tracing.c"> <Filter>Source Files</Filter> </ClCompile> @@ -1355,6 +1367,9 @@ <ClCompile Include="..\Python\mystrtoul.c"> <Filter>Python</Filter> </ClCompile> + <ClCompile Include="..\Python\object_stack.c"> + <Filter>Python</Filter> + </ClCompile> <ClCompile Include="..\Python\optimizer.c"> <Filter>Python</Filter> </ClCompile> diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index b9d76515c383f77..387565515fa0b02 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -189,7 +189,7 @@ _ssl again when building. _sqlite3 - Wraps SQLite 3.44.2, which is itself built by sqlite3.vcxproj + Wraps SQLite 3.45.1, which is itself built by sqlite3.vcxproj Homepage: https://www.sqlite.org/ _tkinter diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index cc9469c7ddd726b..a90620d6ca8b7d1 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -28,6 +28,9 @@ </_TokenOutputs> <_KeywordSources Include="$(PySourcePath)Grammar\python.gram;$(PySourcePath)Grammar\Tokens" /> <_KeywordOutputs Include="$(PySourcePath)Lib\keyword.py" /> + <!-- Taken from _Target._compute_digest in Tools\jit\_targets.py: --> + <_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(GeneratedPyConfigDir)pyconfig.h;$(PySourcePath)Tools\jit\**"/> + <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/> </ItemGroup> <Target Name="_TouchRegenSources" Condition="$(ForceRegen) == 'true'"> @@ -76,10 +79,28 @@ <Exec Command="$(PythonForBuild) Tools\build\generate_global_objects.py" WorkingDirectory="$(PySourcePath)" /> </Target> + + <Target Name="_RegenJIT" + Condition="'$(UseJIT)' == 'true'" + DependsOnTargets="_UpdatePyconfig;FindPythonForBuild" + Inputs="@(_JITSources)" + Outputs="@(_JITOutputs)"> + <PropertyGroup> + <JITArgs Condition="$(Platform) == 'ARM64'">aarch64-pc-windows-msvc</JITArgs> + <JITArgs Condition="$(Platform) == 'Win32'">i686-pc-windows-msvc</JITArgs> + <JITArgs Condition="$(Platform) == 'x64'">x86_64-pc-windows-msvc</JITArgs> + <JITArgs Condition="$(Configuration) == 'Debug'">$(JITArgs) --debug</JITArgs> + </PropertyGroup> + <Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\jit\build.py" $(JITArgs)' + WorkingDirectory="$(GeneratedPyConfigDir)"/> + </Target> - <Target Name="Regen" + <Target Name="_RegenNoPGUpdate" Condition="$(Configuration) != 'PGUpdate'" DependsOnTargets="_TouchRegenSources;_RegenPegen;_RegenAST_H;_RegenTokens;_RegenKeywords;_RegenGlobalObjects"> + </Target> + + <Target Name="Regen" DependsOnTargets="_RegenNoPGUpdate;_RegenJIT"> <Message Text="Generated sources are up to date" Importance="high" /> </Target> diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index ebf7686773ff45c..82b0e4ee352d622 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -129,7 +129,7 @@ set_fstring_expr(struct tok_state* tok, struct token *token, char c) { if (hash_detected) { Py_ssize_t input_length = tok_mode->last_expr_size - tok_mode->last_expr_end; - char *result = (char *)PyObject_Malloc((input_length + 1) * sizeof(char)); + char *result = (char *)PyMem_Malloc((input_length + 1) * sizeof(char)); if (!result) { return -1; } @@ -154,7 +154,7 @@ set_fstring_expr(struct tok_state* tok, struct token *token, char c) { result[j] = '\0'; // Null-terminate the result string res = PyUnicode_DecodeUTF8(result, j, NULL); - PyObject_Free(result); + PyMem_Free(result); } else { res = PyUnicode_DecodeUTF8( tok_mode->last_expr_buffer, diff --git a/Parser/pegen.c b/Parser/pegen.c index 7766253a76066f0..3d3e64559403b16 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -844,7 +844,7 @@ _PyPegen_run_parser(Parser *p) if (res == NULL) { if ((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) && _is_end_of_source(p)) { PyErr_Clear(); - return RAISE_SYNTAX_ERROR("incomplete input"); + return _PyPegen_raise_error(p, PyExc_IncompleteInputError, 0, "incomplete input"); } if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_SyntaxError)) { return NULL; diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 4fb78cf632d70ef..657e9345cf5ab7d 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,7 +1,7 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, - 0,0,0,0,0,243,164,0,0,0,149,0,83,0,83,1, + 0,0,0,0,0,243,166,0,0,0,149,0,83,0,83,1, 75,0,114,0,83,0,83,1,75,1,114,1,92,2,34,0, 83,2,53,1,0,0,0,0,0,0,32,0,92,2,34,0, 83,3,92,0,82,6,0,0,0,0,0,0,0,0,0,0, @@ -11,28 +11,28 @@ unsigned char M_test_frozenmain[] = { 0,0,83,4,5,0,0,0,114,5,83,5,19,0,72,20, 0,0,114,6,92,2,34,0,83,6,92,6,14,0,83,7, 92,5,92,6,5,0,0,0,14,0,51,4,53,1,0,0, - 0,0,0,0,32,0,77,22,0,0,11,0,103,1,41,8, - 233,0,0,0,0,78,122,18,70,114,111,122,101,110,32,72, - 101,108,108,111,32,87,111,114,108,100,122,8,115,121,115,46, - 97,114,103,118,218,6,99,111,110,102,105,103,41,5,218,12, - 112,114,111,103,114,97,109,95,110,97,109,101,218,10,101,120, - 101,99,117,116,97,98,108,101,218,15,117,115,101,95,101,110, - 118,105,114,111,110,109,101,110,116,218,17,99,111,110,102,105, - 103,117,114,101,95,99,95,115,116,100,105,111,218,14,98,117, - 102,102,101,114,101,100,95,115,116,100,105,111,122,7,99,111, - 110,102,105,103,32,122,2,58,32,41,7,218,3,115,121,115, - 218,17,95,116,101,115,116,105,110,116,101,114,110,97,108,99, - 97,112,105,218,5,112,114,105,110,116,218,4,97,114,103,118, - 218,11,103,101,116,95,99,111,110,102,105,103,115,114,3,0, - 0,0,218,3,107,101,121,169,0,243,0,0,0,0,250,18, - 116,101,115,116,95,102,114,111,122,101,110,109,97,105,110,46, - 112,121,250,8,60,109,111,100,117,108,101,62,114,18,0,0, - 0,1,0,0,0,115,99,0,0,0,240,3,1,1,1,243, - 8,0,1,11,219,0,24,225,0,5,208,6,26,212,0,27, - 217,0,5,128,106,144,35,151,40,145,40,212,0,27,216,9, - 26,215,9,38,210,9,38,211,9,40,168,24,209,9,50,128, - 6,240,2,6,12,2,242,0,7,1,42,128,67,241,14,0, - 5,10,136,71,144,67,144,53,152,2,152,54,160,35,153,59, - 152,45,208,10,40,214,4,41,241,15,7,1,42,114,16,0, - 0,0, + 0,0,0,0,32,0,77,22,0,0,11,0,32,0,103,1, + 41,8,233,0,0,0,0,78,122,18,70,114,111,122,101,110, + 32,72,101,108,108,111,32,87,111,114,108,100,122,8,115,121, + 115,46,97,114,103,118,218,6,99,111,110,102,105,103,41,5, + 218,12,112,114,111,103,114,97,109,95,110,97,109,101,218,10, + 101,120,101,99,117,116,97,98,108,101,218,15,117,115,101,95, + 101,110,118,105,114,111,110,109,101,110,116,218,17,99,111,110, + 102,105,103,117,114,101,95,99,95,115,116,100,105,111,218,14, + 98,117,102,102,101,114,101,100,95,115,116,100,105,111,122,7, + 99,111,110,102,105,103,32,122,2,58,32,41,7,218,3,115, + 121,115,218,17,95,116,101,115,116,105,110,116,101,114,110,97, + 108,99,97,112,105,218,5,112,114,105,110,116,218,4,97,114, + 103,118,218,11,103,101,116,95,99,111,110,102,105,103,115,114, + 3,0,0,0,218,3,107,101,121,169,0,243,0,0,0,0, + 250,18,116,101,115,116,95,102,114,111,122,101,110,109,97,105, + 110,46,112,121,250,8,60,109,111,100,117,108,101,62,114,18, + 0,0,0,1,0,0,0,115,99,0,0,0,240,3,1,1, + 1,243,8,0,1,11,219,0,24,225,0,5,208,6,26,212, + 0,27,217,0,5,128,106,144,35,151,40,145,40,212,0,27, + 216,9,26,215,9,38,210,9,38,211,9,40,168,24,209,9, + 50,128,6,240,2,6,12,2,242,0,7,1,42,128,67,241, + 14,0,5,10,136,71,144,67,144,53,152,2,152,54,160,35, + 153,59,152,45,208,10,40,214,4,41,242,15,7,1,42,114, + 16,0,0,0, }; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index e54d5cbacdc96f9..b0074962b737992 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -703,17 +703,34 @@ builtin_format_impl(PyObject *module, PyObject *value, PyObject *format_spec) /*[clinic input] chr as builtin_chr - i: int + i: object / Return a Unicode string of one character with ordinal i; 0 <= i <= 0x10ffff. [clinic start generated code]*/ static PyObject * -builtin_chr_impl(PyObject *module, int i) -/*[clinic end generated code: output=c733afcd200afcb7 input=3f604ef45a70750d]*/ +builtin_chr(PyObject *module, PyObject *i) +/*[clinic end generated code: output=d34f25b8035a9b10 input=f919867f0ba2f496]*/ { - return PyUnicode_FromOrdinal(i); + int overflow; + long v = PyLong_AsLongAndOverflow(i, &overflow); + if (v == -1 && PyErr_Occurred()) { + return NULL; + } + if (overflow) { + v = overflow < 0 ? INT_MIN : INT_MAX; + /* Allow PyUnicode_FromOrdinal() to raise an exception */ + } +#if SIZEOF_INT < SIZEOF_LONG + else if (v < INT_MIN) { + v = INT_MIN; + } + else if (v > INT_MAX) { + v = INT_MAX; + } +#endif + return PyUnicode_FromOrdinal(v); } @@ -2382,11 +2399,6 @@ builtin_round_impl(PyObject *module, PyObject *number, PyObject *ndigits) { PyObject *round, *result; - if (!_PyType_IsReady(Py_TYPE(number))) { - if (PyType_Ready(Py_TYPE(number)) < 0) - return NULL; - } - round = _PyObject_LookupSpecial(number, &_Py_ID(__round__)); if (round == NULL) { if (!PyErr_Occurred()) diff --git a/Python/brc.c b/Python/brc.c new file mode 100644 index 000000000000000..f1fd57a2964cf55 --- /dev/null +++ b/Python/brc.c @@ -0,0 +1,198 @@ +// Implementation of biased reference counting inter-thread queue. +// +// Biased reference counting maintains two refcount fields in each object: +// ob_ref_local and ob_ref_shared. The true refcount is the sum of these two +// fields. In some cases, when refcounting operations are split across threads, +// the ob_ref_shared field can be negative (although the total refcount must +// be at least zero). In this case, the thread that decremented the refcount +// requests that the owning thread give up ownership and merge the refcount +// fields. This file implements the mechanism for doing so. +// +// Each thread state maintains a queue of objects whose refcounts it should +// merge. The thread states are stored in a per-interpreter hash table by +// thread id. The hash table has a fixed size and uses a linked list to store +// thread states within each bucket. +// +// The queueing thread uses the eval breaker mechanism to notify the owning +// thread that it has objects to merge. Additionaly, all queued objects are +// merged during GC. +#include "Python.h" +#include "pycore_object.h" // _Py_ExplicitMergeRefcount +#include "pycore_brc.h" // struct _brc_thread_state +#include "pycore_ceval.h" // _Py_set_eval_breaker_bit +#include "pycore_llist.h" // struct llist_node +#include "pycore_pystate.h" // _PyThreadStateImpl + +#ifdef Py_GIL_DISABLED + +// Get the hashtable bucket for a given thread id. +static struct _brc_bucket * +get_bucket(PyInterpreterState *interp, uintptr_t tid) +{ + return &interp->brc.table[tid % _Py_BRC_NUM_BUCKETS]; +} + +// Find the thread state in a hash table bucket by thread id. +static _PyThreadStateImpl * +find_thread_state(struct _brc_bucket *bucket, uintptr_t thread_id) +{ + struct llist_node *node; + llist_for_each(node, &bucket->root) { + // Get the containing _PyThreadStateImpl from the linked-list node. + _PyThreadStateImpl *ts = llist_data(node, _PyThreadStateImpl, + brc.bucket_node); + if (ts->brc.tid == thread_id) { + return ts; + } + } + return NULL; +} + +// Enqueue an object to be merged by the owning thread. This steals a +// reference to the object. +void +_Py_brc_queue_object(PyObject *ob) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + + uintptr_t ob_tid = _Py_atomic_load_uintptr(&ob->ob_tid); + if (ob_tid == 0) { + // The owning thread may have concurrently decided to merge the + // refcount fields. + Py_DECREF(ob); + return; + } + + struct _brc_bucket *bucket = get_bucket(interp, ob_tid); + PyMutex_Lock(&bucket->mutex); + _PyThreadStateImpl *tstate = find_thread_state(bucket, ob_tid); + if (tstate == NULL) { + // If we didn't find the owning thread then it must have already exited. + // It's safe (and necessary) to merge the refcount. Subtract one when + // merging because we've stolen a reference. + Py_ssize_t refcount = _Py_ExplicitMergeRefcount(ob, -1); + PyMutex_Unlock(&bucket->mutex); + if (refcount == 0) { + _Py_Dealloc(ob); + } + return; + } + + if (_PyObjectStack_Push(&tstate->brc.objects_to_merge, ob) < 0) { + PyMutex_Unlock(&bucket->mutex); + + // Fall back to stopping all threads and manually merging the refcount + // if we can't enqueue the object to be merged. + _PyEval_StopTheWorld(interp); + Py_ssize_t refcount = _Py_ExplicitMergeRefcount(ob, -1); + _PyEval_StartTheWorld(interp); + + if (refcount == 0) { + _Py_Dealloc(ob); + } + return; + } + + // Notify owning thread + _Py_set_eval_breaker_bit(interp, _PY_EVAL_EXPLICIT_MERGE_BIT, 1); + + PyMutex_Unlock(&bucket->mutex); +} + +static void +merge_queued_objects(_PyObjectStack *to_merge) +{ + PyObject *ob; + while ((ob = _PyObjectStack_Pop(to_merge)) != NULL) { + // Subtract one when merging because the queue had a reference. + Py_ssize_t refcount = _Py_ExplicitMergeRefcount(ob, -1); + if (refcount == 0) { + _Py_Dealloc(ob); + } + } +} + +// Process this thread's queue of objects to merge. +void +_Py_brc_merge_refcounts(PyThreadState *tstate) +{ + struct _brc_thread_state *brc = &((_PyThreadStateImpl *)tstate)->brc; + struct _brc_bucket *bucket = get_bucket(tstate->interp, brc->tid); + + // Append all objects into a local stack. We don't want to hold the lock + // while calling destructors. + PyMutex_Lock(&bucket->mutex); + _PyObjectStack_Merge(&brc->local_objects_to_merge, &brc->objects_to_merge); + PyMutex_Unlock(&bucket->mutex); + + // Process the local stack until it's empty + merge_queued_objects(&brc->local_objects_to_merge); +} + +void +_Py_brc_init_state(PyInterpreterState *interp) +{ + struct _brc_state *brc = &interp->brc; + for (Py_ssize_t i = 0; i < _Py_BRC_NUM_BUCKETS; i++) { + llist_init(&brc->table[i].root); + } +} + +void +_Py_brc_init_thread(PyThreadState *tstate) +{ + struct _brc_thread_state *brc = &((_PyThreadStateImpl *)tstate)->brc; + brc->tid = _Py_ThreadId(); + + // Add ourself to the hashtable + struct _brc_bucket *bucket = get_bucket(tstate->interp, brc->tid); + PyMutex_Lock(&bucket->mutex); + llist_insert_tail(&bucket->root, &brc->bucket_node); + PyMutex_Unlock(&bucket->mutex); +} + +void +_Py_brc_remove_thread(PyThreadState *tstate) +{ + struct _brc_thread_state *brc = &((_PyThreadStateImpl *)tstate)->brc; + struct _brc_bucket *bucket = get_bucket(tstate->interp, brc->tid); + + // We need to fully process any objects to merge before removing ourself + // from the hashtable. It is not safe to perform any refcount operations + // after we are removed. After that point, other threads treat our objects + // as abandoned and may merge the objects' refcounts directly. + bool empty = false; + while (!empty) { + // Process the local stack until it's empty + merge_queued_objects(&brc->local_objects_to_merge); + + PyMutex_Lock(&bucket->mutex); + empty = (brc->objects_to_merge.head == NULL); + if (empty) { + llist_remove(&brc->bucket_node); + } + else { + _PyObjectStack_Merge(&brc->local_objects_to_merge, + &brc->objects_to_merge); + } + PyMutex_Unlock(&bucket->mutex); + } + + assert(brc->local_objects_to_merge.head == NULL); + assert(brc->objects_to_merge.head == NULL); +} + +void +_Py_brc_after_fork(PyInterpreterState *interp) +{ + // Unlock all bucket mutexes. Some of the buckets may be locked because + // locks can be handed off to a parked thread (see lock.c). We don't have + // to worry about consistency here, becuase no thread can be actively + // modifying a bucket, but it might be paused (not yet woken up) on a + // PyMutex_Lock while holding that lock. + for (Py_ssize_t i = 0; i < _Py_BRC_NUM_BUCKETS; i++) { + _PyMutex_at_fork_reinit(&interp->brc.table[i].mutex); + } +} + +#endif /* Py_GIL_DISABLED */ diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7674ff81f64cec0..6fb4d719e43991c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1,6 +1,6 @@ // This file contains instruction definitions. -// It is read by Tools/cases_generator/generate_cases.py -// to generate Python/generated_cases.c.h. +// It is read by generators stored in Tools/cases_generator/ +// to generate Python/generated_cases.c.h and others. // Note that there is some dummy C code at the top and bottom of the file // to fool text editors like VS Code into believing this is valid C code. // The actual instruction definitions start at // BEGIN BYTECODES //. @@ -265,9 +265,9 @@ dummy_func( res = NULL; } - macro(END_FOR) = POP_TOP + POP_TOP; + macro(END_FOR) = POP_TOP; - inst(INSTRUMENTED_END_FOR, (receiver, value --)) { + inst(INSTRUMENTED_END_FOR, (receiver, value -- receiver)) { TIER_ONE_ONLY /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ @@ -2550,8 +2550,8 @@ dummy_func( next_instr[oparg].op.code == INSTRUMENTED_END_FOR); Py_DECREF(iter); STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */ + JUMPBY(oparg + 2); DISPATCH(); } // Common case: no jump, leave it to the code generator @@ -2599,8 +2599,8 @@ dummy_func( next_instr[oparg].op.code == INSTRUMENTED_END_FOR); STACK_SHRINK(1); Py_DECREF(iter); - /* Skip END_FOR */ - target = next_instr + oparg + 1; + /* Skip END_FOR and POP_TOP */ + target = next_instr + oparg + 2; } INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); } @@ -2621,8 +2621,8 @@ dummy_func( } Py_DECREF(iter); STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ + JUMPBY(oparg + 2); DISPATCH(); } } @@ -2667,8 +2667,8 @@ dummy_func( } Py_DECREF(iter); STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ + JUMPBY(oparg + 2); DISPATCH(); } } @@ -2709,8 +2709,8 @@ dummy_func( if (r->len <= 0) { STACK_SHRINK(1); Py_DECREF(r); - // Jump over END_FOR instruction. - JUMPBY(oparg + 1); + // Jump over END_FOR and POP_TOP instructions. + JUMPBY(oparg + 2); DISPATCH(); } } @@ -4070,10 +4070,38 @@ dummy_func( DEOPT_IF(!current_executor->vm_data.valid); } + op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { + TIER_TWO_ONLY + value = Py_NewRef(ptr); + } + op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { + TIER_TWO_ONLY value = ptr; } + op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { + TIER_TWO_ONLY + value = Py_NewRef(ptr); + null = NULL; + } + + op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) { + TIER_TWO_ONLY + value = ptr; + null = NULL; + } + + op(_CHECK_GLOBALS, (dict/4 -- )) { + TIER_TWO_ONLY + DEOPT_IF(GLOBALS() != dict); + } + + op(_CHECK_BUILTINS, (dict/4 -- )) { + TIER_TWO_ONLY + DEOPT_IF(BUILTINS() != dict); + } + /* Internal -- for testing executors */ op(_INTERNAL_INCREMENT_OPT_COUNTER, (opt --)) { _PyCounterOptimizerObject *exe = (_PyCounterOptimizerObject *)opt; diff --git a/Python/ceval.c b/Python/ceval.c index 49388cd20377c0c..4f2080090861914 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -11,6 +11,7 @@ #include "pycore_function.h" #include "pycore_instruments.h" #include "pycore_intrinsics.h" +#include "pycore_jit.h" #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_object.h" // _PyObject_GC_TRACK() @@ -955,9 +956,24 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int -// The Tier 2 interpreter is also here! +// Tier 2 is also here! enter_tier_two: +#ifdef _Py_JIT + + ; // ;) + jit_func jitted = current_executor->jit_code; + next_instr = jitted(frame, stack_pointer, tstate); + frame = tstate->current_frame; + Py_DECREF(current_executor); + if (next_instr == NULL) { + goto resume_with_error; + } + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + +#else + #undef LOAD_IP #define LOAD_IP(UNUSED) (void)0 @@ -1073,6 +1089,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int Py_DECREF(current_executor); DISPATCH(); +#endif // _Py_JIT + } #if defined(__GNUC__) # pragma GCC diagnostic pop diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index f3b169241535f37..deb9741291fca73 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -610,8 +610,16 @@ PyEval_SaveThread(void) void PyEval_RestoreThread(PyThreadState *tstate) { +#ifdef MS_WINDOWS + int err = GetLastError(); +#endif + _Py_EnsureTstateNotNULL(tstate); _PyThreadState_Attach(tstate); + +#ifdef MS_WINDOWS + SetLastError(err); +#endif } @@ -972,6 +980,14 @@ _Py_HandlePending(PyThreadState *tstate) } } +#ifdef Py_GIL_DISABLED + /* Objects with refcounts to merge */ + if (_Py_eval_breaker_bit_is_set(interp, _PY_EVAL_EXPLICIT_MERGE_BIT)) { + _Py_set_eval_breaker_bit(interp, _PY_EVAL_EXPLICIT_MERGE_BIT, 0); + _Py_brc_merge_refcounts(tstate); + } +#endif + /* GC scheduled to run */ if (_Py_eval_breaker_bit_is_set(interp, _PY_GC_SCHEDULED_BIT)) { _Py_set_eval_breaker_bit(interp, _PY_GC_SCHEDULED_BIT, 0); diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 8d40e659b54a57e..3898f987cd61ea6 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -233,25 +233,6 @@ PyDoc_STRVAR(builtin_chr__doc__, #define BUILTIN_CHR_METHODDEF \ {"chr", (PyCFunction)builtin_chr, METH_O, builtin_chr__doc__}, -static PyObject * -builtin_chr_impl(PyObject *module, int i); - -static PyObject * -builtin_chr(PyObject *module, PyObject *arg) -{ - PyObject *return_value = NULL; - int i; - - i = PyLong_AsInt(arg); - if (i == -1 && PyErr_Occurred()) { - goto exit; - } - return_value = builtin_chr_impl(module, i); - -exit: - return return_value; -} - PyDoc_STRVAR(builtin_compile__doc__, "compile($module, /, source, filename, mode, flags=0,\n" " dont_inherit=False, optimize=-1, *, _feature_version=-1)\n" @@ -1212,4 +1193,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=31bded5d08647a57 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=643a8d5f900e0c36 input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index 2a6291ccb51b0c9..15e5cf38a37b97a 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -160,7 +160,7 @@ _PyCompile_EnsureArrayLargeEnough(int idx, void **array, int *alloc, if (idx >= new_alloc) { new_alloc = idx + default_alloc; } - arr = PyObject_Calloc(new_alloc, item_size); + arr = PyMem_Calloc(new_alloc, item_size); if (arr == NULL) { PyErr_NoMemory(); return ERROR; @@ -181,7 +181,7 @@ _PyCompile_EnsureArrayLargeEnough(int idx, void **array, int *alloc, } assert(newsize > 0); - void *tmp = PyObject_Realloc(arr, newsize); + void *tmp = PyMem_Realloc(arr, newsize); if (tmp == NULL) { PyErr_NoMemory(); return ERROR; @@ -282,10 +282,10 @@ instr_sequence_insert_instruction(instr_sequence *seq, int pos, static void instr_sequence_fini(instr_sequence *seq) { - PyObject_Free(seq->s_labelmap); + PyMem_Free(seq->s_labelmap); seq->s_labelmap = NULL; - PyObject_Free(seq->s_instrs); + PyMem_Free(seq->s_instrs); seq->s_instrs = NULL; } @@ -690,7 +690,7 @@ compiler_unit_free(struct compiler_unit *u) Py_CLEAR(u->u_metadata.u_cellvars); Py_CLEAR(u->u_metadata.u_fasthidden); Py_CLEAR(u->u_private); - PyObject_Free(u); + PyMem_Free(u); } static int @@ -958,14 +958,15 @@ merge_consts_recursive(PyObject *const_cache, PyObject *o) return NULL; } - // t is borrowed reference - PyObject *t = PyDict_SetDefault(const_cache, key, key); - if (t != key) { - // o is registered in const_cache. Just use it. - Py_XINCREF(t); + PyObject *t; + int res = PyDict_SetDefaultRef(const_cache, key, key, &t); + if (res != 0) { + // o was not inserted into const_cache. t is either the existing value + // or NULL (on error). Py_DECREF(key); return t; } + Py_DECREF(t); // We registered o in const_cache. // When o is a tuple or frozenset, we want to merge its @@ -1262,8 +1263,7 @@ compiler_enter_scope(struct compiler *c, identifier name, struct compiler_unit *u; - u = (struct compiler_unit *)PyObject_Calloc(1, sizeof( - struct compiler_unit)); + u = (struct compiler_unit *)PyMem_Calloc(1, sizeof(struct compiler_unit)); if (!u) { PyErr_NoMemory(); return ERROR; @@ -3075,7 +3075,12 @@ compiler_for(struct compiler *c, stmt_ty s) ADDOP_JUMP(c, NO_LOCATION, JUMP, start); USE_LABEL(c, cleanup); + /* It is important for instrumentation that the `END_FOR` comes first. + * Iteration over a generator will jump to the first of these instructions, + * but a non-generator will jump to a later instruction. + */ ADDOP(c, NO_LOCATION, END_FOR); + ADDOP(c, NO_LOCATION, POP_TOP); compiler_pop_fblock(c, FOR_LOOP, start); @@ -5390,7 +5395,12 @@ compiler_sync_comprehension_generator(struct compiler *c, location loc, ADDOP_JUMP(c, elt_loc, JUMP, start); USE_LABEL(c, anchor); + /* It is important for instrumentation that the `END_FOR` comes first. + * Iteration over a generator will jump to the first of these instructions, + * but a non-generator will jump to a later instruction. + */ ADDOP(c, NO_LOCATION, END_FOR); + ADDOP(c, NO_LOCATION, POP_TOP); } return SUCCESS; @@ -6647,7 +6657,7 @@ ensure_fail_pop(struct compiler *c, pattern_context *pc, Py_ssize_t n) return SUCCESS; } Py_ssize_t needed = sizeof(jump_target_label) * size; - jump_target_label *resized = PyObject_Realloc(pc->fail_pop, needed); + jump_target_label *resized = PyMem_Realloc(pc->fail_pop, needed); if (resized == NULL) { PyErr_NoMemory(); return ERROR; @@ -6686,13 +6696,13 @@ emit_and_reset_fail_pop(struct compiler *c, location loc, USE_LABEL(c, pc->fail_pop[pc->fail_pop_size]); if (codegen_addop_noarg(INSTR_SEQUENCE(c), POP_TOP, loc) < 0) { pc->fail_pop_size = 0; - PyObject_Free(pc->fail_pop); + PyMem_Free(pc->fail_pop); pc->fail_pop = NULL; return ERROR; } } USE_LABEL(c, pc->fail_pop[0]); - PyObject_Free(pc->fail_pop); + PyMem_Free(pc->fail_pop); pc->fail_pop = NULL; return SUCCESS; } @@ -7196,7 +7206,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) Py_DECREF(pc->stores); *pc = old_pc; Py_INCREF(pc->stores); - // Need to NULL this for the PyObject_Free call in the error block. + // Need to NULL this for the PyMem_Free call in the error block. old_pc.fail_pop = NULL; // No match. Pop the remaining copy of the subject and fail: if (codegen_addop_noarg(INSTR_SEQUENCE(c), POP_TOP, LOC(p)) < 0 || @@ -7242,7 +7252,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) diff: compiler_error(c, LOC(p), "alternative patterns bind different names"); error: - PyObject_Free(old_pc.fail_pop); + PyMem_Free(old_pc.fail_pop); Py_DECREF(old_pc.stores); Py_XDECREF(control); return ERROR; @@ -7443,7 +7453,7 @@ compiler_match(struct compiler *c, stmt_ty s) pattern_context pc; pc.fail_pop = NULL; int result = compiler_match_inner(c, s, &pc); - PyObject_Free(pc.fail_pop); + PyMem_Free(pc.fail_pop); return result; } @@ -7518,22 +7528,26 @@ _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj) return ERROR; } - // t is borrowed reference - PyObject *t = PyDict_SetDefault(const_cache, key, key); + PyObject *t; + int res = PyDict_SetDefaultRef(const_cache, key, key, &t); Py_DECREF(key); - if (t == NULL) { + if (res < 0) { return ERROR; } - if (t == key) { // obj is new constant. + if (res == 0) { // inserted: obj is new constant. + Py_DECREF(t); return SUCCESS; } if (PyTuple_CheckExact(t)) { - // t is still borrowed reference - t = PyTuple_GET_ITEM(t, 1); + PyObject *item = PyTuple_GET_ITEM(t, 1); + Py_SETREF(*obj, Py_NewRef(item)); + Py_DECREF(t); + } + else { + Py_SETREF(*obj, t); } - Py_SETREF(*obj, Py_NewRef(t)); return SUCCESS; } diff --git a/Python/condvar.h b/Python/condvar.h index d54db94f2c871d4..dcabed6d55928ce 100644 --- a/Python/condvar.h +++ b/Python/condvar.h @@ -260,13 +260,13 @@ PyMUTEX_UNLOCK(PyMUTEX_T *cs) return 0; } - Py_LOCAL_INLINE(int) PyCOND_INIT(PyCOND_T *cv) { InitializeConditionVariable(cv); return 0; } + Py_LOCAL_INLINE(int) PyCOND_FINI(PyCOND_T *cv) { @@ -279,27 +279,32 @@ PyCOND_WAIT(PyCOND_T *cv, PyMUTEX_T *cs) return SleepConditionVariableSRW(cv, cs, INFINITE, 0) ? 0 : -1; } -/* This implementation makes no distinction about timeouts. Signal - * 2 to indicate that we don't know. - */ +/* return 0 for success, 1 on timeout, -1 on error */ Py_LOCAL_INLINE(int) PyCOND_TIMEDWAIT(PyCOND_T *cv, PyMUTEX_T *cs, long long us) { - return SleepConditionVariableSRW(cv, cs, (DWORD)(us/1000), 0) ? 2 : -1; + BOOL success = SleepConditionVariableSRW(cv, cs, (DWORD)(us/1000), 0); + if (!success) { + if (GetLastError() == ERROR_TIMEOUT) { + return 1; + } + return -1; + } + return 0; } Py_LOCAL_INLINE(int) PyCOND_SIGNAL(PyCOND_T *cv) { - WakeConditionVariable(cv); - return 0; + WakeConditionVariable(cv); + return 0; } Py_LOCAL_INLINE(int) PyCOND_BROADCAST(PyCOND_T *cv) { - WakeAllConditionVariable(cv); - return 0; + WakeAllConditionVariable(cv); + return 0; } diff --git a/Python/context.c b/Python/context.c index 1e90811c374ec63..e44fef705c36e0e 100644 --- a/Python/context.c +++ b/Python/context.c @@ -69,7 +69,7 @@ static struct _Py_context_state * get_context_state(void) { _PyFreeListState *state = _PyFreeListState_GET(); - return &state->context_state; + return &state->contexts; } #endif @@ -1270,7 +1270,7 @@ void _PyContext_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization) { #ifdef WITH_FREELISTS - struct _Py_context_state *state = &freelist_state->context_state; + struct _Py_context_state *state = &freelist_state->contexts; for (; state->numfree > 0; state->numfree--) { PyContext *ctx = state->freelist; state->freelist = (PyContext *)ctx->ctx_weakreflist; @@ -1284,13 +1284,6 @@ _PyContext_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization) } -void -_PyContext_Fini(_PyFreeListState *state) -{ - _PyContext_ClearFreeList(state, 1); -} - - PyStatus _PyContext_Init(PyInterpreterState *interp) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 2b4399b25bae2bc..2d914b82dbf88f4 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3390,15 +3390,66 @@ break; } + case _LOAD_CONST_INLINE: { + PyObject *value; + PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + TIER_TWO_ONLY + value = Py_NewRef(ptr); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + case _LOAD_CONST_INLINE_BORROW: { PyObject *value; PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + TIER_TWO_ONLY value = ptr; stack_pointer[0] = value; stack_pointer += 1; break; } + case _LOAD_CONST_INLINE_WITH_NULL: { + PyObject *value; + PyObject *null; + PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + TIER_TWO_ONLY + value = Py_NewRef(ptr); + null = NULL; + stack_pointer[0] = value; + stack_pointer[1] = null; + stack_pointer += 2; + break; + } + + case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { + PyObject *value; + PyObject *null; + PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + TIER_TWO_ONLY + value = ptr; + null = NULL; + stack_pointer[0] = value; + stack_pointer[1] = null; + stack_pointer += 2; + break; + } + + case _CHECK_GLOBALS: { + PyObject *dict = (PyObject *)CURRENT_OPERAND(); + TIER_TWO_ONLY + if (GLOBALS() != dict) goto deoptimize; + break; + } + + case _CHECK_BUILTINS: { + PyObject *dict = (PyObject *)CURRENT_OPERAND(); + TIER_TWO_ONLY + if (BUILTINS() != dict) goto deoptimize; + break; + } + case _INTERNAL_INCREMENT_OPT_COUNTER: { PyObject *opt; opt = stack_pointer[-1]; diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 2fc90b8877b475b..1a648edf0880c02 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -29,7 +29,6 @@ typedef struct _PyCfgInstruction { int i_opcode; int i_oparg; _PyCompilerSrcLocation i_loc; - unsigned i_loc_propagated : 1; /* location was set by propagate_line_numbers */ struct _PyCfgBasicblock *i_target; /* target block (if jump instruction) */ struct _PyCfgBasicblock *i_except; /* target block when exception is raised */ } cfg_instr; @@ -146,6 +145,16 @@ basicblock_next_instr(basicblock *b) return b->b_iused++; } +static cfg_instr * +basicblock_last_instr(const basicblock *b) { + assert(b->b_iused >= 0); + if (b->b_iused > 0) { + assert(b->b_instr != NULL); + return &b->b_instr[b->b_iused - 1]; + } + return NULL; +} + /* Allocate a new block and return a pointer to it. Returns NULL on error. */ @@ -153,7 +162,7 @@ basicblock_next_instr(basicblock *b) static basicblock * cfg_builder_new_block(cfg_builder *g) { - basicblock *b = (basicblock *)PyObject_Calloc(1, sizeof(basicblock)); + basicblock *b = (basicblock *)PyMem_Calloc(1, sizeof(basicblock)); if (b == NULL) { PyErr_NoMemory(); return NULL; @@ -186,29 +195,35 @@ basicblock_addop(basicblock *b, int opcode, int oparg, location loc) return SUCCESS; } +static int +basicblock_add_jump(basicblock *b, int opcode, basicblock *target, location loc) +{ + cfg_instr *last = basicblock_last_instr(b); + if (last && is_jump(last)) { + return ERROR; + } + + RETURN_IF_ERROR( + basicblock_addop(b, opcode, target->b_label.id, loc)); + last = basicblock_last_instr(b); + assert(last && last->i_opcode == opcode); + last->i_target = target; + return SUCCESS; +} + static inline int -basicblock_append_instructions(basicblock *target, basicblock *source) +basicblock_append_instructions(basicblock *to, basicblock *from) { - for (int i = 0; i < source->b_iused; i++) { - int n = basicblock_next_instr(target); + for (int i = 0; i < from->b_iused; i++) { + int n = basicblock_next_instr(to); if (n < 0) { return ERROR; } - target->b_instr[n] = source->b_instr[i]; + to->b_instr[n] = from->b_instr[i]; } return SUCCESS; } -static cfg_instr * -basicblock_last_instr(const basicblock *b) { - assert(b->b_iused >= 0); - if (b->b_iused > 0) { - assert(b->b_instr != NULL); - return &b->b_instr[b->b_iused - 1]; - } - return NULL; -} - static inline int basicblock_nofallthrough(const basicblock *b) { cfg_instr *last = basicblock_last_instr(b); @@ -277,9 +292,9 @@ static void dump_basicblock(const basicblock *b) { const char *b_return = basicblock_returns(b) ? "return " : ""; - fprintf(stderr, "%d: [EH=%d CLD=%d WRM=%d NO_FT=%d %p] used: %d, depth: %d, %s\n", + fprintf(stderr, "%d: [EH=%d CLD=%d WRM=%d NO_FT=%d %p] used: %d, depth: %d, preds: %d %s\n", b->b_label.id, b->b_except_handler, b->b_cold, b->b_warm, BB_NO_FALLTHROUGH(b), b, b->b_iused, - b->b_startdepth, b_return); + b->b_startdepth, b->b_predecessors, b_return); if (b->b_instr) { int i; for (i = 0; i < b->b_iused; i++) { @@ -422,10 +437,10 @@ _PyCfgBuilder_Free(cfg_builder *g) basicblock *b = g->g_block_list; while (b != NULL) { if (b->b_instr) { - PyObject_Free((void *)b->b_instr); + PyMem_Free((void *)b->b_instr); } basicblock *next = b->b_list; - PyObject_Free((void *)b); + PyMem_Free((void *)b); b = next; } PyMem_Free(g); @@ -560,8 +575,8 @@ normalize_jumps_in_block(cfg_builder *g, basicblock *b) { if (backwards_jump == NULL) { return ERROR; } - basicblock_addop(backwards_jump, JUMP, target->b_label.id, last->i_loc); - backwards_jump->b_instr[0].i_target = target; + RETURN_IF_ERROR( + basicblock_add_jump(backwards_jump, JUMP, target, last->i_loc)); last->i_opcode = reversed_opcode; last->i_target = b->b_next; @@ -888,6 +903,7 @@ label_exception_targets(basicblock *entryblock) { } else if (instr->i_opcode == POP_BLOCK) { handler = pop_except_block(except_stack); + INSTR_SET_OP0(instr, NOP); } else if (is_jump(instr)) { instr->i_except = handler; @@ -1141,13 +1157,7 @@ remove_redundant_jumps(cfg_builder *g) { basicblock *next = next_nonempty_block(b->b_next); if (jump_target == next) { changes++; - if (last->i_loc_propagated) { - b->b_iused--; - } - else { - assert(last->i_loc.lineno != -1); - INSTR_SET_OP0(last, NOP); - } + INSTR_SET_OP0(last, NOP); } } } @@ -1155,15 +1165,26 @@ remove_redundant_jumps(cfg_builder *g) { return changes; } +static inline bool +basicblock_has_no_lineno(basicblock *b) { + for (int i = 0; i < b->b_iused; i++) { + if (b->b_instr[i].i_loc.lineno >= 0) { + return false; + } + } + return true; +} + /* Maximum size of basic block that should be copied in optimizer */ #define MAX_COPY_SIZE 4 -/* If this block ends with an unconditional jump to a small exit block, then +/* If this block ends with an unconditional jump to a small exit block or + * a block that has no line numbers (and no fallthrough), then * remove the jump and extend this block with the target. * Returns 1 if extended, 0 if no change, and -1 on error. */ static int -inline_small_exit_blocks(basicblock *bb) { +basicblock_inline_small_or_no_lineno_blocks(basicblock *bb) { cfg_instr *last = basicblock_last_instr(bb); if (last == NULL) { return 0; @@ -1172,35 +1193,67 @@ inline_small_exit_blocks(basicblock *bb) { return 0; } basicblock *target = last->i_target; - if (basicblock_exits_scope(target) && target->b_iused <= MAX_COPY_SIZE) { + bool small_exit_block = (basicblock_exits_scope(target) && + target->b_iused <= MAX_COPY_SIZE); + bool no_lineno_no_fallthrough = (basicblock_has_no_lineno(target) && + !BB_HAS_FALLTHROUGH(target)); + if (small_exit_block || no_lineno_no_fallthrough) { + assert(is_jump(last)); + int removed_jump_opcode = last->i_opcode; INSTR_SET_OP0(last, NOP); RETURN_IF_ERROR(basicblock_append_instructions(bb, target)); + if (no_lineno_no_fallthrough) { + last = basicblock_last_instr(bb); + if (IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode) && + removed_jump_opcode == JUMP) + { + /* Make sure we don't lose eval breaker checks */ + last->i_opcode = JUMP; + } + } + target->b_predecessors--; return 1; } return 0; } +static int +inline_small_or_no_lineno_blocks(basicblock *entryblock) { + bool changes; + do { + changes = false; + for (basicblock *b = entryblock; b != NULL; b = b->b_next) { + int res = basicblock_inline_small_or_no_lineno_blocks(b); + RETURN_IF_ERROR(res); + if (res) { + changes = true; + } + } + } while(changes); /* every change removes a jump, ensuring convergence */ + return changes; +} + // Attempt to eliminate jumps to jumps by updating inst to jump to // target->i_target using the provided opcode. Return whether or not the // optimization was successful. static bool -jump_thread(cfg_instr *inst, cfg_instr *target, int opcode) +jump_thread(basicblock *bb, cfg_instr *inst, cfg_instr *target, int opcode) { assert(is_jump(inst)); assert(is_jump(target)); + assert(inst == basicblock_last_instr(bb)); // bpo-45773: If inst->i_target == target->i_target, then nothing actually // changes (and we fall into an infinite loop): - if (inst->i_loc.lineno == -1) assert(inst->i_loc_propagated); - if (target->i_loc.lineno == -1) assert(target->i_loc_propagated); - if ((inst->i_loc.lineno == target->i_loc.lineno || - inst->i_loc_propagated || target->i_loc_propagated) && - inst->i_target != target->i_target) - { - inst->i_target = target->i_target; - inst->i_opcode = opcode; - if (inst->i_loc_propagated && !target->i_loc_propagated) { - inst->i_loc = target->i_loc; - } + if (inst->i_target != target->i_target) { + /* Change inst to NOP and append a jump to target->i_target. The + * NOP will be removed later if it's not needed for the lineno. + */ + INSTR_SET_OP0(inst, NOP); + + RETURN_IF_ERROR( + basicblock_add_jump( + bb, opcode, target->i_target, target->i_loc)); + return true; } return false; @@ -1673,29 +1726,29 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) case POP_JUMP_IF_NONE: switch (target->i_opcode) { case JUMP: - i -= jump_thread(inst, target, inst->i_opcode); + i -= jump_thread(bb, inst, target, inst->i_opcode); } break; case POP_JUMP_IF_FALSE: switch (target->i_opcode) { case JUMP: - i -= jump_thread(inst, target, POP_JUMP_IF_FALSE); + i -= jump_thread(bb, inst, target, POP_JUMP_IF_FALSE); } break; case POP_JUMP_IF_TRUE: switch (target->i_opcode) { case JUMP: - i -= jump_thread(inst, target, POP_JUMP_IF_TRUE); + i -= jump_thread(bb, inst, target, POP_JUMP_IF_TRUE); } break; case JUMP: case JUMP_NO_INTERRUPT: switch (target->i_opcode) { case JUMP: - i -= jump_thread(inst, target, JUMP); + i -= jump_thread(bb, inst, target, JUMP); continue; case JUMP_NO_INTERRUPT: - i -= jump_thread(inst, target, opcode); + i -= jump_thread(bb, inst, target, opcode); continue; } break; @@ -1707,7 +1760,7 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) * of FOR_ITER. */ /* - i -= jump_thread(inst, target, FOR_ITER); + i -= jump_thread(bb, inst, target, FOR_ITER); */ } break; @@ -1794,9 +1847,7 @@ optimize_cfg(cfg_builder *g, PyObject *consts, PyObject *const_cache, int firstl { assert(PyDict_CheckExact(const_cache)); RETURN_IF_ERROR(check_cfg(g)); - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - RETURN_IF_ERROR(inline_small_exit_blocks(b)); - } + RETURN_IF_ERROR(inline_small_or_no_lineno_blocks(g->g_entryblock)); RETURN_IF_ERROR(remove_unreachable(g->g_entryblock)); RETURN_IF_ERROR(resolve_line_numbers(g, firstlineno)); RETURN_IF_ERROR(optimize_load_const(const_cache, g, consts)); @@ -1804,9 +1855,6 @@ optimize_cfg(cfg_builder *g, PyObject *consts, PyObject *const_cache, int firstl RETURN_IF_ERROR(optimize_basic_block(const_cache, b, consts)); } RETURN_IF_ERROR(remove_redundant_nops_and_pairs(g->g_entryblock)); - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - RETURN_IF_ERROR(inline_small_exit_blocks(b)); - } RETURN_IF_ERROR(remove_unreachable(g->g_entryblock)); int removed_nops, removed_jumps; @@ -2304,7 +2352,7 @@ convert_pseudo_ops(cfg_builder *g) for (basicblock *b = entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { cfg_instr *instr = &b->b_instr[i]; - if (is_block_push(instr) || instr->i_opcode == POP_BLOCK) { + if (is_block_push(instr)) { INSTR_SET_OP0(instr, NOP); } else if (instr->i_opcode == LOAD_CLOSURE) { @@ -2323,12 +2371,7 @@ convert_pseudo_ops(cfg_builder *g) static inline bool is_exit_or_eval_check_without_lineno(basicblock *b) { if (basicblock_exits_scope(b) || basicblock_has_eval_break(b)) { - for (int i = 0; i < b->b_iused; i++) { - if (b->b_instr[i].i_loc.lineno >= 0) { - return false; - } - } - return true; + return basicblock_has_no_lineno(b); } else { return false; @@ -2410,7 +2453,6 @@ propagate_line_numbers(basicblock *entryblock) { for (int i = 0; i < b->b_iused; i++) { if (b->b_instr[i].i_loc.lineno < 0) { b->b_instr[i].i_loc = prev_location; - b->b_instr[i].i_loc_propagated = 1; } else { prev_location = b->b_instr[i].i_loc; @@ -2420,7 +2462,6 @@ propagate_line_numbers(basicblock *entryblock) { if (b->b_next->b_iused > 0) { if (b->b_next->b_instr[0].i_loc.lineno < 0) { b->b_next->b_instr[0].i_loc = prev_location; - b->b_next->b_instr[0].i_loc_propagated = 1; } } } @@ -2429,7 +2470,6 @@ propagate_line_numbers(basicblock *entryblock) { if (target->b_predecessors == 1) { if (target->b_instr[0].i_loc.lineno < 0) { target->b_instr[0].i_loc = prev_location; - target->b_instr[0].i_loc_propagated = 1; } } } diff --git a/Python/frame.c b/Python/frame.c index 2865b2eab603c27..ddf6ef6ba5465cf 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -139,7 +139,6 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) for (int i = 0; i < frame->stacktop; i++) { Py_XDECREF(frame->localsplus[i]); } - Py_XDECREF(frame->frame_obj); Py_XDECREF(frame->f_locals); Py_DECREF(frame->f_funcobj); } diff --git a/Python/gc.c b/Python/gc.c index 14870505ef1308e..466467602915264 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -15,6 +15,8 @@ #include "pycore_weakref.h" // _PyWeakref_ClearRef() #include "pydtrace.h" +#ifndef Py_GIL_DISABLED + typedef struct _gc_runtime_state GCState; #ifdef Py_DEBUG @@ -964,10 +966,10 @@ finalize_garbage(PyThreadState *tstate, PyGC_Head *collectable) PyGC_Head *gc = GC_NEXT(collectable); PyObject *op = FROM_GC(gc); gc_list_move(gc, &seen); - if (!_PyGCHead_FINALIZED(gc) && + if (!_PyGC_FINALIZED(op) && (finalize = Py_TYPE(op)->tp_finalize) != NULL) { - _PyGCHead_SET_FINALIZED(gc); + _PyGC_SET_FINALIZED(op); Py_INCREF(op); finalize(op); assert(!_PyErr_Occurred(tstate)); @@ -1942,3 +1944,5 @@ PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) done: gcstate->enabled = origenstate; } + +#endif // Py_GIL_DISABLED diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 207a43b68d21f58..93e1168002b6f7d 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -1,9 +1,1713 @@ +// Cyclic garbage collector implementation for free-threaded build. #include "Python.h" -#include "pycore_pystate.h" // _PyFreeListState_GET() -#include "pycore_tstate.h" // _PyThreadStateImpl +#include "pycore_brc.h" // struct _brc_thread_state +#include "pycore_ceval.h" // _Py_set_eval_breaker_bit() +#include "pycore_context.h" +#include "pycore_dict.h" // _PyDict_MaybeUntrack() +#include "pycore_initconfig.h" +#include "pycore_interp.h" // PyInterpreterState.gc +#include "pycore_object.h" +#include "pycore_object_alloc.h" // _PyObject_MallocWithType() +#include "pycore_object_stack.h" +#include "pycore_pyerrors.h" +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_tstate.h" // _PyThreadStateImpl +#include "pycore_weakref.h" // _PyWeakref_ClearRef() +#include "pydtrace.h" #ifdef Py_GIL_DISABLED +typedef struct _gc_runtime_state GCState; + +#ifdef Py_DEBUG +# define GC_DEBUG +#endif + +// Automatically choose the generation that needs collecting. +#define GENERATION_AUTO (-1) + +// A linked list of objects using the `ob_tid` field as the next pointer. +// The linked list pointers are distinct from any real thread ids, because the +// thread ids returned by _Py_ThreadId() are also pointers to distinct objects. +// No thread will confuse its own id with a linked list pointer. +struct worklist { + uintptr_t head; +}; + +struct worklist_iter { + uintptr_t *ptr; // pointer to current object + uintptr_t *next; // next value of ptr +}; + +struct visitor_args { + size_t offset; // offset of PyObject from start of block +}; + +// Per-collection state +struct collection_state { + struct visitor_args base; + PyInterpreterState *interp; + GCState *gcstate; + Py_ssize_t collected; + Py_ssize_t uncollectable; + Py_ssize_t long_lived_total; + struct worklist unreachable; + struct worklist legacy_finalizers; + struct worklist wrcb_to_call; + struct worklist objs_to_decref; +}; + +// iterate over a worklist +#define WORKSTACK_FOR_EACH(stack, op) \ + for ((op) = (PyObject *)(stack)->head; (op) != NULL; (op) = (PyObject *)(op)->ob_tid) + +// iterate over a worklist with support for removing the current object +#define WORKSTACK_FOR_EACH_ITER(stack, iter, op) \ + for (worklist_iter_init((iter), &(stack)->head), (op) = (PyObject *)(*(iter)->ptr); \ + (op) != NULL; \ + worklist_iter_init((iter), (iter)->next), (op) = (PyObject *)(*(iter)->ptr)) + +static void +worklist_push(struct worklist *worklist, PyObject *op) +{ + assert(op->ob_tid == 0); + op->ob_tid = worklist->head; + worklist->head = (uintptr_t)op; +} + +static PyObject * +worklist_pop(struct worklist *worklist) +{ + PyObject *op = (PyObject *)worklist->head; + if (op != NULL) { + worklist->head = op->ob_tid; + op->ob_tid = 0; + } + return op; +} + +static void +worklist_iter_init(struct worklist_iter *iter, uintptr_t *next) +{ + iter->ptr = next; + PyObject *op = (PyObject *)*(iter->ptr); + if (op) { + iter->next = &op->ob_tid; + } +} + +static void +worklist_remove(struct worklist_iter *iter) +{ + PyObject *op = (PyObject *)*(iter->ptr); + *(iter->ptr) = op->ob_tid; + op->ob_tid = 0; + iter->next = iter->ptr; +} + +static inline int +gc_is_unreachable(PyObject *op) +{ + return (op->ob_gc_bits & _PyGC_BITS_UNREACHABLE) != 0; +} + +static void +gc_set_unreachable(PyObject *op) +{ + op->ob_gc_bits |= _PyGC_BITS_UNREACHABLE; +} + +static void +gc_clear_unreachable(PyObject *op) +{ + op->ob_gc_bits &= ~_PyGC_BITS_UNREACHABLE; +} + +// Initialize the `ob_tid` field to zero if the object is not already +// initialized as unreachable. +static void +gc_maybe_init_refs(PyObject *op) +{ + if (!gc_is_unreachable(op)) { + gc_set_unreachable(op); + op->ob_tid = 0; + } +} + +static inline Py_ssize_t +gc_get_refs(PyObject *op) +{ + return (Py_ssize_t)op->ob_tid; +} + +static inline void +gc_add_refs(PyObject *op, Py_ssize_t refs) +{ + assert(_PyObject_GC_IS_TRACKED(op)); + op->ob_tid += refs; +} + +static inline void +gc_decref(PyObject *op) +{ + op->ob_tid -= 1; +} + +static Py_ssize_t +merge_refcount(PyObject *op, Py_ssize_t extra) +{ + assert(_PyInterpreterState_GET()->stoptheworld.world_stopped); + + Py_ssize_t refcount = Py_REFCNT(op); + refcount += extra; + +#ifdef Py_REF_DEBUG + _Py_AddRefTotal(_PyInterpreterState_GET(), extra); +#endif + + // No atomics necessary; all other threads in this interpreter are paused. + op->ob_tid = 0; + op->ob_ref_local = 0; + op->ob_ref_shared = _Py_REF_SHARED(refcount, _Py_REF_MERGED); + return refcount; +} + +static void +gc_restore_tid(PyObject *op) +{ + mi_segment_t *segment = _mi_ptr_segment(op); + if (_Py_REF_IS_MERGED(op->ob_ref_shared)) { + op->ob_tid = 0; + } + else { + // NOTE: may change ob_tid if the object was re-initialized by + // a different thread or its segment was abandoned and reclaimed. + // The segment thread id might be zero, in which case we should + // ensure the refcounts are now merged. + op->ob_tid = segment->thread_id; + if (op->ob_tid == 0) { + merge_refcount(op, 0); + } + } +} + +static void +gc_restore_refs(PyObject *op) +{ + if (gc_is_unreachable(op)) { + gc_restore_tid(op); + gc_clear_unreachable(op); + } +} + +// Given a mimalloc memory block return the PyObject stored in it or NULL if +// the block is not allocated or the object is not tracked or is immortal. +static PyObject * +op_from_block(void *block, void *arg, bool include_frozen) +{ + struct visitor_args *a = arg; + if (block == NULL) { + return NULL; + } + PyObject *op = (PyObject *)((char*)block + a->offset); + assert(PyObject_IS_GC(op)); + if (!_PyObject_GC_IS_TRACKED(op)) { + return NULL; + } + if (!include_frozen && (op->ob_gc_bits & _PyGC_BITS_FROZEN) != 0) { + return NULL; + } + return op; +} + +static int +gc_visit_heaps_lock_held(PyInterpreterState *interp, mi_block_visit_fun *visitor, + struct visitor_args *arg) +{ + // Offset of PyObject header from start of memory block. + Py_ssize_t offset_base = 0; + if (_PyMem_DebugEnabled()) { + // The debug allocator adds two words at the beginning of each block. + offset_base += 2 * sizeof(size_t); + } + + // Objects with Py_TPFLAGS_PREHEADER have two extra fields + Py_ssize_t offset_pre = offset_base + 2 * sizeof(PyObject*); + + // visit each thread's heaps for GC objects + for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { + struct _mimalloc_thread_state *m = &((_PyThreadStateImpl *)p)->mimalloc; + + arg->offset = offset_base; + if (!mi_heap_visit_blocks(&m->heaps[_Py_MIMALLOC_HEAP_GC], true, + visitor, arg)) { + return -1; + } + arg->offset = offset_pre; + if (!mi_heap_visit_blocks(&m->heaps[_Py_MIMALLOC_HEAP_GC_PRE], true, + visitor, arg)) { + return -1; + } + } + + // visit blocks in the per-interpreter abandoned pool (from dead threads) + mi_abandoned_pool_t *pool = &interp->mimalloc.abandoned_pool; + arg->offset = offset_base; + if (!_mi_abandoned_pool_visit_blocks(pool, _Py_MIMALLOC_HEAP_GC, true, + visitor, arg)) { + return -1; + } + arg->offset = offset_pre; + if (!_mi_abandoned_pool_visit_blocks(pool, _Py_MIMALLOC_HEAP_GC_PRE, true, + visitor, arg)) { + return -1; + } + return 0; +} + +// Visits all GC objects in the interpreter's heaps. +// NOTE: It is not safe to allocate or free any mimalloc managed memory while +// this function is running. +static int +gc_visit_heaps(PyInterpreterState *interp, mi_block_visit_fun *visitor, + struct visitor_args *arg) +{ + // Other threads in the interpreter must be paused so that we can safely + // traverse their heaps. + assert(interp->stoptheworld.world_stopped); + + int err; + HEAD_LOCK(&_PyRuntime); + err = gc_visit_heaps_lock_held(interp, visitor, arg); + HEAD_UNLOCK(&_PyRuntime); + return err; +} + +static void +merge_queued_objects(_PyThreadStateImpl *tstate, struct collection_state *state) +{ + struct _brc_thread_state *brc = &tstate->brc; + _PyObjectStack_Merge(&brc->local_objects_to_merge, &brc->objects_to_merge); + + PyObject *op; + while ((op = _PyObjectStack_Pop(&brc->local_objects_to_merge)) != NULL) { + // Subtract one when merging because the queue had a reference. + Py_ssize_t refcount = merge_refcount(op, -1); + + if (!_PyObject_GC_IS_TRACKED(op) && refcount == 0) { + // GC objects with zero refcount are handled subsequently by the + // GC as if they were cyclic trash, but we have to handle dead + // non-GC objects here. Add one to the refcount so that we can + // decref and deallocate the object once we start the world again. + op->ob_ref_shared += (1 << _Py_REF_SHARED_SHIFT); +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyInterpreterState_GET()); +#endif + worklist_push(&state->objs_to_decref, op); + } + } +} + +static void +merge_all_queued_objects(PyInterpreterState *interp, struct collection_state *state) +{ + HEAD_LOCK(&_PyRuntime); + for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { + merge_queued_objects((_PyThreadStateImpl *)p, state); + } + HEAD_UNLOCK(&_PyRuntime); +} + +// Subtract an incoming reference from the computed "gc_refs" refcount. +static int +visit_decref(PyObject *op, void *arg) +{ + if (_PyObject_GC_IS_TRACKED(op) && !_Py_IsImmortal(op)) { + // If update_refs hasn't reached this object yet, mark it + // as (tentatively) unreachable and initialize ob_tid to zero. + gc_maybe_init_refs(op); + gc_decref(op); + } + return 0; +} + +// Compute the number of external references to objects in the heap +// by subtracting internal references from the refcount. The difference is +// computed in the ob_tid field (we restore it later). +static bool +update_refs(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + + // Exclude immortal objects from garbage collection + if (_Py_IsImmortal(op)) { + op->ob_tid = 0; + _PyObject_GC_UNTRACK(op); + gc_clear_unreachable(op); + return true; + } + + // Untrack tuples and dicts as necessary in this pass. + if (PyTuple_CheckExact(op)) { + _PyTuple_MaybeUntrack(op); + if (!_PyObject_GC_IS_TRACKED(op)) { + gc_restore_refs(op); + return true; + } + } + else if (PyDict_CheckExact(op)) { + _PyDict_MaybeUntrack(op); + if (!_PyObject_GC_IS_TRACKED(op)) { + gc_restore_refs(op); + return true; + } + } + + Py_ssize_t refcount = Py_REFCNT(op); + _PyObject_ASSERT(op, refcount >= 0); + + // We repurpose ob_tid to compute "gc_refs", the number of external + // references to the object (i.e., from outside the GC heaps). This means + // that ob_tid is no longer a valid thread id until it is restored by + // scan_heap_visitor(). Until then, we cannot use the standard reference + // counting functions or allow other threads to run Python code. + gc_maybe_init_refs(op); + + // Add the actual refcount to ob_tid. + gc_add_refs(op, refcount); + + // Subtract internal references from ob_tid. Objects with ob_tid > 0 + // are directly reachable from outside containers, and so can't be + // collected. + Py_TYPE(op)->tp_traverse(op, visit_decref, NULL); + return true; +} + +static int +visit_clear_unreachable(PyObject *op, _PyObjectStack *stack) +{ + if (gc_is_unreachable(op)) { + _PyObject_ASSERT(op, _PyObject_GC_IS_TRACKED(op)); + gc_clear_unreachable(op); + return _PyObjectStack_Push(stack, op); + } + return 0; +} + +// Transitively clear the unreachable bit on all objects reachable from op. +static int +mark_reachable(PyObject *op) +{ + _PyObjectStack stack = { NULL }; + do { + traverseproc traverse = Py_TYPE(op)->tp_traverse; + if (traverse(op, (visitproc)&visit_clear_unreachable, &stack) < 0) { + _PyObjectStack_Clear(&stack); + return -1; + } + op = _PyObjectStack_Pop(&stack); + } while (op != NULL); + return 0; +} + +#ifdef GC_DEBUG +static bool +validate_gc_objects(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + + _PyObject_ASSERT(op, gc_is_unreachable(op)); + _PyObject_ASSERT_WITH_MSG(op, gc_get_refs(op) >= 0, + "refcount is too small"); + return true; +} +#endif + +static bool +mark_heap_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + + if (gc_is_unreachable(op) && gc_get_refs(op) != 0) { + // Object is reachable but currently marked as unreachable. + // Mark it as reachable and traverse its pointers to find + // any other object that may be directly reachable from it. + gc_clear_unreachable(op); + + // Transitively mark reachable objects by clearing the unreachable flag. + if (mark_reachable(op) < 0) { + return false; + } + } + + return true; +} + +/* Return true if object has a pre-PEP 442 finalization method. */ +static int +has_legacy_finalizer(PyObject *op) +{ + return Py_TYPE(op)->tp_del != NULL; +} + +static bool +scan_heap_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + + struct collection_state *state = (struct collection_state *)args; + if (gc_is_unreachable(op)) { + // Merge and add one to the refcount to prevent deallocation while we + // are holding on to it in a worklist. + merge_refcount(op, 1); + + if (has_legacy_finalizer(op)) { + // would be unreachable, but has legacy finalizer + gc_clear_unreachable(op); + worklist_push(&state->legacy_finalizers, op); + } + else { + worklist_push(&state->unreachable, op); + } + } + else { + // object is reachable, restore `ob_tid`; we're done with these objects + gc_restore_tid(op); + state->long_lived_total++; + } + + return true; +} + +static int +move_legacy_finalizer_reachable(struct collection_state *state); + +static int +deduce_unreachable_heap(PyInterpreterState *interp, + struct collection_state *state) +{ + // Identify objects that are directly reachable from outside the GC heap + // by computing the difference between the refcount and the number of + // incoming references. + gc_visit_heaps(interp, &update_refs, &state->base); + +#ifdef GC_DEBUG + // Check that all objects are marked as unreachable and that the computed + // reference count difference (stored in `ob_tid`) is non-negative. + gc_visit_heaps(interp, &validate_gc_objects, &state->base); +#endif + + // Transitively mark reachable objects by clearing the + // _PyGC_BITS_UNREACHABLE flag. + if (gc_visit_heaps(interp, &mark_heap_visitor, &state->base) < 0) { + return -1; + } + + // Identify remaining unreachable objects and push them onto a stack. + // Restores ob_tid for reachable objects. + gc_visit_heaps(interp, &scan_heap_visitor, &state->base); + + if (state->legacy_finalizers.head) { + // There may be objects reachable from legacy finalizers that are in + // the unreachable set. We need to mark them as reachable. + if (move_legacy_finalizer_reachable(state) < 0) { + return -1; + } + } + + return 0; +} + +static int +move_legacy_finalizer_reachable(struct collection_state *state) +{ + // Clear the reachable bit on all objects transitively reachable + // from the objects with legacy finalizers. + PyObject *op; + WORKSTACK_FOR_EACH(&state->legacy_finalizers, op) { + if (mark_reachable(op) < 0) { + return -1; + } + } + + // Move the reachable objects from the unreachable worklist to the legacy + // finalizer worklist. + struct worklist_iter iter; + WORKSTACK_FOR_EACH_ITER(&state->unreachable, &iter, op) { + if (!gc_is_unreachable(op)) { + worklist_remove(&iter); + worklist_push(&state->legacy_finalizers, op); + } + } + + return 0; +} + +// Clear all weakrefs to unreachable objects. Weakrefs with callbacks are +// enqueued in `wrcb_to_call`, but not invoked yet. +static void +clear_weakrefs(struct collection_state *state) +{ + PyObject *op; + WORKSTACK_FOR_EACH(&state->unreachable, op) { + if (PyWeakref_Check(op)) { + // Clear weakrefs that are themselves unreachable to ensure their + // callbacks will not be executed later from a `tp_clear()` + // inside delete_garbage(). That would be unsafe: it could + // resurrect a dead object or access a an already cleared object. + // See bpo-38006 for one example. + _PyWeakref_ClearRef((PyWeakReference *)op); + } + + if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(op))) { + continue; + } + + // NOTE: This is never triggered for static types so we can avoid the + // (slightly) more costly _PyObject_GET_WEAKREFS_LISTPTR(). + PyWeakReference **wrlist = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(op); + + // `op` may have some weakrefs. March over the list, clear + // all the weakrefs, and enqueue the weakrefs with callbacks + // that must be called into wrcb_to_call. + for (PyWeakReference *wr = *wrlist; wr != NULL; wr = *wrlist) { + // _PyWeakref_ClearRef clears the weakref but leaves + // the callback pointer intact. Obscure: it also + // changes *wrlist. + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); + _PyWeakref_ClearRef(wr); + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); + + // We do not invoke callbacks for weakrefs that are themselves + // unreachable. This is partly for historical reasons: weakrefs + // predate safe object finalization, and a weakref that is itself + // unreachable may have a callback that resurrects other + // unreachable objects. + if (wr->wr_callback == NULL || gc_is_unreachable((PyObject *)wr)) { + continue; + } + + // Create a new reference so that wr can't go away before we can + // process it again. + merge_refcount((PyObject *)wr, 1); + + // Enqueue weakref to be called later. + worklist_push(&state->wrcb_to_call, (PyObject *)wr); + } + } +} + +static void +call_weakref_callbacks(struct collection_state *state) +{ + // Invoke the callbacks we decided to honor. + PyObject *op; + while ((op = worklist_pop(&state->wrcb_to_call)) != NULL) { + _PyObject_ASSERT(op, PyWeakref_Check(op)); + + PyWeakReference *wr = (PyWeakReference *)op; + PyObject *callback = wr->wr_callback; + _PyObject_ASSERT(op, callback != NULL); + + /* copy-paste of weakrefobject.c's handle_callback() */ + PyObject *temp = PyObject_CallOneArg(callback, (PyObject *)wr); + if (temp == NULL) { + PyErr_WriteUnraisable(callback); + } + else { + Py_DECREF(temp); + } + + gc_restore_tid(op); + Py_DECREF(op); // drop worklist reference + } +} + + +static GCState * +get_gc_state(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + return &interp->gc; +} + + +void +_PyGC_InitState(GCState *gcstate) +{ + // TODO: move to pycore_runtime_init.h once the incremental GC lands. + gcstate->generations[0].threshold = 2000; +} + + +PyStatus +_PyGC_Init(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + + gcstate->garbage = PyList_New(0); + if (gcstate->garbage == NULL) { + return _PyStatus_NO_MEMORY(); + } + + gcstate->callbacks = PyList_New(0); + if (gcstate->callbacks == NULL) { + return _PyStatus_NO_MEMORY(); + } + + return _PyStatus_OK(); +} + +static void +debug_cycle(const char *msg, PyObject *op) +{ + PySys_FormatStderr("gc: %s <%s %p>\n", + msg, Py_TYPE(op)->tp_name, op); +} + +/* Run first-time finalizers (if any) on all the objects in collectable. + * Note that this may remove some (or even all) of the objects from the + * list, due to refcounts falling to 0. + */ +static void +finalize_garbage(struct collection_state *state) +{ + // NOTE: the unreachable worklist holds a strong reference to the object + // to prevent it from being deallocated while we are holding on to it. + PyObject *op; + WORKSTACK_FOR_EACH(&state->unreachable, op) { + if (!_PyGC_FINALIZED(op)) { + destructor finalize = Py_TYPE(op)->tp_finalize; + if (finalize != NULL) { + _PyGC_SET_FINALIZED(op); + finalize(op); + assert(!_PyErr_Occurred(_PyThreadState_GET())); + } + } + } +} + +// Break reference cycles by clearing the containers involved. +static void +delete_garbage(struct collection_state *state) +{ + PyThreadState *tstate = _PyThreadState_GET(); + GCState *gcstate = state->gcstate; + + assert(!_PyErr_Occurred(tstate)); + + PyObject *op; + while ((op = worklist_pop(&state->objs_to_decref)) != NULL) { + Py_DECREF(op); + } + + while ((op = worklist_pop(&state->unreachable)) != NULL) { + _PyObject_ASSERT(op, gc_is_unreachable(op)); + + // Clear the unreachable flag. + gc_clear_unreachable(op); + + if (!_PyObject_GC_IS_TRACKED(op)) { + // Object might have been untracked by some other tp_clear() call. + Py_DECREF(op); // drop the reference from the worklist + continue; + } + + state->collected++; + + if (gcstate->debug & _PyGC_DEBUG_SAVEALL) { + assert(gcstate->garbage != NULL); + if (PyList_Append(gcstate->garbage, op) < 0) { + _PyErr_Clear(tstate); + } + } + else { + inquiry clear = Py_TYPE(op)->tp_clear; + if (clear != NULL) { + (void) clear(op); + if (_PyErr_Occurred(tstate)) { + PyErr_FormatUnraisable("Exception ignored in tp_clear of %s", + Py_TYPE(op)->tp_name); + } + } + } + + Py_DECREF(op); // drop the reference from the worklist + } +} + +static void +handle_legacy_finalizers(struct collection_state *state) +{ + GCState *gcstate = state->gcstate; + assert(gcstate->garbage != NULL); + + PyObject *op; + while ((op = worklist_pop(&state->legacy_finalizers)) != NULL) { + state->uncollectable++; + + if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) { + debug_cycle("uncollectable", op); + } + + if ((gcstate->debug & _PyGC_DEBUG_SAVEALL) || has_legacy_finalizer(op)) { + if (PyList_Append(gcstate->garbage, op) < 0) { + PyErr_Clear(); + } + } + Py_DECREF(op); // drop worklist reference + } +} + +// Show stats for objects in each generations +static void +show_stats_each_generations(GCState *gcstate) +{ + // TODO +} + +// Traversal callback for handle_resurrected_objects. +static int +visit_decref_unreachable(PyObject *op, void *data) +{ + if (gc_is_unreachable(op) && _PyObject_GC_IS_TRACKED(op)) { + op->ob_ref_local -= 1; + } + return 0; +} + +// Handle objects that may have resurrected after a call to 'finalize_garbage'. +static int +handle_resurrected_objects(struct collection_state *state) +{ + // First, find externally reachable objects by computing the reference + // count difference in ob_ref_local. We can't use ob_tid here because + // that's already used to store the unreachable worklist. + PyObject *op; + struct worklist_iter iter; + WORKSTACK_FOR_EACH_ITER(&state->unreachable, &iter, op) { + assert(gc_is_unreachable(op)); + assert(_Py_REF_IS_MERGED(op->ob_ref_shared)); + + if (!_PyObject_GC_IS_TRACKED(op)) { + // Object was untracked by a finalizer. Schedule it for a Py_DECREF + // after we finish with the stop-the-world pause. + gc_clear_unreachable(op); + worklist_remove(&iter); + worklist_push(&state->objs_to_decref, op); + continue; + } + + Py_ssize_t refcount = (op->ob_ref_shared >> _Py_REF_SHARED_SHIFT); + if (refcount > INT32_MAX) { + // The refcount is too big to fit in `ob_ref_local`. Mark the + // object as immortal and bail out. + gc_clear_unreachable(op); + worklist_remove(&iter); + _Py_SetImmortal(op); + continue; + } + + op->ob_ref_local += (uint32_t)refcount; + + // Subtract one to account for the reference from the worklist. + op->ob_ref_local -= 1; + + traverseproc traverse = Py_TYPE(op)->tp_traverse; + (void) traverse(op, + (visitproc)visit_decref_unreachable, + NULL); + } + + // Find resurrected objects + bool any_resurrected = false; + WORKSTACK_FOR_EACH(&state->unreachable, op) { + int32_t gc_refs = (int32_t)op->ob_ref_local; + op->ob_ref_local = 0; // restore ob_ref_local + + _PyObject_ASSERT(op, gc_refs >= 0); + + if (gc_is_unreachable(op) && gc_refs > 0) { + // Clear the unreachable flag on any transitively reachable objects + // from this one. + any_resurrected = true; + gc_clear_unreachable(op); + if (mark_reachable(op) < 0) { + return -1; + } + } + } + + if (any_resurrected) { + // Remove resurrected objects from the unreachable list. + WORKSTACK_FOR_EACH_ITER(&state->unreachable, &iter, op) { + if (!gc_is_unreachable(op)) { + _PyObject_ASSERT(op, Py_REFCNT(op) > 1); + worklist_remove(&iter); + merge_refcount(op, -1); // remove worklist reference + } + } + } + +#ifdef GC_DEBUG + WORKSTACK_FOR_EACH(&state->unreachable, op) { + _PyObject_ASSERT(op, gc_is_unreachable(op)); + _PyObject_ASSERT(op, _PyObject_GC_IS_TRACKED(op)); + _PyObject_ASSERT(op, op->ob_ref_local == 0); + _PyObject_ASSERT(op, _Py_REF_IS_MERGED(op->ob_ref_shared)); + } +#endif + + return 0; +} + + +/* Invoke progress callbacks to notify clients that garbage collection + * is starting or stopping + */ +static void +invoke_gc_callback(PyThreadState *tstate, const char *phase, + int generation, Py_ssize_t collected, + Py_ssize_t uncollectable) +{ + assert(!_PyErr_Occurred(tstate)); + + /* we may get called very early */ + GCState *gcstate = &tstate->interp->gc; + if (gcstate->callbacks == NULL) { + return; + } + + /* The local variable cannot be rebound, check it for sanity */ + assert(PyList_CheckExact(gcstate->callbacks)); + PyObject *info = NULL; + if (PyList_GET_SIZE(gcstate->callbacks) != 0) { + info = Py_BuildValue("{sisnsn}", + "generation", generation, + "collected", collected, + "uncollectable", uncollectable); + if (info == NULL) { + PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); + return; + } + } + + PyObject *phase_obj = PyUnicode_FromString(phase); + if (phase_obj == NULL) { + Py_XDECREF(info); + PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); + return; + } + + PyObject *stack[] = {phase_obj, info}; + for (Py_ssize_t i=0; i<PyList_GET_SIZE(gcstate->callbacks); i++) { + PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i); + Py_INCREF(cb); /* make sure cb doesn't go away */ + r = PyObject_Vectorcall(cb, stack, 2, NULL); + if (r == NULL) { + PyErr_WriteUnraisable(cb); + } + else { + Py_DECREF(r); + } + Py_DECREF(cb); + } + Py_DECREF(phase_obj); + Py_XDECREF(info); + assert(!_PyErr_Occurred(tstate)); +} + +static void +cleanup_worklist(struct worklist *worklist) +{ + PyObject *op; + while ((op = worklist_pop(worklist)) != NULL) { + gc_restore_tid(op); + gc_clear_unreachable(op); + Py_DECREF(op); + } +} + +static bool +gc_should_collect(GCState *gcstate) +{ + int count = _Py_atomic_load_int_relaxed(&gcstate->generations[0].count); + int threshold = gcstate->generations[0].threshold; + if (count <= threshold || threshold == 0 || !gcstate->enabled) { + return false; + } + // Avoid quadratic behavior by scaling threshold to the number of live + // objects. A few tests rely on immediate scheduling of the GC so we ignore + // the scaled threshold if generations[1].threshold is set to zero. + return (count > gcstate->long_lived_total / 4 || + gcstate->generations[1].threshold == 0); +} + +static void +gc_collect_internal(PyInterpreterState *interp, struct collection_state *state) +{ + _PyEval_StopTheWorld(interp); + // merge refcounts for all queued objects + merge_all_queued_objects(interp, state); + + // Find unreachable objects + int err = deduce_unreachable_heap(interp, state); + if (err < 0) { + _PyEval_StartTheWorld(interp); + goto error; + } + + // Print debugging information. + if (interp->gc.debug & _PyGC_DEBUG_COLLECTABLE) { + PyObject *op; + WORKSTACK_FOR_EACH(&state->unreachable, op) { + debug_cycle("collectable", op); + } + } + + // Clear weakrefs and enqueue callbacks (but do not call them). + clear_weakrefs(state); + _PyEval_StartTheWorld(interp); + + // Deallocate any object from the refcount merge step + cleanup_worklist(&state->objs_to_decref); + + // Call weakref callbacks and finalizers after unpausing other threads to + // avoid potential deadlocks. + call_weakref_callbacks(state); + finalize_garbage(state); + + // Handle any objects that may have resurrected after the finalization. + _PyEval_StopTheWorld(interp); + err = handle_resurrected_objects(state); + _PyEval_StartTheWorld(interp); + + if (err < 0) { + goto error; + } + + // Call tp_clear on objects in the unreachable set. This will cause + // the reference cycles to be broken. It may also cause some objects + // to be freed. + delete_garbage(state); + + // Append objects with legacy finalizers to the "gc.garbage" list. + handle_legacy_finalizers(state); + return; + +error: + cleanup_worklist(&state->unreachable); + cleanup_worklist(&state->legacy_finalizers); + cleanup_worklist(&state->wrcb_to_call); + cleanup_worklist(&state->objs_to_decref); + PyErr_NoMemory(); + PyErr_FormatUnraisable("Out of memory during garbage collection"); +} + +/* This is the main function. Read this to understand how the + * collection process works. */ +static Py_ssize_t +gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) +{ + int i; + Py_ssize_t m = 0; /* # objects collected */ + Py_ssize_t n = 0; /* # unreachable objects that couldn't be collected */ + _PyTime_t t1 = 0; /* initialize to prevent a compiler warning */ + GCState *gcstate = &tstate->interp->gc; + + // gc_collect_main() must not be called before _PyGC_Init + // or after _PyGC_Fini() + assert(gcstate->garbage != NULL); + assert(!_PyErr_Occurred(tstate)); + + int expected = 0; + if (!_Py_atomic_compare_exchange_int(&gcstate->collecting, &expected, 1)) { + // Don't start a garbage collection if one is already in progress. + return 0; + } + + if (reason == _Py_GC_REASON_HEAP && !gc_should_collect(gcstate)) { + // Don't collect if the threshold is not exceeded. + _Py_atomic_store_int(&gcstate->collecting, 0); + return 0; + } + + assert(generation >= 0 && generation < NUM_GENERATIONS); + +#ifdef Py_STATS + if (_Py_stats) { + _Py_stats->object_stats.object_visits = 0; + } +#endif + GC_STAT_ADD(generation, collections, 1); + + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(tstate, "start", generation, 0, 0); + } + + if (gcstate->debug & _PyGC_DEBUG_STATS) { + PySys_WriteStderr("gc: collecting generation %d...\n", generation); + show_stats_each_generations(gcstate); + t1 = _PyTime_GetPerfCounter(); + } + + if (PyDTrace_GC_START_ENABLED()) { + PyDTrace_GC_START(generation); + } + + /* update collection and allocation counters */ + if (generation+1 < NUM_GENERATIONS) { + gcstate->generations[generation+1].count += 1; + } + for (i = 0; i <= generation; i++) { + gcstate->generations[i].count = 0; + } + + PyInterpreterState *interp = tstate->interp; + + struct collection_state state = { + .interp = interp, + .gcstate = gcstate, + }; + + gc_collect_internal(interp, &state); + + m = state.collected; + n = state.uncollectable; + gcstate->long_lived_total = state.long_lived_total; + + if (gcstate->debug & _PyGC_DEBUG_STATS) { + double d = _PyTime_AsSecondsDouble(_PyTime_GetPerfCounter() - t1); + PySys_WriteStderr( + "gc: done, %zd unreachable, %zd uncollectable, %.4fs elapsed\n", + n+m, n, d); + } + + // Clear free lists in all threads + _PyGC_ClearAllFreeLists(interp); + + if (_PyErr_Occurred(tstate)) { + if (reason == _Py_GC_REASON_SHUTDOWN) { + _PyErr_Clear(tstate); + } + else { + PyErr_FormatUnraisable("Exception ignored in garbage collection"); + } + } + + /* Update stats */ + struct gc_generation_stats *stats = &gcstate->generation_stats[generation]; + stats->collections++; + stats->collected += m; + stats->uncollectable += n; + + GC_STAT_ADD(generation, objects_collected, m); +#ifdef Py_STATS + if (_Py_stats) { + GC_STAT_ADD(generation, object_visits, + _Py_stats->object_stats.object_visits); + _Py_stats->object_stats.object_visits = 0; + } +#endif + + if (PyDTrace_GC_DONE_ENABLED()) { + PyDTrace_GC_DONE(n + m); + } + + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(tstate, "stop", generation, m, n); + } + + assert(!_PyErr_Occurred(tstate)); + _Py_atomic_store_int(&gcstate->collecting, 0); + return n + m; +} + +struct get_referrers_args { + struct visitor_args base; + PyObject *objs; + struct worklist results; +}; + +static int +referrersvisit(PyObject* obj, void *arg) +{ + PyObject *objs = arg; + Py_ssize_t i; + for (i = 0; i < PyTuple_GET_SIZE(objs); i++) { + if (PyTuple_GET_ITEM(objs, i) == obj) { + return 1; + } + } + return 0; +} + +static bool +visit_get_referrers(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, true); + if (op == NULL) { + return true; + } + + struct get_referrers_args *arg = (struct get_referrers_args *)args; + if (Py_TYPE(op)->tp_traverse(op, referrersvisit, arg->objs)) { + op->ob_tid = 0; // we will restore the refcount later + worklist_push(&arg->results, op); + } + + return true; +} + +PyObject * +_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs) +{ + PyObject *result = PyList_New(0); + if (!result) { + return NULL; + } + + _PyEval_StopTheWorld(interp); + + // Append all objects to a worklist. This abuses ob_tid. We will restore + // it later. NOTE: We can't append to the PyListObject during + // gc_visit_heaps() because PyList_Append() may reclaim an abandoned + // mimalloc segments while we are traversing them. + struct get_referrers_args args = { .objs = objs }; + gc_visit_heaps(interp, &visit_get_referrers, &args.base); + + bool error = false; + PyObject *op; + while ((op = worklist_pop(&args.results)) != NULL) { + gc_restore_tid(op); + if (op != objs && PyList_Append(result, op) < 0) { + error = true; + break; + } + } + + // In case of error, clear the remaining worklist + while ((op = worklist_pop(&args.results)) != NULL) { + gc_restore_tid(op); + } + + _PyEval_StartTheWorld(interp); + + if (error) { + Py_DECREF(result); + return NULL; + } + + return result; +} + +struct get_objects_args { + struct visitor_args base; + struct worklist objects; +}; + +static bool +visit_get_objects(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, true); + if (op == NULL) { + return true; + } + + struct get_objects_args *arg = (struct get_objects_args *)args; + op->ob_tid = 0; // we will restore the refcount later + worklist_push(&arg->objects, op); + + return true; +} + +PyObject * +_PyGC_GetObjects(PyInterpreterState *interp, Py_ssize_t generation) +{ + PyObject *result = PyList_New(0); + if (!result) { + return NULL; + } + + _PyEval_StopTheWorld(interp); + + // Append all objects to a worklist. This abuses ob_tid. We will restore + // it later. NOTE: We can't append to the list during gc_visit_heaps() + // because PyList_Append() may reclaim an abandoned mimalloc segment + // while we are traversing it. + struct get_objects_args args = { 0 }; + gc_visit_heaps(interp, &visit_get_objects, &args.base); + + bool error = false; + PyObject *op; + while ((op = worklist_pop(&args.objects)) != NULL) { + gc_restore_tid(op); + if (op != result && PyList_Append(result, op) < 0) { + error = true; + break; + } + } + + // In case of error, clear the remaining worklist + while ((op = worklist_pop(&args.objects)) != NULL) { + gc_restore_tid(op); + } + + _PyEval_StartTheWorld(interp); + + if (error) { + Py_DECREF(result); + return NULL; + } + + return result; +} + +static bool +visit_freeze(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, true); + if (op != NULL) { + op->ob_gc_bits |= _PyGC_BITS_FROZEN; + } + return true; +} + +void +_PyGC_Freeze(PyInterpreterState *interp) +{ + struct visitor_args args; + _PyEval_StopTheWorld(interp); + gc_visit_heaps(interp, &visit_freeze, &args); + _PyEval_StartTheWorld(interp); +} + +static bool +visit_unfreeze(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, true); + if (op != NULL) { + op->ob_gc_bits &= ~_PyGC_BITS_FROZEN; + } + return true; +} + +void +_PyGC_Unfreeze(PyInterpreterState *interp) +{ + struct visitor_args args; + _PyEval_StopTheWorld(interp); + gc_visit_heaps(interp, &visit_unfreeze, &args); + _PyEval_StartTheWorld(interp); +} + +struct count_frozen_args { + struct visitor_args base; + Py_ssize_t count; +}; + +static bool +visit_count_frozen(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, true); + if (op != NULL && (op->ob_gc_bits & _PyGC_BITS_FROZEN) != 0) { + struct count_frozen_args *arg = (struct count_frozen_args *)args; + arg->count++; + } + return true; +} + +Py_ssize_t +_PyGC_GetFreezeCount(PyInterpreterState *interp) +{ + struct count_frozen_args args = { .count = 0 }; + _PyEval_StopTheWorld(interp); + gc_visit_heaps(interp, &visit_count_frozen, &args.base); + _PyEval_StartTheWorld(interp); + return args.count; +} + +/* C API for controlling the state of the garbage collector */ +int +PyGC_Enable(void) +{ + GCState *gcstate = get_gc_state(); + int old_state = gcstate->enabled; + gcstate->enabled = 1; + return old_state; +} + +int +PyGC_Disable(void) +{ + GCState *gcstate = get_gc_state(); + int old_state = gcstate->enabled; + gcstate->enabled = 0; + return old_state; +} + +int +PyGC_IsEnabled(void) +{ + GCState *gcstate = get_gc_state(); + return gcstate->enabled; +} + +/* Public API to invoke gc.collect() from C */ +Py_ssize_t +PyGC_Collect(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + GCState *gcstate = &tstate->interp->gc; + + if (!gcstate->enabled) { + return 0; + } + + Py_ssize_t n; + PyObject *exc = _PyErr_GetRaisedException(tstate); + n = gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_MANUAL); + _PyErr_SetRaisedException(tstate, exc); + + return n; +} + +Py_ssize_t +_PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) +{ + return gc_collect_main(tstate, generation, reason); +} + +Py_ssize_t +_PyGC_CollectNoFail(PyThreadState *tstate) +{ + /* Ideally, this function is only called on interpreter shutdown, + and therefore not recursively. Unfortunately, when there are daemon + threads, a daemon thread can start a cyclic garbage collection + during interpreter shutdown (and then never finish it). + See http://bugs.python.org/issue8713#msg195178 for an example. + */ + return gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_SHUTDOWN); +} + +void +_PyGC_DumpShutdownStats(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + if (!(gcstate->debug & _PyGC_DEBUG_SAVEALL) + && gcstate->garbage != NULL && PyList_GET_SIZE(gcstate->garbage) > 0) { + const char *message; + if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) { + message = "gc: %zd uncollectable objects at shutdown"; + } + else { + message = "gc: %zd uncollectable objects at shutdown; " \ + "use gc.set_debug(gc.DEBUG_UNCOLLECTABLE) to list them"; + } + /* PyErr_WarnFormat does too many things and we are at shutdown, + the warnings module's dependencies (e.g. linecache) may be gone + already. */ + if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, + "gc", NULL, message, + PyList_GET_SIZE(gcstate->garbage))) + { + PyErr_WriteUnraisable(NULL); + } + if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) { + PyObject *repr = NULL, *bytes = NULL; + repr = PyObject_Repr(gcstate->garbage); + if (!repr || !(bytes = PyUnicode_EncodeFSDefault(repr))) { + PyErr_WriteUnraisable(gcstate->garbage); + } + else { + PySys_WriteStderr( + " %s\n", + PyBytes_AS_STRING(bytes) + ); + } + Py_XDECREF(repr); + Py_XDECREF(bytes); + } + } +} + + +void +_PyGC_Fini(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + Py_CLEAR(gcstate->garbage); + Py_CLEAR(gcstate->callbacks); + + /* We expect that none of this interpreters objects are shared + with other interpreters. + See https://github.com/python/cpython/issues/90228. */ +} + +/* for debugging */ + +#ifdef Py_DEBUG +static int +visit_validate(PyObject *op, void *parent_raw) +{ + PyObject *parent = _PyObject_CAST(parent_raw); + if (_PyObject_IsFreed(op)) { + _PyObject_ASSERT_FAILED_MSG(parent, + "PyObject_GC_Track() object is not valid"); + } + return 0; +} +#endif + + +/* extension modules might be compiled with GC support so these + functions must always be available */ + +void +PyObject_GC_Track(void *op_raw) +{ + PyObject *op = _PyObject_CAST(op_raw); + if (_PyObject_GC_IS_TRACKED(op)) { + _PyObject_ASSERT_FAILED_MSG(op, + "object already tracked " + "by the garbage collector"); + } + _PyObject_GC_TRACK(op); + +#ifdef Py_DEBUG + /* Check that the object is valid: validate objects traversed + by tp_traverse() */ + traverseproc traverse = Py_TYPE(op)->tp_traverse; + (void)traverse(op, visit_validate, op); +#endif +} + +void +PyObject_GC_UnTrack(void *op_raw) +{ + PyObject *op = _PyObject_CAST(op_raw); + /* Obscure: the Py_TRASHCAN mechanism requires that we be able to + * call PyObject_GC_UnTrack twice on an object. + */ + if (_PyObject_GC_IS_TRACKED(op)) { + _PyObject_GC_UNTRACK(op); + } +} + +int +PyObject_IS_GC(PyObject *obj) +{ + return _PyObject_IS_GC(obj); +} + +void +_Py_ScheduleGC(PyInterpreterState *interp) +{ + _Py_set_eval_breaker_bit(interp, _PY_GC_SCHEDULED_BIT, 1); +} + +void +_PyObject_GC_Link(PyObject *op) +{ + PyThreadState *tstate = _PyThreadState_GET(); + GCState *gcstate = &tstate->interp->gc; + gcstate->generations[0].count++; + + if (gc_should_collect(gcstate) && + !_Py_atomic_load_int_relaxed(&gcstate->collecting)) + { + _Py_ScheduleGC(tstate->interp); + } +} + +void +_Py_RunGC(PyThreadState *tstate) +{ + gc_collect_main(tstate, 0, _Py_GC_REASON_HEAP); +} + +static PyObject * +gc_alloc(PyTypeObject *tp, size_t basicsize, size_t presize) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (basicsize > PY_SSIZE_T_MAX - presize) { + return _PyErr_NoMemory(tstate); + } + size_t size = presize + basicsize; + char *mem = _PyObject_MallocWithType(tp, size); + if (mem == NULL) { + return _PyErr_NoMemory(tstate); + } + if (presize) { + ((PyObject **)mem)[0] = NULL; + ((PyObject **)mem)[1] = NULL; + } + PyObject *op = (PyObject *)(mem + presize); + _PyObject_GC_Link(op); + return op; +} + +PyObject * +_PyObject_GC_New(PyTypeObject *tp) +{ + size_t presize = _PyType_PreHeaderSize(tp); + PyObject *op = gc_alloc(tp, _PyObject_SIZE(tp), presize); + if (op == NULL) { + return NULL; + } + _PyObject_Init(op, tp); + return op; +} + +PyVarObject * +_PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems) +{ + PyVarObject *op; + + if (nitems < 0) { + PyErr_BadInternalCall(); + return NULL; + } + size_t presize = _PyType_PreHeaderSize(tp); + size_t size = _PyObject_VAR_SIZE(tp, nitems); + op = (PyVarObject *)gc_alloc(tp, size, presize); + if (op == NULL) { + return NULL; + } + _PyObject_InitVar(op, tp, nitems); + return op; +} + +PyObject * +PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *tp, size_t extra_size) +{ + size_t presize = _PyType_PreHeaderSize(tp); + PyObject *op = gc_alloc(tp, _PyObject_SIZE(tp) + extra_size, presize); + if (op == NULL) { + return NULL; + } + memset(op, 0, _PyObject_SIZE(tp) + extra_size); + _PyObject_Init(op, tp); + return op; +} + +PyVarObject * +_PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems) +{ + const size_t basicsize = _PyObject_VAR_SIZE(Py_TYPE(op), nitems); + const size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type); + _PyObject_ASSERT((PyObject *)op, !_PyObject_GC_IS_TRACKED(op)); + if (basicsize > (size_t)PY_SSIZE_T_MAX - presize) { + return (PyVarObject *)PyErr_NoMemory(); + } + char *mem = (char *)op - presize; + mem = (char *)_PyObject_ReallocWithType(Py_TYPE(op), mem, presize + basicsize); + if (mem == NULL) { + return (PyVarObject *)PyErr_NoMemory(); + } + op = (PyVarObject *) (mem + presize); + Py_SET_SIZE(op, nitems); + return op; +} + +void +PyObject_GC_Del(void *op) +{ + size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type); + if (_PyObject_GC_IS_TRACKED(op)) { +#ifdef Py_DEBUG + PyObject *exc = PyErr_GetRaisedException(); + if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, + "gc", NULL, "Object of type %s is not untracked before destruction", + ((PyObject*)op)->ob_type->tp_name)) { + PyErr_WriteUnraisable(NULL); + } + PyErr_SetRaisedException(exc); +#endif + } + GCState *gcstate = get_gc_state(); + if (gcstate->generations[0].count > 0) { + gcstate->generations[0].count--; + } + PyObject_Free(((char *)op)-presize); +} + +int +PyObject_GC_IsTracked(PyObject* obj) +{ + if (_PyObject_IS_GC(obj) && _PyObject_GC_IS_TRACKED(obj)) { + return 1; + } + return 0; +} + +int +PyObject_GC_IsFinalized(PyObject *obj) +{ + if (_PyObject_IS_GC(obj) && _PyGC_FINALIZED(obj)) { + return 1; + } + return 0; +} + +struct custom_visitor_args { + struct visitor_args base; + gcvisitobjects_t callback; + void *arg; +}; + +static bool +custom_visitor_wrapper(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + + struct custom_visitor_args *wrapper = (struct custom_visitor_args *)args; + if (!wrapper->callback(op, wrapper->arg)) { + return false; + } + + return true; +} + +void +PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct custom_visitor_args wrapper = { + .callback = callback, + .arg = arg, + }; + + _PyEval_StopTheWorld(interp); + gc_visit_heaps(interp, &custom_visitor_wrapper, &wrapper.base); + _PyEval_StartTheWorld(interp); +} + /* Clear all free lists * All free lists are cleared during the collection of the highest generation. * Allocated items in the free list may keep a pymalloc arena occupied. @@ -14,15 +1718,13 @@ void _PyGC_ClearAllFreeLists(PyInterpreterState *interp) { - _PyDict_ClearFreeList(interp); - HEAD_LOCK(&_PyRuntime); _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)interp->threads.head; while (tstate != NULL) { - _Py_ClearFreeLists(&tstate->freelist_state, 0); + _PyObject_ClearFreeLists(&tstate->freelist_state, 0); tstate = (_PyThreadStateImpl *)tstate->base.next; } HEAD_UNLOCK(&_PyRuntime); } -#endif +#endif // Py_GIL_DISABLED diff --git a/Python/gc_gil.c b/Python/gc_gil.c index 04c1c184250c609..5f1365f509deb0a 100644 --- a/Python/gc_gil.c +++ b/Python/gc_gil.c @@ -11,9 +11,7 @@ void _PyGC_ClearAllFreeLists(PyInterpreterState *interp) { - _PyDict_ClearFreeList(interp); - - _Py_ClearFreeLists(&interp->freelist_state, 0); + _PyObject_ClearFreeLists(&interp->freelist_state, 0); } #endif diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c4bb3aeec5e2246..16f1db30620d722 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2342,17 +2342,9 @@ next_instr += 1; INSTRUCTION_STATS(END_FOR); PyObject *value; - // _POP_TOP value = stack_pointer[-1]; - { - Py_DECREF(value); - } - // _POP_TOP - value = stack_pointer[-2]; - { - Py_DECREF(value); - } - stack_pointer += -2; + Py_DECREF(value); + stack_pointer += -1; DISPATCH(); } @@ -2505,8 +2497,8 @@ next_instr[oparg].op.code == INSTRUMENTED_END_FOR); Py_DECREF(iter); STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */ + JUMPBY(oparg + 2); DISPATCH(); } // Common case: no jump, leave it to the code generator @@ -2567,8 +2559,8 @@ } Py_DECREF(iter); STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ + JUMPBY(oparg + 2); DISPATCH(); } } @@ -2608,8 +2600,8 @@ if (r->len <= 0) { STACK_SHRINK(1); Py_DECREF(r); - // Jump over END_FOR instruction. - JUMPBY(oparg + 1); + // Jump over END_FOR and POP_TOP instructions. + JUMPBY(oparg + 2); DISPATCH(); } } @@ -2655,8 +2647,8 @@ } Py_DECREF(iter); STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ + JUMPBY(oparg + 2); DISPATCH(); } } @@ -2952,9 +2944,8 @@ } PyErr_SetRaisedException(NULL); } - Py_DECREF(receiver); Py_DECREF(value); - stack_pointer += -2; + stack_pointer += -1; DISPATCH(); } @@ -3005,8 +2996,8 @@ next_instr[oparg].op.code == INSTRUMENTED_END_FOR); STACK_SHRINK(1); Py_DECREF(iter); - /* Skip END_FOR */ - target = next_instr + oparg + 1; + /* Skip END_FOR and POP_TOP */ + target = next_instr + oparg + 2; } INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); DISPATCH(); diff --git a/Python/getargs.c b/Python/getargs.c index 0c4ce282f487648..08e97ee3e627b5a 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -8,6 +8,7 @@ #include "pycore_modsupport.h" // export _PyArg_NoKeywords() #include "pycore_pylifecycle.h" // _PyArg_Fini #include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_pyerrors.h" // _Py_CalculateSuggestions() /* Export Stable ABIs (abi only) */ PyAPI_FUNC(int) _PyArg_Parse_SizeT(PyObject *, const char *, ...); @@ -1424,12 +1425,31 @@ error_unexpected_keyword_arg(PyObject *kwargs, PyObject *kwnames, PyObject *kwtu int match = PySequence_Contains(kwtuple, keyword); if (match <= 0) { if (!match) { - PyErr_Format(PyExc_TypeError, - "'%S' is an invalid keyword " - "argument for %.200s%s", - keyword, - (fname == NULL) ? "this function" : fname, - (fname == NULL) ? "" : "()"); + PyObject *kwlist = PySequence_List(kwtuple); + if (!kwlist) { + return; + } + PyObject *suggestion_keyword = _Py_CalculateSuggestions(kwlist, keyword); + Py_DECREF(kwlist); + + if (suggestion_keyword) { + PyErr_Format(PyExc_TypeError, + "%.200s%s got an unexpected keyword argument '%S'." + " Did you mean '%S'?", + (fname == NULL) ? "this function" : fname, + (fname == NULL) ? "" : "()", + keyword, + suggestion_keyword); + Py_DECREF(suggestion_keyword); + } + else { + PyErr_Format(PyExc_TypeError, + "%.200s%s got an unexpected keyword argument '%S'", + (fname == NULL) ? "this function" : fname, + (fname == NULL) ? "" : "()", + keyword); + } + } return; } @@ -1457,6 +1477,9 @@ PyArg_ValidateKeywordArguments(PyObject *kwargs) return 1; } +static PyObject * +new_kwtuple(const char * const *keywords, int total, int pos); + #define IS_END_OF_FORMAT(c) (c == '\0' || c == ';' || c == ':') static int @@ -1722,12 +1745,35 @@ vgetargskeywords(PyObject *args, PyObject *kwargs, const char *format, } } if (!match) { - PyErr_Format(PyExc_TypeError, - "'%U' is an invalid keyword " - "argument for %.200s%s", - key, - (fname == NULL) ? "this function" : fname, - (fname == NULL) ? "" : "()"); + PyObject *_pykwtuple = new_kwtuple(kwlist, len, pos); + if (!_pykwtuple) { + return cleanreturn(0, &freelist); + } + PyObject *pykwlist = PySequence_List(_pykwtuple); + Py_DECREF(_pykwtuple); + if (!pykwlist) { + return cleanreturn(0, &freelist); + } + PyObject *suggestion_keyword = _Py_CalculateSuggestions(pykwlist, key); + Py_DECREF(pykwlist); + + if (suggestion_keyword) { + PyErr_Format(PyExc_TypeError, + "%.200s%s got an unexpected keyword argument '%S'." + " Did you mean '%S'?", + (fname == NULL) ? "this function" : fname, + (fname == NULL) ? "" : "()", + key, + suggestion_keyword); + Py_DECREF(suggestion_keyword); + } + else { + PyErr_Format(PyExc_TypeError, + "%.200s%s got an unexpected keyword argument '%S'", + (fname == NULL) ? "this function" : fname, + (fname == NULL) ? "" : "()", + key); + } return cleanreturn(0, &freelist); } } diff --git a/Python/import.c b/Python/import.c index 2dd95d8364a0be0..2fd0c08a6bb5aec 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3544,7 +3544,7 @@ _imp_get_frozen_object_impl(PyObject *module, PyObject *name, struct frozen_info info = {0}; Py_buffer buf = {0}; if (PyObject_CheckBuffer(dataobj)) { - if (PyObject_GetBuffer(dataobj, &buf, PyBUF_READ) != 0) { + if (PyObject_GetBuffer(dataobj, &buf, PyBUF_SIMPLE) != 0) { return NULL; } info.data = (const char *)buf.buf; diff --git a/Python/jit.c b/Python/jit.c new file mode 100644 index 000000000000000..22949c082da05a2 --- /dev/null +++ b/Python/jit.c @@ -0,0 +1,369 @@ +#ifdef _Py_JIT + +#include "Python.h" + +#include "pycore_abstract.h" +#include "pycore_call.h" +#include "pycore_ceval.h" +#include "pycore_dict.h" +#include "pycore_intrinsics.h" +#include "pycore_long.h" +#include "pycore_opcode_metadata.h" +#include "pycore_opcode_utils.h" +#include "pycore_optimizer.h" +#include "pycore_pyerrors.h" +#include "pycore_setobject.h" +#include "pycore_sliceobject.h" +#include "pycore_jit.h" + +#include "jit_stencils.h" + +// Memory management stuff: //////////////////////////////////////////////////// + +#ifndef MS_WINDOWS + #include <sys/mman.h> +#endif + +static size_t +get_page_size(void) +{ +#ifdef MS_WINDOWS + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwPageSize; +#else + return sysconf(_SC_PAGESIZE); +#endif +} + +static void +jit_error(const char *message) +{ +#ifdef MS_WINDOWS + int hint = GetLastError(); +#else + int hint = errno; +#endif + PyErr_Format(PyExc_RuntimeWarning, "JIT %s (%d)", message, hint); +} + +static char * +jit_alloc(size_t size) +{ + assert(size); + assert(size % get_page_size() == 0); +#ifdef MS_WINDOWS + int flags = MEM_COMMIT | MEM_RESERVE; + char *memory = VirtualAlloc(NULL, size, flags, PAGE_READWRITE); + int failed = memory == NULL; +#else + int flags = MAP_ANONYMOUS | MAP_PRIVATE; + char *memory = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, -1, 0); + int failed = memory == MAP_FAILED; +#endif + if (failed) { + jit_error("unable to allocate memory"); + return NULL; + } + return memory; +} + +static int +jit_free(char *memory, size_t size) +{ + assert(size); + assert(size % get_page_size() == 0); +#ifdef MS_WINDOWS + int failed = !VirtualFree(memory, 0, MEM_RELEASE); +#else + int failed = munmap(memory, size); +#endif + if (failed) { + jit_error("unable to free memory"); + return -1; + } + return 0; +} + +static int +mark_executable(char *memory, size_t size) +{ + if (size == 0) { + return 0; + } + assert(size % get_page_size() == 0); + // Do NOT ever leave the memory writable! Also, don't forget to flush the + // i-cache (I cannot begin to tell you how horrible that is to debug): +#ifdef MS_WINDOWS + if (!FlushInstructionCache(GetCurrentProcess(), memory, size)) { + jit_error("unable to flush instruction cache"); + return -1; + } + int old; + int failed = !VirtualProtect(memory, size, PAGE_EXECUTE_READ, &old); +#else + __builtin___clear_cache((char *)memory, (char *)memory + size); + int failed = mprotect(memory, size, PROT_EXEC | PROT_READ); +#endif + if (failed) { + jit_error("unable to protect executable memory"); + return -1; + } + return 0; +} + +static int +mark_readable(char *memory, size_t size) +{ + if (size == 0) { + return 0; + } + assert(size % get_page_size() == 0); +#ifdef MS_WINDOWS + DWORD old; + int failed = !VirtualProtect(memory, size, PAGE_READONLY, &old); +#else + int failed = mprotect(memory, size, PROT_READ); +#endif + if (failed) { + jit_error("unable to protect readable memory"); + return -1; + } + return 0; +} + +// JIT compiler stuff: ///////////////////////////////////////////////////////// + +// Warning! AArch64 requires you to get your hands dirty. These are your gloves: + +// value[value_start : value_start + len] +static uint32_t +get_bits(uint64_t value, uint8_t value_start, uint8_t width) +{ + assert(width <= 32); + return (value >> value_start) & ((1ULL << width) - 1); +} + +// *loc[loc_start : loc_start + width] = value[value_start : value_start + width] +static void +set_bits(uint32_t *loc, uint8_t loc_start, uint64_t value, uint8_t value_start, + uint8_t width) +{ + assert(loc_start + width <= 32); + // Clear the bits we're about to patch: + *loc &= ~(((1ULL << width) - 1) << loc_start); + assert(get_bits(*loc, loc_start, width) == 0); + // Patch the bits: + *loc |= get_bits(value, value_start, width) << loc_start; + assert(get_bits(*loc, loc_start, width) == get_bits(value, value_start, width)); +} + +// See https://developer.arm.com/documentation/ddi0602/2023-09/Base-Instructions +// for instruction encodings: +#define IS_AARCH64_ADD_OR_SUB(I) (((I) & 0x11C00000) == 0x11000000) +#define IS_AARCH64_ADRP(I) (((I) & 0x9F000000) == 0x90000000) +#define IS_AARCH64_BRANCH(I) (((I) & 0x7C000000) == 0x14000000) +#define IS_AARCH64_LDR_OR_STR(I) (((I) & 0x3B000000) == 0x39000000) +#define IS_AARCH64_MOV(I) (((I) & 0x9F800000) == 0x92800000) + +// Fill all of stencil's holes in the memory pointed to by base, using the +// values in patches. +static void +patch(char *base, const Stencil *stencil, uint64_t *patches) +{ + for (uint64_t i = 0; i < stencil->holes_size; i++) { + const Hole *hole = &stencil->holes[i]; + void *location = base + hole->offset; + uint64_t value = patches[hole->value] + (uint64_t)hole->symbol + hole->addend; + uint32_t *loc32 = (uint32_t *)location; + uint64_t *loc64 = (uint64_t *)location; + // LLD is a great reference for performing relocations... just keep in + // mind that Tools/jit/build.py does filtering and preprocessing for us! + // Here's a good place to start for each platform: + // - aarch64-apple-darwin: + // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.cpp + // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.h + // - aarch64-unknown-linux-gnu: + // - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/AArch64.cpp + // - i686-pc-windows-msvc: + // - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp + // - x86_64-apple-darwin: + // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/X86_64.cpp + // - x86_64-pc-windows-msvc: + // - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp + // - x86_64-unknown-linux-gnu: + // - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/X86_64.cpp + switch (hole->kind) { + case HoleKind_IMAGE_REL_I386_DIR32: + // 32-bit absolute address. + // Check that we're not out of range of 32 unsigned bits: + assert(value < (1ULL << 32)); + *loc32 = (uint32_t)value; + continue; + case HoleKind_ARM64_RELOC_UNSIGNED: + case HoleKind_IMAGE_REL_AMD64_ADDR64: + case HoleKind_R_AARCH64_ABS64: + case HoleKind_X86_64_RELOC_UNSIGNED: + case HoleKind_R_X86_64_64: + // 64-bit absolute address. + *loc64 = value; + continue; + case HoleKind_R_AARCH64_CALL26: + case HoleKind_R_AARCH64_JUMP26: + // 28-bit relative branch. + assert(IS_AARCH64_BRANCH(*loc32)); + value -= (uint64_t)location; + // Check that we're not out of range of 28 signed bits: + assert((int64_t)value >= -(1 << 27)); + assert((int64_t)value < (1 << 27)); + // Since instructions are 4-byte aligned, only use 26 bits: + assert(get_bits(value, 0, 2) == 0); + set_bits(loc32, 0, value, 2, 26); + continue; + case HoleKind_R_AARCH64_MOVW_UABS_G0_NC: + // 16-bit low part of an absolute address. + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 0 of 3"): + assert(get_bits(*loc32, 21, 2) == 0); + set_bits(loc32, 5, value, 0, 16); + continue; + case HoleKind_R_AARCH64_MOVW_UABS_G1_NC: + // 16-bit middle-low part of an absolute address. + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 1 of 3"): + assert(get_bits(*loc32, 21, 2) == 1); + set_bits(loc32, 5, value, 16, 16); + continue; + case HoleKind_R_AARCH64_MOVW_UABS_G2_NC: + // 16-bit middle-high part of an absolute address. + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 2 of 3"): + assert(get_bits(*loc32, 21, 2) == 2); + set_bits(loc32, 5, value, 32, 16); + continue; + case HoleKind_R_AARCH64_MOVW_UABS_G3: + // 16-bit high part of an absolute address. + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 3 of 3"): + assert(get_bits(*loc32, 21, 2) == 3); + set_bits(loc32, 5, value, 48, 16); + continue; + case HoleKind_ARM64_RELOC_GOT_LOAD_PAGE21: + // 21-bit count of pages between this page and an absolute address's + // page... I know, I know, it's weird. Pairs nicely with + // ARM64_RELOC_GOT_LOAD_PAGEOFF12 (below). + assert(IS_AARCH64_ADRP(*loc32)); + // Number of pages between this page and the value's page: + value = (value >> 12) - ((uint64_t)location >> 12); + // Check that we're not out of range of 21 signed bits: + assert((int64_t)value >= -(1 << 20)); + assert((int64_t)value < (1 << 20)); + // value[0:2] goes in loc[29:31]: + set_bits(loc32, 29, value, 0, 2); + // value[2:21] goes in loc[5:26]: + set_bits(loc32, 5, value, 2, 19); + continue; + case HoleKind_ARM64_RELOC_GOT_LOAD_PAGEOFF12: + // 12-bit low part of an absolute address. Pairs nicely with + // ARM64_RELOC_GOT_LOAD_PAGE21 (above). + assert(IS_AARCH64_LDR_OR_STR(*loc32) || IS_AARCH64_ADD_OR_SUB(*loc32)); + // There might be an implicit shift encoded in the instruction: + uint8_t shift = 0; + if (IS_AARCH64_LDR_OR_STR(*loc32)) { + shift = (uint8_t)get_bits(*loc32, 30, 2); + // If both of these are set, the shift is supposed to be 4. + // That's pretty weird, and it's never actually been observed... + assert(get_bits(*loc32, 23, 1) == 0 || get_bits(*loc32, 26, 1) == 0); + } + value = get_bits(value, 0, 12); + assert(get_bits(value, 0, shift) == 0); + set_bits(loc32, 10, value, shift, 12); + continue; + } + Py_UNREACHABLE(); + } +} + +static void +copy_and_patch(char *base, const Stencil *stencil, uint64_t *patches) +{ + memcpy(base, stencil->body, stencil->body_size); + patch(base, stencil, patches); +} + +static void +emit(const StencilGroup *group, uint64_t patches[]) +{ + copy_and_patch((char *)patches[HoleValue_CODE], &group->code, patches); + copy_and_patch((char *)patches[HoleValue_DATA], &group->data, patches); +} + +// Compiles executor in-place. Don't forget to call _PyJIT_Free later! +int +_PyJIT_Compile(_PyExecutorObject *executor, _PyUOpInstruction *trace, size_t length) +{ + // Loop once to find the total compiled size: + size_t code_size = 0; + size_t data_size = 0; + for (size_t i = 0; i < length; i++) { + _PyUOpInstruction *instruction = &trace[i]; + const StencilGroup *group = &stencil_groups[instruction->opcode]; + code_size += group->code.body_size; + data_size += group->data.body_size; + } + // Round up to the nearest page (code and data need separate pages): + size_t page_size = get_page_size(); + assert((page_size & (page_size - 1)) == 0); + code_size += page_size - (code_size & (page_size - 1)); + data_size += page_size - (data_size & (page_size - 1)); + char *memory = jit_alloc(code_size + data_size); + if (memory == NULL) { + return -1; + } + // Loop again to emit the code: + char *code = memory; + char *data = memory + code_size; + for (size_t i = 0; i < length; i++) { + _PyUOpInstruction *instruction = &trace[i]; + const StencilGroup *group = &stencil_groups[instruction->opcode]; + // Think of patches as a dictionary mapping HoleValue to uint64_t: + uint64_t patches[] = GET_PATCHES(); + patches[HoleValue_CODE] = (uint64_t)code; + patches[HoleValue_CONTINUE] = (uint64_t)code + group->code.body_size; + patches[HoleValue_DATA] = (uint64_t)data; + patches[HoleValue_EXECUTOR] = (uint64_t)executor; + patches[HoleValue_OPARG] = instruction->oparg; + patches[HoleValue_OPERAND] = instruction->operand; + patches[HoleValue_TARGET] = instruction->target; + patches[HoleValue_TOP] = (uint64_t)memory; + patches[HoleValue_ZERO] = 0; + emit(group, patches); + code += group->code.body_size; + data += group->data.body_size; + } + if (mark_executable(memory, code_size) || + mark_readable(memory + code_size, data_size)) + { + jit_free(memory, code_size + data_size); + return -1; + } + executor->jit_code = memory; + executor->jit_size = code_size + data_size; + return 0; +} + +void +_PyJIT_Free(_PyExecutorObject *executor) +{ + char *memory = (char *)executor->jit_code; + size_t size = executor->jit_size; + if (memory) { + executor->jit_code = NULL; + executor->jit_size = 0; + if (jit_free(memory, size)) { + PyErr_WriteUnraisable(NULL); + } + } +} + +#endif // _Py_JIT diff --git a/Python/object_stack.c b/Python/object_stack.c new file mode 100644 index 000000000000000..ced4460da00f442 --- /dev/null +++ b/Python/object_stack.c @@ -0,0 +1,108 @@ +// Stack of Python objects + +#include "Python.h" +#include "pycore_freelist.h" +#include "pycore_pystate.h" +#include "pycore_object_stack.h" + +extern _PyObjectStackChunk *_PyObjectStackChunk_New(void); +extern void _PyObjectStackChunk_Free(_PyObjectStackChunk *); + +static struct _Py_object_stack_state * +get_state(void) +{ + _PyFreeListState *state = _PyFreeListState_GET(); + return &state->object_stacks; +} + +_PyObjectStackChunk * +_PyObjectStackChunk_New(void) +{ + _PyObjectStackChunk *buf; + struct _Py_object_stack_state *state = get_state(); + if (state->numfree > 0) { + buf = state->free_list; + state->free_list = buf->prev; + state->numfree--; + } + else { + // NOTE: we use PyMem_RawMalloc() here because this is used by the GC + // during mimalloc heap traversal. In that context, it is not safe to + // allocate mimalloc memory, such as via PyMem_Malloc(). + buf = PyMem_RawMalloc(sizeof(_PyObjectStackChunk)); + if (buf == NULL) { + return NULL; + } + } + buf->prev = NULL; + buf->n = 0; + return buf; +} + +void +_PyObjectStackChunk_Free(_PyObjectStackChunk *buf) +{ + assert(buf->n == 0); + struct _Py_object_stack_state *state = get_state(); + if (state->numfree >= 0 && + state->numfree < _PyObjectStackChunk_MAXFREELIST) + { + buf->prev = state->free_list; + state->free_list = buf; + state->numfree++; + } + else { + PyMem_RawFree(buf); + } +} + +void +_PyObjectStack_Clear(_PyObjectStack *queue) +{ + while (queue->head != NULL) { + _PyObjectStackChunk *buf = queue->head; + buf->n = 0; + queue->head = buf->prev; + _PyObjectStackChunk_Free(buf); + } +} + +void +_PyObjectStack_Merge(_PyObjectStack *dst, _PyObjectStack *src) +{ + if (src->head == NULL) { + return; + } + + if (dst->head != NULL) { + // First, append dst to the bottom of src + _PyObjectStackChunk *last = src->head; + while (last->prev != NULL) { + last = last->prev; + } + last->prev = dst->head; + } + + // Now that src has all the chunks, set dst to src + dst->head = src->head; + src->head = NULL; +} + +void +_PyObjectStackChunk_ClearFreeList(_PyFreeListState *free_lists, int is_finalization) +{ + if (!is_finalization) { + // Ignore requests to clear the free list during GC. We use object + // stacks during GC, so emptying the free-list is counterproductive. + return; + } + + struct _Py_object_stack_state *state = &free_lists->object_stacks; + while (state->numfree > 0) { + _PyObjectStackChunk *buf = state->free_list; + state->free_list = buf->prev; + state->numfree--; + PyMem_RawFree(buf); + } + state->numfree = -1; +} diff --git a/Python/optimizer.c b/Python/optimizer.c index 1551a5ef61f892f..d71ca0aef0e11ac 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -7,6 +7,7 @@ #include "pycore_optimizer.h" // _Py_uop_analyze_and_optimize() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_uop_ids.h" +#include "pycore_jit.h" #include "cpython/optimizer.h" #include <stdbool.h> #include <stdint.h> @@ -107,16 +108,14 @@ PyUnstable_Replace_Executor(PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutor } static int -error_optimize( +never_optimize( _PyOptimizerObject* self, - PyCodeObject *code, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _PyExecutorObject **exec, int Py_UNUSED(stack_entries)) { - assert(0); - PyErr_Format(PyExc_SystemError, "Should never call error_optimize"); - return -1; + return 0; } PyTypeObject _PyDefaultOptimizer_Type = { @@ -129,7 +128,7 @@ PyTypeObject _PyDefaultOptimizer_Type = { _PyOptimizerObject _PyOptimizer_Default = { PyObject_HEAD_INIT(&_PyDefaultOptimizer_Type) - .optimize = error_optimize, + .optimize = never_optimize, .resume_threshold = INT16_MAX, .backedge_threshold = INT16_MAX, }; @@ -173,7 +172,7 @@ _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject } _PyOptimizerObject *opt = interp->optimizer; _PyExecutorObject *executor = NULL; - int err = opt->optimize(opt, code, start, &executor, (int)(stack_pointer - _PyFrame_Stackbase(frame))); + int err = opt->optimize(opt, frame, start, &executor, (int)(stack_pointer - _PyFrame_Stackbase(frame))); if (err <= 0) { assert(executor == NULL); return err; @@ -227,6 +226,9 @@ static PyMethodDef executor_methods[] = { static void uop_dealloc(_PyExecutorObject *self) { _Py_ExecutorClear(self); +#ifdef _Py_JIT + _PyJIT_Free(self); +#endif PyObject_Free(self); } @@ -359,7 +361,8 @@ BRANCH_TO_GUARD[4][2] = { ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); \ goto done; \ } \ - trace_stack[trace_stack_depth].code = code; \ + assert(func->func_code == (PyObject *)code); \ + trace_stack[trace_stack_depth].func = func; \ trace_stack[trace_stack_depth].instr = instr; \ trace_stack_depth++; #define TRACE_STACK_POP() \ @@ -367,7 +370,8 @@ BRANCH_TO_GUARD[4][2] = { Py_FatalError("Trace stack underflow\n"); \ } \ trace_stack_depth--; \ - code = trace_stack[trace_stack_depth].code; \ + func = trace_stack[trace_stack_depth].func; \ + code = (PyCodeObject *)trace_stack[trace_stack_depth].func->func_code; \ instr = trace_stack[trace_stack_depth].instr; /* Returns 1 on success, @@ -376,20 +380,23 @@ BRANCH_TO_GUARD[4][2] = { */ static int translate_bytecode_to_trace( - PyCodeObject *code, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _PyUOpInstruction *trace, int buffer_size, _PyBloomFilter *dependencies) { bool progress_needed = true; + PyCodeObject *code = (PyCodeObject *)frame->f_executable; + PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + assert(PyFunction_Check(func)); PyCodeObject *initial_code = code; _Py_BloomFilter_Add(dependencies, initial_code); _Py_CODEUNIT *initial_instr = instr; int trace_length = 0; int max_length = buffer_size; struct { - PyCodeObject *code; + PyFunctionObject *func; _Py_CODEUNIT *instr; } trace_stack[TRACE_STACK_SIZE]; int trace_stack_depth = 0; @@ -572,9 +579,10 @@ translate_bytecode_to_trace( uop = _PyUOp_Replacements[uop]; assert(uop != 0); if (uop == _FOR_ITER_TIER_TWO) { - target += 1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1 + extended; - assert(_PyCode_CODE(code)[target-1].op.code == END_FOR || - _PyCode_CODE(code)[target-1].op.code == INSTRUMENTED_END_FOR); + target += 1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 2 + extended; + assert(_PyCode_CODE(code)[target-2].op.code == END_FOR || + _PyCode_CODE(code)[target-2].op.code == INSTRUMENTED_END_FOR); + assert(_PyCode_CODE(code)[target-1].op.code == POP_TOP); } break; default: @@ -588,6 +596,9 @@ translate_bytecode_to_trace( ADD_TO_TRACE(uop, oparg, operand, target); if (uop == _POP_FRAME) { TRACE_STACK_POP(); + /* Set the operand to the function object returned to, + * to assist optimization passes */ + trace[trace_length-1].operand = (uintptr_t)func; DPRINTF(2, "Returning to %s (%s:%d) at byte offset %d\n", PyUnicode_AsUTF8(code->co_qualname), @@ -603,10 +614,10 @@ translate_bytecode_to_trace( // Add one to account for the actual opcode/oparg pair: + 1; uint32_t func_version = read_u32(&instr[func_version_offset].cache); - PyFunctionObject *func = _PyFunction_LookupByVersion(func_version); + PyFunctionObject *new_func = _PyFunction_LookupByVersion(func_version); DPRINTF(3, "Function object: %p\n", func); - if (func != NULL) { - PyCodeObject *new_code = (PyCodeObject *)PyFunction_GET_CODE(func); + if (new_func != NULL) { + PyCodeObject *new_code = (PyCodeObject *)PyFunction_GET_CODE(new_func); if (new_code == code) { // Recursive call, bail (we could be here forever). DPRINTF(2, "Bailing on recursive call to %s (%s:%d)\n", @@ -629,7 +640,11 @@ translate_bytecode_to_trace( instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1; TRACE_STACK_PUSH(); _Py_BloomFilter_Add(dependencies, new_code); + /* Set the operand to the callee's code object, + * to assist optimization passes */ + trace[trace_length-1].operand = (uintptr_t)new_func; code = new_code; + func = new_func; instr = _PyCode_CODE(code); DPRINTF(2, "Continuing in %s (%s:%d) at byte offset %d\n", @@ -782,6 +797,14 @@ make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) executor->trace[i].operand); } } +#endif +#ifdef _Py_JIT + executor->jit_code = NULL; + executor->jit_size = 0; + if (_PyJIT_Compile(executor, executor->trace, Py_SIZE(executor))) { + Py_DECREF(executor); + return NULL; + } #endif return executor; } @@ -789,7 +812,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) static int uop_optimize( _PyOptimizerObject *self, - PyCodeObject *code, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _PyExecutorObject **exec_ptr, int curr_stackentries) @@ -797,7 +820,7 @@ uop_optimize( _PyBloomFilter dependencies; _Py_BloomFilter_Init(&dependencies); _PyUOpInstruction buffer[UOP_MAX_TRACE_LENGTH]; - int err = translate_bytecode_to_trace(code, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies); + int err = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies); if (err <= 0) { // Error or nothing translated return err; @@ -805,9 +828,10 @@ uop_optimize( OPT_STAT_INC(traces_created); char *uop_optimize = Py_GETENV("PYTHONUOPSOPTIMIZE"); if (uop_optimize == NULL || *uop_optimize > '0') { - err = _Py_uop_analyze_and_optimize(code, buffer, UOP_MAX_TRACE_LENGTH, curr_stackentries); - if (err < 0) { - return -1; + err = _Py_uop_analyze_and_optimize(frame, buffer, + UOP_MAX_TRACE_LENGTH, curr_stackentries, &dependencies); + if (err <= 0) { + return err; } } _PyExecutorObject *executor = make_executor_from_uops(buffer, &dependencies); @@ -868,12 +892,13 @@ PyTypeObject _PyCounterExecutor_Type = { static int counter_optimize( _PyOptimizerObject* self, - PyCodeObject *code, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _PyExecutorObject **exec_ptr, int Py_UNUSED(curr_stackentries) ) { + PyCodeObject *code = (PyCodeObject *)frame->f_executable; int oparg = instr->op.arg; while (instr->op.code == EXTENDED_ARG) { instr++; diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 7db51f0d90a4530..2cfbf4b349d0f52 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -1,10 +1,12 @@ #include "Python.h" #include "opcode.h" +#include "pycore_dict.h" #include "pycore_interp.h" #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_uop_metadata.h" +#include "pycore_dict.h" #include "pycore_long.h" #include "cpython/optimizer.h" #include <stdbool.h> @@ -12,6 +14,249 @@ #include <stddef.h> #include "pycore_optimizer.h" +static int +get_mutations(PyObject* dict) { + assert(PyDict_CheckExact(dict)); + PyDictObject *d = (PyDictObject *)dict; + return (d->ma_version_tag >> DICT_MAX_WATCHERS) & ((1 << DICT_WATCHED_MUTATION_BITS)-1); +} + +static void +increment_mutations(PyObject* dict) { + assert(PyDict_CheckExact(dict)); + PyDictObject *d = (PyDictObject *)dict; + d->ma_version_tag += (1 << DICT_MAX_WATCHERS); +} + +static int +globals_watcher_callback(PyDict_WatchEvent event, PyObject* dict, + PyObject* key, PyObject* new_value) +{ + if (event == PyDict_EVENT_CLONED) { + return 0; + } + uint64_t watched_mutations = get_mutations(dict); + if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), dict); + increment_mutations(dict); + } + else { + PyDict_Unwatch(1, dict); + } + return 0; +} + + +static void +global_to_const(_PyUOpInstruction *inst, PyObject *obj) +{ + assert(inst->opcode == _LOAD_GLOBAL_MODULE || inst->opcode == _LOAD_GLOBAL_BUILTINS); + assert(PyDict_CheckExact(obj)); + PyDictObject *dict = (PyDictObject *)obj; + assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); + assert(inst->operand <= UINT16_MAX); + PyObject *res = entries[inst->operand].me_value; + if (res == NULL) { + return; + } + if (_Py_IsImmortal(res)) { + inst->opcode = (inst->oparg & 1) ? _LOAD_CONST_INLINE_BORROW_WITH_NULL : _LOAD_CONST_INLINE_BORROW; + } + else { + inst->opcode = (inst->oparg & 1) ? _LOAD_CONST_INLINE_WITH_NULL : _LOAD_CONST_INLINE; + } + inst->operand = (uint64_t)res; +} + +static int +incorrect_keys(_PyUOpInstruction *inst, PyObject *obj) +{ + if (!PyDict_CheckExact(obj)) { + return 1; + } + PyDictObject *dict = (PyDictObject *)obj; + if (dict->ma_keys->dk_version != inst->operand) { + return 1; + } + return 0; +} + +/* The first two dict watcher IDs are reserved for CPython, + * so we don't need to check that they haven't been used */ +#define BUILTINS_WATCHER_ID 0 +#define GLOBALS_WATCHER_ID 1 + +/* Returns 1 if successfully optimized + * 0 if the trace is not suitable for optimization (yet) + * -1 if there was an error. */ +static int +remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, + int buffer_size, _PyBloomFilter *dependencies) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyObject *builtins = frame->f_builtins; + if (builtins != interp->builtins) { + return 1; + } + PyObject *globals = frame->f_globals; + assert(PyFunction_Check(((PyFunctionObject *)frame->f_funcobj))); + assert(((PyFunctionObject *)frame->f_funcobj)->func_builtins == builtins); + assert(((PyFunctionObject *)frame->f_funcobj)->func_globals == globals); + /* In order to treat globals as constants, we need to + * know that the globals dict is the one we expected, and + * that it hasn't changed + * In order to treat builtins as constants, we need to + * know that the builtins dict is the one we expected, and + * that it hasn't changed and that the global dictionary's + * keys have not changed */ + + /* These values represent stacks of booleans (one bool per bit). + * Pushing a frame shifts left, popping a frame shifts right. */ + uint32_t builtins_checked = 0; + uint32_t builtins_watched = 0; + uint32_t globals_checked = 0; + uint32_t globals_watched = 0; + if (interp->dict_state.watchers[1] == NULL) { + interp->dict_state.watchers[1] = globals_watcher_callback; + } + for (int pc = 0; pc < buffer_size; pc++) { + _PyUOpInstruction *inst = &buffer[pc]; + int opcode = inst->opcode; + switch(opcode) { + case _GUARD_BUILTINS_VERSION: + if (incorrect_keys(inst, builtins)) { + return 0; + } + if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { + continue; + } + if ((builtins_watched & 1) == 0) { + PyDict_Watch(BUILTINS_WATCHER_ID, builtins); + builtins_watched |= 1; + } + if (builtins_checked & 1) { + buffer[pc].opcode = NOP; + } + else { + buffer[pc].opcode = _CHECK_BUILTINS; + buffer[pc].operand = (uintptr_t)builtins; + builtins_checked |= 1; + } + break; + case _GUARD_GLOBALS_VERSION: + if (incorrect_keys(inst, globals)) { + return 0; + } + uint64_t watched_mutations = get_mutations(globals); + if (watched_mutations >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + continue; + } + if ((globals_watched & 1) == 0) { + PyDict_Watch(GLOBALS_WATCHER_ID, globals); + _Py_BloomFilter_Add(dependencies, globals); + globals_watched |= 1; + } + if (globals_checked & 1) { + buffer[pc].opcode = NOP; + } + else { + buffer[pc].opcode = _CHECK_GLOBALS; + buffer[pc].operand = (uintptr_t)globals; + globals_checked |= 1; + } + break; + case _LOAD_GLOBAL_BUILTINS: + if (globals_checked & builtins_checked & globals_watched & builtins_watched & 1) { + global_to_const(inst, builtins); + } + break; + case _LOAD_GLOBAL_MODULE: + if (globals_checked & globals_watched & 1) { + global_to_const(inst, globals); + } + break; + case _PUSH_FRAME: + { + globals_checked <<= 1; + globals_watched <<= 1; + builtins_checked <<= 1; + builtins_watched <<= 1; + PyFunctionObject *func = (PyFunctionObject *)buffer[pc].operand; + if (func == NULL) { + return 1; + } + assert(PyFunction_Check(func)); + globals = func->func_globals; + builtins = func->func_builtins; + if (builtins != interp->builtins) { + return 1; + } + break; + } + case _POP_FRAME: + { + globals_checked >>= 1; + globals_watched >>= 1; + builtins_checked >>= 1; + builtins_watched >>= 1; + PyFunctionObject *func = (PyFunctionObject *)buffer[pc].operand; + assert(PyFunction_Check(func)); + globals = func->func_globals; + builtins = func->func_builtins; + break; + } + case _JUMP_TO_TOP: + case _EXIT_TRACE: + return 1; + } + } + return 0; +} + +static void +peephole_opt(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, int buffer_size) +{ + PyCodeObject *co = (PyCodeObject *)frame->f_executable; + for (int pc = 0; pc < buffer_size; pc++) { + int opcode = buffer[pc].opcode; + switch(opcode) { + case _LOAD_CONST: { + assert(co != NULL); + PyObject *val = PyTuple_GET_ITEM(co->co_consts, buffer[pc].oparg); + buffer[pc].opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; + buffer[pc].operand = (uintptr_t)val; + break; + } + case _CHECK_PEP_523: + { + /* Setting the eval frame function invalidates + * all executors, so no need to check dynamically */ + if (_PyInterpreterState_GET()->eval_frame == NULL) { + buffer[pc].opcode = _NOP; + } + break; + } + case _PUSH_FRAME: + case _POP_FRAME: + { + PyFunctionObject *func = (PyFunctionObject *)buffer[pc].operand; + if (func == NULL) { + co = NULL; + } + else { + assert(PyFunction_Check(func)); + co = (PyCodeObject *)func->func_code; + } + break; + } + case _JUMP_TO_TOP: + case _EXIT_TRACE: + return; + } + } +} + static void remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) { @@ -50,15 +295,20 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) } } - int _Py_uop_analyze_and_optimize( - PyCodeObject *co, + _PyInterpreterFrame *frame, _PyUOpInstruction *buffer, int buffer_size, - int curr_stacklen + int curr_stacklen, + _PyBloomFilter *dependencies ) { + int err = remove_globals(frame, buffer, buffer_size, dependencies); + if (err <= 0) { + return err; + } + peephole_opt(frame, buffer, buffer_size); remove_unneeded_uops(buffer, buffer_size); - return 0; + return 1; } diff --git a/Python/parking_lot.c b/Python/parking_lot.c index d44c1b4b93b4d2b..8ba50fc1353ebdd 100644 --- a/Python/parking_lot.c +++ b/Python/parking_lot.c @@ -158,11 +158,15 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, _PyTime_t timeout) if (sema->counter == 0) { if (timeout >= 0) { struct timespec ts; - +#if defined(HAVE_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP) + _PyTime_AsTimespec_clamp(timeout, &ts); + err = pthread_cond_timedwait_relative_np(&sema->cond, &sema->mutex, &ts); +#else _PyTime_t deadline = _PyTime_Add(_PyTime_GetSystemClock(), timeout); _PyTime_AsTimespec_clamp(deadline, &ts); err = pthread_cond_timedwait(&sema->cond, &sema->mutex, &ts); +#endif // HAVE_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP } else { err = pthread_cond_wait(&sema->cond, &sema->mutex); @@ -240,6 +244,7 @@ dequeue(Bucket *bucket, const void *address) if (wait->addr == (uintptr_t)address) { llist_remove(node); --bucket->num_waiters; + wait->is_unparking = true; return wait; } } @@ -258,6 +263,7 @@ dequeue_all(Bucket *bucket, const void *address, struct llist_node *dst) llist_remove(node); llist_insert_tail(dst, node); --bucket->num_waiters; + wait->is_unparking = true; } } } @@ -333,8 +339,6 @@ _PyParkingLot_Unpark(const void *addr, _Py_unpark_fn_t *fn, void *arg) _PyRawMutex_Lock(&bucket->mutex); struct wait_entry *waiter = dequeue(bucket, addr); if (waiter) { - waiter->is_unparking = true; - int has_more_waiters = (bucket->num_waiters > 0); fn(arg, waiter->park_arg, has_more_waiters); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 0d5eec06e9b458a..61c9d4f9ea95754 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -32,6 +32,8 @@ #include "pycore_typevarobject.h" // _Py_clear_generic_types() #include "pycore_unicodeobject.h" // _PyUnicode_InitTypes() #include "pycore_weakref.h" // _PyWeakref_GET_REF() +#include "cpython/optimizer.h" // _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS +#include "pycore_obmalloc.h" // _PyMem_init_obmalloc() #include "opcode.h" @@ -605,6 +607,16 @@ init_interp_create_gil(PyThreadState *tstate, int gil) _PyEval_InitGIL(tstate, own_gil); } +static int +builtins_dict_watcher(PyDict_WatchEvent event, PyObject *dict, PyObject *key, PyObject *new_value) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (event != PyDict_EVENT_CLONED && interp->rare_events.builtin_dict < _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { + _Py_Executors_InvalidateAll(interp); + } + RARE_EVENT_INTERP_INC(interp, builtin_dict); + return 0; +} static PyStatus pycore_create_interpreter(_PyRuntimeState *runtime, @@ -639,6 +651,13 @@ pycore_create_interpreter(_PyRuntimeState *runtime, return status; } + // initialize the interp->obmalloc state. This must be done after + // the settings are loaded (so that feature_flags are set) but before + // any calls are made to obmalloc functions. + if (_PyMem_init_obmalloc(interp) < 0) { + return _PyStatus_NO_MEMORY(); + } + PyThreadState *tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_INTERP); if (tstate == NULL) { @@ -1226,12 +1245,19 @@ init_interp_main(PyThreadState *tstate) // Turn on experimental tier 2 (uops-based) optimizer if (is_main_interp) { +#ifndef _Py_JIT + // No JIT, maybe use the tier two interpreter: char *envvar = Py_GETENV("PYTHON_UOPS"); int enabled = envvar != NULL && *envvar > '0'; if (_Py_get_xoption(&config->xoptions, L"uops") != NULL) { enabled = 1; } if (enabled) { +#else + // Always enable tier two for JIT builds (ignoring the environment + // variable and command-line option above): + if (true) { +#endif PyObject *opt = PyUnstable_Optimizer_NewUOpOptimizer(); if (opt == NULL) { return _PyStatus_ERR("can't initialize optimizer"); @@ -1266,6 +1292,12 @@ init_interp_main(PyThreadState *tstate) } } + + interp->dict_state.watchers[0] = &builtins_dict_watcher; + if (PyDict_Watch(0, interp->builtins) != 0) { + return _PyStatus_ERR("failed to set builtin dict watcher"); + } + assert(!_PyErr_Occurred(tstate)); return _PyStatus_OK(); @@ -1592,6 +1624,15 @@ static void finalize_modules(PyThreadState *tstate) { PyInterpreterState *interp = tstate->interp; + + // Invalidate all executors and turn off tier 2 optimizer + _Py_Executors_InvalidateAll(interp); + Py_XDECREF(interp->optimizer); + interp->optimizer = &_PyOptimizer_Default; + + // Stop watching __builtin__ modifications + PyDict_Unwatch(0, interp->builtins); + PyObject *modules = _PyImport_GetModules(interp); if (modules == NULL) { // Already done @@ -1749,16 +1790,14 @@ finalize_interp_types(PyInterpreterState *interp) // a dict internally. _PyUnicode_ClearInterned(interp); - _PyDict_Fini(interp); _PyUnicode_Fini(interp); +#ifndef Py_GIL_DISABLED + // With Py_GIL_DISABLED: + // the freelists for the current thread state have already been cleared. _PyFreeListState *state = _PyFreeListState_GET(); - _PyTuple_Fini(state); - _PyList_Fini(state); - _PyFloat_Fini(state); - _PySlice_Fini(state); - _PyContext_Fini(state); - _PyAsyncGen_Fini(state); + _PyObject_ClearFreeLists(state, 1); +#endif #ifdef Py_DEBUG _PyStaticObjects_CheckRefcnt(interp); @@ -2126,6 +2165,14 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) goto error; } + // initialize the interp->obmalloc state. This must be done after + // the settings are loaded (so that feature_flags are set) but before + // any calls are made to obmalloc functions. + if (_PyMem_init_obmalloc(interp) < 0) { + status = _PyStatus_NO_MEMORY(); + goto error; + } + tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_INTERP); if (tstate == NULL) { status = _PyStatus_NO_MEMORY(); diff --git a/Python/pystate.c b/Python/pystate.c index 23ddc781434ac88..937c43033b068d4 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -10,6 +10,7 @@ #include "pycore_frame.h" #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_object.h" // _PyType_InitCache() +#include "pycore_object_stack.h" // _PyObjectStackChunk_ClearFreeList() #include "pycore_parking_lot.h" // _PyParkingLot_AfterFork() #include "pycore_pyerrors.h" // _PyErr_Clear() #include "pycore_pylifecycle.h" // _PyAST_Fini() @@ -17,6 +18,7 @@ #include "pycore_pystate.h" #include "pycore_runtime_init.h" // _PyRuntimeState_INIT #include "pycore_sysmodule.h" // _PySys_Audit() +#include "pycore_obmalloc.h" // _PyMem_obmalloc_state_on_heap() /* -------------------------------------------------------------------------- CAUTION @@ -65,7 +67,7 @@ _Py_thread_local PyThreadState *_Py_tss_tstate = NULL; #endif static inline PyThreadState * -current_fast_get(_PyRuntimeState *Py_UNUSED(runtime)) +current_fast_get(void) { #ifdef HAVE_THREAD_LOCAL return _Py_tss_tstate; @@ -99,14 +101,14 @@ current_fast_clear(_PyRuntimeState *Py_UNUSED(runtime)) } #define tstate_verify_not_active(tstate) \ - if (tstate == current_fast_get((tstate)->interp->runtime)) { \ + if (tstate == current_fast_get()) { \ _Py_FatalErrorFormat(__func__, "tstate %p is still current", tstate); \ } PyThreadState * _PyThreadState_GetCurrent(void) { - return current_fast_get(&_PyRuntime); + return current_fast_get(); } @@ -358,10 +360,9 @@ holds_gil(PyThreadState *tstate) // XXX Fall back to tstate->interp->runtime->ceval.gil.last_holder // (and tstate->interp->runtime->ceval.gil.locked). assert(tstate != NULL); - _PyRuntimeState *runtime = tstate->interp->runtime; /* Must be the tstate for this thread */ - assert(tstate == gilstate_tss_get(runtime)); - return tstate == current_fast_get(runtime); + assert(tstate == gilstate_tss_get(tstate->interp->runtime)); + return tstate == current_fast_get(); } @@ -516,6 +517,8 @@ _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime) return _PyStatus_NO_MEMORY(); } + _PyThread_AfterFork(&runtime->threads); + return _PyStatus_OK(); } #endif @@ -552,6 +555,11 @@ free_interpreter(PyInterpreterState *interp) // The main interpreter is statically allocated so // should not be freed. if (interp != &_PyRuntime._main_interpreter) { + if (_PyMem_obmalloc_state_on_heap(interp)) { + // interpreter has its own obmalloc state, free it + PyMem_RawFree(interp->obmalloc); + interp->obmalloc = NULL; + } PyMem_RawFree(interp); } } @@ -594,14 +602,6 @@ init_interpreter(PyInterpreterState *interp, assert(next != NULL || (interp == runtime->interpreters.main)); interp->next = next; - /* Initialize obmalloc, but only for subinterpreters, - since the main interpreter is initialized statically. */ - if (interp != &runtime->_main_interpreter) { - poolp temp[OBMALLOC_USED_POOLS_SIZE] = \ - _obmalloc_pools_INIT(interp->obmalloc.pools); - memcpy(&interp->obmalloc.pools.used, temp, sizeof(temp)); - } - PyStatus status = _PyObject_InitState(interp); if (_PyStatus_EXCEPTION(status)) { return status; @@ -611,6 +611,9 @@ init_interpreter(PyInterpreterState *interp, _PyGC_InitState(&interp->gc); PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); +#ifdef Py_GIL_DISABLED + _Py_brc_init_state(interp); +#endif for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { interp->monitors.tools[i] = 0; } @@ -724,7 +727,7 @@ PyInterpreterState * PyInterpreterState_New(void) { // tstate can be NULL - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); PyInterpreterState *interp; PyStatus status = _PyInterpreterState_New(tstate, &interp); @@ -883,7 +886,7 @@ PyInterpreterState_Clear(PyInterpreterState *interp) // Use the current Python thread state to call audit hooks and to collect // garbage. It can be different than the current Python thread state // of 'interp'. - PyThreadState *current_tstate = current_fast_get(interp->runtime); + PyThreadState *current_tstate = current_fast_get(); _PyImport_ClearCore(interp); interpreter_clear(interp, current_tstate); } @@ -909,7 +912,7 @@ PyInterpreterState_Delete(PyInterpreterState *interp) // XXX Clearing the "current" thread state should happen before // we start finalizing the interpreter (or the current thread state). - PyThreadState *tcur = current_fast_get(runtime); + PyThreadState *tcur = current_fast_get(); if (tcur != NULL && interp == tcur->interp) { /* Unset current thread. After this, many C API calls become crashy. */ _PyThreadState_Detach(tcur); @@ -1011,7 +1014,7 @@ _PyInterpreterState_SetRunningMain(PyInterpreterState *interp) if (_PyInterpreterState_FailIfRunningMain(interp) < 0) { return -1; } - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); _Py_EnsureTstateNotNULL(tstate); if (tstate->interp != interp) { PyErr_SetString(PyExc_RuntimeError, @@ -1026,7 +1029,7 @@ void _PyInterpreterState_SetNotRunningMain(PyInterpreterState *interp) { PyThreadState *tstate = interp->threads.main; - assert(tstate == current_fast_get(&_PyRuntime)); + assert(tstate == current_fast_get()); if (tstate->on_delete != NULL) { // The threading module was imported for the first time in this @@ -1179,7 +1182,7 @@ PyInterpreterState_GetDict(PyInterpreterState *interp) PyInterpreterState* PyInterpreterState_Get(void) { - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); _Py_EnsureTstateNotNULL(tstate); PyInterpreterState *interp = tstate->interp; if (interp == NULL) { @@ -1336,6 +1339,11 @@ init_threadstate(_PyThreadStateImpl *_tstate, tstate->datastack_limit = NULL; tstate->what_event = -1; +#ifdef Py_GIL_DISABLED + // Initialize biased reference counting inter-thread queue + _Py_brc_init_thread(tstate); +#endif + if (interp->stoptheworld.requested || _PyRuntime.stoptheworld.requested) { // Start in the suspended state if there is an ongoing stop-the-world. tstate->state = _Py_THREAD_SUSPENDED; @@ -1460,21 +1468,11 @@ clear_datastack(PyThreadState *tstate) } } -void -_Py_ClearFreeLists(_PyFreeListState *state, int is_finalization) -{ - _PyFloat_ClearFreeList(state, is_finalization); - _PyTuple_ClearFreeList(state, is_finalization); - _PyList_ClearFreeList(state, is_finalization); - _PyContext_ClearFreeList(state, is_finalization); - _PyAsyncGen_ClearFreeLists(state, is_finalization); -} - void PyThreadState_Clear(PyThreadState *tstate) { assert(tstate->_status.initialized && !tstate->_status.cleared); - assert(current_fast_get(&_PyRuntime)->interp == tstate->interp); + assert(current_fast_get()->interp == tstate->interp); // XXX assert(!tstate->_status.bound || tstate->_status.unbound); tstate->_status.finalizing = 1; // just in case @@ -1554,9 +1552,11 @@ PyThreadState_Clear(PyThreadState *tstate) } #ifdef Py_GIL_DISABLED // Each thread should clear own freelists in free-threading builds. - _PyFreeListState *freelist_state = &((_PyThreadStateImpl*)tstate)->freelist_state; - _Py_ClearFreeLists(freelist_state, 1); - _PySlice_ClearCache(freelist_state); + _PyFreeListState *freelist_state = _PyFreeListState_GET(); + _PyObject_ClearFreeLists(freelist_state, 1); + + // Remove ourself from the biased reference counting table of threads. + _Py_brc_remove_thread(tstate); #endif _PyThreadState_ClearMimallocHeaps(tstate); @@ -1656,7 +1656,7 @@ _PyThreadState_DeleteCurrent(PyThreadState *tstate) void PyThreadState_DeleteCurrent(void) { - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); _PyThreadState_DeleteCurrent(tstate); } @@ -1732,7 +1732,7 @@ _PyThreadState_GetDict(PyThreadState *tstate) PyObject * PyThreadState_GetDict(void) { - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); if (tstate == NULL) { return NULL; } @@ -1853,7 +1853,7 @@ _PyThreadState_Attach(PyThreadState *tstate) #endif _Py_EnsureTstateNotNULL(tstate); - if (current_fast_get(&_PyRuntime) != NULL) { + if (current_fast_get() != NULL) { Py_FatalError("non-NULL old thread state"); } @@ -1883,7 +1883,7 @@ detach_thread(PyThreadState *tstate, int detached_state) { // XXX assert(tstate_is_alive(tstate) && tstate_is_bound(tstate)); assert(tstate->state == _Py_THREAD_ATTACHED); - assert(tstate == current_fast_get(&_PyRuntime)); + assert(tstate == current_fast_get()); if (tstate->critical_section != 0) { _PyCriticalSection_SuspendAll(tstate); } @@ -2055,7 +2055,6 @@ start_the_world(struct _stoptheworld_state *stw) HEAD_LOCK(runtime); stw->requested = 0; stw->world_stopped = 0; - stw->requester = NULL; // Switch threads back to the detached state. PyInterpreterState *i; PyThreadState *t; @@ -2066,6 +2065,7 @@ start_the_world(struct _stoptheworld_state *stw) _PyParkingLot_UnparkAll(&t->state); } } + stw->requester = NULL; HEAD_UNLOCK(runtime); if (stw->is_global) { _PyRWMutex_Unlock(&runtime->stoptheworld_mutex); @@ -2168,14 +2168,14 @@ PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) PyThreadState * PyThreadState_GetUnchecked(void) { - return current_fast_get(&_PyRuntime); + return current_fast_get(); } PyThreadState * PyThreadState_Get(void) { - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); _Py_EnsureTstateNotNULL(tstate); return tstate; } @@ -2183,7 +2183,7 @@ PyThreadState_Get(void) PyThreadState * _PyThreadState_Swap(_PyRuntimeState *runtime, PyThreadState *newts) { - PyThreadState *oldts = current_fast_get(runtime); + PyThreadState *oldts = current_fast_get(); if (oldts != NULL) { _PyThreadState_Detach(oldts); } @@ -2278,7 +2278,7 @@ PyObject * _PyThread_CurrentFrames(void) { _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = current_fast_get(runtime); + PyThreadState *tstate = current_fast_get(); if (_PySys_Audit(tstate, "sys._current_frames", NULL) < 0) { return NULL; } @@ -2339,7 +2339,7 @@ PyObject * _PyThread_CurrentExceptions(void) { _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = current_fast_get(runtime); + PyThreadState *tstate = current_fast_get(); _Py_EnsureTstateNotNULL(tstate); @@ -2481,12 +2481,22 @@ PyGILState_Check(void) return 1; } - PyThreadState *tstate = current_fast_get(runtime); + PyThreadState *tstate = current_fast_get(); if (tstate == NULL) { return 0; } - return (tstate == gilstate_tss_get(runtime)); +#ifdef MS_WINDOWS + int err = GetLastError(); +#endif + + PyThreadState *tcur = gilstate_tss_get(runtime); + +#ifdef MS_WINDOWS + SetLastError(err); +#endif + + return (tstate == tcur); } PyGILState_STATE @@ -2579,7 +2589,7 @@ PyGILState_Release(PyGILState_STATE oldstate) * races; see bugs 225673 and 1061968 (that nasty bug has a * habit of coming back). */ - assert(current_fast_get(runtime) == tstate); + assert(current_fast_get() == tstate); _PyThreadState_DeleteCurrent(tstate); } /* Release the lock if necessary */ @@ -2608,11 +2618,16 @@ _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame) { if (eval_frame == _PyEval_EvalFrameDefault) { - interp->eval_frame = NULL; + eval_frame = NULL; } - else { - interp->eval_frame = eval_frame; + if (eval_frame == interp->eval_frame) { + return; + } + if (eval_frame != NULL) { + _Py_Executors_InvalidateAll(interp); } + RARE_EVENT_INC(set_eval_frame_func); + interp->eval_frame = eval_frame; } @@ -2640,9 +2655,8 @@ _PyInterpreterState_GetConfigCopy(PyConfig *config) const PyConfig* _Py_GetConfig(void) { - _PyRuntimeState *runtime = &_PyRuntime; assert(PyGILState_Check()); - PyThreadState *tstate = current_fast_get(runtime); + PyThreadState *tstate = current_fast_get(); _Py_EnsureTstateNotNULL(tstate); return _PyInterpreterState_GetConfig(tstate->interp); } diff --git a/Python/specialize.c b/Python/specialize.c index 13e0440dd9dd0d7..e38e3556a6d6425 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -267,6 +267,16 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) } } +static void +print_rare_event_stats(FILE *out, RareEventStats *stats) +{ + fprintf(out, "Rare event (set_class): %" PRIu64 "\n", stats->set_class); + fprintf(out, "Rare event (set_bases): %" PRIu64 "\n", stats->set_bases); + fprintf(out, "Rare event (set_eval_frame_func): %" PRIu64 "\n", stats->set_eval_frame_func); + fprintf(out, "Rare event (builtin_dict): %" PRIu64 "\n", stats->builtin_dict); + fprintf(out, "Rare event (func_modification): %" PRIu64 "\n", stats->func_modification); +} + static void print_stats(FILE *out, PyStats *stats) { @@ -275,6 +285,7 @@ print_stats(FILE *out, PyStats *stats) print_object_stats(out, &stats->object_stats); print_gc_stats(out, stats->gc_stats); print_optimization_stats(out, &stats->optimization_stats); + print_rare_event_stats(out, &stats->rare_event_stats); } void @@ -529,6 +540,7 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_CALL_METHOD_WRAPPER 28 #define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29 #define SPEC_FAIL_CALL_INIT_NOT_SIMPLE 30 +#define SPEC_FAIL_CALL_METACLASS 31 /* COMPARE_OP */ #define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12 @@ -1746,6 +1758,10 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) SPEC_FAIL_CALL_STR : SPEC_FAIL_CALL_CLASS_NO_VECTORCALL); return -1; } + if (Py_TYPE(tp) != &PyType_Type) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_METACLASS); + return -1; + } if (tp->tp_new == PyBaseObject_Type.tp_new) { PyFunctionObject *init = get_init_for_simple_managed_python_class(tp); if (type_get_version(tp, CALL) == 0) { diff --git a/Python/structmember.c b/Python/structmember.c index 7a5a6a49d231167..c9f03a464078d0b 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -197,45 +197,74 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) WARN("Truncation of value to int"); break; } - case Py_T_UINT:{ - unsigned long ulong_val = PyLong_AsUnsignedLong(v); - if ((ulong_val == (unsigned long)-1) && PyErr_Occurred()) { - /* XXX: For compatibility, accept negative int values - as well. */ - PyErr_Clear(); - ulong_val = PyLong_AsLong(v); - if ((ulong_val == (unsigned long)-1) && - PyErr_Occurred()) + case Py_T_UINT: { + /* XXX: For compatibility, accept negative int values + as well. */ + int overflow; + long long_val = PyLong_AsLongAndOverflow(v, &overflow); + if (long_val == -1 && PyErr_Occurred()) { + return -1; + } + if (overflow < 0) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C long"); + return -1; + } + else if (!overflow) { + *(unsigned int *)addr = (unsigned int)(unsigned long)long_val; + if (long_val < 0) { + WARN("Writing negative value into unsigned field"); + } + else if ((unsigned long)long_val > UINT_MAX) { + WARN("Truncation of value to unsigned short"); + } + } + else { + unsigned long ulong_val = PyLong_AsUnsignedLong(v); + if (ulong_val == (unsigned long)-1 && PyErr_Occurred()) { return -1; - *(unsigned int *)addr = (unsigned int)ulong_val; - WARN("Writing negative value into unsigned field"); - } else - *(unsigned int *)addr = (unsigned int)ulong_val; - if (ulong_val > UINT_MAX) - WARN("Truncation of value to unsigned int"); - break; + } + *(unsigned int*)addr = (unsigned int)ulong_val; + if (ulong_val > UINT_MAX) { + WARN("Truncation of value to unsigned int"); + } } + break; + } case Py_T_LONG:{ *(long*)addr = PyLong_AsLong(v); if ((*(long*)addr == -1) && PyErr_Occurred()) return -1; break; } - case Py_T_ULONG:{ - *(unsigned long*)addr = PyLong_AsUnsignedLong(v); - if ((*(unsigned long*)addr == (unsigned long)-1) - && PyErr_Occurred()) { - /* XXX: For compatibility, accept negative int values - as well. */ - PyErr_Clear(); - *(unsigned long*)addr = PyLong_AsLong(v); - if ((*(unsigned long*)addr == (unsigned long)-1) - && PyErr_Occurred()) + case Py_T_ULONG: { + /* XXX: For compatibility, accept negative int values + as well. */ + int overflow; + long long_val = PyLong_AsLongAndOverflow(v, &overflow); + if (long_val == -1 && PyErr_Occurred()) { + return -1; + } + if (overflow < 0) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C long"); + return -1; + } + else if (!overflow) { + *(unsigned long *)addr = (unsigned long)long_val; + if (long_val < 0) { + WARN("Writing negative value into unsigned field"); + } + } + else { + unsigned long ulong_val = PyLong_AsUnsignedLong(v); + if (ulong_val == (unsigned long)-1 && PyErr_Occurred()) { return -1; - WARN("Writing negative value into unsigned field"); + } + *(unsigned long*)addr = ulong_val; } break; - } + } case Py_T_PYSSIZET:{ *(Py_ssize_t*)addr = PyLong_AsSsize_t(v); if ((*(Py_ssize_t*)addr == (Py_ssize_t)-1) diff --git a/Python/symtable.c b/Python/symtable.c index 743029956e32faf..d69516351efba22 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -758,6 +758,8 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, { PyObject *k, *v; Py_ssize_t pos = 0; + int remove_dunder_class = 0; + while (PyDict_Next(comp->ste_symbols, &pos, &k, &v)) { // skip comprehension parameter long comp_flags = PyLong_AS_LONG(v); @@ -779,6 +781,19 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, if (!existing) { // name does not exist in scope, copy from comprehension assert(scope != FREE || PySet_Contains(comp_free, k) == 1); + if (scope == FREE && ste->ste_type == ClassBlock && + _PyUnicode_EqualToASCIIString(k, "__class__")) { + // if __class__ is unbound in the enclosing class scope and free + // in the comprehension scope, it needs special handling; just + // letting it be marked as free in class scope will break due to + // drop_class_free + scope = GLOBAL_IMPLICIT; + only_flags &= ~DEF_FREE; + if (PySet_Discard(comp_free, k) < 0) { + return 0; + } + remove_dunder_class = 1; + } PyObject *v_flags = PyLong_FromLong(only_flags); if (v_flags == NULL) { return 0; @@ -803,6 +818,10 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, } } } + comp->ste_free = PySet_Size(comp_free) > 0; + if (remove_dunder_class && PyDict_DelItemString(comp->ste_symbols, "__class__") < 0) { + return 0; + } return 1; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index f558a00a6916ebc..437d7f8dfc49580 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1878,7 +1878,15 @@ _PySys_GetSizeOf(PyObject *o) return (size_t)-1; } - return (size_t)size + _PyType_PreHeaderSize(Py_TYPE(o)); + size_t presize = 0; + if (!Py_IS_TYPE(o, &PyType_Type) || + PyType_HasFeature((PyTypeObject *)o, Py_TPFLAGS_HEAPTYPE)) + { + /* Add the size of the pre-header if "o" is not a static type */ + presize = _PyType_PreHeaderSize(Py_TYPE(o)); + } + + return (size_t)size + presize; } static PyObject * diff --git a/Python/thread_nt.h b/Python/thread_nt.h index 14b9cddc24c0ec3..ad467e0e7840e7b 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -242,10 +242,6 @@ PyThread_detach_thread(PyThread_handle_t handle) { return (CloseHandle(hThread) == 0); } -void -PyThread_update_thread_after_fork(PyThread_ident_t* ident, PyThread_handle_t* handle) { -} - /* * Return the thread Id instead of a handle. The Id is said to uniquely identify the * thread in the system @@ -444,16 +440,7 @@ PyThread_set_key_value(int key, void *value) void * PyThread_get_key_value(int key) { - /* because TLS is used in the Py_END_ALLOW_THREAD macro, - * it is necessary to preserve the windows error state, because - * it is assumed to be preserved across the call to the macro. - * Ideally, the macro should be fixed, but it is simpler to - * do it here. - */ - DWORD error = GetLastError(); - void *result = TlsGetValue(key); - SetLastError(error); - return result; + return TlsGetValue(key); } void @@ -525,14 +512,5 @@ void * PyThread_tss_get(Py_tss_t *key) { assert(key != NULL); - /* because TSS is used in the Py_END_ALLOW_THREAD macro, - * it is necessary to preserve the windows error state, because - * it is assumed to be preserved across the call to the macro. - * Ideally, the macro should be fixed, but it is simpler to - * do it here. - */ - DWORD error = GetLastError(); - void *result = TlsGetValue(key->_key); - SetLastError(error); - return result; + return TlsGetValue(key->_key); } diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index fb3b79fc160502f..556e3de0b071f81 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -339,16 +339,6 @@ PyThread_detach_thread(PyThread_handle_t th) { return pthread_detach((pthread_t) th); } -void -PyThread_update_thread_after_fork(PyThread_ident_t* ident, PyThread_handle_t* handle) { - // The thread id might have been updated in the forked child - pthread_t th = pthread_self(); - *ident = (PyThread_ident_t) th; - *handle = (PyThread_handle_t) th; - assert(th == (pthread_t) *ident); - assert(th == (pthread_t) *handle); -} - /* XXX This implementation is considered (to quote Tim Peters) "inherently hosed" because: - It does not guarantee the promise that a non-zero integer is returned. diff --git a/Tools/build/generate_global_objects.py b/Tools/build/generate_global_objects.py index ded19ee489e79b6..33d1b323fc17534 100644 --- a/Tools/build/generate_global_objects.py +++ b/Tools/build/generate_global_objects.py @@ -123,6 +123,14 @@ '__rdivmod__', '__buffer__', '__release_buffer__', + + #Workarounds for GH-108918 + 'alias', + 'args', + 'exc_type', + 'exc_value', + 'self', + 'traceback', ] NON_GENERATED_IMMORTAL_OBJECTS = [ diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py index 317d48fee3a9d40..442487f2d2546b4 100644 --- a/Tools/build/generate_sbom.py +++ b/Tools/build/generate_sbom.py @@ -8,6 +8,7 @@ import subprocess import sys import typing +import zipfile from urllib.request import urlopen CPYTHON_ROOT_DIR = pathlib.Path(__file__).parent.parent.parent @@ -16,10 +17,16 @@ # the license expression is a valid SPDX license expression: # See: https://spdx.org/licenses ALLOWED_LICENSE_EXPRESSIONS = { - "MIT", - "CC0-1.0", "Apache-2.0", + "Apache-2.0 OR BSD-2-Clause", "BSD-2-Clause", + "BSD-3-Clause", + "CC0-1.0", + "ISC", + "LGPL-2.1-only", + "MIT", + "MPL-2.0", + "Python-2.0.1", } # Properties which are required for our purposes. @@ -31,14 +38,13 @@ "checksums", "licenseConcluded", "externalRefs", - "originator", "primaryPackagePurpose", ]) class PackageFiles(typing.NamedTuple): """Structure for describing the files of a package""" - include: list[str] + include: list[str] | None exclude: list[str] | None = None @@ -118,62 +124,209 @@ def filter_gitignored_paths(paths: list[str]) -> list[str]: return sorted([line.split()[-1] for line in git_check_ignore_lines if line.startswith("::")]) +def fetch_package_metadata_from_pypi(project: str, version: str, filename: str | None = None) -> tuple[str, str] | None: + """ + Fetches the SHA256 checksum and download location from PyPI. + If we're given a filename then we match with that, otherwise we use wheels. + """ + # Get pip's download location from PyPI. Check that the checksum is correct too. + try: + raw_text = urlopen(f"https://pypi.org/pypi/{project}/{version}/json").read() + release_metadata = json.loads(raw_text) + url: dict[str, typing.Any] + + # Look for a matching artifact filename and then check + # its remote checksum to the local one. + for url in release_metadata["urls"]: + # pip can only use Python-only dependencies, so there's + # no risk of picking the 'incorrect' wheel here. + if ( + (filename is None and url["packagetype"] == "bdist_wheel") + or (filename is not None and url["filename"] == filename) + ): + break + else: + raise ValueError(f"No matching filename on PyPI for '{filename}'") + + # Successfully found the download URL for the matching artifact. + download_url = url["url"] + checksum_sha256 = url["digests"]["sha256"] + return download_url, checksum_sha256 + + except (OSError, ValueError) as e: + # Fail if we're running in CI where we should have an internet connection. + error_if( + "CI" in os.environ, + f"Couldn't fetch metadata for project '{project}' from PyPI: {e}" + ) + return None + + +def find_ensurepip_pip_wheel() -> pathlib.Path | None: + """Try to find the pip wheel bundled in ensurepip. If missing return None""" + + ensurepip_bundled_dir = CPYTHON_ROOT_DIR / "Lib/ensurepip/_bundled" + + pip_wheels = [] + try: + for wheel_filename in os.listdir(ensurepip_bundled_dir): + if wheel_filename.startswith("pip-"): + pip_wheels.append(wheel_filename) + else: + print(f"Unexpected wheel in ensurepip: '{wheel_filename}'") + sys.exit(1) + + # Ignore this error, likely caused by downstream distributors + # deleting the 'ensurepip/_bundled' directory. + except FileNotFoundError: + pass + + if len(pip_wheels) == 0: + return None + elif len(pip_wheels) > 1: + print("Multiple pip wheels detected in 'Lib/ensurepip/_bundled'") + sys.exit(1) + # Otherwise return the one pip wheel. + return ensurepip_bundled_dir / pip_wheels[0] + + +def maybe_remove_pip_and_deps_from_sbom(sbom_data: dict[str, typing.Any]) -> None: + """ + Removes pip and its dependencies from the SBOM data + if the pip wheel is removed from ensurepip. This is done + by redistributors of Python and pip. + """ + + # If there's a wheel we don't remove anything. + if find_ensurepip_pip_wheel() is not None: + return + + # Otherwise we traverse the relationships + # to find dependent packages to remove. + sbom_pip_spdx_id = spdx_id("SPDXRef-PACKAGE-pip") + sbom_spdx_ids_to_remove = {sbom_pip_spdx_id} + + # Find all package SPDXIDs that pip depends on. + for sbom_relationship in sbom_data["relationships"]: + if ( + sbom_relationship["relationshipType"] == "DEPENDS_ON" + and sbom_relationship["spdxElementId"] == sbom_pip_spdx_id + ): + sbom_spdx_ids_to_remove.add(sbom_relationship["relatedSpdxElement"]) + + # Remove all the packages and relationships. + sbom_data["packages"] = [ + sbom_package for sbom_package in sbom_data["packages"] + if sbom_package["SPDXID"] not in sbom_spdx_ids_to_remove + ] + sbom_data["relationships"] = [ + sbom_relationship for sbom_relationship in sbom_data["relationships"] + if sbom_relationship["relatedSpdxElement"] not in sbom_spdx_ids_to_remove + ] + + def discover_pip_sbom_package(sbom_data: dict[str, typing.Any]) -> None: """pip is a part of a packaging ecosystem (Python, surprise!) so it's actually automatable to discover the metadata we need like the version and checksums - so let's do that on behalf of our friends at the PyPA. + so let's do that on behalf of our friends at the PyPA. This function also + discovers vendored packages within pip and fetches their metadata. """ global PACKAGE_TO_FILES - ensurepip_bundled_dir = CPYTHON_ROOT_DIR / "Lib/ensurepip/_bundled" - pip_wheels = [] - - # Find the hopefully one pip wheel in the bundled directory. - for wheel_filename in os.listdir(ensurepip_bundled_dir): - if wheel_filename.startswith("pip-"): - pip_wheels.append(wheel_filename) - if len(pip_wheels) != 1: - print("Zero or multiple pip wheels detected in 'Lib/ensurepip/_bundled'") - sys.exit(1) - pip_wheel_filename = pip_wheels[0] + pip_wheel_filepath = find_ensurepip_pip_wheel() + if pip_wheel_filepath is None: + return # There's no pip wheel, nothing to discover. # Add the wheel filename to the list of files so the SBOM file # and relationship generator can work its magic on the wheel too. PACKAGE_TO_FILES["pip"] = PackageFiles( - include=[f"Lib/ensurepip/_bundled/{pip_wheel_filename}"] + include=[str(pip_wheel_filepath.relative_to(CPYTHON_ROOT_DIR))] ) # Wheel filename format puts the version right after the project name. - pip_version = pip_wheel_filename.split("-")[1] + pip_version = pip_wheel_filepath.name.split("-")[1] pip_checksum_sha256 = hashlib.sha256( - (ensurepip_bundled_dir / pip_wheel_filename).read_bytes() + pip_wheel_filepath.read_bytes() ).hexdigest() - # Get pip's download location from PyPI. Check that the checksum is correct too. - try: - raw_text = urlopen(f"https://pypi.org/pypi/pip/{pip_version}/json").read() - pip_release_metadata = json.loads(raw_text) - url: dict[str, typing.Any] + pip_metadata = fetch_package_metadata_from_pypi( + project="pip", + version=pip_version, + filename=pip_wheel_filepath.name, + ) + # We couldn't fetch any metadata from PyPI, + # so we give up on verifying if we're not in CI. + if pip_metadata is None: + return + + pip_download_url, pip_actual_sha256 = pip_metadata + if pip_actual_sha256 != pip_checksum_sha256: + raise ValueError("Unexpected") + + # Parse 'pip/_vendor/vendor.txt' from the wheel for sub-dependencies. + with zipfile.ZipFile(pip_wheel_filepath) as whl: + vendor_txt_data = whl.read("pip/_vendor/vendor.txt").decode() + + # With this version regex we're assuming that pip isn't using pre-releases. + # If any version doesn't match we get a failure below, so we're safe doing this. + version_pin_re = re.compile(r"^([a-zA-Z0-9_.-]+)==([0-9.]*[0-9])$") + sbom_pip_dependency_spdx_ids = set() + for line in vendor_txt_data.splitlines(): + line = line.partition("#")[0].strip() # Strip comments and whitespace. + if not line: # Skip empty lines. + continue + + # Non-empty lines we must be able to match. + match = version_pin_re.match(line) + error_if(match is None, f"Couldn't parse line from pip vendor.txt: '{line}'") + assert match is not None # Make mypy happy. + + # Parse out and normalize the project name. + project_name, project_version = match.groups() + project_name = project_name.lower() + + # At this point if pip's metadata fetch succeeded we should + # expect this request to also succeed. + project_metadata = ( + fetch_package_metadata_from_pypi(project_name, project_version) + ) + assert project_metadata is not None + project_download_url, project_checksum_sha256 = project_metadata + + # Update our SBOM data with what we received from PyPI. + # Don't overwrite any existing values. + sbom_project_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{project_name}") + sbom_pip_dependency_spdx_ids.add(sbom_project_spdx_id) + for package in sbom_data["packages"]: + if package["SPDXID"] != sbom_project_spdx_id: + continue - # Look for a matching artifact filename and then check - # its remote checksum to the local one. - for url in pip_release_metadata["urls"]: - if url["filename"] == pip_wheel_filename: + # Only thing missing from this blob is the `licenseConcluded`, + # that needs to be triaged by human maintainers if the list changes. + package.update({ + "SPDXID": sbom_project_spdx_id, + "name": project_name, + "versionInfo": project_version, + "downloadLocation": project_download_url, + "checksums": [ + {"algorithm": "SHA256", "checksumValue": project_checksum_sha256} + ], + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": f"pkg:pypi/{project_name}@{project_version}", + "referenceType": "purl", + }, + ], + "primaryPackagePurpose": "SOURCE" + }) break - else: - raise ValueError(f"No matching filename on PyPI for '{pip_wheel_filename}'") - if url["digests"]["sha256"] != pip_checksum_sha256: - raise ValueError(f"Local pip checksum doesn't match artifact on PyPI") - - # Successfully found the download URL for the matching artifact. - pip_download_url = url["url"] - except (OSError, ValueError) as e: - print(f"Couldn't fetch pip's metadata from PyPI: {e}") - sys.exit(1) + PACKAGE_TO_FILES[project_name] = PackageFiles(include=None) # Remove pip from the existing SBOM packages if it's there # and then overwrite its entry with our own generated one. + sbom_pip_spdx_id = spdx_id("SPDXRef-PACKAGE-pip") sbom_data["packages"] = [ sbom_package for sbom_package in sbom_data["packages"] @@ -181,11 +334,11 @@ def discover_pip_sbom_package(sbom_data: dict[str, typing.Any]) -> None: ] sbom_data["packages"].append( { - "SPDXID": spdx_id("SPDXRef-PACKAGE-pip"), + "SPDXID": sbom_pip_spdx_id, "name": "pip", "versionInfo": pip_version, "originator": "Organization: Python Packaging Authority", - "licenseConcluded": "MIT", + "licenseConcluded": "NOASSERTION", "downloadLocation": pip_download_url, "checksums": [ {"algorithm": "SHA256", "checksumValue": pip_checksum_sha256} @@ -205,19 +358,36 @@ def discover_pip_sbom_package(sbom_data: dict[str, typing.Any]) -> None: "primaryPackagePurpose": "SOURCE", } ) + for sbom_dep_spdx_id in sorted(sbom_pip_dependency_spdx_ids): + sbom_data["relationships"].append({ + "spdxElementId": sbom_pip_spdx_id, + "relatedSpdxElement": sbom_dep_spdx_id, + "relationshipType": "DEPENDS_ON" + }) def main() -> None: sbom_path = CPYTHON_ROOT_DIR / "Misc/sbom.spdx.json" sbom_data = json.loads(sbom_path.read_bytes()) + # Check if pip should be removed if the wheel is missing. + # We can't reset the SBOM relationship data until checking this. + maybe_remove_pip_and_deps_from_sbom(sbom_data) + + # We regenerate all of this information. Package information + # should be preserved though since that is edited by humans. + sbom_data["files"] = [] + sbom_data["relationships"] = [] + # Insert pip's SBOM metadata from the wheel. discover_pip_sbom_package(sbom_data) # Ensure all packages in this tool are represented also in the SBOM file. + actual_names = {package["name"] for package in sbom_data["packages"]} + expected_names = set(PACKAGE_TO_FILES) error_if( - {package["name"] for package in sbom_data["packages"]} != set(PACKAGE_TO_FILES), - "Packages defined in SBOM tool don't match those defined in SBOM file.", + actual_names != expected_names, + f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}", ) # Make a bunch of assertions about the SBOM data to ensure it's consistent. @@ -227,9 +397,10 @@ def main() -> None: "name" not in package, "Package is missing the 'name' field" ) + missing_required_keys = REQUIRED_PROPERTIES_PACKAGE - set(package.keys()) error_if( - set(package.keys()) != REQUIRED_PROPERTIES_PACKAGE, - f"Package '{package['name']}' is missing required fields", + bool(missing_required_keys), + f"Package '{package['name']}' is missing required fields: {missing_required_keys}", ) error_if( package["SPDXID"] != spdx_id(f"SPDXRef-PACKAGE-{package['name']}"), @@ -253,19 +424,15 @@ def main() -> None: # License must be on the approved list for SPDX. license_concluded = package["licenseConcluded"] error_if( - license_concluded not in ALLOWED_LICENSE_EXPRESSIONS, - f"License identifier '{license_concluded}' not in SBOM tool allowlist" + license_concluded != "NOASSERTION", + f"License identifier must be 'NOASSERTION'" ) - # Regenerate file information from current data. - sbom_files = [] - sbom_relationships = [] - # We call 'sorted()' here a lot to avoid filesystem scan order issues. for name, files in sorted(PACKAGE_TO_FILES.items()): package_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{name}") exclude = files.exclude or () - for include in sorted(files.include): + for include in sorted(files.include or ()): # Find all the paths and then filter them through .gitignore. paths = glob.glob(include, root_dir=CPYTHON_ROOT_DIR, recursive=True) paths = filter_gitignored_paths(paths) @@ -285,7 +452,7 @@ def main() -> None: checksum_sha256 = hashlib.sha256(data).hexdigest() file_spdx_id = spdx_id(f"SPDXRef-FILE-{path}") - sbom_files.append({ + sbom_data["files"].append({ "SPDXID": file_spdx_id, "fileName": path, "checksums": [ @@ -295,15 +462,13 @@ def main() -> None: }) # Tie each file back to its respective package. - sbom_relationships.append({ + sbom_data["relationships"].append({ "spdxElementId": package_spdx_id, "relatedSpdxElement": file_spdx_id, "relationshipType": "CONTAINS", }) # Update the SBOM on disk - sbom_data["files"] = sbom_files - sbom_data["relationships"] = sbom_relationships sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True)) diff --git a/Tools/build/stable_abi.py b/Tools/build/stable_abi.py index 85c437d521a15ad..83146622c74f941 100644 --- a/Tools/build/stable_abi.py +++ b/Tools/build/stable_abi.py @@ -278,6 +278,13 @@ def gen_ctypes_test(manifest, args, outfile): from _testcapi import get_feature_macros feature_macros = get_feature_macros() + + # Stable ABI is incompatible with Py_TRACE_REFS builds due to PyObject + # layout differences. + # See https://github.com/python/cpython/issues/88299#issuecomment-1113366226 + if feature_macros['Py_TRACE_REFS']: + raise unittest.SkipTest("incompatible with Py_TRACE_REFS.") + ctypes_test = import_module('ctypes') class TestStableABIAvailability(unittest.TestCase): @@ -308,16 +315,11 @@ def test_windows_feature_macros(self): {'function', 'data'}, include_abi_only=True, ) - optional_items = {} + feature_macros = list(manifest.select({'feature_macro'})) + optional_items = {m.name: [] for m in feature_macros} for item in items: - if item.name in ( - # Some symbols aren't exported on all platforms. - # This is a bug: https://bugs.python.org/issue44133 - 'PyModule_Create2', 'PyModule_FromDefAndSpec2', - ): - continue if item.ifdef: - optional_items.setdefault(item.ifdef, []).append(item.name) + optional_items[item.ifdef].append(item.name) else: write(f' "{item.name}",') write(")") @@ -328,7 +330,6 @@ def test_windows_feature_macros(self): write(f" {name!r},") write(" )") write("") - feature_macros = list(manifest.select({'feature_macro'})) feature_names = sorted(m.name for m in feature_macros) write(f"EXPECTED_FEATURE_MACROS = set({pprint.pformat(feature_names)})") diff --git a/Tools/build/verify_ensurepip_wheels.py b/Tools/build/verify_ensurepip_wheels.py index 29897425da6c036..a37da2f70757e58 100755 --- a/Tools/build/verify_ensurepip_wheels.py +++ b/Tools/build/verify_ensurepip_wheels.py @@ -14,7 +14,6 @@ from pathlib import Path from urllib.request import urlopen -PACKAGE_NAMES = ("pip",) ENSURE_PIP_ROOT = Path(__file__).parent.parent.parent / "Lib/ensurepip" WHEEL_DIR = ENSURE_PIP_ROOT / "_bundled" ENSURE_PIP_INIT_PY_TEXT = (ENSURE_PIP_ROOT / "__init__.py").read_text(encoding="utf-8") @@ -97,8 +96,5 @@ def verify_wheel(package_name: str) -> bool: if __name__ == "__main__": - exit_status = 0 - for package_name in PACKAGE_NAMES: - if not verify_wheel(package_name): - exit_status = 1 + exit_status = int(not verify_wheel("pip")) raise SystemExit(exit_status) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index e3a1b5d532bda2a..1d9576d083d8dc7 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -197,6 +197,7 @@ Objects/exceptions.c - _PyExc_AttributeError - Objects/exceptions.c - _PyExc_SyntaxError - Objects/exceptions.c - _PyExc_IndentationError - Objects/exceptions.c - _PyExc_TabError - +Objects/exceptions.c - _PyExc_IncompleteInputError - Objects/exceptions.c - _PyExc_LookupError - Objects/exceptions.c - _PyExc_IndexError - Objects/exceptions.c - _PyExc_KeyError - @@ -261,6 +262,7 @@ Objects/exceptions.c - PyExc_AttributeError - Objects/exceptions.c - PyExc_SyntaxError - Objects/exceptions.c - PyExc_IndentationError - Objects/exceptions.c - PyExc_TabError - +Objects/exceptions.c - PyExc_IncompleteInputError - Objects/exceptions.c - PyExc_LookupError - Objects/exceptions.c - PyExc_IndexError - Objects/exceptions.c - PyExc_KeyError - @@ -332,6 +334,7 @@ Modules/_testclinic.c - DeprStarNew - Modules/_testclinic.c - DeprKwdInit - Modules/_testclinic.c - DeprKwdInitNoInline - Modules/_testclinic.c - DeprKwdNew - +Modules/_testclinic.c - TestClass - ################################## diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 2f9e80d6ab67371..c75aff8c1723c10 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -325,7 +325,8 @@ Objects/obmalloc.c - _PyMem_Debug - Objects/obmalloc.c - _PyMem_Raw - Objects/obmalloc.c - _PyObject - Objects/obmalloc.c - last_final_leaks - -Objects/obmalloc.c - usedpools - +Objects/obmalloc.c - obmalloc_state_main - +Objects/obmalloc.c - obmalloc_state_initialized - Objects/typeobject.c - name_op - Objects/typeobject.c - slotdefs - Objects/unicodeobject.c - stripfuncnames - diff --git a/Tools/cases_generator/README.md b/Tools/cases_generator/README.md index ed802e44f31ad5f..7fec8a882336cdf 100644 --- a/Tools/cases_generator/README.md +++ b/Tools/cases_generator/README.md @@ -5,16 +5,30 @@ Documentation for the instruction definitions in `Python/bytecodes.c` What's currently here: +- `analyzer.py`: code for converting `AST` generated by `Parser` + to more high-level structure for easier interaction - `lexer.py`: lexer for C, originally written by Mark Shannon - `plexer.py`: OO interface on top of lexer.py; main class: `PLexer` -- `parsing.py`: Parser for instruction definition DSL; main class `Parser` -- `generate_cases.py`: driver script to read `Python/bytecodes.c` and +- `parsing.py`: Parser for instruction definition DSL; main class: `Parser` +- `parser.py` helper for interactions with `parsing.py` +- `tierN_generator.py`: a couple of driver scripts to read `Python/bytecodes.c` and write `Python/generated_cases.c.h` (and several other files) -- `analysis.py`: `Analyzer` class used to read the input files -- `flags.py`: abstractions related to metadata flags for instructions -- `formatting.py`: `Formatter` class used to write the output files -- `instructions.py`: classes to analyze and write instructions -- `stacking.py`: code to handle generalized stack effects +- `stack.py`: code to handle generalized stack effects +- `cwriter.py`: code which understands tokens and how to format C code; + main class: `CWriter` +- `generators_common.py`: helpers for generators +- `opcode_id_generator.py`: generate a list of opcodes and write them to + `Include/opcode_ids.h` +- `opcode_metadata_generator.py`: reads the instruction definitions and + write the metadata to `Include/internal/pycore_opcode_metadata.h` +- `py_metadata_generator.py`: reads the instruction definitions and + write the metadata to `Lib/_opcode_metadata.py` +- `target_generator.py`: generate targets for computed goto dispatch and + write them to `Python/opcode_targets.h` +- `uop_id_generator.py`: generate a list of uop IDs and write them to + `Include/internal/pycore_uop_ids.h` +- `uop_metadata_generator.py`: reads the instruction definitions and + write the metadata to `Include/internal/pycore_uop_metadata.h` Note that there is some dummy C code at the top and bottom of `Python/bytecodes.c` diff --git a/Tools/cases_generator/opcode_id_generator.py b/Tools/cases_generator/opcode_id_generator.py index dbea3d0b622c87f..5a3009a5c04c27b 100644 --- a/Tools/cases_generator/opcode_id_generator.py +++ b/Tools/cases_generator/opcode_id_generator.py @@ -1,6 +1,6 @@ """Generate the list of opcode IDs. Reads the instruction definitions from bytecodes.c. -Writes the IDs to opcode._ids.h by default. +Writes the IDs to opcode_ids.h by default. """ import argparse diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 1826a0b645c3b86..3e9fa3e26daa539 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -1,6 +1,6 @@ -"""Generate uop metedata. +"""Generate opcode metadata. Reads the instruction definitions from bytecodes.c. -Writes the metadata to pycore_uop_metadata.h by default. +Writes the metadata to pycore_opcode_metadata.h by default. """ import argparse diff --git a/Tools/cases_generator/py_metadata_generator.py b/Tools/cases_generator/py_metadata_generator.py index 43811fdacc8a9e1..0dbcd599f9d4d98 100644 --- a/Tools/cases_generator/py_metadata_generator.py +++ b/Tools/cases_generator/py_metadata_generator.py @@ -1,6 +1,6 @@ -"""Generate uop metedata. +"""Generate opcode metadata for Python. Reads the instruction definitions from bytecodes.c. -Writes the metadata to pycore_uop_metadata.h by default. +Writes the metadata to _opcode_metadata.py by default. """ import argparse diff --git a/Tools/cases_generator/uop_metadata_generator.py b/Tools/cases_generator/uop_metadata_generator.py index d4f3a096d2acc18..9083ecc48bdf5b8 100644 --- a/Tools/cases_generator/uop_metadata_generator.py +++ b/Tools/cases_generator/uop_metadata_generator.py @@ -1,4 +1,4 @@ -"""Generate uop metedata. +"""Generate uop metadata. Reads the instruction definitions from bytecodes.c. Writes the metadata to pycore_uop_metadata.h by default. """ diff --git a/Tools/clinic/.ruff.toml b/Tools/clinic/.ruff.toml index cbb3a9a8f3a8c2c..c019572d0cb1864 100644 --- a/Tools/clinic/.ruff.toml +++ b/Tools/clinic/.ruff.toml @@ -1,5 +1,7 @@ target-version = "py310" fix = true + +[lint] select = [ "F", # Enable all pyflakes rules "UP", # Enable all pyupgrade rules by default diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 770878a3f8d2c78..db57d17899af939 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -960,7 +960,7 @@ def parser_body( return_error = ('return NULL;' if simple_return else 'goto exit;') parser_code = [libclinic.normalize_snippet(""" - if (nargs) {{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) {{ PyErr_SetString(PyExc_TypeError, "{name}() takes no arguments"); %s }} @@ -4368,7 +4368,9 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st if (ptr == NULL) {{{{ goto exit; }}}} - PyBuffer_FillInfo(&{paramname}, {argname}, (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&{paramname}, {argname}, (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) {{{{ + goto exit; + }}}} }}}} else {{{{ /* any bytes-like object */ if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_SIMPLE) != 0) {{{{ diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 5ef55524c11be20..483f28b46dfec74 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -70,6 +70,14 @@ def _type_unsigned_int_ptr(): def _sizeof_void_p(): return gdb.lookup_type('void').pointer().sizeof +def _managed_dict_offset(): + # See pycore_object.h + pyobj = gdb.lookup_type("PyObject") + if any(field.name == "ob_ref_local" for field in pyobj.fields()): + return -1 * _sizeof_void_p() + else: + return -3 * _sizeof_void_p() + Py_TPFLAGS_MANAGED_DICT = (1 << 4) Py_TPFLAGS_HEAPTYPE = (1 << 9) @@ -457,7 +465,7 @@ def get_attr_dict(self): if dictoffset < 0: if int_from_int(typeobj.field('tp_flags')) & Py_TPFLAGS_MANAGED_DICT: assert dictoffset == -1 - dictoffset = -3 * _sizeof_void_p() + dictoffset = _managed_dict_offset() else: type_PyVarObject_ptr = gdb.lookup_type('PyVarObject').pointer() tsize = int_from_int(self._gdbval.cast(type_PyVarObject_ptr)['ob_size']) @@ -485,9 +493,8 @@ def get_keys_values(self): has_values = int_from_int(typeobj.field('tp_flags')) & Py_TPFLAGS_MANAGED_DICT if not has_values: return None - charptrptr_t = _type_char_ptr().pointer() - ptr = self._gdbval.cast(charptrptr_t) - 3 - char_ptr = ptr.dereference() + ptr = self._gdbval.cast(_type_char_ptr()) + _managed_dict_offset() + char_ptr = ptr.cast(_type_char_ptr().pointer()).dereference() if (int(char_ptr) & 1) == 0: return None char_ptr += 1 diff --git a/Tools/jit/README.md b/Tools/jit/README.md new file mode 100644 index 000000000000000..04a6c0780bf9727 --- /dev/null +++ b/Tools/jit/README.md @@ -0,0 +1,46 @@ +The JIT Compiler +================ + +This version of CPython can be built with an experimental just-in-time compiler. While most everything you already know about building and using CPython is unchanged, you will probably need to install a compatible version of LLVM first. + +## Installing LLVM + +The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). + +LLVM version 16 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-16`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. + +It's easy to install all of the required tools: + +### Linux + +Install LLVM 16 on Ubuntu/Debian: + +```sh +wget https://apt.llvm.org/llvm.sh +chmod +x llvm.sh +sudo ./llvm.sh 16 +``` + +### macOS + +Install LLVM 16 with [Homebrew](https://brew.sh): + +```sh +brew install llvm@16 +``` + +Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them. + +### Windows + +Install LLVM 16 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=16), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** + +## Building + +For `PCbuild`-based builds, pass the new `--experimental-jit` option to `build.bat`. + +For all other builds, pass the new `--enable-experimental-jit` option to `configure`. + +Otherwise, just configure and build as you normally would. Cross-compiling "just works", since the JIT is built for the host platform. + +[^why-llvm]: Clang is specifically needed because it's the only C compiler with support for guaranteed tail calls (`musttail`), which are required by CPython's continuation-passing-style approach to JIT compilation. Since LLVM also includes other functionalities we need (namely, object file parsing and disassembly), it's convenient to only support one toolchain at this time. diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py new file mode 100644 index 000000000000000..603bbef59ba2e63 --- /dev/null +++ b/Tools/jit/_llvm.py @@ -0,0 +1,99 @@ +"""Utilities for invoking LLVM tools.""" +import asyncio +import functools +import os +import re +import shlex +import subprocess +import typing + +_LLVM_VERSION = 16 +_LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\s+") + +_P = typing.ParamSpec("_P") +_R = typing.TypeVar("_R") +_C = typing.Callable[_P, typing.Awaitable[_R]] + + +def _async_cache(f: _C[_P, _R]) -> _C[_P, _R]: + cache = {} + lock = asyncio.Lock() + + @functools.wraps(f) + async def wrapper( + *args: _P.args, **kwargs: _P.kwargs # pylint: disable = no-member + ) -> _R: + async with lock: + if args not in cache: + cache[args] = await f(*args, **kwargs) + return cache[args] + + return wrapper + + +_CORES = asyncio.BoundedSemaphore(os.cpu_count() or 1) + + +async def _run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str | None: + command = [tool, *args] + async with _CORES: + if echo: + print(shlex.join(command)) + try: + process = await asyncio.create_subprocess_exec( + *command, stdout=subprocess.PIPE + ) + except FileNotFoundError: + return None + out, _ = await process.communicate() + if process.returncode: + raise RuntimeError(f"{tool} exited with return code {process.returncode}") + return out.decode() + + +@_async_cache +async def _check_tool_version(name: str, *, echo: bool = False) -> bool: + output = await _run(name, ["--version"], echo=echo) + return bool(output and _LLVM_VERSION_PATTERN.search(output)) + + +@_async_cache +async def _get_brew_llvm_prefix(*, echo: bool = False) -> str | None: + output = await _run("brew", ["--prefix", f"llvm@{_LLVM_VERSION}"], echo=echo) + return output and output.removesuffix("\n") + + +@_async_cache +async def _find_tool(tool: str, *, echo: bool = False) -> str | None: + # Unversioned executables: + path = tool + if await _check_tool_version(path, echo=echo): + return path + # Versioned executables: + path = f"{tool}-{_LLVM_VERSION}" + if await _check_tool_version(path, echo=echo): + return path + # Homebrew-installed executables: + prefix = await _get_brew_llvm_prefix(echo=echo) + if prefix is not None: + path = os.path.join(prefix, "bin", tool) + if await _check_tool_version(path, echo=echo): + return path + # Nothing found: + return None + + +async def maybe_run( + tool: str, args: typing.Iterable[str], echo: bool = False +) -> str | None: + """Run an LLVM tool if it can be found. Otherwise, return None.""" + path = await _find_tool(tool, echo=echo) + return path and await _run(path, args, echo=echo) + + +async def run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str: + """Run an LLVM tool if it can be found. Otherwise, raise RuntimeError.""" + output = await maybe_run(tool, args, echo=echo) + if output is None: + raise RuntimeError(f"Can't find {tool}-{_LLVM_VERSION}!") + return output diff --git a/Tools/jit/_schema.py b/Tools/jit/_schema.py new file mode 100644 index 000000000000000..8eeb78e6cd69eee --- /dev/null +++ b/Tools/jit/_schema.py @@ -0,0 +1,99 @@ +"""Schema for the JSON produced by llvm-readobj --elf-output-style=JSON.""" +import typing + +HoleKind: typing.TypeAlias = typing.Literal[ + "ARM64_RELOC_GOT_LOAD_PAGE21", + "ARM64_RELOC_GOT_LOAD_PAGEOFF12", + "ARM64_RELOC_UNSIGNED", + "IMAGE_REL_AMD64_ADDR64", + "IMAGE_REL_I386_DIR32", + "R_AARCH64_ABS64", + "R_AARCH64_CALL26", + "R_AARCH64_JUMP26", + "R_AARCH64_MOVW_UABS_G0_NC", + "R_AARCH64_MOVW_UABS_G1_NC", + "R_AARCH64_MOVW_UABS_G2_NC", + "R_AARCH64_MOVW_UABS_G3", + "R_X86_64_64", + "X86_64_RELOC_UNSIGNED", +] + + +class COFFRelocation(typing.TypedDict): + """A COFF object file relocation record.""" + + Type: dict[typing.Literal["Value"], HoleKind] + Symbol: str + Offset: int + + +class ELFRelocation(typing.TypedDict): + """An ELF object file relocation record.""" + + Addend: int + Offset: int + Symbol: dict[typing.Literal["Value"], str] + Type: dict[typing.Literal["Value"], HoleKind] + + +class MachORelocation(typing.TypedDict): + """A Mach-O object file relocation record.""" + + Offset: int + Section: typing.NotRequired[dict[typing.Literal["Value"], str]] + Symbol: typing.NotRequired[dict[typing.Literal["Value"], str]] + Type: dict[typing.Literal["Value"], HoleKind] + + +class _COFFSymbol(typing.TypedDict): + Name: str + Value: int + + +class _ELFSymbol(typing.TypedDict): + Name: dict[typing.Literal["Value"], str] + Value: int + + +class _MachOSymbol(typing.TypedDict): + Name: dict[typing.Literal["Value"], str] + Value: int + + +class COFFSection(typing.TypedDict): + """A COFF object file section.""" + + Characteristics: dict[ + typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]] + ] + Number: int + RawDataSize: int + Relocations: list[dict[typing.Literal["Relocation"], COFFRelocation]] + SectionData: typing.NotRequired[dict[typing.Literal["Bytes"], list[int]]] + Symbols: list[dict[typing.Literal["Symbol"], _COFFSymbol]] + + +class ELFSection(typing.TypedDict): + """An ELF object file section.""" + + Flags: dict[typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]]] + Index: int + Info: int + Relocations: list[dict[typing.Literal["Relocation"], ELFRelocation]] + SectionData: dict[typing.Literal["Bytes"], list[int]] + Symbols: list[dict[typing.Literal["Symbol"], _ELFSymbol]] + Type: dict[typing.Literal["Value"], str] + + +class MachOSection(typing.TypedDict): + """A Mach-O object file section.""" + + Address: int + Attributes: dict[typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]]] + Index: int + Name: dict[typing.Literal["Value"], str] + Relocations: typing.NotRequired[ + list[dict[typing.Literal["Relocation"], MachORelocation]] + ] + SectionData: typing.NotRequired[dict[typing.Literal["Bytes"], list[int]]] + Symbols: typing.NotRequired[list[dict[typing.Literal["Symbol"], _MachOSymbol]]] diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py new file mode 100644 index 000000000000000..71c678e04fbfd5a --- /dev/null +++ b/Tools/jit/_stencils.py @@ -0,0 +1,220 @@ +"""Core data structures for compiled code templates.""" +import dataclasses +import enum +import sys + +import _schema + + +@enum.unique +class HoleValue(enum.Enum): + """ + Different "base" values that can be patched into holes (usually combined with the + address of a symbol and/or an addend). + """ + + # The base address of the machine code for the current uop (exposed as _JIT_ENTRY): + CODE = enum.auto() + # The base address of the machine code for the next uop (exposed as _JIT_CONTINUE): + CONTINUE = enum.auto() + # The base address of the read-only data for this uop: + DATA = enum.auto() + # The address of the current executor (exposed as _JIT_EXECUTOR): + EXECUTOR = enum.auto() + # The base address of the "global" offset table located in the read-only data. + # Shouldn't be present in the final stencils, since these are all replaced with + # equivalent DATA values: + GOT = enum.auto() + # The current uop's oparg (exposed as _JIT_OPARG): + OPARG = enum.auto() + # The current uop's operand (exposed as _JIT_OPERAND): + OPERAND = enum.auto() + # The current uop's target (exposed as _JIT_TARGET): + TARGET = enum.auto() + # The base address of the machine code for the first uop (exposed as _JIT_TOP): + TOP = enum.auto() + # A hardcoded value of zero (used for symbol lookups): + ZERO = enum.auto() + + +@dataclasses.dataclass +class Hole: + """ + A "hole" in the stencil to be patched with a computed runtime value. + + Analogous to relocation records in an object file. + """ + + offset: int + kind: _schema.HoleKind + # Patch with this base value: + value: HoleValue + # ...plus the address of this symbol: + symbol: str | None + # ...plus this addend: + addend: int + # Convenience method: + replace = dataclasses.replace + + def as_c(self) -> str: + """Dump this hole as an initialization of a C Hole struct.""" + parts = [ + f"{self.offset:#x}", + f"HoleKind_{self.kind}", + f"HoleValue_{self.value.name}", + f"&{self.symbol}" if self.symbol else "NULL", + _format_addend(self.addend), + ] + return f"{{{', '.join(parts)}}}" + + +@dataclasses.dataclass +class Stencil: + """ + A contiguous block of machine code or data to be copied-and-patched. + + Analogous to a section or segment in an object file. + """ + + body: bytearray = dataclasses.field(default_factory=bytearray, init=False) + holes: list[Hole] = dataclasses.field(default_factory=list, init=False) + disassembly: list[str] = dataclasses.field(default_factory=list, init=False) + + def pad(self, alignment: int) -> None: + """Pad the stencil to the given alignment.""" + offset = len(self.body) + padding = -offset % alignment + self.disassembly.append(f"{offset:x}: {' '.join(['00'] * padding)}") + self.body.extend([0] * padding) + + def emit_aarch64_trampoline(self, hole: Hole) -> None: + """Even with the large code model, AArch64 Linux insists on 28-bit jumps.""" + base = len(self.body) + where = slice(hole.offset, hole.offset + 4) + instruction = int.from_bytes(self.body[where], sys.byteorder) + instruction &= 0xFC000000 + instruction |= ((base - hole.offset) >> 2) & 0x03FFFFFF + self.body[where] = instruction.to_bytes(4, sys.byteorder) + self.disassembly += [ + f"{base + 4 * 0: x}: d2800008 mov x8, #0x0", + f"{base + 4 * 0:016x}: R_AARCH64_MOVW_UABS_G0_NC {hole.symbol}", + f"{base + 4 * 1:x}: f2a00008 movk x8, #0x0, lsl #16", + f"{base + 4 * 1:016x}: R_AARCH64_MOVW_UABS_G1_NC {hole.symbol}", + f"{base + 4 * 2:x}: f2c00008 movk x8, #0x0, lsl #32", + f"{base + 4 * 2:016x}: R_AARCH64_MOVW_UABS_G2_NC {hole.symbol}", + f"{base + 4 * 3:x}: f2e00008 movk x8, #0x0, lsl #48", + f"{base + 4 * 3:016x}: R_AARCH64_MOVW_UABS_G3 {hole.symbol}", + f"{base + 4 * 4:x}: d61f0100 br x8", + ] + for code in [ + 0xD2800008.to_bytes(4, sys.byteorder), + 0xF2A00008.to_bytes(4, sys.byteorder), + 0xF2C00008.to_bytes(4, sys.byteorder), + 0xF2E00008.to_bytes(4, sys.byteorder), + 0xD61F0100.to_bytes(4, sys.byteorder), + ]: + self.body.extend(code) + for i, kind in enumerate( + [ + "R_AARCH64_MOVW_UABS_G0_NC", + "R_AARCH64_MOVW_UABS_G1_NC", + "R_AARCH64_MOVW_UABS_G2_NC", + "R_AARCH64_MOVW_UABS_G3", + ] + ): + self.holes.append(hole.replace(offset=base + 4 * i, kind=kind)) + + +@dataclasses.dataclass +class StencilGroup: + """ + Code and data corresponding to a given micro-opcode. + + Analogous to an entire object file. + """ + + code: Stencil = dataclasses.field(default_factory=Stencil, init=False) + data: Stencil = dataclasses.field(default_factory=Stencil, init=False) + symbols: dict[int | str, tuple[HoleValue, int]] = dataclasses.field( + default_factory=dict, init=False + ) + _got: dict[str, int] = dataclasses.field(default_factory=dict, init=False) + + def process_relocations(self, *, alignment: int = 1) -> None: + """Fix up all GOT and internal relocations for this stencil group.""" + self.code.pad(alignment) + self.data.pad(8) + for stencil in [self.code, self.data]: + holes = [] + for hole in stencil.holes: + if hole.value is HoleValue.GOT: + assert hole.symbol is not None + hole.value = HoleValue.DATA + hole.addend += self._global_offset_table_lookup(hole.symbol) + hole.symbol = None + elif hole.symbol in self.symbols: + hole.value, addend = self.symbols[hole.symbol] + hole.addend += addend + hole.symbol = None + elif ( + hole.kind in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26"} + and hole.value is HoleValue.ZERO + ): + self.code.emit_aarch64_trampoline(hole) + continue + holes.append(hole) + stencil.holes[:] = holes + self.code.pad(alignment) + self._emit_global_offset_table() + self.code.holes.sort(key=lambda hole: hole.offset) + self.data.holes.sort(key=lambda hole: hole.offset) + + def _global_offset_table_lookup(self, symbol: str) -> int: + return len(self.data.body) + self._got.setdefault(symbol, 8 * len(self._got)) + + def _emit_global_offset_table(self) -> None: + got = len(self.data.body) + for s, offset in self._got.items(): + if s in self.symbols: + value, addend = self.symbols[s] + symbol = None + else: + value, symbol = symbol_to_value(s) + addend = 0 + self.data.holes.append( + Hole(got + offset, "R_X86_64_64", value, symbol, addend) + ) + value_part = value.name if value is not HoleValue.ZERO else "" + if value_part and not symbol and not addend: + addend_part = "" + else: + addend_part = f"&{symbol}" if symbol else "" + addend_part += _format_addend(addend, signed=symbol is not None) + if value_part: + value_part += "+" + self.data.disassembly.append( + f"{len(self.data.body):x}: {value_part}{addend_part}" + ) + self.data.body.extend([0] * 8) + + +def symbol_to_value(symbol: str) -> tuple[HoleValue, str | None]: + """ + Convert a symbol name to a HoleValue and a symbol name. + + Some symbols (starting with "_JIT_") are special and are converted to their + own HoleValues. + """ + if symbol.startswith("_JIT_"): + try: + return HoleValue[symbol.removeprefix("_JIT_")], None + except KeyError: + pass + return HoleValue.ZERO, symbol + + +def _format_addend(addend: int, signed: bool = False) -> str: + addend %= 1 << 64 + if addend & (1 << 63): + addend -= 1 << 64 + return f"{addend:{'+#x' if signed else '#x'}}" diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py new file mode 100644 index 000000000000000..51b091eb2464131 --- /dev/null +++ b/Tools/jit/_targets.py @@ -0,0 +1,394 @@ +"""Target-specific code generation, parsing, and processing.""" +import asyncio +import dataclasses +import hashlib +import json +import os +import pathlib +import re +import sys +import tempfile +import typing + +import _llvm +import _schema +import _stencils +import _writer + +if sys.version_info < (3, 11): + raise RuntimeError("Building the JIT compiler requires Python 3.11 or newer!") + +TOOLS_JIT_BUILD = pathlib.Path(__file__).resolve() +TOOLS_JIT = TOOLS_JIT_BUILD.parent +TOOLS = TOOLS_JIT.parent +CPYTHON = TOOLS.parent +PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h" +TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c" + + +_S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection) +_R = typing.TypeVar( + "_R", _schema.COFFRelocation, _schema.ELFRelocation, _schema.MachORelocation +) + + +@dataclasses.dataclass +class _Target(typing.Generic[_S, _R]): + triple: str + _: dataclasses.KW_ONLY + alignment: int = 1 + prefix: str = "" + debug: bool = False + force: bool = False + verbose: bool = False + + def _compute_digest(self, out: pathlib.Path) -> str: + hasher = hashlib.sha256() + hasher.update(self.triple.encode()) + hasher.update(self.alignment.to_bytes()) + hasher.update(self.prefix.encode()) + # These dependencies are also reflected in _JITSources in regen.targets: + hasher.update(PYTHON_EXECUTOR_CASES_C_H.read_bytes()) + hasher.update((out / "pyconfig.h").read_bytes()) + for dirpath, _, filenames in sorted(os.walk(TOOLS_JIT)): + for filename in filenames: + hasher.update(pathlib.Path(dirpath, filename).read_bytes()) + return hasher.hexdigest() + + async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: + group = _stencils.StencilGroup() + args = ["--disassemble", "--reloc", f"{path}"] + output = await _llvm.maybe_run("llvm-objdump", args, echo=self.verbose) + if output is not None: + group.code.disassembly.extend( + line.expandtabs().strip() + for line in output.splitlines() + if not line.isspace() + ) + args = [ + "--elf-output-style=JSON", + "--expand-relocs", + # "--pretty-print", + "--section-data", + "--section-relocations", + "--section-symbols", + "--sections", + f"{path}", + ] + output = await _llvm.run("llvm-readobj", args, echo=self.verbose) + # --elf-output-style=JSON is only *slightly* broken on Mach-O... + output = output.replace("PrivateExtern\n", "\n") + output = output.replace("Extern\n", "\n") + # ...and also COFF: + output = output[output.index("[", 1, None) :] + output = output[: output.rindex("]", None, -1) + 1] + sections: list[dict[typing.Literal["Section"], _S]] = json.loads(output) + for wrapped_section in sections: + self._handle_section(wrapped_section["Section"], group) + assert group.symbols["_JIT_ENTRY"] == (_stencils.HoleValue.CODE, 0) + if group.data.body: + line = f"0: {str(bytes(group.data.body)).removeprefix('b')}" + group.data.disassembly.append(line) + group.process_relocations() + return group + + def _handle_section(self, section: _S, group: _stencils.StencilGroup) -> None: + raise NotImplementedError(type(self)) + + def _handle_relocation( + self, base: int, relocation: _R, raw: bytes + ) -> _stencils.Hole: + raise NotImplementedError(type(self)) + + async def _compile( + self, opname: str, c: pathlib.Path, tempdir: pathlib.Path + ) -> _stencils.StencilGroup: + o = tempdir / f"{opname}.o" + args = [ + f"--target={self.triple}", + "-DPy_BUILD_CORE", + "-D_DEBUG" if self.debug else "-DNDEBUG", + f"-D_JIT_OPCODE={opname}", + "-D_PyJIT_ACTIVE", + "-D_Py_JIT", + "-I.", + f"-I{CPYTHON / 'Include'}", + f"-I{CPYTHON / 'Include' / 'internal'}", + f"-I{CPYTHON / 'Include' / 'internal' / 'mimalloc'}", + f"-I{CPYTHON / 'Python'}", + "-O3", + "-c", + "-fno-asynchronous-unwind-tables", + # SET_FUNCTION_ATTRIBUTE on 32-bit Windows debug builds: + "-fno-jump-tables", + # Position-independent code adds indirection to every load and jump: + "-fno-pic", + # Don't make calls to weird stack-smashing canaries: + "-fno-stack-protector", + # We have three options for code model: + # - "small": the default, assumes that code and data reside in the + # lowest 2GB of memory (128MB on aarch64) + # - "medium": assumes that code resides in the lowest 2GB of memory, + # and makes no assumptions about data (not available on aarch64) + # - "large": makes no assumptions about either code or data + "-mcmodel=large", + "-o", + f"{o}", + "-std=c11", + f"{c}", + ] + await _llvm.run("clang", args, echo=self.verbose) + return await self._parse(o) + + async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: + generated_cases = PYTHON_EXECUTOR_CASES_C_H.read_text() + opnames = sorted(re.findall(r"\n {8}case (\w+): \{\n", generated_cases)) + tasks = [] + with tempfile.TemporaryDirectory() as tempdir: + work = pathlib.Path(tempdir).resolve() + async with asyncio.TaskGroup() as group: + for opname in opnames: + coro = self._compile(opname, TOOLS_JIT_TEMPLATE_C, work) + tasks.append(group.create_task(coro, name=opname)) + return {task.get_name(): task.result() for task in tasks} + + def build(self, out: pathlib.Path, *, comment: str = "") -> None: + """Build jit_stencils.h in the given directory.""" + digest = f"// {self._compute_digest(out)}\n" + jit_stencils = out / "jit_stencils.h" + if ( + not self.force + and jit_stencils.exists() + and jit_stencils.read_text().startswith(digest) + ): + return + stencil_groups = asyncio.run(self._build_stencils()) + with jit_stencils.open("w") as file: + file.write(digest) + if comment: + file.write(f"// {comment}\n") + file.write("") + for line in _writer.dump(stencil_groups): + file.write(f"{line}\n") + + +class _COFF( + _Target[_schema.COFFSection, _schema.COFFRelocation] +): # pylint: disable = too-few-public-methods + def _handle_section( + self, section: _schema.COFFSection, group: _stencils.StencilGroup + ) -> None: + flags = {flag["Name"] for flag in section["Characteristics"]["Flags"]} + if "SectionData" in section: + section_data_bytes = section["SectionData"]["Bytes"] + else: + # Zeroed BSS data, seen with printf debugging calls: + section_data_bytes = [0] * section["RawDataSize"] + if "IMAGE_SCN_MEM_EXECUTE" in flags: + value = _stencils.HoleValue.CODE + stencil = group.code + elif "IMAGE_SCN_MEM_READ" in flags: + value = _stencils.HoleValue.DATA + stencil = group.data + else: + return + base = len(stencil.body) + group.symbols[section["Number"]] = value, base + stencil.body.extend(section_data_bytes) + for wrapped_symbol in section["Symbols"]: + symbol = wrapped_symbol["Symbol"] + offset = base + symbol["Value"] + name = symbol["Name"] + name = name.removeprefix(self.prefix) + group.symbols[name] = value, offset + for wrapped_relocation in section["Relocations"]: + relocation = wrapped_relocation["Relocation"] + hole = self._handle_relocation(base, relocation, stencil.body) + stencil.holes.append(hole) + + def _handle_relocation( + self, base: int, relocation: _schema.COFFRelocation, raw: bytes + ) -> _stencils.Hole: + match relocation: + case { + "Offset": offset, + "Symbol": s, + "Type": {"Value": "IMAGE_REL_AMD64_ADDR64" as kind}, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.symbol_to_value(s) + addend = int.from_bytes(raw[offset : offset + 8], "little") + case { + "Offset": offset, + "Symbol": s, + "Type": {"Value": "IMAGE_REL_I386_DIR32" as kind}, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.symbol_to_value(s) + addend = int.from_bytes(raw[offset : offset + 4], "little") + case _: + raise NotImplementedError(relocation) + return _stencils.Hole(offset, kind, value, symbol, addend) + + +class _ELF( + _Target[_schema.ELFSection, _schema.ELFRelocation] +): # pylint: disable = too-few-public-methods + def _handle_section( + self, section: _schema.ELFSection, group: _stencils.StencilGroup + ) -> None: + section_type = section["Type"]["Value"] + flags = {flag["Name"] for flag in section["Flags"]["Flags"]} + if section_type == "SHT_RELA": + assert "SHF_INFO_LINK" in flags, flags + assert not section["Symbols"] + value, base = group.symbols[section["Info"]] + if value is _stencils.HoleValue.CODE: + stencil = group.code + else: + assert value is _stencils.HoleValue.DATA + stencil = group.data + for wrapped_relocation in section["Relocations"]: + relocation = wrapped_relocation["Relocation"] + hole = self._handle_relocation(base, relocation, stencil.body) + stencil.holes.append(hole) + elif section_type == "SHT_PROGBITS": + if "SHF_ALLOC" not in flags: + return + if "SHF_EXECINSTR" in flags: + value = _stencils.HoleValue.CODE + stencil = group.code + else: + value = _stencils.HoleValue.DATA + stencil = group.data + group.symbols[section["Index"]] = value, len(stencil.body) + for wrapped_symbol in section["Symbols"]: + symbol = wrapped_symbol["Symbol"] + offset = len(stencil.body) + symbol["Value"] + name = symbol["Name"]["Value"] + name = name.removeprefix(self.prefix) + group.symbols[name] = value, offset + stencil.body.extend(section["SectionData"]["Bytes"]) + assert not section["Relocations"] + else: + assert section_type in { + "SHT_GROUP", + "SHT_LLVM_ADDRSIG", + "SHT_NULL", + "SHT_STRTAB", + "SHT_SYMTAB", + }, section_type + + def _handle_relocation( + self, base: int, relocation: _schema.ELFRelocation, raw: bytes + ) -> _stencils.Hole: + match relocation: + case { + "Addend": addend, + "Offset": offset, + "Symbol": {"Value": s}, + "Type": {"Value": kind}, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.symbol_to_value(s) + case _: + raise NotImplementedError(relocation) + return _stencils.Hole(offset, kind, value, symbol, addend) + + +class _MachO( + _Target[_schema.MachOSection, _schema.MachORelocation] +): # pylint: disable = too-few-public-methods + def _handle_section( + self, section: _schema.MachOSection, group: _stencils.StencilGroup + ) -> None: + assert section["Address"] >= len(group.code.body) + assert "SectionData" in section + flags = {flag["Name"] for flag in section["Attributes"]["Flags"]} + name = section["Name"]["Value"] + name = name.removeprefix(self.prefix) + if "SomeInstructions" in flags: + value = _stencils.HoleValue.CODE + stencil = group.code + start_address = 0 + group.symbols[name] = value, section["Address"] - start_address + else: + value = _stencils.HoleValue.DATA + stencil = group.data + start_address = len(group.code.body) + group.symbols[name] = value, len(group.code.body) + base = section["Address"] - start_address + group.symbols[section["Index"]] = value, base + stencil.body.extend( + [0] * (section["Address"] - len(group.code.body) - len(group.data.body)) + ) + stencil.body.extend(section["SectionData"]["Bytes"]) + assert "Symbols" in section + for wrapped_symbol in section["Symbols"]: + symbol = wrapped_symbol["Symbol"] + offset = symbol["Value"] - start_address + name = symbol["Name"]["Value"] + name = name.removeprefix(self.prefix) + group.symbols[name] = value, offset + assert "Relocations" in section + for wrapped_relocation in section["Relocations"]: + relocation = wrapped_relocation["Relocation"] + hole = self._handle_relocation(base, relocation, stencil.body) + stencil.holes.append(hole) + + def _handle_relocation( + self, base: int, relocation: _schema.MachORelocation, raw: bytes + ) -> _stencils.Hole: + symbol: str | None + match relocation: + case { + "Offset": offset, + "Symbol": {"Value": s}, + "Type": { + "Value": "ARM64_RELOC_GOT_LOAD_PAGE21" + | "ARM64_RELOC_GOT_LOAD_PAGEOFF12" as kind + }, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.HoleValue.GOT, s + addend = 0 + case { + "Offset": offset, + "Section": {"Value": s}, + "Type": {"Value": kind}, + } | { + "Offset": offset, + "Symbol": {"Value": s}, + "Type": {"Value": kind}, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.symbol_to_value(s) + addend = 0 + case _: + raise NotImplementedError(relocation) + # Turn Clang's weird __bzero calls into normal bzero calls: + if symbol == "__bzero": + symbol = "bzero" + return _stencils.Hole(offset, kind, value, symbol, addend) + + +def get_target(host: str) -> _COFF | _ELF | _MachO: + """Build a _Target for the given host "triple" and options.""" + if re.fullmatch(r"aarch64-apple-darwin.*", host): + return _MachO(host, alignment=8, prefix="_") + if re.fullmatch(r"aarch64-.*-linux-gnu", host): + return _ELF(host, alignment=8) + if re.fullmatch(r"i686-pc-windows-msvc", host): + return _COFF(host, prefix="_") + if re.fullmatch(r"x86_64-apple-darwin.*", host): + return _MachO(host, prefix="_") + if re.fullmatch(r"x86_64-pc-windows-msvc", host): + return _COFF(host) + if re.fullmatch(r"x86_64-.*-linux-gnu", host): + return _ELF(host) + raise ValueError(host) diff --git a/Tools/jit/_writer.py b/Tools/jit/_writer.py new file mode 100644 index 000000000000000..8a2a42e75cfb9b3 --- /dev/null +++ b/Tools/jit/_writer.py @@ -0,0 +1,95 @@ +"""Utilities for writing StencilGroups out to a C header file.""" +import typing + +import _schema +import _stencils + + +def _dump_header() -> typing.Iterator[str]: + yield "typedef enum {" + for kind in typing.get_args(_schema.HoleKind): + yield f" HoleKind_{kind}," + yield "} HoleKind;" + yield "" + yield "typedef enum {" + for value in _stencils.HoleValue: + yield f" HoleValue_{value.name}," + yield "} HoleValue;" + yield "" + yield "typedef struct {" + yield " const uint64_t offset;" + yield " const HoleKind kind;" + yield " const HoleValue value;" + yield " const void *symbol;" + yield " const uint64_t addend;" + yield "} Hole;" + yield "" + yield "typedef struct {" + yield " const size_t body_size;" + yield " const unsigned char * const body;" + yield " const size_t holes_size;" + yield " const Hole * const holes;" + yield "} Stencil;" + yield "" + yield "typedef struct {" + yield " const Stencil code;" + yield " const Stencil data;" + yield "} StencilGroup;" + yield "" + + +def _dump_footer(opnames: typing.Iterable[str]) -> typing.Iterator[str]: + yield "#define INIT_STENCIL(STENCIL) { \\" + yield " .body_size = Py_ARRAY_LENGTH(STENCIL##_body) - 1, \\" + yield " .body = STENCIL##_body, \\" + yield " .holes_size = Py_ARRAY_LENGTH(STENCIL##_holes) - 1, \\" + yield " .holes = STENCIL##_holes, \\" + yield "}" + yield "" + yield "#define INIT_STENCIL_GROUP(OP) { \\" + yield " .code = INIT_STENCIL(OP##_code), \\" + yield " .data = INIT_STENCIL(OP##_data), \\" + yield "}" + yield "" + yield "static const StencilGroup stencil_groups[512] = {" + for opname in opnames: + yield f" [{opname}] = INIT_STENCIL_GROUP({opname})," + yield "};" + yield "" + yield "#define GET_PATCHES() { \\" + for value in _stencils.HoleValue: + yield f" [HoleValue_{value.name}] = (uint64_t)0xBADBADBADBADBADB, \\" + yield "}" + + +def _dump_stencil(opname: str, group: _stencils.StencilGroup) -> typing.Iterator[str]: + yield f"// {opname}" + for part, stencil in [("code", group.code), ("data", group.data)]: + for line in stencil.disassembly: + yield f"// {line}" + if stencil.body: + size = len(stencil.body) + 1 + yield f"static const unsigned char {opname}_{part}_body[{size}] = {{" + for i in range(0, len(stencil.body), 8): + row = " ".join(f"{byte:#04x}," for byte in stencil.body[i : i + 8]) + yield f" {row}" + yield "};" + else: + yield f"static const unsigned char {opname}_{part}_body[1];" + if stencil.holes: + size = len(stencil.holes) + 1 + yield f"static const Hole {opname}_{part}_holes[{size}] = {{" + for hole in stencil.holes: + yield f" {hole.as_c()}," + yield "};" + else: + yield f"static const Hole {opname}_{part}_holes[1];" + yield "" + + +def dump(groups: dict[str, _stencils.StencilGroup]) -> typing.Iterator[str]: + """Yield a JIT compiler line-by-line as a C header file.""" + yield from _dump_header() + for opname, group in groups.items(): + yield from _dump_stencil(opname, group) + yield from _dump_footer(groups) diff --git a/Tools/jit/build.py b/Tools/jit/build.py new file mode 100644 index 000000000000000..4d4ace14ebf26c4 --- /dev/null +++ b/Tools/jit/build.py @@ -0,0 +1,28 @@ +"""Build an experimental just-in-time compiler for CPython.""" +import argparse +import pathlib +import shlex +import sys + +import _targets + +if __name__ == "__main__": + comment = f"$ {shlex.join([sys.executable] + sys.argv)}" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "target", type=_targets.get_target, help="a PEP 11 target triple to compile for" + ) + parser.add_argument( + "-d", "--debug", action="store_true", help="compile for a debug build of Python" + ) + parser.add_argument( + "-f", "--force", action="store_true", help="force the entire JIT to be rebuilt" + ) + parser.add_argument( + "-v", "--verbose", action="store_true", help="echo commands as they are run" + ) + args = parser.parse_args() + args.target.debug = args.debug + args.target.force = args.force + args.target.verbose = args.verbose + args.target.build(pathlib.Path.cwd(), comment=comment) diff --git a/Tools/jit/mypy.ini b/Tools/jit/mypy.ini new file mode 100644 index 000000000000000..768d0028516abd0 --- /dev/null +++ b/Tools/jit/mypy.ini @@ -0,0 +1,5 @@ +[mypy] +files = Tools/jit +pretty = True +python_version = 3.11 +strict = True diff --git a/Tools/jit/template.c b/Tools/jit/template.c new file mode 100644 index 000000000000000..12303a550d8879e --- /dev/null +++ b/Tools/jit/template.c @@ -0,0 +1,98 @@ +#include "Python.h" + +#include "pycore_call.h" +#include "pycore_ceval.h" +#include "pycore_dict.h" +#include "pycore_emscripten_signal.h" +#include "pycore_intrinsics.h" +#include "pycore_jit.h" +#include "pycore_long.h" +#include "pycore_opcode_metadata.h" +#include "pycore_opcode_utils.h" +#include "pycore_range.h" +#include "pycore_setobject.h" +#include "pycore_sliceobject.h" + +#include "ceval_macros.h" + +#undef CURRENT_OPARG +#define CURRENT_OPARG() (_oparg) + +#undef CURRENT_OPERAND +#define CURRENT_OPERAND() (_operand) + +#undef DEOPT_IF +#define DEOPT_IF(COND, INSTNAME) \ + do { \ + if ((COND)) { \ + goto deoptimize; \ + } \ + } while (0) + +#undef ENABLE_SPECIALIZATION +#define ENABLE_SPECIALIZATION (0) + +#undef GOTO_ERROR +#define GOTO_ERROR(LABEL) \ + do { \ + goto LABEL ## _tier_two; \ + } while (0) + +#undef LOAD_IP +#define LOAD_IP(UNUSED) \ + do { \ + } while (0) + +#define PATCH_VALUE(TYPE, NAME, ALIAS) \ + extern void ALIAS; \ + TYPE NAME = (TYPE)(uint64_t)&ALIAS; + +#define PATCH_JUMP(ALIAS) \ + extern void ALIAS; \ + __attribute__((musttail)) \ + return ((jit_func)&ALIAS)(frame, stack_pointer, tstate); + +_Py_CODEUNIT * +_JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate) +{ + // Locals that the instruction implementations expect to exist: + PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR) + int oparg; + int opcode = _JIT_OPCODE; + _PyUOpInstruction *next_uop; + // Other stuff we need handy: + PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG) + PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND) + PATCH_VALUE(uint32_t, _target, _JIT_TARGET) + // The actual instruction definitions (only one will be used): + if (opcode == _JUMP_TO_TOP) { + CHECK_EVAL_BREAKER(); + PATCH_JUMP(_JIT_TOP); + } + switch (opcode) { +#include "executor_cases.c.h" + default: + Py_UNREACHABLE(); + } + PATCH_JUMP(_JIT_CONTINUE); + // Labels that the instruction implementations expect to exist: +unbound_local_error_tier_two: + _PyEval_FormatExcCheckArg( + tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)); + goto error_tier_two; +pop_4_error_tier_two: + STACK_SHRINK(1); +pop_3_error_tier_two: + STACK_SHRINK(1); +pop_2_error_tier_two: + STACK_SHRINK(1); +pop_1_error_tier_two: + STACK_SHRINK(1); +error_tier_two: + _PyFrame_SetStackPointer(frame, stack_pointer); + return NULL; +deoptimize: + _PyFrame_SetStackPointer(frame, stack_pointer); + return _PyCode_CODE(_PyFrame_GetCode(frame)) + _target; +} diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index b89f86a35d61153..c0a63b40ff4155f 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -3,5 +3,5 @@ mypy==1.8.0 # needed for peg_generator: -types-psutil==5.9.5.17 -types-setuptools==69.0.0.0 +types-psutil==5.9.5.20240106 +types-setuptools==69.0.0.20240125 diff --git a/Tools/requirements-hypothesis.txt b/Tools/requirements-hypothesis.txt index 0e6e16ae198162c..064731a236ee863 100644 --- a/Tools/requirements-hypothesis.txt +++ b/Tools/requirements-hypothesis.txt @@ -1,4 +1,4 @@ # Requirements file for hypothesis that # we use to run our property-based tests in CI. -hypothesis==6.92.2 +hypothesis==6.97.4 diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 1e9dc07bae89812..9b7e7b999ea7c7d 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -412,6 +412,14 @@ def get_histogram(self, prefix: str) -> list[tuple[int, int]]: rows.sort() return rows + def get_rare_events(self) -> list[tuple[str, int]]: + prefix = "Rare event " + return [ + (key[len(prefix) + 1:-1], val) + for key, val in self._data.items() + if key.startswith(prefix) + ] + class Count(int): def markdown(self) -> str: @@ -1064,6 +1072,17 @@ def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None) ) +def rare_event_section() -> Section: + def calc_rare_event_table(stats: Stats) -> Table: + return [(x, Count(y)) for x, y in stats.get_rare_events()] + + return Section( + "Rare events", + "Counts of rare/unlikely events", + [Table(("Event", "Count:"), calc_rare_event_table, JoinMode.CHANGE)], + ) + + def meta_stats_section() -> Section: def calc_rows(stats: Stats) -> Rows: return [("Number of data files", Count(stats.get("__nfiles__")))] @@ -1085,6 +1104,7 @@ def calc_rows(stats: Stats) -> Rows: object_stats_section(), gc_stats_section(), optimization_section(), + rare_event_section(), meta_stats_section(), ] @@ -1162,7 +1182,7 @@ def output_stats(inputs: list[Path], json_output=str | None): case 1: data = load_raw_data(Path(inputs[0])) if json_output is not None: - with open(json_output, 'w', encoding='utf-8') as f: + with open(json_output, "w", encoding="utf-8") as f: save_raw_data(data, f) # type: ignore stats = Stats(data) output_markdown(sys.stdout, LAYOUT, stats) diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index 120e3883adc795d..baa16102068aa08 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -47,8 +47,9 @@ OPENSSL_RECENT_VERSIONS = [ "1.1.1w", - "3.0.11", - "3.1.3", + "3.0.13", + "3.1.5", + "3.2.1", ] LIBRESSL_OLD_VERSIONS = [ diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index beb857f69e40da1..23b38c8e93638ad 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -83,7 +83,7 @@ embuilder --pic build zlib bzip2 MINIMAL_PIC ``` -#### Compile a build Python interpreter +### Compile and build Python interpreter From within the container, run the following command: diff --git a/configure b/configure index b1153df4d7ec523..705a778cafced35 100755 --- a/configure +++ b/configure @@ -920,6 +920,8 @@ LLVM_AR PROFILE_TASK DEF_MAKE_RULE DEF_MAKE_ALL_RULE +JIT_STENCILS_H +REGEN_JIT_COMMAND ABIFLAGS LN MKDIR_P @@ -1074,6 +1076,7 @@ with_pydebug with_trace_refs enable_pystats with_assertions +enable_experimental_jit enable_optimizations with_lto enable_bolt @@ -1801,6 +1804,9 @@ Optional Features: --disable-gil enable experimental support for running without the GIL (default is no) --enable-pystats enable internal statistics gathering (default is no) + --enable-experimental-jit + build the experimental just-in-time compiler + (default is no) --enable-optimizations enable expensive, stable optimizations (PGO, etc.) (default is no) --enable-bolt enable usage of the llvm-bolt post-link optimizer @@ -6799,6 +6805,8 @@ case $host/$ac_cv_cc_name in #( aarch64-*-linux-gnu/clang) : PY_SUPPORT_TIER=2 ;; #( powerpc64le-*-linux-gnu/gcc) : + PY_SUPPORT_TIER=2 ;; #( + wasm32-unknown-wasi/clang) : PY_SUPPORT_TIER=2 ;; #( x86_64-*-linux-gnu/clang) : PY_SUPPORT_TIER=2 ;; #( @@ -6811,10 +6819,6 @@ case $host/$ac_cv_cc_name in #( PY_SUPPORT_TIER=3 ;; #( s390x-*-linux-gnu/gcc) : PY_SUPPORT_TIER=3 ;; #( - wasm32-unknown-emscripten/clang) : - PY_SUPPORT_TIER=3 ;; #( - wasm32-unknown-wasi/clang) : - PY_SUPPORT_TIER=3 ;; #( x86_64-*-freebsd*/clang) : PY_SUPPORT_TIER=3 ;; #( *) : @@ -7997,6 +8001,34 @@ else printf "%s\n" "no" >&6; } fi +# Check for --enable-experimental-jit: +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-experimental-jit" >&5 +printf %s "checking for --enable-experimental-jit... " >&6; } +# Check whether --enable-experimental-jit was given. +if test ${enable_experimental_jit+y} +then : + enableval=$enable_experimental_jit; +else $as_nop + enable_experimental_jit=no +fi + +if test "x$enable_experimental_jit" = xno +then : + +else $as_nop + as_fn_append CFLAGS_NODIST " -D_Py_JIT" + REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host" + JIT_STENCILS_H="jit_stencils.h" + if test "x$Py_DEBUG" = xtrue +then : + as_fn_append REGEN_JIT_COMMAND " --debug" +fi +fi + + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_experimental_jit" >&5 +printf "%s\n" "$enable_experimental_jit" >&6; } + # Enable optimization flags @@ -17444,6 +17476,12 @@ if test "x$ac_cv_func_getgid" = xyes then : printf "%s\n" "#define HAVE_GETGID 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "getgrent" "ac_cv_func_getgrent" +if test "x$ac_cv_func_getgrent" = xyes +then : + printf "%s\n" "#define HAVE_GETGRENT 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "getgrgid" "ac_cv_func_getgrgid" if test "x$ac_cv_func_getgrgid" = xyes @@ -17606,6 +17644,12 @@ if test "x$ac_cv_func_getwd" = xyes then : printf "%s\n" "#define HAVE_GETWD 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "grantpt" "ac_cv_func_grantpt" +if test "x$ac_cv_func_grantpt" = xyes +then : + printf "%s\n" "#define HAVE_GRANTPT 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "if_nameindex" "ac_cv_func_if_nameindex" if test "x$ac_cv_func_if_nameindex" = xyes @@ -17792,6 +17836,12 @@ if test "x$ac_cv_func_posix_fallocate" = xyes then : printf "%s\n" "#define HAVE_POSIX_FALLOCATE 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "posix_openpt" "ac_cv_func_posix_openpt" +if test "x$ac_cv_func_posix_openpt" = xyes +then : + printf "%s\n" "#define HAVE_POSIX_OPENPT 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "posix_spawn" "ac_cv_func_posix_spawn" if test "x$ac_cv_func_posix_spawn" = xyes @@ -17828,6 +17878,12 @@ if test "x$ac_cv_func_preadv2" = xyes then : printf "%s\n" "#define HAVE_PREADV2 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "pthread_cond_timedwait_relative_np" "ac_cv_func_pthread_cond_timedwait_relative_np" +if test "x$ac_cv_func_pthread_cond_timedwait_relative_np" = xyes +then : + printf "%s\n" "#define HAVE_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "pthread_condattr_setclock" "ac_cv_func_pthread_condattr_setclock" if test "x$ac_cv_func_pthread_condattr_setclock" = xyes @@ -17846,6 +17902,18 @@ if test "x$ac_cv_func_pthread_kill" = xyes then : printf "%s\n" "#define HAVE_PTHREAD_KILL 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "ptsname" "ac_cv_func_ptsname" +if test "x$ac_cv_func_ptsname" = xyes +then : + printf "%s\n" "#define HAVE_PTSNAME 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "ptsname_r" "ac_cv_func_ptsname_r" +if test "x$ac_cv_func_ptsname_r" = xyes +then : + printf "%s\n" "#define HAVE_PTSNAME_R 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "pwrite" "ac_cv_func_pwrite" if test "x$ac_cv_func_pwrite" = xyes @@ -18254,6 +18322,12 @@ if test "x$ac_cv_func_unlinkat" = xyes then : printf "%s\n" "#define HAVE_UNLINKAT 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "unlockpt" "ac_cv_func_unlockpt" +if test "x$ac_cv_func_unlockpt" = xyes +then : + printf "%s\n" "#define HAVE_UNLOCKPT 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "utimensat" "ac_cv_func_utimensat" if test "x$ac_cv_func_utimensat" = xyes @@ -28901,7 +28975,8 @@ then : if true then : - if test "$ac_cv_func_getgrgid" = yes -o "$ac_cv_func_getgrgid_r" = yes + if test "$ac_cv_func_getgrent" = "yes" && + { test "$ac_cv_func_getgrgid" = "yes" || test "$ac_cv_func_getgrgid_r" = "yes"; } then : py_cv_module_grp=yes else $as_nop diff --git a/configure.ac b/configure.ac index 9587e6d63499aac..dee7ed552b370f0 100644 --- a/configure.ac +++ b/configure.ac @@ -973,14 +973,13 @@ AS_CASE([$host/$ac_cv_cc_name], [aarch64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux ARM64, glibc, gcc+clang [aarch64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], [powerpc64le-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux on PPC64 little endian, glibc, gcc + [wasm32-unknown-wasi/clang], [PY_SUPPORT_TIER=2], dnl WebAssembly System Interface, clang [x86_64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], dnl Linux on AMD64, any vendor, glibc, clang [aarch64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=3], dnl Windows ARM64, MSVC [armv7l-*-linux-gnueabihf/gcc], [PY_SUPPORT_TIER=3], dnl ARMv7 LE with hardware floats, any vendor, glibc, gcc [powerpc64le-*-linux-gnu/clang], [PY_SUPPORT_TIER=3], dnl Linux on PPC64 little endian, glibc, clang [s390x-*-linux-gnu/gcc], [PY_SUPPORT_TIER=3], dnl Linux on 64bit s390x (big endian), glibc, gcc - [wasm32-unknown-emscripten/clang], [PY_SUPPORT_TIER=3], dnl WebAssembly Emscripten - [wasm32-unknown-wasi/clang], [PY_SUPPORT_TIER=3], dnl WebAssembly System Interface [x86_64-*-freebsd*/clang], [PY_SUPPORT_TIER=3], dnl FreeBSD on AMD64 [PY_SUPPORT_TIER=0] ) @@ -1579,6 +1578,28 @@ else AC_MSG_RESULT([no]) fi +# Check for --enable-experimental-jit: +AC_MSG_CHECKING([for --enable-experimental-jit]) +AC_ARG_ENABLE([experimental-jit], + [AS_HELP_STRING([--enable-experimental-jit], + [build the experimental just-in-time compiler (default is no)])], + [], + [enable_experimental_jit=no]) +AS_VAR_IF([enable_experimental_jit], + [no], + [], + [AS_VAR_APPEND([CFLAGS_NODIST], [" -D_Py_JIT"]) + AS_VAR_SET([REGEN_JIT_COMMAND], + ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host"]) + AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"]) + AS_VAR_IF([Py_DEBUG], + [true], + [AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])], + [])]) +AC_SUBST([REGEN_JIT_COMMAND]) +AC_SUBST([JIT_STENCILS_H]) +AC_MSG_RESULT([$enable_experimental_jit]) + # Enable optimization flags AC_SUBST([DEF_MAKE_ALL_RULE]) AC_SUBST([DEF_MAKE_RULE]) @@ -4767,17 +4788,17 @@ AC_CHECK_FUNCS([ \ copy_file_range ctermid dup dup3 execv explicit_bzero explicit_memset \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fork fork1 fpathconf fstatat ftime ftruncate futimens futimes futimesat \ - gai_strerror getegid getentropy geteuid getgid getgrgid getgrgid_r \ + gai_strerror getegid getentropy geteuid getgid getgrent getgrgid getgrgid_r \ getgrnam_r getgrouplist getgroups gethostname getitimer getloadavg getlogin \ getpeername getpgid getpid getppid getpriority _getpty \ getpwent getpwnam_r getpwuid getpwuid_r getresgid getresuid getrusage getsid getspent \ - getspnam getuid getwd if_nameindex initgroups kill killpg lchown linkat \ + getspnam getuid getwd grantpt if_nameindex initgroups kill killpg lchown linkat \ lockf lstat lutimes madvise mbrtowc memrchr mkdirat mkfifo mkfifoat \ mknod mknodat mktime mmap mremap nice openat opendir pathconf pause pipe \ - pipe2 plock poll posix_fadvise posix_fallocate posix_spawn posix_spawnp \ + pipe2 plock poll posix_fadvise posix_fallocate posix_openpt posix_spawn posix_spawnp \ posix_spawn_file_actions_addclosefrom_np \ - pread preadv preadv2 pthread_condattr_setclock pthread_init pthread_kill \ - pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ + pread preadv preadv2 pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ + pthread_kill ptsname ptsname_r pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ rtpSpawn sched_get_priority_max sched_rr_get_interval sched_setaffinity \ sched_setparam sched_setscheduler sem_clockwait sem_getvalue sem_open \ sem_timedwait sem_unlink sendfile setegid seteuid setgid sethostname \ @@ -4786,7 +4807,7 @@ AC_CHECK_FUNCS([ \ sigfillset siginterrupt sigpending sigrelse sigtimedwait sigwait \ sigwaitinfo snprintf splice strftime strlcpy strsignal symlinkat sync \ sysconf system tcgetpgrp tcsetpgrp tempnam timegm times tmpfile \ - tmpnam tmpnam_r truncate ttyname umask uname unlinkat utimensat utimes vfork \ + tmpnam tmpnam_r truncate ttyname umask uname unlinkat unlockpt utimensat utimes vfork \ wait wait3 wait4 waitid waitpid wcscoll wcsftime wcsxfrm wmemcmp writev \ ]) @@ -7293,7 +7314,9 @@ PY_STDLIB_MOD([_socket], -a "$ac_cv_header_netinet_in_h" = "yes"])) dnl platform specific extensions -PY_STDLIB_MOD([grp], [], [test "$ac_cv_func_getgrgid" = yes -o "$ac_cv_func_getgrgid_r" = yes]) +PY_STDLIB_MOD([grp], [], + [test "$ac_cv_func_getgrent" = "yes" && + { test "$ac_cv_func_getgrgid" = "yes" || test "$ac_cv_func_getgrgid_r" = "yes"; }]) PY_STDLIB_MOD([pwd], [], [test "$ac_cv_func_getpwuid" = yes -o "$ac_cv_func_getpwuid_r" = yes]) PY_STDLIB_MOD([resource], [], [test "$ac_cv_header_sys_resource_h" = yes]) PY_STDLIB_MOD([_scproxy], diff --git a/pyconfig.h.in b/pyconfig.h.in index d8a9f68951afbd1..2b4bb1a2b528662 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -474,6 +474,9 @@ /* Define to 1 if you have the `getgid' function. */ #undef HAVE_GETGID +/* Define to 1 if you have the `getgrent' function. */ +#undef HAVE_GETGRENT + /* Define to 1 if you have the `getgrgid' function. */ #undef HAVE_GETGRGID @@ -601,6 +604,9 @@ bcopy. */ #undef HAVE_GLIBC_MEMMOVE_BUG +/* Define to 1 if you have the `grantpt' function. */ +#undef HAVE_GRANTPT + /* Define to 1 if you have the <grp.h> header file. */ #undef HAVE_GRP_H @@ -899,6 +905,9 @@ /* Define to 1 if you have the `posix_fallocate' function. */ #undef HAVE_POSIX_FALLOCATE +/* Define to 1 if you have the `posix_openpt' function. */ +#undef HAVE_POSIX_OPENPT + /* Define to 1 if you have the `posix_spawn' function. */ #undef HAVE_POSIX_SPAWN @@ -930,6 +939,10 @@ /* Define to 1 if you have the `pthread_condattr_setclock' function. */ #undef HAVE_PTHREAD_CONDATTR_SETCLOCK +/* Define to 1 if you have the `pthread_cond_timedwait_relative_np' function. + */ +#undef HAVE_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP + /* Defined for Solaris 2.6 bug in pthread header. */ #undef HAVE_PTHREAD_DESTRUCTOR @@ -951,6 +964,12 @@ /* Define if platform requires stubbed pthreads support */ #undef HAVE_PTHREAD_STUBS +/* Define to 1 if you have the `ptsname' function. */ +#undef HAVE_PTSNAME + +/* Define to 1 if you have the `ptsname_r' function. */ +#undef HAVE_PTSNAME_R + /* Define to 1 if you have the <pty.h> header file. */ #undef HAVE_PTY_H @@ -1459,6 +1478,9 @@ /* Define to 1 if you have the `unlinkat' function. */ #undef HAVE_UNLINKAT +/* Define to 1 if you have the `unlockpt' function. */ +#undef HAVE_UNLOCKPT + /* Define to 1 if you have the `unshare' function. */ #undef HAVE_UNSHARE