Merge branch 'main' into tk9_for_python314

.github/CODEOWNERS

@@ -33,11 +33,11 @@ Objects/frameobject.c         @markshannon
 Objects/call.c                @markshannon
 Python/ceval*.c               @markshannon
 Python/ceval*.h               @markshannon
+Python/codegen.c              @markshannon @iritkatriel
 Python/compile.c              @markshannon @iritkatriel
 Python/assemble.c             @markshannon @iritkatriel
 Python/flowgraph.c            @markshannon @iritkatriel
 Python/instruction_sequence.c @iritkatriel
-Python/ast_opt.c              @isidentical
 Python/bytecodes.c            @markshannon
 Python/optimizer*.c           @markshannon
 Python/optimizer_analysis.c   @Fidget-Spinner
@@ -157,10 +157,12 @@ Include/internal/pycore_time.h  @pganssle @abalkin
 /Tools/cases_generator/        @markshannon
 
 # AST
-Python/ast.c                  @isidentical @JelleZijlstra
-Parser/asdl.py                @isidentical @JelleZijlstra
-Parser/asdl_c.py              @isidentical @JelleZijlstra
-Lib/ast.py                    @isidentical @JelleZijlstra
+Python/ast.c                  @isidentical @JelleZijlstra @eclips4
+Python/ast_opt.c              @isidentical @eclips4
+Parser/asdl.py                @isidentical @JelleZijlstra @eclips4
+Parser/asdl_c.py              @isidentical @JelleZijlstra @eclips4
+Lib/ast.py                    @isidentical @JelleZijlstra @eclips4
+Lib/test/test_ast/            @eclips4
 
 # Mock
 /Lib/unittest/mock.py         @cjw296

.github/workflows/build.yml

@@ -195,13 +195,14 @@ jobs:
 
   build_ubuntu_ssltests:
     name: 'Ubuntu SSL tests with OpenSSL'
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.os }}
     timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     strategy:
       fail-fast: false
       matrix:
+        os: [ubuntu-22.04]
         openssl_ver: [3.0.15, 3.1.7, 3.2.3, 3.3.2]
     env:
       OPENSSL_VER: ${{ matrix.openssl_ver }}
@@ -231,7 +232,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
-        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+        key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
     - name: Install OpenSSL
       if: steps.cache-openssl.outputs.cache-hit != 'true'
       run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
@@ -410,7 +411,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
-        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+        key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
     - name: Install OpenSSL
       if: steps.cache-openssl.outputs.cache-hit != 'true'
       run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux

.github/workflows/jit.yml

@@ -110,17 +110,15 @@ jobs:
       - name: Native Windows
         if: runner.os == 'Windows' && matrix.architecture != 'ARM64'
         run: |
-          choco upgrade llvm -y
-          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
+          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0
           ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }}
           ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
       # No PGO or tests (yet):
       - name: Emulated Windows
         if: runner.os == 'Windows' && matrix.architecture == 'ARM64'
         run: |
-          choco upgrade llvm -y
-          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
+          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0
           ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }}
 
       - name: Native macOS
@@ -159,7 +157,7 @@ jobs:
           CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \
             CPP="$CC --preprocess" \
             HOSTRUNNER=qemu-${{ matrix.architecture }} \
-            ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
+            ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
           make all --jobs 4
           ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 

.github/workflows/reusable-macos.yml

@@ -35,7 +35,7 @@ jobs:
         path: config.cache
         key: ${{ github.job }}-${{ inputs.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}
     - name: Install Homebrew dependencies
-      run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk
+      run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk make
     - name: Configure CPython
       run: |
         GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \
@@ -44,14 +44,27 @@ jobs:
           --config-cache \
           --with-pydebug \
           --enable-slower-safety \
+          --enable-safety \
           ${{ inputs.free-threading && '--disable-gil' || '' }} \
           --prefix=/opt/python-dev \
           --with-openssl="$(brew --prefix openssl@3.0)"
     - name: Build CPython
-      run: set -o pipefail; make -j8 2>&1 | tee compiler_output.txt
+      if : ${{ inputs.free-threading || inputs.os != 'macos-13' }}
+      run: gmake -j8
+    - name: Build CPython for compiler warning check
+      if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }}
+      run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt
     - name: Display build info
       run: make pythoninfo
     - name: Check compiler warnings
-      run: python3 Tools/build/check_warnings.py --compiler-output-file-path=compiler_output.txt --warning-ignore-file-path=Tools/build/.warningignore_macos --compiler-output-type=clang
+      if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }}
+      run: >-
+        python3 Tools/build/check_warnings.py
+        --compiler-output-file-path=compiler_output_macos.txt
+        --warning-ignore-file-path=Tools/build/.warningignore_macos
+        --compiler-output-type=clang
+        --fail-on-regression
+        --fail-on-improvement
+        --path-prefix="./"
     - name: Tests
       run: make test

.github/workflows/reusable-ubuntu.yml

@@ -14,7 +14,11 @@ jobs:
   build_ubuntu_reusable:
     name: 'build and test'
     timeout-minutes: 60
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-22.04]
     env:
       FORCE_COLOR: 1
       OPENSSL_VER: 3.0.15
@@ -36,7 +40,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
-        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+        key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
     - name: Install OpenSSL
       if: steps.cache-openssl.outputs.cache-hit != 'true'
       run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
@@ -67,20 +71,33 @@ jobs:
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: >-
         ../cpython-ro-srcdir/configure
-        CFLAGS="-fdiagnostics-format=json"
         --config-cache
         --with-pydebug
         --enable-slower-safety
+        --enable-safety
         --with-openssl=$OPENSSL_DIR
         ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }}
     - name: Build CPython out-of-tree
+      if: ${{ inputs.free-threading }}
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
-      run: set -o pipefail; make -j4 2>&1 | tee compiler_output.txt
+      run: make -j4
+    - name: Build CPython out-of-tree (for compiler warning check)
+      if: ${{ !inputs.free-threading}}
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: set -o pipefail; make -j4 --output-sync 2>&1 | tee compiler_output_ubuntu.txt
     - name: Display build info
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: make pythoninfo
     - name: Check compiler warnings
-      run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu --compiler-output-type=json
+      if: ${{ !inputs.free-threading }}
+      run: >-
+        python Tools/build/check_warnings.py
+        --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output_ubuntu.txt
+        --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu
+        --compiler-output-type=gcc
+        --fail-on-regression
+        --fail-on-improvement
+        --path-prefix="../cpython-ro-srcdir/"
     - name: Remount sources writable for tests
       # some tests write to srcdir, lack of pyc files slows down testing
       run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.10
+    rev: v0.6.7
     hooks:
       - id: ruff
         name: Run Ruff (lint) on Doc/
@@ -10,6 +10,10 @@ repos:
         name: Run Ruff (lint) on Lib/test/
         args: [--exit-non-zero-on-fix]
         files: ^Lib/test/
+      - id: ruff
+        name: Run Ruff (lint) on Tools/build/check_warnings.py
+        args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml]
+        files: ^Tools/build/check_warnings.py
       - id: ruff
         name: Run Ruff (lint) on Argument Clinic
         args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml]
@@ -22,6 +26,11 @@ repos:
   - repo: https://github.com/psf/black-pre-commit-mirror
     rev: 24.8.0
     hooks:
+      - id: black
+        name: Run Black on Tools/build/check_warnings.py
+        files: ^Tools/build/check_warnings.py
+        language_version: python3.12
+        args: [--line-length=79]
       - id: black
         name: Run Black on Tools/jit/
         files: ^Tools/jit/

Android/README.md

@@ -1,19 +1,19 @@
 # Python for Android
 
 These instructions are only needed if you're planning to compile Python for
-Android yourself. Most users should *not* need to do this. If you're looking to
-use Python on Android, one of the following tools will provide a much more
-approachable user experience:
-
-* [Briefcase](https://briefcase.readthedocs.io), from the BeeWare project
-* [Buildozer](https://buildozer.readthedocs.io), from the Kivy project
-* [Chaquopy](https://chaquo.com/chaquopy/)
+Android yourself. Most users should *not* need to do this. Instead, use one of
+the tools listed in `Doc/using/android.rst`, which will provide a much easier
+experience.
 
 
 ## Prerequisites
 
-Export the `ANDROID_HOME` environment variable to point at your Android SDK. If
-you don't already have the SDK, here's how to install it:
+First, make sure you have all the usual tools and libraries needed to build
+Python for your development machine.
+
+Second, you'll need an Android SDK. If you already have the SDK installed,
+export the `ANDROID_HOME` environment variable to point at its location.
+Otherwise, here's how to install it:
 
 * Download the "Command line tools" from <https://developer.android.com/studio>.
 * Create a directory `android-sdk/cmdline-tools`, and unzip the command line
@@ -37,11 +37,6 @@ development tools, which currently means Linux or macOS. This involves doing a
 cross-build where you use a "build" Python (for your development machine) to
 help produce a "host" Python for Android.
 
-First, make sure you have all the usual tools and libraries needed to build
-Python for your development machine. The only Android tool you need to install
-is the command line tools package above: the build script will download the
-rest.
-
 The easiest way to do a build is to use the `android.py` script. You can either
 have it perform the entire build process from start to finish in one step, or
 you can do it in discrete steps that mirror running `configure` and `make` for
@@ -80,17 +75,20 @@ call. For example, if you want a pydebug build that also caches the results from
 
 ## Testing
 
-The tests can be run on Linux, macOS, or Windows, although on Windows you'll
-have to build the `cross-build/HOST` subdirectory on one of the other platforms
-and copy it over.
+The test suite can be run on Linux, macOS, or Windows:
 
-The test suite can usually be run on a device with 2 GB of RAM, though for some
-configurations or test orders you may need to increase this. As of Android
+* On Linux, the emulator needs access to the KVM virtualization interface, and
+  a DISPLAY environment variable pointing at an X server.
+* On Windows, you won't be able to do the build on the same machine, so you'll
+  have to copy the `cross-build/HOST` directory from somewhere else.
+
+The test suite can usually be run on a device with 2 GB of RAM, but this is
+borderline, so you may need to increase it to 4 GB. As of Android
 Studio Koala, 2 GB is the default for all emulators, although the user interface
-may indicate otherwise. The effective setting is `hw.ramSize` in
-~/.android/avd/*.avd/hardware-qemu.ini, whereas Android Studio displays the
-value from config.ini. Changing the value in Android Studio will update both of
-these files.
+may indicate otherwise. Locate the emulator's directory under `~/.android/avd`,
+and find `hw.ramSize` in both config.ini and hardware-qemu.ini. Either set these
+manually to the same value, or use the Android Studio Device Manager, which will
+update both files.
 
 Before running the test suite, follow the instructions in the previous section
 to build the architecture you want to test. Then run the test script in one of
@@ -131,3 +129,8 @@ Every time you run `android.py test`, changes in pure-Python files in the
 repository's `Lib` directory will be picked up immediately. Changes in C files,
 and architecture-specific files such as sysconfigdata, will not take effect
 until you re-run `android.py make-host` or `build`.
+
+
+## Using in your own app
+
+See `Doc/using/android.rst`.

Android/android.py

@@ -259,8 +259,8 @@ def setup_testbed():
              f"{temp_dir}/{outer_jar}", "gradle-wrapper.jar"])
 
 
-# run_testbed will build the app automatically, but it hides the Gradle output
-# by default, so it's useful to have this as a separate command for the buildbot.
+# run_testbed will build the app automatically, but it's useful to have this as
+# a separate command to allow running the app outside of this script.
 def build_testbed(context):
     setup_sdk()
     setup_testbed()
@@ -376,6 +376,8 @@ async def find_pid(serial):
     shown_error = False
     while True:
         try:
+            # `pidof` requires API level 24 or higher. The level 23 emulator
+            # includes it, but it doesn't work (it returns all processes).
             pid = (await async_check_output(
                 adb, "-s", serial, "shell", "pidof", "-s", APP_ID
             )).strip()
@@ -407,6 +409,7 @@ async def logcat_task(context, initial_devices):
     serial = await wait_for(find_device(context, initial_devices), startup_timeout)
     pid = await wait_for(find_pid(serial), startup_timeout)
 
+    # `--pid` requires API level 24 or higher.
     args = [adb, "-s", serial, "logcat", "--pid", pid,  "--format", "tag"]
     hidden_output = []
     async with async_process(
@@ -421,11 +424,15 @@ async def logcat_task(context, initial_devices):
                 # such messages, but other components might.
                 level, message = None, line
 
+            # Exclude high-volume messages which are rarely useful.
+            if context.verbose < 2 and "from python test_syslog" in message:
+                continue
+
             # Put high-level messages on stderr so they're highlighted in the
             # buildbot logs. This will include Python's own stderr.
             stream = (
                 sys.stderr
-                if level in ["E", "F"]  # ERROR and FATAL (aka ASSERT)
+                if level in ["W", "E", "F"]  # WARNING, ERROR, FATAL (aka ASSERT)
                 else sys.stdout
             )
 
@@ -573,8 +580,9 @@ def parse_args():
     test = subcommands.add_parser(
         "test", help="Run the test suite")
     test.add_argument(
-        "-v", "--verbose", action="store_true",
-        help="Show Gradle output, and non-Python logcat messages")
+        "-v", "--verbose", action="count", default=0,
+        help="Show Gradle output, and non-Python logcat messages. "
+        "Use twice to include high-volume messages which are rarely useful.")
     device_group = test.add_mutually_exclusive_group(required=True)
     device_group.add_argument(
         "--connected", metavar="SERIAL", help="Run on a connected device. "
@@ -591,6 +599,13 @@ def parse_args():
 
 def main():
     install_signal_handler()
+
+    # Under the buildbot, stdout is not a TTY, but we must still flush after
+    # every line to make sure our output appears in the correct order relative
+    # to the output of our subprocesses.
+    for stream in [sys.stdout, sys.stderr]:
+        stream.reconfigure(line_buffering=True)
+
     context = parse_args()
     dispatch = {"configure-build": configure_build_python,
                 "make-build": make_build_python,