Merge branch 'main' into docs-floating-point

Doc/library/functions.rst

@@ -1509,7 +1509,7 @@ are always available.  They are listed here in alphabetical order.
    (where :func:`open` is declared), :mod:`os`, :mod:`os.path`, :mod:`tempfile`,
    and :mod:`shutil`.
 
-   .. audit-event:: open file,mode,flags open
+   .. audit-event:: open path,mode,flags open
 
    The ``mode`` and ``flags`` arguments may have been modified or inferred from
    the original call.

Include/cpython/pystate.h

@@ -68,8 +68,6 @@ struct _ts {
        pycore_ceval.h. */
     uintptr_t eval_breaker;
 
-    PyObject *asyncio_running_loop; // Strong reference
-
     struct {
         /* Has been initialized to a safe state.
 

Include/internal/pycore_object.h

@@ -15,6 +15,30 @@ extern "C" {
 #include "pycore_pyatomic_ft_wrappers.h"  // FT_ATOMIC_STORE_PTR_RELAXED
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
 
+
+#define _Py_IMMORTAL_REFCNT_LOOSE ((_Py_IMMORTAL_REFCNT >> 1) + 1)
+
+// gh-121528, gh-118997: Similar to _Py_IsImmortal() but be more loose when
+// comparing the reference count to stay compatible with C extensions built
+// with the stable ABI 3.11 or older. Such extensions implement INCREF/DECREF
+// as refcnt++ and refcnt-- without taking in account immortal objects. For
+// example, the reference count of an immortal object can change from
+// _Py_IMMORTAL_REFCNT to _Py_IMMORTAL_REFCNT+1 (INCREF) or
+// _Py_IMMORTAL_REFCNT-1 (DECREF).
+//
+// This function should only be used in assertions. Otherwise, _Py_IsImmortal()
+// must be used instead.
+static inline int _Py_IsImmortalLoose(PyObject *op)
+{
+#if defined(Py_GIL_DISABLED)
+    return _Py_IsImmortal(op);
+#else
+    return (op->ob_refcnt >= _Py_IMMORTAL_REFCNT_LOOSE);
+#endif
+}
+#define _Py_IsImmortalLoose(op) _Py_IsImmortalLoose(_PyObject_CAST(op))
+
+
 /* Check if an object is consistent. For example, ensure that the reference
    counter is greater than or equal to 1, and ensure that ob_type is not NULL.
 
@@ -134,7 +158,7 @@ PyAPI_FUNC(void) _Py_SetImmortalUntracked(PyObject *op);
 static inline void _Py_SetMortal(PyObject *op, Py_ssize_t refcnt)
 {
     if (op) {
-        assert(_Py_IsImmortal(op));
+        assert(_Py_IsImmortalLoose(op));
 #ifdef Py_GIL_DISABLED
         op->ob_tid = _Py_UNOWNED_TID;
         op->ob_ref_local = 0;
@@ -266,7 +290,7 @@ _PyObject_Init(PyObject *op, PyTypeObject *typeobj)
 {
     assert(op != NULL);
     Py_SET_TYPE(op, typeobj);
-    assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortal(typeobj));
+    assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortalLoose(typeobj));
     Py_INCREF(typeobj);
     _Py_NewReference(op);
 }

Include/internal/pycore_tstate.h

@@ -21,6 +21,8 @@ typedef struct _PyThreadStateImpl {
     // semi-public fields are in PyThreadState.
     PyThreadState base;
 
+    PyObject *asyncio_running_loop; // Strong reference
+
     struct _qsbr_thread_state *qsbr;  // only used by free-threaded build
     struct llist_node mem_free_queue; // delayed free queue
 

Lib/_pyrepl/__main__.py

@@ -1,3 +1,6 @@
+# Important: don't add things to this module, as they will end up in the REPL's
+# default globals.  Use _pyrepl.main instead.
+
 if __name__ == "__main__":
     from .main import interactive_console as __pyrepl_interactive_console
     __pyrepl_interactive_console()

Lib/idlelib/News3.txt

@@ -4,6 +4,9 @@ Released on 2024-10-xx
 =========================
 
 
+gh-78889: Stop Shell freezes by blocking user access to non-method
+sys.stdout.shell attributes, which are all private.
+
 gh-78955: Use user-selected color theme for Help => IDLE Doc.
 
 gh-96905: In idlelib code, stop redefining built-ins 'dict' and 'object'.

Lib/idlelib/run.py

@@ -443,6 +443,9 @@ class StdioFile(io.TextIOBase):
 
     def __init__(self, shell, tags, encoding='utf-8', errors='strict'):
         self.shell = shell
+        # GH-78889: accessing unpickleable attributes freezes Shell.
+        # IDLE only needs methods; allow 'width' for possible use.
+        self.shell._RPCProxy__attributes = {'width': 1}
         self.tags = tags
         self._encoding = encoding
         self._errors = errors

Lib/symtable.py

@@ -249,14 +249,26 @@ def is_local_symbol(ident):
                 if is_local_symbol(st.name):
                     match st.type:
                         case _symtable.TYPE_FUNCTION:
+                            # generators are of type TYPE_FUNCTION with a ".0"
+                            # parameter as a first parameter (which makes them
+                            # distinguishable from a function named 'genexpr')
+                            if st.name == 'genexpr' and '.0' in st.varnames:
+                                continue
                             d[st.name] = 1
                         case _symtable.TYPE_TYPE_PARAMETERS:
                             # Get the function-def block in the annotation
                             # scope 'st' with the same identifier, if any.
                             scope_name = st.name
                             for c in st.children:
                                 if c.name == scope_name and c.type == _symtable.TYPE_FUNCTION:
-                                    d[st.name] = 1
+                                    # A generic generator of type TYPE_FUNCTION
+                                    # cannot be a direct child of 'st' (but it
+                                    # can be a descendant), e.g.:
+                                    #
+                                    # class A:
+                                    #   type genexpr[genexpr] = (x for x in [])
+                                    assert scope_name != 'genexpr' or '.0' not in c.varnames
+                                    d[scope_name] = 1
                                     break
             self.__methods = tuple(d)
         return self.__methods

Lib/test/crashers/bogus_code_obj.py

@@ -12,8 +12,8 @@
 
 """
 
-import types
+def f():
+    pass
 
-co = types.CodeType(0, 0, 0, 0, 0, 0, b'\x04\x00\x71\x00',
-                    (), (), (), '', '', 1, b'')
-exec(co)
+f.__code__ = f.__code__.replace(co_code=b"")
+f()

Lib/test/support/__init__.py

@@ -2614,14 +2614,18 @@ def force_not_colorized(func):
     def wrapper(*args, **kwargs):
         import _colorize
         original_fn = _colorize.can_colorize
-        variables = {"PYTHON_COLORS": None, "FORCE_COLOR": None}
+        variables: dict[str, str | None] = {
+            "PYTHON_COLORS": None, "FORCE_COLOR": None, "NO_COLOR": None
+        }
         try:
             for key in variables:
                 variables[key] = os.environ.pop(key, None)
+            os.environ["NO_COLOR"] = "1"
             _colorize.can_colorize = lambda: False
             return func(*args, **kwargs)
         finally:
             _colorize.can_colorize = original_fn
+            del os.environ["NO_COLOR"]
             for key, value in variables.items():
                 if value is not None:
                     os.environ[key] = value

Lib/test/test_pyrepl/support.py

@@ -1,3 +1,4 @@
+import os
 from code import InteractiveConsole
 from functools import partial
 from typing import Iterable
@@ -100,8 +101,18 @@ def handle_all_events(
 )
 
 
+def make_clean_env() -> dict[str, str]:
+    clean_env = os.environ.copy()
+    for k in clean_env.copy():
+        if k.startswith("PYTHON"):
+            clean_env.pop(k)
+    clean_env.pop("FORCE_COLOR", None)
+    clean_env.pop("NO_COLOR", None)
+    return clean_env
+
+
 class FakeConsole(Console):
-    def __init__(self, events, encoding="utf-8"):
+    def __init__(self, events, encoding="utf-8") -> None:
         self.events = iter(events)
         self.encoding = encoding
         self.screen = []

Lib/test/test_pyrepl/test_pyrepl.py

@@ -2,6 +2,7 @@
 import itertools
 import os
 import pathlib
+import re
 import rlcompleter
 import select
 import subprocess
@@ -21,7 +22,8 @@
     more_lines,
     multiline_input,
     code_to_events,
-    clean_screen
+    clean_screen,
+    make_clean_env,
 )
 from _pyrepl.console import Event
 from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig
@@ -487,6 +489,26 @@ def prepare_reader(self, events):
         reader.can_colorize = False
         return reader
 
+    def test_stdin_is_tty(self):
+        # Used during test log analysis to figure out if a TTY was available.
+        try:
+            if os.isatty(sys.stdin.fileno()):
+                return
+        except OSError as ose:
+            self.skipTest(f"stdin tty check failed: {ose}")
+        else:
+            self.skipTest("stdin is not a tty")
+
+    def test_stdout_is_tty(self):
+        # Used during test log analysis to figure out if a TTY was available.
+        try:
+            if os.isatty(sys.stdout.fileno()):
+                return
+        except OSError as ose:
+            self.skipTest(f"stdout tty check failed: {ose}")
+        else:
+            self.skipTest("stdout is not a tty")
+
     def test_basic(self):
         reader = self.prepare_reader(code_to_events("1+1\n"))
 
@@ -888,12 +910,7 @@ def setUp(self):
         # Cleanup from PYTHON* variables to isolate from local
         # user settings, see #121359.  Such variables should be
         # added later in test methods to patched os.environ.
-        clean_env = os.environ.copy()
-        for k in clean_env.copy():
-            if k.startswith("PYTHON"):
-                clean_env.pop(k)
-
-        patcher = patch('os.environ', new=clean_env)
+        patcher = patch('os.environ', new=make_clean_env())
         self.addCleanup(patcher.stop)
         patcher.start()
 
@@ -920,6 +937,84 @@ def test_exposed_globals_in_repl(self):
 
         self.assertTrue(case1 or case2 or case3 or case4, output)
 
+    def _assertMatchOK(
+            self, var: str, expected: str | re.Pattern, actual: str
+    ) -> None:
+        if isinstance(expected, re.Pattern):
+            self.assertTrue(
+                expected.match(actual),
+                f"{var}={actual} does not match {expected.pattern}",
+            )
+        else:
+            self.assertEqual(
+                actual,
+                expected,
+                f"expected {var}={expected}, got {var}={actual}",
+            )
+
+    @force_not_colorized
+    def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False):
+        clean_env = make_clean_env()
+        clean_env["NO_COLOR"] = "1"  # force_not_colorized doesn't touch subprocesses
+
+        with tempfile.TemporaryDirectory() as td:
+            blue = pathlib.Path(td) / "blue"
+            blue.mkdir()
+            mod = blue / "calx.py"
+            mod.write_text("FOO = 42", encoding="utf-8")
+            commands = [
+                "print(f'{" + var + "=}')" for var in expectations
+            ] + ["exit"]
+            if as_file and as_module:
+                self.fail("as_file and as_module are mutually exclusive")
+            elif as_file:
+                output, exit_code = self.run_repl(
+                    commands,
+                    cmdline_args=[str(mod)],
+                    env=clean_env,
+                )
+            elif as_module:
+                output, exit_code = self.run_repl(
+                    commands,
+                    cmdline_args=["-m", "blue.calx"],
+                    env=clean_env,
+                    cwd=td,
+                )
+            else:
+                self.fail("Choose one of as_file or as_module")
+
+        if "can't use pyrepl" in output:
+            self.skipTest("pyrepl not available")
+
+        self.assertEqual(exit_code, 0)
+        for var, expected in expectations.items():
+            with self.subTest(var=var, expected=expected):
+                if m := re.search(rf"[\r\n]{var}=(.+?)[\r\n]", output):
+                    self._assertMatchOK(var, expected, actual=m.group(1))
+                else:
+                    self.fail(f"{var}= not found in output")
+
+        self.assertNotIn("Exception", output)
+        self.assertNotIn("Traceback", output)
+
+    def test_inspect_keeps_globals_from_inspected_file(self):
+        expectations = {
+            "FOO": "42",
+            "__name__": "'__main__'",
+            "__package__": "None",
+            # "__file__" is missing in -i, like in the basic REPL
+        }
+        self._run_repl_globals_test(expectations, as_file=True)
+
+    def test_inspect_keeps_globals_from_inspected_module(self):
+        expectations = {
+            "FOO": "42",
+            "__name__": "'__main__'",
+            "__package__": "'blue'",
+            "__file__": re.compile(r"^'.*calx.py'$"),
+        }
+        self._run_repl_globals_test(expectations, as_module=True)
+
     def test_dumb_terminal_exits_cleanly(self):
         env = os.environ.copy()
         env.update({"TERM": "dumb"})
@@ -981,16 +1076,27 @@ def test_not_wiping_history_file(self):
         self.assertIn("spam", output)
         self.assertNotEqual(pathlib.Path(hfile.name).stat().st_size, 0)
 
-    def run_repl(self, repl_input: str | list[str], env: dict | None = None) -> tuple[str, int]:
+    def run_repl(
+        self,
+        repl_input: str | list[str],
+        env: dict | None = None,
+        *,
+        cmdline_args: list[str] | None = None,
+        cwd: str | None = None,
+    ) -> tuple[str, int]:
+        assert pty
         master_fd, slave_fd = pty.openpty()
         cmd = [sys.executable, "-i", "-u"]
         if env is None:
             cmd.append("-I")
+        if cmdline_args is not None:
+            cmd.extend(cmdline_args)
         process = subprocess.Popen(
             cmd,
             stdin=slave_fd,
             stdout=slave_fd,
             stderr=slave_fd,
+            cwd=cwd,
             text=True,
             close_fds=True,
             env=env if env else os.environ,