Merge remote-tracking branch 'upstream/main' into pythongh-123085/inf…

…erred-compiled

Doc/Makefile

@@ -222,16 +222,6 @@ dist:
 	cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-a4.tar.bz2
 	@echo "Build finished and archived!"
 
-	# archive the letter latex
-	@echo "Building LaTeX (US paper)..."
-	rm -rf build/latex
-	$(MAKE) latex PAPER=letter
-	-sed -i 's/: all-$$(FMT)/:/' build/latex/Makefile
-	(cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`nproc`+1)) --output-sync LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2)
-	cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-letter.zip
-	cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-letter.tar.bz2
-	@echo "Build finished and archived!"
-
 	# copy the epub build
 	@echo "Building EPUB..."
 	rm -rf build/epub

Doc/library/inspect.rst

@@ -1043,7 +1043,8 @@ function.
    .. attribute:: BoundArguments.kwargs
 
       A dict of keyword arguments values.  Dynamically computed from the
-      :attr:`arguments` attribute.
+      :attr:`arguments` attribute.  Arguments that can be passed positionally
+      are included in :attr:`args` instead.
 
    .. attribute:: BoundArguments.signature
 

Doc/tools/templates/download.html

@@ -25,12 +25,7 @@ <h1>{% trans %}Download Python {{ release }} Documentation{% endtrans %}</h1>
     <th>{% trans %}Packed as .tar.bz2{% endtrans %}</th>
   </tr>
   <tr>
-    <td>{% trans %}PDF (US-Letter paper size){% endtrans %}</td>
-    <td>{% trans download_size="17" %}<a href="{{ dlbase }}/python-{{ release }}-docs-pdf-letter.zip">Download</a> (ca. {{ download_size }} MiB){% endtrans %}</td>
-    <td>{% trans download_size="17" %}<a href="{{ dlbase }}/python-{{ release }}-docs-pdf-letter.tar.bz2">Download</a> (ca. {{ download_size }} MiB){% endtrans %}</td>
-  </tr>
-  <tr>
-    <td>{% trans %}PDF (A4 paper size){% endtrans %}</td>
+    <td>{% trans %}PDF{% endtrans %}</td>
     <td>{% trans download_size="17" %}<a href="{{ dlbase }}/python-{{ release }}-docs-pdf-a4.zip">Download</a> (ca. {{ download_size }} MiB){% endtrans %}</td>
     <td>{% trans download_size="17" %}<a href="{{ dlbase }}/python-{{ release }}-docs-pdf-a4.tar.bz2">Download</a> (ca. {{ download_size }} MiB){% endtrans %}</td>
   </tr>

Doc/whatsnew/3.13.rst

@@ -159,6 +159,7 @@ C API improvements:
 * The :doc:`PyTime C API </c-api/time>` has been added,
   providing access to system clocks.
 * :c:type:`PyMutex` is a new lightweight mutex that occupies a single byte.
+* :doc:`C-API support </c-api/monitoring>` was added for generating :pep:`669` monitoring events.
 
 New typing features:
 
@@ -2175,6 +2176,9 @@ New Features
   if the operation needs to block.
   (Contributed by Sam Gross in :gh:`108724`.)
 
+* Add C API functions for generating :pep:`669` monitoring events.
+  (Contributed by Irit Katriel in :gh:`111997`).
+
 Build Changes
 =============
 

Include/internal/pycore_frame.h

@@ -60,7 +60,7 @@ enum _frameowner {
 };
 
 typedef struct _PyInterpreterFrame {
-    PyObject *f_executable; /* Strong reference (code object or None) */
+    _PyStackRef f_executable; /* Deferred or strong reference (code object or None) */
     struct _PyInterpreterFrame *previous;
     PyObject *f_funcobj; /* Strong reference. Only valid if not on C stack */
     PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */
@@ -79,8 +79,9 @@ typedef struct _PyInterpreterFrame {
     ((int)((IF)->instr_ptr - _PyCode_CODE(_PyFrame_GetCode(IF))))
 
 static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) {
-    assert(PyCode_Check(f->f_executable));
-    return (PyCodeObject *)f->f_executable;
+    PyObject *executable = PyStackRef_AsPyObjectBorrow(f->f_executable);
+    assert(PyCode_Check(executable));
+    return (PyCodeObject *)executable;
 }
 
 static inline _PyStackRef *_PyFrame_Stackbase(_PyInterpreterFrame *f) {
@@ -130,7 +131,7 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *
     dest->previous = NULL;
 
 #ifdef Py_GIL_DISABLED
-    PyCodeObject *co = (PyCodeObject *)dest->f_executable;
+    PyCodeObject *co = _PyFrame_GetCode(dest);
     for (int i = stacktop; i < co->co_nlocalsplus + co->co_stacksize; i++) {
         dest->localsplus[i] = PyStackRef_NULL;
     }
@@ -148,7 +149,7 @@ _PyFrame_Initialize(
 {
     frame->previous = previous;
     frame->f_funcobj = (PyObject *)func;
-    frame->f_executable = Py_NewRef(code);
+    frame->f_executable = PyStackRef_FromPyObjectNew(code);
     frame->f_builtins = func->func_builtins;
     frame->f_globals = func->func_globals;
     frame->f_locals = locals;
@@ -321,7 +322,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int
     assert(tstate->datastack_top < tstate->datastack_limit);
     frame->previous = previous;
     frame->f_funcobj = Py_None;
-    frame->f_executable = Py_NewRef(code);
+    frame->f_executable = PyStackRef_FromPyObjectNew(code);
 #ifdef Py_DEBUG
     frame->f_builtins = NULL;
     frame->f_globals = NULL;

Include/internal/pycore_gc.h

@@ -381,8 +381,11 @@ extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp);
 extern void _Py_ScheduleGC(PyThreadState *tstate);
 extern void _Py_RunGC(PyThreadState *tstate);
 
+union _PyStackRef;
+
 // GC visit callback for tracked interpreter frames
 extern int _PyGC_VisitFrameStack(struct _PyInterpreterFrame *frame, visitproc visit, void *arg);
+extern int _PyGC_VisitStackRef(union _PyStackRef *ref, visitproc visit, void *arg);
 
 #ifdef __cplusplus
 }

Include/internal/pycore_stackref.h

@@ -227,6 +227,13 @@ PyStackRef_DUP(_PyStackRef stackref)
 #   define PyStackRef_DUP(stackref) PyStackRef_FromPyObjectSteal(Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)))
 #endif
 
+// Convert a possibly deferred reference to a strong reference.
+static inline _PyStackRef
+PyStackRef_AsStrongReference(_PyStackRef stackref)
+{
+    return PyStackRef_FromPyObjectSteal(PyStackRef_AsPyObjectSteal(stackref));
+}
+
 static inline void
 _PyObjectStack_FromStackRefStack(PyObject **dst, const _PyStackRef *src, size_t length)
 {
@@ -261,6 +268,11 @@ PyStackRef_ExceptionInstanceCheck(_PyStackRef stackref)
     return PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(stackref));
 }
 
+static inline bool
+PyStackRef_CodeCheck(_PyStackRef stackref)
+{
+    return PyCode_Check(PyStackRef_AsPyObjectBorrow(stackref));
+}
 
 static inline bool
 PyStackRef_FunctionCheck(_PyStackRef stackref)

Lib/random.py

@@ -1013,7 +1013,7 @@ def _parse_args(arg_list: list[str] | None):
         help="print a random integer between 1 and N inclusive")
     group.add_argument(
         "-f", "--float", type=float, metavar="N",
-        help="print a random floating-point number between 1 and N inclusive")
+        help="print a random floating-point number between 0 and N inclusive")
     group.add_argument(
         "--test", type=int, const=10_000, nargs="?",
         help=argparse.SUPPRESS)
@@ -1038,7 +1038,7 @@ def main(arg_list: list[str] | None = None) -> int | str:
         return randint(1, args.integer)
 
     if args.float is not None:
-        return uniform(1, args.float)
+        return uniform(0, args.float)
 
     if args.test:
         _test(args.test)
@@ -1055,7 +1055,7 @@ def main(arg_list: list[str] | None = None) -> int | str:
             try:
                 # Is it a float?
                 val = float(val)
-                return uniform(1, val)
+                return uniform(0, val)
             except ValueError:
                 # Split in case of space-separated string: "a b c"
                 return choice(val.split())

Lib/test/test_generated_cases.py

@@ -1110,6 +1110,44 @@ def test_push_then_error(self):
         """
         self.run_cases_test(input, output)
 
+    def test_scalar_array_inconsistency(self):
+
+        input = """
+        op(FIRST, ( -- a)) {
+            a = 1;
+        }
+
+        op(SECOND, (a[1] -- b)) {
+            b = 1;
+        }
+
+        macro(TEST) = FIRST + SECOND;
+        """
+
+        output = """
+        """
+        with self.assertRaises(SyntaxError):
+            self.run_cases_test(input, output)
+
+    def test_array_size_inconsistency(self):
+
+        input = """
+        op(FIRST, ( -- a[2])) {
+            a[0] = 1;
+        }
+
+        op(SECOND, (a[1] -- b)) {
+            b = 1;
+        }
+
+        macro(TEST) = FIRST + SECOND;
+        """
+
+        output = """
+        """
+        with self.assertRaises(SyntaxError):
+            self.run_cases_test(input, output)
+
 
 class TestGeneratedAbstractCases(unittest.TestCase):
     def setUp(self) -> None:

Lib/test/test_generators.py

@@ -6,6 +6,7 @@
 import unittest
 import weakref
 import inspect
+import textwrap
 import types
 
 from test import support
@@ -112,6 +113,27 @@ def g3(): return (yield from f())
                 gen.send(2)
             self.assertEqual(cm.exception.value, 2)
 
+    def test_generator_resurrect(self):
+        # Test that a resurrected generator still has a valid gi_code
+        resurrected = []
+
+        # Resurrect a generator in a finalizer
+        exec(textwrap.dedent("""
+            def gen():
+                try:
+                    yield
+                except:
+                    resurrected.append(g)
+
+            g = gen()
+            next(g)
+        """), {"resurrected": resurrected})
+
+        support.gc_collect()
+
+        self.assertEqual(len(resurrected), 1)
+        self.assertIsInstance(resurrected[0].gi_code, types.CodeType)
+
 
 class GeneratorTest(unittest.TestCase):
 

Lib/test/test_importlib/resources/util.py

@@ -145,7 +145,7 @@ def test_useless_loader(self):
     data01={
         '__init__.py': '',
         'binary.file': bytes(range(4)),
-        'utf-16.file': 'Hello, UTF-16 world!\n'.encode('utf-16'),
+        'utf-16.file': '\ufeffHello, UTF-16 world!\n'.encode('utf-16-le'),
         'utf-8.file': 'Hello, UTF-8 world!\n'.encode('utf-8'),
         'subdirectory': {
             '__init__.py': '',
@@ -160,7 +160,7 @@ def test_useless_loader(self):
     },
     namespacedata01={
         'binary.file': bytes(range(4)),
-        'utf-16.file': 'Hello, UTF-16 world!\n'.encode('utf-16'),
+        'utf-16.file': '\ufeffHello, UTF-16 world!\n'.encode('utf-16-le'),
         'utf-8.file': 'Hello, UTF-8 world!\n'.encode('utf-8'),
         'subdirectory': {
             'binary.file': bytes(range(12, 16)),

Lib/test/test_random.py

@@ -1433,8 +1433,8 @@ def test_main(self):
             ("'a a' 'b b' 'c c'", "b b"),
             ("--integer 5", 4),
             ("5", 4),
-            ("--float 2.5", 2.266632777287572),
-            ("2.5", 2.266632777287572),
+            ("--float 2.5", 2.1110546288126204),
+            ("2.5", 2.1110546288126204),
         ]:
             random.seed(0)
             self.assertEqual(random.main(shlex.split(command)), expected)

Misc/NEWS.d/next/Core_and_Builtins/2024-09-10-20-25-00.gh-issue-123923.A7uxqa.rst

@@ -0,0 +1,4 @@
+The ``f_executable`` field in the internal :c:struct:`_PyInterpreterFrame`
+struct now uses a tagged pointer.  Profilers and debuggers that uses this
+field should clear the least significant bit to recover the
+:c:expr:`PyObject*` pointer.

Misc/NEWS.d/next/Library/2024-09-11-19-12-23.gh-issue-123968.OwHON_.rst

@@ -0,0 +1 @@
+Fix the command-line interface for the :mod:`random` module to select floats between 0 and N, not 1 and N.

Modules/_testexternalinspection.c

@@ -510,7 +510,7 @@ parse_frame_object(
         return 0;
     }
 
-    void* address_of_code_object;
+    uintptr_t address_of_code_object;
     bytes_read = read_memory(
             pid,
             (void*)(address + offsets->interpreter_frame.executable),
@@ -520,10 +520,11 @@ parse_frame_object(
         return -1;
     }
 
-    if (address_of_code_object == NULL) {
+    if (address_of_code_object == 0) {
         return 0;
     }
-    return parse_code_object(pid, result, offsets, address_of_code_object, previous_frame);
+    address_of_code_object &= ~Py_TAG_BITS;
+    return parse_code_object(pid, result, offsets, (void *)address_of_code_object, previous_frame);
 }
 
 static PyObject*

Objects/frameobject.c

@@ -1625,8 +1625,6 @@ frame_dealloc(PyFrameObject *f)
     }
 
     Py_TRASHCAN_BEGIN(f, frame_dealloc);
-    PyObject *co = NULL;
-
     /* GH-106092: If f->f_frame was on the stack and we reached the maximum
      * nesting depth for deallocations, the trashcan may have delayed this
      * deallocation until after f->f_frame is freed. Avoid dereferencing
@@ -1635,9 +1633,7 @@ frame_dealloc(PyFrameObject *f)
 
     /* Kill all local variables including specials, if we own them */
     if (f->f_frame == frame && frame->owner == FRAME_OWNED_BY_FRAME_OBJECT) {
-        /* Don't clear code object until the end */
-        co = frame->f_executable;
-        frame->f_executable = NULL;
+        PyStackRef_CLEAR(frame->f_executable);
         Py_CLEAR(frame->f_funcobj);
         Py_CLEAR(frame->f_locals);
         _PyStackRef *locals = _PyFrame_GetLocalsArray(frame);
@@ -1652,7 +1648,6 @@ frame_dealloc(PyFrameObject *f)
     Py_CLEAR(f->f_extra_locals);
     Py_CLEAR(f->f_locals_cache);
     PyObject_GC_Del(f);
-    Py_XDECREF(co);
     Py_TRASHCAN_END;
 }
 

Objects/genobject.c

@@ -55,6 +55,14 @@ gen_traverse(PyGenObject *gen, visitproc visit, void *arg)
             return err;
         }
     }
+    else {
+        // We still need to visit the code object when the frame is cleared to
+        // ensure that it's kept alive if the reference is deferred.
+        int err = _PyGC_VisitStackRef(&gen->gi_iframe.f_executable, visit, arg);
+        if (err) {
+            return err;
+        }
+    }
     /* No need to visit cr_origin, because it's just tuples/str/int, so can't
        participate in a reference cycle. */
     Py_VISIT(gen->gi_exc_state.exc_value);
@@ -139,6 +147,9 @@ gen_dealloc(PyGenObject *gen)
            and GC_Del. */
         Py_CLEAR(((PyAsyncGenObject*)gen)->ag_origin_or_finalizer);
     }
+    if (PyCoro_CheckExact(gen)) {
+        Py_CLEAR(((PyCoroObject *)gen)->cr_origin_or_finalizer);
+    }
     if (gen->gi_frame_state != FRAME_CLEARED) {
         _PyInterpreterFrame *frame = &gen->gi_iframe;
         gen->gi_frame_state = FRAME_CLEARED;
@@ -147,10 +158,7 @@ gen_dealloc(PyGenObject *gen)
         _PyErr_ClearExcState(&gen->gi_exc_state);
     }
     assert(gen->gi_exc_state.exc_value == NULL);
-    if (_PyGen_GetCode(gen)->co_flags & CO_COROUTINE) {
-        Py_CLEAR(((PyCoroObject *)gen)->cr_origin_or_finalizer);
-    }
-    Py_DECREF(_PyGen_GetCode(gen));
+    PyStackRef_CLEAR(gen->gi_iframe.f_executable);
     Py_CLEAR(gen->gi_name);
     Py_CLEAR(gen->gi_qualname);