[pull] main from python:main

Doc/c-api/typeobj.rst

@@ -3051,7 +3051,7 @@ Buffer Object Structures
 
    * Resource cleanup when the counter reaches zero must be done atomically,
      as the final release may race with concurrent releases from other
-     threads and dellocation must only happen once.
+     threads and deallocation must only happen once.
 
    The exporter MUST use the :c:member:`~Py_buffer.internal` field to keep
    track of buffer-specific resources. This field is guaranteed to remain

Lib/asyncio/base_subprocess.py

@@ -165,6 +165,9 @@ def kill(self):
     else:
         def send_signal(self, signal):
             self._check_proc()
+            if self._returncode is not None:
+                # The process already exited
+                return
             try:
                 os.kill(self._proc.pid, signal)
             except ProcessLookupError:

Lib/asyncio/unix_events.py

@@ -219,7 +219,7 @@ async def _make_subprocess_transport(self, protocol, args, shell,
         return transp
 
     def _child_watcher_callback(self, pid, returncode, transp):
-        self.call_soon_threadsafe(transp._process_exited, returncode)
+        transp._process_exited(returncode)
 
     async def create_unix_connection(
             self, protocol_factory, path=None, *,
@@ -930,6 +930,49 @@ def add_child_handler(self, pid, callback, *args):
     def _do_waitpid(self, loop, expected_pid, callback, args):
         assert expected_pid > 0
 
+        if hasattr(os, 'waitid'):
+            # Wait for the child process using waitid() on platforms which support it.
+            # WNOWAIT is used to avoid reaping the child process, allowing the event loop to
+            # reap the child process with waitpid() later in event loop thread.
+            # This makes the reaping of the child and notification of the return code
+            # atomic with respect to the event loop thread.
+            try:
+                os.waitid(os.P_PID, expected_pid, os.WEXITED | os.WNOWAIT)
+            except ChildProcessError:
+                # The child process is already reaped
+                pass
+            if loop.is_closed():
+                # loop is already closed, reap the zombie here so that it is not leaked.
+                pid, _ = self._reap(loop, expected_pid)
+                logger.warning("Loop %r that handles pid %r is closed",
+                               loop, pid)
+            else:
+                try:
+                    loop.call_soon_threadsafe(
+                        self._reap_and_notify, loop, expected_pid,
+                        callback, args)
+                except RuntimeError:
+                    # The event loop was closed concurrently.
+                    pid, _ = self._reap(loop, expected_pid)
+                    logger.warning("Loop %r that handles pid %r is closed",
+                                   loop, pid)
+        else:
+            # Fallback for platforms that don't support waitid(): we have to
+            # reap the child here, which is racy with respect to send_signal()
+            pid, returncode = self._reap(loop, expected_pid)
+            if loop.is_closed():
+                logger.warning("Loop %r that handles pid %r is closed",
+                               loop, pid)
+            else:
+                loop.call_soon_threadsafe(callback, pid, returncode, *args)
+
+        self._threads.pop(expected_pid)
+
+    def _reap_and_notify(self, loop, expected_pid, callback, args):
+        pid, returncode = self._reap(loop, expected_pid)
+        callback(pid, returncode, *args)
+
+    def _reap(self, loop, expected_pid):
         try:
             pid, status = os.waitpid(expected_pid, 0)
         except ChildProcessError:
@@ -945,13 +988,7 @@ def _do_waitpid(self, loop, expected_pid, callback, args):
             if loop.get_debug():
                 logger.debug('process %s exited with returncode %s',
                              expected_pid, returncode)
-
-        if loop.is_closed():
-            logger.warning("Loop %r that handles pid %r is closed", loop, pid)
-        else:
-            loop.call_soon_threadsafe(callback, pid, returncode, *args)
-
-        self._threads.pop(expected_pid)
+        return pid, returncode
 
 def can_use_pidfd():
     if not hasattr(os, 'pidfd_open'):

Lib/compression/zstd/_zstdfile.py

@@ -34,7 +34,7 @@ def __init__(self, file, /, mode='r', *,
                  level=None, options=None, zstd_dict=None):
         """Open a Zstandard compressed file in binary mode.
 
-        *file* can be either an file-like object, or a file name to open.
+        *file* can be either a file-like object, or a file name to open.
 
         *mode* can be 'r' for reading (default), 'w' for (over)writing, 'x'
         for creating exclusively, or 'a' for appending.  These can

Lib/concurrent/futures/_base.py

@@ -703,5 +703,5 @@ def __exit__(self, exc_type, exc_val, exc_tb):
 
 class BrokenExecutor(RuntimeError):
     """
-    Raised when a executor has become non-functional after a severe failure.
+    Raised when an executor has become non-functional after a severe failure.
     """

Lib/profiling/sampling/cli.py

@@ -87,8 +87,8 @@ def __call__(self, parser, namespace, values, option_string=None):
 
 
 # Constants for socket synchronization
-_SYNC_TIMEOUT_SEC = 5.0
-_PROCESS_KILL_TIMEOUT_SEC = 2.0
+_SYNC_TIMEOUT_SEC = 15.0
+_PROCESS_KILL_TIMEOUT_SEC = 5.0
 _READY_MESSAGE = b"ready"
 _RECV_BUFFER_SIZE = 1024
 _BINARY_PROFILE_HEADER_SIZE = 64

Lib/test/list_tests.py

@@ -6,7 +6,7 @@
 from functools import cmp_to_key
 
 from test import seq_tests
-from test.support import ALWAYS_EQ, NEVER_EQ
+from test.support import ALWAYS_EQ, NEVER_EQ, skip_if_huge_c_stack
 from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow
 
 
@@ -60,6 +60,7 @@ def test_repr(self):
         self.assertEqual(str(a2), "[0, 1, 2, [...], 3]")
         self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]")
 
+    @skip_if_huge_c_stack(200_000)
     @skip_wasi_stack_overflow()
     @skip_emscripten_stack_overflow()
     def test_repr_deep(self):

Lib/test/mapping_tests.py

@@ -629,6 +629,7 @@ def __repr__(self):
         d = self._full_mapping({1: BadRepr()})
         self.assertRaises(Exc, repr, d)
 
+    @support.skip_if_huge_c_stack()
     @support.skip_wasi_stack_overflow()
     @support.skip_emscripten_stack_overflow()
     @support.skip_if_sanitizer("requires deep stack", ub=True)

Lib/test/support/__init__.py

@@ -45,7 +45,7 @@
     "check__all__", "skip_if_buggy_ucrt_strfptime",
     "check_disallow_instantiation", "check_sanitizer", "skip_if_sanitizer",
     "requires_limited_api", "requires_specialization", "thread_unsafe",
-    "skip_if_unlimited_stack_size",
+    "skip_if_unlimited_stack_size", "skip_if_huge_c_stack",
     # sys
     "MS_WINDOWS", "is_jython", "is_android", "is_emscripten", "is_wasi",
     "is_apple_mobile", "check_impl_detail", "unix_shell", "setswitchinterval",
@@ -2839,6 +2839,32 @@ def exceeds_recursion_limit():
     return 150_000
 
 
+def skip_if_huge_c_stack(depth=150_000):
+    """Skip decorator for tests which cannot overflow the C stack.
+
+    Tests exhausting the C stack with *depth* recursive calls cannot
+    trigger the recursion protection if the C stack is too large (e.g.
+    with a large or unlimited RLIMIT_STACK), and either fail, or run
+    for a very long time, or crash, or consume all memory.
+    """
+    try:
+        from _testinternalcapi import get_c_recursion_remaining
+    except ImportError:
+        # Fall back to checking for an unlimited stack size.
+        huge = False
+        if not (is_emscripten or is_wasi) and os.name != "nt":
+            import resource
+            soft, hard = resource.getrlimit(resource.RLIMIT_STACK)
+            huge = soft == hard and soft in (-1, 0xFFFF_FFFF_FFFF_FFFF)
+    else:
+        remaining = get_c_recursion_remaining()
+        # A negative value means integer overflow in the estimate
+        # (e.g. with an unlimited RLIMIT_STACK).
+        huge = remaining >= depth or remaining < 0
+    return unittest.skipIf(
+        huge, f"the C stack is large enough for {depth} recursive calls")
+
+
 # Windows doesn't have os.uname() but it doesn't support s390x.
 is_s390x = hasattr(os, 'uname') and os.uname().machine == 's390x'
 skip_on_s390x = unittest.skipIf(is_s390x, 'skipped on s390x')

Lib/test/test_ast/test_ast.py

@@ -26,7 +26,9 @@
 
 from test import support
 from test.support import os_helper
-from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow, skip_if_unlimited_stack_size
+from test.support import (skip_emscripten_stack_overflow,
+                          skip_wasi_stack_overflow,
+                          skip_if_unlimited_stack_size, skip_if_huge_c_stack)
 from test.support.ast_helper import ASTTestMixin
 from test.support.import_helper import ensure_lazy_imports
 from test.test_ast.utils import to_tuple
@@ -1023,7 +1025,7 @@ def next(self):
         enum._test_simple_enum(_Precedence, _ast_unparse._Precedence)
 
     @support.cpython_only
-    @skip_if_unlimited_stack_size
+    @support.skip_if_huge_c_stack(100_000 if sys.platform == "android" else 500_000)
     @skip_wasi_stack_overflow()
     @skip_emscripten_stack_overflow()
     def test_ast_recursion_limit(self):
@@ -2096,7 +2098,7 @@ def test_level_as_none(self):
         exec(code, ns)
         self.assertIn('sleep', ns)
 
-    @skip_if_unlimited_stack_size
+    @skip_if_huge_c_stack()
     @skip_emscripten_stack_overflow()
     def test_recursion_direct(self):
         e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1))
@@ -2105,7 +2107,7 @@ def test_recursion_direct(self):
             with support.infinite_recursion():
                 compile(ast.Expression(e), "<test>", "eval")
 
-    @skip_if_unlimited_stack_size
+    @skip_if_huge_c_stack()
     @skip_emscripten_stack_overflow()
     def test_recursion_indirect(self):
         e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1))

Lib/test/test_asyncio/test_subprocess.py

@@ -974,6 +974,45 @@ def test_watcher_implementation(self):
             else:
                 self.assertIsInstance(watcher, unix_events._ThreadedChildWatcher)
 
+        @unittest.skipUnless(hasattr(os, 'waitid'), 'needs os.waitid()')
+        def test_send_signal_never_targets_reaped_pid(self):
+            # gh-127049: there must be no window between the child watcher
+            # reaping the child and asyncio publishing the exit in which
+            # send_signal() signals the freed (possibly recycled) PID.
+            reaped_pids = set()
+            stale_kills = []
+            orig_waitpid = os.waitpid
+            orig_kill = os.kill
+
+            def waitpid(pid, options):
+                res = orig_waitpid(pid, options)
+                if res[0] != 0:
+                    reaped_pids.add(res[0])
+                return res
+
+            def kill(pid, sig):
+                if pid in reaped_pids:
+                    stale_kills.append(pid)
+                    return
+                orig_kill(pid, sig)
+
+            async def run():
+                with mock.patch('os.waitpid', waitpid), \
+                        mock.patch('os.kill', kill):
+                    proc = await asyncio.create_subprocess_exec(
+                        *PROGRAM_BLOCKED)
+                    proc.kill()
+                    deadline = self.loop.time() + support.SHORT_TIMEOUT
+                    while proc.returncode is None:
+                        if self.loop.time() > deadline:
+                            self.fail('child exit was not published in time')
+                        proc.kill()
+                        await asyncio.sleep(0)
+                    await proc.wait()
+                self.assertEqual(stale_kills, [])
+
+            self.loop.run_until_complete(run())
+
 
     class SubprocessThreadedWatcherTests(SubprocessWatcherMixin,
                                          test_utils.TestCase):
@@ -987,6 +1026,35 @@ def tearDown(self):
             unix_events.can_use_pidfd = self._original_can_use_pidfd
             return super().tearDown()
 
+        @unittest.skipUnless(hasattr(os, 'waitid'), 'needs os.waitid()')
+        def test_pid_not_reaped_before_exit_published(self):
+            # gh-127049: the watcher thread must wait for the child
+            # without reaping it: the PID must stay reserved until the
+            # event loop thread reaps it and publishes the exit as one
+            # atomic step.
+            async def run():
+                proc = await asyncio.create_subprocess_exec(
+                    *PROGRAM_BLOCKED)
+                thread = self.loop._watcher._threads.get(proc.pid)
+                self.assertIsNotNone(thread)
+                proc.kill()
+                # Wait for the watcher thread to observe the exit while
+                # the event loop cannot process the notification yet.
+                thread.join(support.SHORT_TIMEOUT)
+                self.assertFalse(thread.is_alive())
+                # The exit has not been published yet...
+                self.assertIsNone(proc.returncode)
+                # ...so the child must still be an unreaped zombie and
+                # signalling its PID must still be safe.  waitid() raises
+                # ChildProcessError if the PID was already reaped (and
+                # possibly recycled by the kernel).
+                os.waitid(os.P_PID, proc.pid,
+                          os.WEXITED | os.WNOWAIT | os.WNOHANG)
+                proc.kill()
+                self.assertEqual(await proc.wait(), -signal.SIGKILL)
+
+            self.loop.run_until_complete(run())
+
     @unittest.skipUnless(
         unix_events.can_use_pidfd(),
         "operating system does not support pidfds",

Lib/test/test_call.py

@@ -3,7 +3,7 @@
                           set_recursion_limit, skip_on_s390x,
                           skip_emscripten_stack_overflow,
                           skip_wasi_stack_overflow, skip_if_sanitizer,
-                          import_helper)
+                          skip_if_huge_c_stack, import_helper)
 try:
     import _testcapi
 except ImportError:
@@ -1062,6 +1062,7 @@ def get_sp():
     @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack")
     @skip_if_sanitizer("requires deep stack", thread=True)
     @unittest.skipIf(_testcapi is None, "requires _testcapi")
+    @skip_if_huge_c_stack(90_000)
     @skip_emscripten_stack_overflow()
     @skip_wasi_stack_overflow()
     def test_super_deep(self):

Lib/test/test_class.py

@@ -555,6 +555,7 @@ class Custom:
         self.assertFalse(hasattr(o, "__call__"))
         self.assertFalse(hasattr(c, "__call__"))
 
+    @support.skip_if_huge_c_stack()
     @support.skip_emscripten_stack_overflow()
     @support.skip_wasi_stack_overflow()
     def testSFBug532646(self):

Lib/test/test_compile.py

@@ -724,6 +724,7 @@ def test_yet_more_evil_still_undecodable(self):
 
     @support.cpython_only
     @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
+    @support.skip_if_huge_c_stack(100_000 if sys.platform == "android" else 500_000)
     @support.skip_emscripten_stack_overflow()
     def test_compiler_recursion_limit(self):
         # Compiler frames are small

Lib/test/test_copy.py

@@ -377,6 +377,7 @@ def test_deepcopy_list(self):
         self.assertIsNot(x, y)
         self.assertIsNot(x[0], y[0])
 
+    @support.skip_if_huge_c_stack()
     @support.skip_emscripten_stack_overflow()
     @support.skip_wasi_stack_overflow()
     def test_deepcopy_reflexive_list(self):
@@ -406,6 +407,7 @@ def test_deepcopy_tuple_of_immutables(self):
         y = copy.deepcopy(x)
         self.assertIs(x, y)
 
+    @support.skip_if_huge_c_stack()
     @support.skip_emscripten_stack_overflow()
     @support.skip_wasi_stack_overflow()
     def test_deepcopy_reflexive_tuple(self):
@@ -449,6 +451,7 @@ def test_deepcopy_frozendict(self):
         self.assertIsNot(x[0], y[0])
         self.assertIs(y[0]['foo'], y)
 
+    @support.skip_if_huge_c_stack()
     @support.skip_emscripten_stack_overflow()
     @support.skip_wasi_stack_overflow()
     def test_deepcopy_reflexive_dict(self):
@@ -609,6 +612,7 @@ def __eq__(self, other):
         self.assertIsNot(y, x)
         self.assertIsNot(y.foo, x.foo)
 
+    @support.skip_if_huge_c_stack()
     def test_deepcopy_reflexive_inst(self):
         class C:
             pass
@@ -671,6 +675,7 @@ def __eq__(self, other):
         self.assertEqual(y, x)
         self.assertIsNot(y.foo, x.foo)
 
+    @support.skip_if_huge_c_stack()
     def test_reconstruct_reflexive(self):
         class C(object):
             pass

Lib/test/test_ctypes/test_as_parameter.py

@@ -5,7 +5,8 @@
                     c_short, c_int, c_long, c_longlong,
                     c_byte, c_wchar, c_float, c_double,
                     ArgumentError)
-from test.support import import_helper, skip_if_sanitizer, skip_emscripten_stack_overflow
+from test.support import (import_helper, skip_if_sanitizer,
+                          skip_emscripten_stack_overflow, skip_if_huge_c_stack)
 _ctypes_test = import_helper.import_module("_ctypes_test")
 
 
@@ -193,6 +194,7 @@ class S8I(Structure):
                              (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9))
 
     @skip_if_sanitizer('requires deep stack', thread=True)
+    @skip_if_huge_c_stack()
     @skip_emscripten_stack_overflow()
     def test_recursive_as_param(self):
         class A: