[pull] main from python:main

Doc/c-api/type.rst

@@ -37,6 +37,10 @@ Type Objects
 
    Clear the internal lookup cache. Return the current version tag.
 
+   .. versionchanged:: 3.16
+      This function is now a no-op as the type cache is now implemented
+      per-type. It still returns the current version tag.
+
 .. c:function:: unsigned long PyType_GetFlags(PyTypeObject* type)
 
    Return the :c:member:`~PyTypeObject.tp_flags` member of *type*. This function is primarily

Doc/library/asyncio-subprocess.rst

@@ -240,10 +240,34 @@ their completion.
       Note, that the data read is buffered in memory, so do not use
       this method if the data size is large or unlimited.
 
+      If this coroutine is cancelled (for example, when a timeout is
+      set with :func:`~asyncio.wait_for`), the output that was already
+      read is not lost: call :meth:`!communicate` again to read the
+      remaining output and get the complete data::
+
+         try:
+             stdout, stderr = await asyncio.wait_for(
+                 proc.communicate(), timeout=5.0)
+         except TimeoutError:
+             proc.kill()
+             stdout, stderr = await proc.communicate()
+
+      Passing *input* after a previous :meth:`!communicate` call was
+      cancelled raises :exc:`ValueError`; pass ``input=None`` to
+      continue the communication, the original *input* is used.
+
       .. versionchanged:: 3.12
 
          *stdin* gets closed when ``input=None`` too.
 
+      .. versionchanged:: next
+
+         If :meth:`!communicate` is cancelled, the output that was
+         already read is now preserved and returned by a subsequent
+         :meth:`!communicate` call.  Passing *input* to a
+         :meth:`!communicate` call following a cancelled one now raises
+         :exc:`ValueError`.
+
    .. method:: send_signal(signal)
 
       Sends the signal *signal* to the child process.

Doc/library/csv.rst

@@ -312,6 +312,22 @@ The :mod:`!csv` module defines the following classes:
       is given, it is interpreted as a string containing possible valid
       delimiter characters.
 
+      The dialect is deduced by parsing the sample with every plausible
+      combination of parameters
+      and choosing the combination which splits the sample into rows
+      with the most consistent number of fields,
+      so the returned dialect is consistent with how :func:`reader`
+      will parse the sample.
+      Raise :exc:`Error` if no combination fits the sample,
+      in particular if it is a single column,
+      so there is no delimiter to find.
+
+      .. versionchanged:: next
+         The dialect is now deduced by trial parsing
+         and the results may differ from those of earlier Python versions.
+         The *escapechar* parameter can now be detected,
+         and the requested *delimiters* are not restricted to ASCII.
+
 
    .. method:: has_header(sample)
 

Doc/whatsnew/3.16.rst

@@ -101,6 +101,21 @@ codecs
   even when a built-in codec of the same name exists.
   (Contributed by Serhiy Storchaka in :gh:`152997`.)
 
+csv
+---
+
+* :meth:`csv.Sniffer.sniff` now deduces the dialect by trial parsing
+  instead of heuristics based on matching isolated fragments
+  and on character frequencies,
+  so the result is consistent with how :func:`csv.reader` will parse the sample.
+  It can now detect the *escapechar* parameter,
+  accepts non-ASCII delimiters in the *delimiters* argument,
+  no longer misdetects the delimiter
+  when the sample contains delimiter characters inside quoted fields,
+  and no longer takes quadratic time on quoted samples.
+  The results may differ from those of earlier Python versions.
+  (Contributed by Serhiy Storchaka in :gh:`83273`.)
+
 curses
 ------
 
@@ -800,7 +815,8 @@ New features
 Porting to Python 3.16
 ----------------------
 
-* TODO
+* :c:func:`PyType_ClearCache` is now a no-op as the type cache is now
+  implemented per-type. It still returns the current version tag.
 
 Deprecated C APIs
 -----------------

Include/cpython/object.h

@@ -246,6 +246,8 @@ struct _typeobject {
       * This function must escape to any code that can result in
       * the GC being run, such as Py_DECREF.  */
     _Py_iteritemfunc _tp_iteritem;
+
+    void *_tp_cache;
 };
 
 #define _Py_ATTR_CACHE_UNUSED (30000)  // (see tp_versions_used)

Include/cpython/pystats.h

@@ -96,7 +96,9 @@ typedef struct _object_stats {
     uint64_t type_cache_misses;
     uint64_t type_cache_dunder_hits;
     uint64_t type_cache_dunder_misses;
-    uint64_t type_cache_collisions;
+    uint64_t type_cache_too_big;
+    uint64_t type_cache_invalidations;
+    uint64_t type_cache_resizes;
     /* Temporary value used during GC */
     uint64_t object_visits;
 } ObjectStats;

Include/internal/pycore_interp_structs.h

@@ -560,23 +560,6 @@ struct _types_runtime_state {
 };
 
 
-// Type attribute lookup cache: speed up attribute and method lookups,
-// see _PyType_Lookup().
-struct type_cache_entry {
-    unsigned int version;  // initialized from type->tp_version_tag
-#ifdef Py_GIL_DISABLED
-   _PySeqLock sequence;
-#endif
-    PyObject *name;        // reference to exactly a str or None
-    PyObject *value;       // borrowed reference or NULL
-};
-
-#define MCACHE_SIZE_EXP 12
-
-struct type_cache {
-    struct type_cache_entry hashtable[1 << MCACHE_SIZE_EXP];
-};
-
 typedef struct {
     PyTypeObject *type;
     int isbuiltin;
@@ -591,6 +574,10 @@ typedef struct {
        are also some diagnostic uses for the list of weakrefs,
        so we still keep it. */
     PyObject *tp_weaklist;
+    /* Per-interpreter attribute lookup cache (struct type_cache *).
+       For static builtin types the cache must be per-interpreter
+       because tp_dict and the values it stores are per-interpreter. */
+    void *_tp_cache;
 } managed_static_type_state;
 
 #define TYPE_VERSION_CACHE_SIZE (1<<12)  /* Must be a power of 2 */
@@ -601,8 +588,6 @@ struct types_state {
        where all those lower numbers are used for core static types. */
     unsigned int next_version_tag;
 
-    struct type_cache type_cache;
-
     /* Every static builtin type is initialized for each interpreter
        during its own initialization, including for the main interpreter
        during global runtime initialization.  This is done by calling

Include/internal/pycore_object.h

@@ -285,8 +285,6 @@ _PyType_HasFeature(PyTypeObject *type, unsigned long feature) {
     return ((type->tp_flags) & feature) != 0;
 }
 
-extern void _PyType_InitCache(PyInterpreterState *interp);
-
 extern PyStatus _PyObject_InitState(PyInterpreterState *interp);
 extern void _PyObject_FiniState(PyInterpreterState *interp);
 extern bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj);

Include/internal/pycore_typecache.h

@@ -0,0 +1,47 @@
+#ifndef PY_INTERNAL_TYPECACHE_H
+#define PY_INTERNAL_TYPECACHE_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+#include "pycore_stackref.h"
+
+
+#define _Py_TYPECACHE_MINSIZE (1 << 3)
+#define _Py_TYPECACHE_MAXSIZE (1 << 16)
+
+struct type_cache_entry {
+    PyObject *name;     // name of the attribute or method, interned string, NULL if the entry is empty
+    PyObject *value;    // borrowed reference or NULL
+};
+
+// Per-type attribute lookup cache: speed up attribute and method lookups,
+// see _PyTypeCache_Lookup().
+struct type_cache {
+    uint32_t mask;                                              // mask for indexing into hashtable, i.e. size of hashtable is mask + 1
+    uint32_t available;                                         // number of available entries in hashtable
+    uint32_t used;                                              // number of used entries in hashtable
+    unsigned int version_tag;                                   // initialized from type->tp_version_tag
+    struct type_cache_entry hashtable[_Py_TYPECACHE_MINSIZE];   // hashtable entries
+};
+
+struct _PyTypeCacheLookupResult {
+    _PyStackRef value;      // value is a stack reference to the cached attribute or method, or NULL if not found
+    int cache_hit;          // 1 if the cache entry is valid and matches the type's version tag, 0 otherwise
+    unsigned int version_tag;   // version tag of the type when the value was cached
+};
+
+
+extern void _PyTypeCache_InitType(PyTypeObject *type);
+extern void _PyTypeCache_Insert(PyTypeObject *type, PyObject *name, PyObject *value);
+PyAPI_FUNC(struct _PyTypeCacheLookupResult) _PyTypeCache_Lookup(PyTypeObject *type, PyObject *name);
+PyAPI_FUNC(void) _PyTypeCache_Invalidate(PyTypeObject *type);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* PY_INTERNAL_TYPECACHE_H */

Include/internal/pycore_typeobject.h

@@ -41,7 +41,6 @@ extern PyStatus _PyTypes_InitTypes(PyInterpreterState *);
 extern void _PyTypes_FiniTypes(PyInterpreterState *);
 extern void _PyTypes_FiniExtTypes(PyInterpreterState *interp);
 extern void _PyTypes_Fini(PyInterpreterState *);
-extern void _PyTypes_AfterFork(void);
 extern void _PyTypes_FiniCachedDescriptors(PyInterpreterState *);
 
 static inline PyObject **

Lib/asyncio/subprocess.py

@@ -124,6 +124,11 @@ def __init__(self, transport, protocol, loop):
         self.stdout = protocol.stdout
         self.stderr = protocol.stderr
         self.pid = transport.get_pid()
+        self._communication_started = False
+        self._input = None
+        self._input_written = False
+        self._stdout_buf = bytearray()
+        self._stderr_buf = bytearray()
 
     def __repr__(self):
         return f'<{self.__class__.__name__} {self.pid}>'
@@ -148,8 +153,9 @@ def kill(self):
     async def _feed_stdin(self, input):
         debug = self._loop.get_debug()
         try:
-            if input is not None:
+            if input is not None and not self._input_written:
                 self.stdin.write(input)
+                self._input_written = True
                 if debug:
                     logger.debug(
                         '%r communicate: feed stdin (%s bytes)', self, len(input))
@@ -172,22 +178,33 @@ async def _read_stream(self, fd):
         transport = self._transport.get_pipe_transport(fd)
         if fd == 2:
             stream = self.stderr
+            buf = self._stderr_buf
         else:
             assert fd == 1
             stream = self.stdout
+            buf = self._stdout_buf
         if self._loop.get_debug():
             name = 'stdout' if fd == 1 else 'stderr'
             logger.debug('%r communicate: read %s', self, name)
-        output = await stream.read()
+        # Append each block to the persistent buffer as soon as it is
+        # read so that no output is lost if this coroutine is cancelled.
+        while block := await stream.read(stream._limit):
+            buf += block
         if self._loop.get_debug():
             name = 'stdout' if fd == 1 else 'stderr'
             logger.debug('%r communicate: close %s', self, name)
         transport.close()
-        return output
 
     async def communicate(self, input=None):
+        if self._communication_started:
+            if input:
+                raise ValueError(
+                    "Cannot send input after starting communication")
+        else:
+            self._input = input
+            self._communication_started = True
         if self.stdin is not None:
-            stdin = self._feed_stdin(input)
+            stdin = self._feed_stdin(self._input)
         else:
             stdin = self._noop()
         if self.stdout is not None:
@@ -198,8 +215,16 @@ async def communicate(self, input=None):
             stderr = self._read_stream(2)
         else:
             stderr = self._noop()
-        stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr)
+        await tasks.gather(stdin, stdout, stderr)
         await self.wait()
+        if self.stdout is not None:
+            stdout = self._stdout_buf.take_bytes()
+        else:
+            stdout = None
+        if self.stderr is not None:
+            stderr = self._stderr_buf.take_bytes()
+        else:
+            stderr = None
         return (stdout, stderr)