diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 4159ff6e5965fbd..a04ef4e422949b9 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -243,3 +243,62 @@ Unless using :pep:`523`, you will not need this. Return the currently executing line number, or -1 if there is no line number. .. versionadded:: 3.12 + + +.. c:var:: const PyTypeObject *PyUnstable_ExecutableKinds + + An array of executable kinds (executor types) for frames, used for internal + debugging and tracing. + + Tools like debuggers and profilers can use this to identify the type of execution + context associated with a frame (such as to filter out internal frames). + The entries are indexed by the following constants: + + .. list-table:: + :header-rows: 1 + :widths: auto + + * - Constant + - Description + * - .. c:macro:: PyUnstable_EXECUTABLE_KIND_SKIP + - The frame is internal (For example: inlined) and should be skipped by tools. + * - .. c:macro:: PyUnstable_EXECUTABLE_KIND_PY_FUNCTION + - The frame corresponds to a standard Python function. + * - .. c:macro:: PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION + - The frame corresponds to a function defined in native code. + * - .. c:macro:: PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR + - The frame corresponds to a method on a class instance. + + However, Python's C API lacks a function to read the executable kind from + a frame. Instead, use this recipe: + + .. code-block:: c + + int + get_executable_kind(PyFrameObject *frame) + { + _PyInterpreterFrame *f = frame->f_frame; + PyObject *exec = PyStackRef_AsPyObjectBorrow(f->f_executable); + + if (PyCode_Check(exec)) { + return PyUnstable_EXECUTABLE_KIND_PY_FUNCTION; + } + if (PyMethod_Check(exec)) { + return PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION; + } + if (Py_IS_TYPE(exec, &PyMethodDescr_Type)) { + return PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR; + } + + return PyUnstable_EXECUTABLE_KIND_SKIP; + } + + .. versionadded:: 3.13 + + +.. c:macro:: PyUnstable_EXECUTABLE_KINDS + + The number of entries in :c:data:`PyUnstable_ExecutableKinds`. + + .. versionadded:: 3.13 + diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 6256bf7a1454a9a..bba5c7f8ecf7751 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -343,11 +343,125 @@ the same library that the Python runtime is using. :py:mod:`!ast` Python module, which exports these constants under the same names. + .. rubric:: Low-level flags + + The following flags and masks serve narrow needs of the standard + library and interactive interpreters. Code outside the standard + library rarely has a reason to use them. They are considered + implementation details and may change at any time. + + .. c:macro:: PyCF_ALLOW_INCOMPLETE_INPUT + + This flag is a private interface between the compiler and the + :mod:`codeop` module. Do not use it; its behavior is unsupported + and may change without warning. + + With this flag set, when compilation fails because the source text + ends where more input is expected, for example in the middle of an + indented block or an unterminated string literal, the error raised + is the undocumented ``_IncompleteInputError``, a subclass of + :exc:`SyntaxError`. The :mod:`codeop` module sets this flag, + together with :c:macro:`PyCF_DONT_IMPLY_DEDENT`, to tell input + that is incomplete apart from input with a real syntax error, so + that interactive interpreters know when to prompt for another + line instead of reporting an error. + + .. versionadded:: 3.11 + + .. c:macro:: PyCF_DONT_IMPLY_DEDENT + + By default, when compiling with the :c:var:`Py_single_input` start + symbol, reaching the end of the source text implicitly closes any + open indented blocks. With this flag set, open blocks are only + closed if the last line of the source ends with a newline; otherwise, + compilation fails with a :exc:`SyntaxError`: + + .. code-block:: c + + PyCompilerFlags flags = { + .cf_flags = 0, + .cf_feature_version = PY_MINOR_VERSION, + }; + const char *source = "if a:\n pass"; + + /* The "if" block is closed implicitly; + this returns a code object: */ + Py_CompileStringFlags(source, "", Py_single_input, &flags); + + /* With the flag, this fails with a SyntaxError, + because the last line does not end with a newline: */ + flags.cf_flags = PyCF_DONT_IMPLY_DEDENT; + Py_CompileStringFlags(source, "", Py_single_input, &flags); + + The :mod:`codeop` module uses this flag to detect incomplete + interactive input. While the user is still typing inside an + indented block, the source does not yet end with a newline, so it + fails to compile and the user is prompted for another line. + + .. c:macro:: PyCF_IGNORE_COOKIE + + Read the source text as UTF-8, ignoring its :pep:`263` encoding + declaration ("coding cookie"), if any: + + .. code-block:: c + + PyCompilerFlags flags = { + .cf_flags = 0, + .cf_feature_version = PY_MINOR_VERSION, + }; + const char *source = "# coding: latin-1\ns = '\xe9'\n"; + + /* The coding cookie is honored: byte 0xE9 is decoded as + Latin-1, and this returns a code object that sets s to "é": */ + Py_CompileStringFlags(source, "", Py_file_input, &flags); + + /* With the flag, the cookie is ignored and compilation fails + with a SyntaxError, because 0xE9 is not valid UTF-8: */ + flags.cf_flags = PyCF_IGNORE_COOKIE; + Py_CompileStringFlags(source, "", Py_file_input, &flags); + + The :func:`compile`, :func:`eval` and :func:`exec` built-in functions + set this flag when the source is a :class:`str` object, because they + pass the text to the parser encoded as UTF-8. + + .. c:macro:: PyCF_SOURCE_IS_UTF8 + + Mark the source text as known to be UTF-8 encoded. + The :func:`compile`, :func:`eval` and :func:`exec` built-in functions + set this flag, but it currently has no effect. + The "``PyCF``" flags above can be combined with "``CO_FUTURE``" flags such as :c:macro:`CO_FUTURE_ANNOTATIONS` to enable features normally selectable using :ref:`future statements `. See :ref:`c_codeobject_flags` for a complete list. + The following masks combine several flags: + + .. c:macro:: PyCF_MASK + + Bitmask of all ``CO_FUTURE`` flags (see :ref:`c_codeobject_flags`), + which select features normally enabled by + :ref:`future statements `. + When code compiled with a ``PyCompilerFlags *flags`` argument + contains a ``from __future__ import`` statement, the flag for the + imported feature is added to *flags*, so that code executed later + in the same context inherits it. + + .. c:macro:: PyCF_MASK_OBSOLETE + + Do not use this mask in new code. It is kept only so that old + code passing its flags to :func:`compile` keeps working. + + Bitmask of flags for obsolete future features that no longer + have any effect. + + .. c:macro:: PyCF_COMPILE_MASK + + Bitmask of all ``PyCF`` flags that change how the source is + compiled, such as :c:macro:`PyCF_ONLY_AST`. + The :func:`compile` built-in function uses this mask to validate + its *flags* argument. + .. _start-symbols: diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 60cd214b4501010..fb0ffc73b48c40c 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -400,9 +400,9 @@ For simple text calendars this module provides the following functions. *month* (``1``--``12``), *day* (``1``--``31``). -.. function:: weekheader(n) +.. function:: weekheader(width) - Return a header containing abbreviated weekday names. *n* specifies the width in + Return a header containing abbreviated weekday names. *width* specifies the width in characters for one weekday. @@ -430,12 +430,12 @@ For simple text calendars this module provides the following functions. of the :class:`TextCalendar` class. -.. function:: prcal(year, w=0, l=0, c=6, m=3) +.. function:: prcal(theyear, w=0, l=0, c=6, m=3) Prints the calendar for an entire year as returned by :func:`calendar`. -.. function:: calendar(year, w=2, l=1, c=6, m=3) +.. function:: calendar(theyear, w=2, l=1, c=6, m=3) Returns a 3-column calendar for an entire year as a multi-line string using the :meth:`~TextCalendar.formatyear` of the :class:`TextCalendar` class. diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 80868a8b2418f35..34c8636abaa925d 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -694,6 +694,46 @@ through the :attr:`~_CFuncPtr.errcheck` attribute; see the reference manual for details. +Specifying function pointers using type annotations +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. decorator:: wrap_dll_function(dll) + :module: ctypes.util + + A :term:`decorator` that generates :attr:`~ctypes._CFuncPtr.argtypes` and + :attr:`~ctypes._CFuncPtr.restype` from a function signature, using the + :attr:`~function.__name__` of the function and its :term:`type annotations `. + + The decorated function should look like this:: + + @wrap_dll_function(dll_to_wrap) + def function_ptr_name(arg_name: ctypes_type, ...) -> ctypes_type: + # There should be no body + pass + + The body of the decorated function is ignored, and any parameters that are + missing type annotations are skipped. The names of the parameters are ignored + and do not have to match the underlying C implementation. + + If the decorated function does not have a return type annotation, a + :exc:`ValueError` is raised. If the name of the function does not exist + in *dll*, an :exc:`AttributeError` is raised. + + For example:: + + import ctypes + from ctypes.util import wrap_dll_function + + @wrap_dll_function(ctypes.pythonapi) + def PyObject_GetAttrString(op: ctypes.py_object, attr: ctypes.c_char_p) -> ctypes.py_object: + pass + + PyObject_GetAttrString(42, b"real") + + + .. versionadded:: next + + .. _ctypes-passing-pointers: Passing pointers (or: passing parameters by reference) diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index 25edb40e35a630a..3ed4768b6a14136 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -85,6 +85,11 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. with inter-line and intra-line change highlights. The table can be generated in either full or contextual difference mode. + .. warning:: + + The trailing newlines get stripped before the diff, so the result can be + incomplete. See :gh:`71896` for details. + The constructor for this class is: diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index a4d37beb436899a..8dbebf7048b5835 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -611,9 +611,10 @@ are always available. They are listed here in alphabetical order. untrusted user-supplied input will lead to security vulnerabilities. The *source* argument is parsed and evaluated as a Python expression - (technically speaking, a condition list) using the *globals* and *locals* - mappings as global and local namespace. If the *globals* dictionary is - present and does not contain a value for the key ``__builtins__``, a + (technically speaking, an :ref:`expression list `) + using the *globals* and *locals* mappings as global and local namespace. + If the *globals* dictionary is present and does not contain a value for the + key ``__builtins__``, a reference to the dictionary of the built-in module :mod:`builtins` is inserted under that key before *source* is parsed. Overriding ``__builtins__`` can be used to restrict or change the available @@ -633,6 +634,9 @@ are always available. They are listed here in alphabetical order. >>> eval('x+1') 2 + >>> eval("1, 2") + (1, 2) + This function can also be used to execute arbitrary code objects (such as those created by :func:`compile`). In this case, pass a code object instead of a string. If the code object has been compiled with ``'exec'`` as the diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index a0f7379b12a8a62..eddc98824d48106 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -635,8 +635,7 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. function:: ismethoddescriptor(object) Return ``True`` if the object is a method descriptor, but not if - :func:`ismethod`, :func:`isclass`, :func:`isfunction` or :func:`isbuiltin` - are true. + :func:`isclass`, :func:`ismethod` or :func:`isfunction` is true. This, for example, is true of ``int.__add__``. An object passing this test has a :meth:`~object.__get__` method, but not a :meth:`~object.__set__` @@ -644,10 +643,10 @@ attributes (see :ref:`import-mod-attrs` for module attributes): attributes varies. A :attr:`~definition.__name__` attribute is usually sensible, and :attr:`~definition.__doc__` often is. - Methods implemented via descriptors that also pass one of the other tests - return ``False`` from the :func:`ismethoddescriptor` test, simply because the - other tests promise more -- you can, e.g., count on having the - :attr:`~method.__func__` attribute (etc) when an object passes + Method descriptors that also pass any of the other tests (:func:`!isclass`, + :func:`!ismethod` or :func:`!isfunction`) make this function return ``False``, + simply because those other tests promise more -- you can, for example, count + on having the :attr:`~method.__func__` attribute when an object passes :func:`ismethod`. .. versionchanged:: 3.13 @@ -658,16 +657,28 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. function:: isdatadescriptor(object) - Return ``True`` if the object is a data descriptor. + Return ``True`` if the object is a data descriptor, but not if + :func:`isclass`, :func:`ismethod` or :func:`isfunction` is true. - Data descriptors have a :attr:`~object.__set__` or a :attr:`~object.__delete__` method. - Examples are properties (defined in Python), getsets, and members. The - latter two are defined in C and there are more specific tests available for - those types, which is robust across Python implementations. Typically, data - descriptors will also have :attr:`~definition.__name__` and :attr:`!__doc__` attributes - (properties, getsets, and members have both of these attributes), but this is - not guaranteed. + Data descriptors always have a :meth:`~object.__set__` method and/or + a :meth:`~object.__delete__` method. Optionally, they may also have a + :meth:`~object.__get__` method. + Examples of data descriptors are :func:`properties `, getsets and + member descriptors. Note that for the latter two (defined only in C extension + modules), more specific tests are available: :func:`isgetsetdescriptor` and + :func:`ismemberdescriptor`, respectively. + + While data descriptors may also have :attr:`~definition.__name__` and + :attr:`!__doc__` attributes (as properties, getsets and member descriptors + do), this is not necessarily the case in general. + + .. versionchanged:: 3.8 + This function now reports objects with only a :meth:`~object.__set__` method + as being data descriptors (the presence of :meth:`~object.__get__` is no + longer required for that). Moreover, objects with :meth:`~object.__delete__`, + but not :meth:`~object.__set__`, are now properly recognized as data + descriptors as well, which was not the case previously. .. function:: isgetsetdescriptor(object) diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 5c29fff146eef00..f6cfbb5126158e1 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -116,12 +116,13 @@ behavior of the module. Previously, Windows registry settings were ignored. -.. function:: read_mime_types(filename) +.. function:: read_mime_types(file) - Load the type map given in the file *filename*, if it exists. The type map is - returned as a dictionary mapping filename extensions, including the leading dot - (``'.'``), to strings of the form ``'type/subtype'``. If the file *filename* - does not exist or cannot be read, ``None`` is returned. + Load the type map given in the file named by *file*, if it exists. *file* + must be a string specifying the name of the file to read. The type map is + returned as a dictionary mapping file extensions, including the leading dot + (``'.'``), to strings of the form ``'type/subtype'``. If the file does not + exist or cannot be read, ``None`` is returned. .. function:: add_type(type, ext, strict=True) diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index e48de587598e5fe..5bec0c57b12214b 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -128,7 +128,7 @@ difference between the two commands is when running without any arguments: help (``pymanager exec ...`` provides equivalent behaviour to ``py ...``). Each of these commands also has a windowed version that avoids creating a -console window. These are ``pyw``, ``pythonw`` and ``pymanagerw``. A ``python3`` +console window. These are ``pyw``, ``pythonw`` and ``pywmanager``. A ``python3`` command is also included that mimics the ``python`` command. It is intended to catch accidental uses of the typical POSIX command on Windows, but is not meant to be widely used or recommended. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 8cba187bf31dd16..df2ca138db32b50 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1225,7 +1225,7 @@ importlib.metadata would return an empty ``PackageMetadata`` object as if the file was present but empty. Now, a ``MetadataNotFound`` exception is raised. See `importlib_metadata#493 `_ - for background and rationale and and :gh:`143387` for rationale on the + for background and rationale and :gh:`143387` for rationale on the compatibility concerns. (Contributed by Jason R. Coombs.) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 5cb1a86dac46233..e6565b186a598c5 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -237,6 +237,9 @@ ctypes from an annotation-based syntax, similar to how the :mod:`dataclasses` module is used. (Contributed by Peter Bierma in :gh:`104533`.) +* Add :func:`ctypes.util.wrap_dll_function` for generating function pointers + through a function signature. + (Contributed by Peter Bierma in :gh:`153903`.) encodings diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 7d952a1e52561a6..6df1c01f151f68e 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1544,6 +1544,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_filters)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_finalizing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_find_and_load)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_find_and_load_lazy_submodule)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_fix_up_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_flags_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_get_sourcefile)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 8a8bbc3b6d05bf7..873344fbdcb67b0 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -267,6 +267,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_filters) STRUCT_FOR_ID(_finalizing) STRUCT_FOR_ID(_find_and_load) + STRUCT_FOR_ID(_find_and_load_lazy_submodule) STRUCT_FOR_ID(_fix_up_module) STRUCT_FOR_ID(_flags_) STRUCT_FOR_ID(_get_sourcefile) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index a1078828afa572e..669e328c266d00f 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -39,8 +39,13 @@ extern PyObject * _PyImport_GetAbsName( // Symbol is exported for the JIT on Windows builds. PyAPI_FUNC(PyObject *) _PyImport_LoadLazyImportTstate( PyThreadState *tstate, PyObject *lazy_import); -extern PyObject * _PyImport_TryLoadLazySubmodule( - PyObject *mod_name, PyObject *attr_name); +typedef enum { + _Py_LAZY_SUBMODULE_ERROR = -1, + _Py_LAZY_SUBMODULE_NOT_FOUND = 0, + _Py_LAZY_SUBMODULE_LOADED = 1, +} _PyLazySubmoduleImportResult; +extern _PyLazySubmoduleImportResult _PyImport_TryLoadLazySubmodule( + PyObject *mod_name, PyObject *attr_name, PyObject **result); extern PyObject * _PyImport_LazyImportModuleLevelObject( PyThreadState *tstate, PyObject *name, PyObject *builtins, PyObject *globals, PyObject *locals, PyObject *fromlist, int level); diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 366d2d300fb4780..378f27ca17b5079 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1542,6 +1542,7 @@ extern "C" { INIT_ID(_filters), \ INIT_ID(_finalizing), \ INIT_ID(_find_and_load), \ + INIT_ID(_find_and_load_lazy_submodule), \ INIT_ID(_fix_up_module), \ INIT_ID(_flags_), \ INIT_ID(_get_sourcefile), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 00d6297432b5fca..daf6840aa47f328 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -848,6 +848,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_find_and_load_lazy_submodule); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_fix_up_module); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 7f0565d0b8ddc76..708fdd595971e85 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -99,7 +99,7 @@ def run(self): console.write(banner) - if not sys.flags.isolated and (startup_path := os.getenv("PYTHONSTARTUP")): + if not sys.flags.ignore_environment and (startup_path := os.getenv("PYTHONSTARTUP")): sys.audit("cpython.run_startup", startup_path) try: import tokenize @@ -200,7 +200,7 @@ def interrupt(self) -> None: sys.audit("cpython.run_stdin") - if os.getenv('PYTHON_BASIC_REPL'): + if not sys.flags.ignore_environment and os.getenv('PYTHON_BASIC_REPL'): CAN_USE_PYREPL = False else: try: diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index 51c0d441bc8df0e..b035bfb99feee0a 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -574,6 +574,24 @@ def inner(decorated_class): return process_the_struct(class_or_none) + +def wrap_dll_function(dll): + def decorator(func): + name = func.__name__ + ptr = getattr(dll, name) + annotations = annotationlib.get_annotations(func) + + try: + restype = annotations.pop("return") + except KeyError as error: + raise ValueError(f"{name!r} missing return type annotation") from error + + ptr.restype = restype + ptr.argtypes = tuple(annotations.values()) + return ptr + + return decorator + ################################################################ # test code diff --git a/Lib/difflib.py b/Lib/difflib.py index ae8b284b4d36474..95ba8fd782c6c3c 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -2016,6 +2016,8 @@ def make_table(self,fromlines,tolines,fromdesc='',todesc='',context=False, # change tabs to spaces before it gets more difficult after we insert # markup + # it also removes trailing newlines, causing some diffs to be missed + # see: gh-71896 fromlines,tolines = self._tab_newline_replace(fromlines,tolines) # create diffs iterator which generates side by side from/to data diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index eb1686a5c8217c6..081bb98ec2a9723 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -1254,7 +1254,7 @@ def _sanity_check(name, package, level): _ERR_MSG_PREFIX = 'No module named ' -def _find_and_load_unlocked(name, import_): +def _find_and_load_unlocked(name, import_, *, lazy_submodule=False): path = None sys.audit( "import", @@ -1277,6 +1277,8 @@ def _find_and_load_unlocked(name, import_): try: path = parent_module.__path__ except AttributeError: + if lazy_submodule: + return None msg = f'{_ERR_MSG_PREFIX}{name!r}; {parent!r} is not a package' raise ModuleNotFoundError(msg, name=name) from None parent_spec = parent_module.__spec__ @@ -1289,6 +1291,8 @@ def _find_and_load_unlocked(name, import_): child = name.rpartition('.')[2] spec = _find_spec(name, path) if spec is None: + if lazy_submodule: + return None raise ModuleNotFoundError(f'{_ERR_MSG_PREFIX}{name!r}', name=name) else: if parent_spec: @@ -1320,7 +1324,7 @@ def _find_and_load_unlocked(name, import_): _NEEDS_LOADING = object() -def _find_and_load(name, import_): +def _find_and_load(name, import_, *, lazy_submodule=False): """Find and load the module.""" # Optimization: we avoid unneeded module locking if the module @@ -1337,7 +1341,8 @@ def _find_and_load(name, import_): with lock_manager: module = sys.modules.get(name, _NEEDS_LOADING) if module is _NEEDS_LOADING: - return _find_and_load_unlocked(name, import_) + return _find_and_load_unlocked( + name, import_, lazy_submodule=lazy_submodule) # Optimization: only call _bootstrap._lock_unlock_module() if # module.__spec__._initializing is True. @@ -1351,7 +1356,7 @@ def _find_and_load(name, import_): # to preserve normal semantics: the caller gets the exception from # the actual import failure rather than a synthetic error. if sys.modules.get(name) is not module: - return _find_and_load(name, import_) + return _find_and_load(name, import_, lazy_submodule=lazy_submodule) if module is None: message = f'import of {name} halted; None in sys.modules' @@ -1360,6 +1365,10 @@ def _find_and_load(name, import_): return module +def _find_and_load_lazy_submodule(name, import_): + return _find_and_load(name, import_, lazy_submodule=True) + + def _gcd_import(name, package=None, level=0): """Import and return the module based on its name, the package the call is being made from, and the level adjustment. diff --git a/Lib/inspect.py b/Lib/inspect.py index af6aa3eb37a53bb..2a14e43b66f2fac 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -196,18 +196,18 @@ def ispackage(object): def ismethoddescriptor(object): """Return true if the object is a method descriptor. - But not if ismethod() or isclass() or isfunction() are true. + But not if ismethod(), isclass() or isfunction() is true. - This is new in Python 2.2, and, for example, is true of int.__add__. - An object passing this test has a __get__ attribute, but not a - __set__ attribute or a __delete__ attribute. Beyond that, the set - of attributes varies; __name__ is usually sensible, and __doc__ - often is. + An object passing this test (for example, int.__add__) has a __get__ + attribute, but not a __set__ attribute or a __delete__ attribute. + Beyond that, the set of attributes varies; __name__ is usually + sensible, and __doc__ often is. Methods implemented via descriptors that also pass one of the other - tests return false from the ismethoddescriptor() test, simply because - the other tests promise more -- you can, e.g., count on having the - __func__ attribute (etc) when an object passes ismethod().""" + tests (ismethod(), isclass(), isfunction()) make this function return + false, simply because those other tests promise more -- you can, for + example, count on having the __func__ attribute when an object passes + ismethod().""" if isclass(object) or ismethod(object) or isfunction(object): # mutual exclusion return False @@ -219,8 +219,13 @@ def ismethoddescriptor(object): def isdatadescriptor(object): """Return true if the object is a data descriptor. + But not if ismethod(), isclass() or isfunction() is true. + Data descriptors have a __set__ or a __delete__ attribute. Examples are - properties (defined in Python) and getsets and members (defined in C). + properties, getsets, and members. For the latter two (defined only in C + extension modules) more specific tests are available as well: + isgetsetdescriptor() and ismemberdescriptor(), respectively. + Typically, data descriptors will also have __name__ and __doc__ attributes (properties, getsets, and members have both of these attributes), but this is not guaranteed.""" diff --git a/Lib/test/test_ctypes/test_funcptr.py b/Lib/test/test_ctypes/test_funcptr.py index be641da30eadae1..94d03ad553d2227 100644 --- a/Lib/test/test_ctypes/test_funcptr.py +++ b/Lib/test/test_ctypes/test_funcptr.py @@ -2,6 +2,7 @@ import unittest from ctypes import (CDLL, Structure, CFUNCTYPE, sizeof, _CFuncPtr, c_void_p, c_char_p, c_char, c_int, c_uint, c_long) +from ctypes.util import wrap_dll_function from test.support import import_helper _ctypes_test = import_helper.import_module("_ctypes_test") from ._support import (_CData, PyCFuncPtrType, Py_TPFLAGS_DISALLOW_INSTANTIATION, @@ -130,6 +131,26 @@ def c_string(init): def test_abstract(self): self.assertRaises(TypeError, _CFuncPtr, 13, "name", 42, "iid") + def test_wrap_dll_function(self): + @wrap_dll_function(ctypes.pythonapi) + def PyObject_GetAttr(op: ctypes.py_object, attr: ctypes.py_object) -> ctypes.py_object: + pass + + class Foo: + a = "abc" + + self.assertEqual(PyObject_GetAttr(Foo, "a"), "abc") + + with self.assertRaises(AttributeError): + @wrap_dll_function(ctypes.pythonapi) + def noexist(): + pass + + with self.assertRaisesRegex(ValueError, "'PyObject_GetAttrString' missing return type annotation"): + @wrap_dll_function(ctypes.pythonapi) + def PyObject_GetAttrString(op: ctypes.py_object, attr: ctypes.c_char_p): + pass + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index 3ba79be789cf3e5..915eae5ec233d51 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -2799,9 +2799,11 @@ def test_complexchar(self): # its single character. A narrow build just forms fewer cells. cc = curses.complexchar def storable(s): + # ValueError if s has combining marks on a narrow build. + # OverflowError if s is a multibyte character on a narrow build. try: cc(s) - except ValueError: + except (ValueError, OverflowError): return False return True diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 46c9b2c1d8c9fc4..4f99b7c91c654e4 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -284,6 +284,29 @@ def test_make_file_usascii_charset_with_nonascii_input(self): self.assertIn('charset="us-ascii"', output) self.assertIn('ımplıcıt', output) + def test_strip_trailing_newlines_before_diff(self): + # characterization test for the current buggy behavior + # see: gh-71896 + html_diff = difflib.HtmlDiff() + from_lines = [ + "Line 1: no newline after", + "Line 2: one newline after\n", + "Line 3: several newlines after\n\n\n\n\n", + ] + to_lines = [ + "Line 1: no newline after", + "Line 2: one newline after", # actually no \n + "Line 3: several newlines after", # actually no \n + ] + output = html_diff.make_table(from_lines, to_lines) + # we (currently) expect no line change, so all equal + self.assertNotIn('class="diff_add"', output) + self.assertNotIn('class="diff_chg"', output) + self.assertNotIn('class="diff_sub"', output) + self.assertEqual(output.count('>Line 1: no newline after<'), 2) + self.assertEqual(output.count('>Line 2: one newline after<'), 2) + self.assertEqual(output.count('>Line 3: several newlines after<'), 2) + class TestDiffer(unittest.TestCase): def test_close_matches_aligned(self): # Of the 4 closely matching pairs, we want 1 to match with 3, diff --git a/Lib/test/test_getopt.py b/Lib/test/test_getopt.py index 8d0d5084abbb597..fac2ca3db81d703 100644 --- a/Lib/test/test_getopt.py +++ b/Lib/test/test_getopt.py @@ -149,6 +149,18 @@ def test_getopt(self): ('-a', ''), ('--alpha', '')]) self.assertEqual(args, ['arg1', 'arg2']) + # Allow string for single long argument + opts, args = getopt.getopt(cmdline, 'a::', 'alpha=?') + self.assertEqual(opts, [('-a', '1'), ('--alpha', '2'), ('--alpha', ''), + ('-a', ''), ('--alpha', '')]) + self.assertEqual(args, ['arg1', 'arg2']) + + # Pass everything after -- as args + cmdline = ['-a1', '--alpha=2', '--', '-b', '--beta=5'] + opts, args = getopt.getopt(cmdline, 'a:b', ['alpha=', 'beta']) + self.assertEqual(opts, [('-a', '1'), ('--alpha', '2')]) + self.assertEqual(args, ['-b', '--beta=5']) + self.assertError(getopt.getopt, cmdline, 'a:b', ['alpha', 'beta']) def test_gnu_getopt(self): @@ -191,6 +203,25 @@ def test_gnu_getopt(self): self.assertEqual(args, ['arg1', '-b', '1', '--alpha', '--beta=2', '--beta', '3', 'arg2']) + # Allow string for single long argument + opts, args = getopt.gnu_getopt(cmdline, 'ab:', 'alpha') + self.assertEqual(opts, [('-a', '')]) + self.assertEqual(args, ['arg1', '-b', '1', '--alpha', '--beta=2', + '--beta', '3', 'arg2']) + + # Pass everything after -- as args + cmdline = ['-a1', '--alpha=2', '--', '-b', '--beta=5'] + opts, args = getopt.gnu_getopt(cmdline, 'a:b', ['alpha=', 'beta']) + self.assertEqual(opts, [('-a', '1'), ('--alpha', '2')]) + self.assertEqual(args, ['-b', '--beta=5']) + + # In order arguments + cmdline = ["gamma", "--alpha=3"] + opts, args = getopt.gnu_getopt(cmdline, '-', ["alpha="]) + self.assertEqual(opts, [(None, ['gamma']), ('--alpha', '3')]) + self.assertEqual(args, []) + + def test_issue4629(self): longopts, shortopts = getopt.getopt(['--help='], '', ['help=']) self.assertEqual(longopts, [('--help', '')]) @@ -198,6 +229,10 @@ def test_issue4629(self): self.assertEqual(longopts, [('--help', 'x')]) self.assertRaises(getopt.GetoptError, getopt.getopt, ['--help='], '', ['help']) + def test_getopt_error_str(self): + error = getopt.GetoptError('option -a not recognized', 'a') + self.assertEqual(str(error), 'option -a not recognized') + def test_libref_examples(): """ Examples from the Library Reference: Doc/lib/libgetopt.tex diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index cf1d3eb793a8b94..899ecec1ba2afa4 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -676,52 +676,35 @@ def test_lazy_modules_tracks_lazy_imports(self): @support.requires_subprocess() class ErrorHandlingTests(LazyImportTestCase): - """Tests for error handling during lazy import reification. + """Tests for error handling during lazy import reification.""" - PEP 810: Errors during reification should show exception chaining with - both the lazy import definition location and the access location. - """ - - def test_import_error_shows_chained_traceback(self): + def test_missing_lazy_submodule_raises_attribute_error(self): """Accessing a nonexistent lazy submodule via parent attr raises AttributeError.""" code = textwrap.dedent(""" - import sys lazy import test.test_lazy_import.data.nonexistent_module try: - x = test.test_lazy_import.data.nonexistent_module - except AttributeError as e: - print("OK") + _ = test.test_lazy_import.data.nonexistent_module + except AttributeError: + pass + else: + raise AssertionError("AttributeError was not raised") """) - result = subprocess.run( - [sys.executable, "-c", code], - capture_output=True, - text=True - ) - self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") - self.assertIn("OK", result.stdout) + assert_python_ok("-c", code) - def test_attribute_error_on_from_import_shows_chained_traceback(self): + def test_missing_lazy_from_import_shows_chained_traceback(self): """Accessing missing attribute from lazy from-import should chain errors.""" - # Tests 'lazy from module import nonexistent' behavior code = textwrap.dedent(""" - import sys lazy from test.test_lazy_import.data.basic2 import nonexistent_name try: - x = nonexistent_name + _ = nonexistent_name except ImportError as e: - # PEP 810: Enhanced error reporting through exception chaining assert e.__cause__ is not None, "Expected chained exception" - print("OK") + else: + raise AssertionError("ImportError was not raised") """) - result = subprocess.run( - [sys.executable, "-c", code], - capture_output=True, - text=True - ) - self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") - self.assertIn("OK", result.stdout) + assert_python_ok("-c", code) def test_reification_retries_on_failure(self): """Failed reification should allow retry on subsequent access. @@ -731,53 +714,92 @@ def test_reification_retries_on_failure(self): """ code = textwrap.dedent(""" import sys - import types lazy import test.test_lazy_import.data.broken_module - # First access - should fail try: - x = test.test_lazy_import.data.broken_module - except AttributeError: - pass + _ = test.test_lazy_import.data.broken_module + except ValueError as exc: + assert str(exc) == "This module always fails to import", exc + else: + raise AssertionError("ValueError was not raised") + + assert "test.test_lazy_import.data.broken_module" not in sys.modules - # The lazy object should still be a lazy proxy (not reified) - g = globals() - lazy_obj = g['test'] - # The root 'test' binding should still allow retry - # Second access - should also fail (retry the import) try: - x = test.test_lazy_import.data.broken_module - except AttributeError: - print("OK - retry worked") + _ = test.test_lazy_import.data.broken_module + except ValueError as exc: + assert str(exc) == "This module always fails to import", exc + else: + raise AssertionError("ValueError was not raised") """) - result = subprocess.run( - [sys.executable, "-c", code], - capture_output=True, - text=True - ) - self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") - self.assertIn("OK", result.stdout) + assert_python_ok("-c", code) - def test_error_during_module_execution_propagates(self): - """Errors in module code during reification should propagate correctly.""" + def test_lazy_submodule_traceback_hides_importlib_frames(self): code = textwrap.dedent(""" - import sys + import traceback + lazy import test.test_lazy_import.data.broken_module try: _ = test.test_lazy_import.data.broken_module - print("FAIL - should have raised") - except AttributeError: - print("OK") + except ValueError as exc: + frames = traceback.extract_tb(exc.__traceback__) + assert [frame.name for frame in frames] == ["", ""], frames + assert frames[0].filename == "", frames + assert frames[1].filename.endswith("broken_module.py"), frames + else: + raise AssertionError("ValueError was not raised") """) - result = subprocess.run( - [sys.executable, "-c", code], - capture_output=True, - text=True - ) - self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") - self.assertIn("OK", result.stdout) + assert_python_ok("-c", code) + + def test_module_not_found_during_module_execution_propagates(self): + code = textwrap.dedent(""" + lazy import test.test_lazy_import.data.missing_dependency + + try: + _ = test.test_lazy_import.data.missing_dependency + except ModuleNotFoundError as exc: + assert exc.name == "missing_dependency_for_lazy_import_test", exc.name + assert str(exc) == "No module named 'missing_dependency_for_lazy_import_test'", exc + else: + raise AssertionError("ModuleNotFoundError was not raised") + """) + assert_python_ok("-c", code) + + def test_self_named_module_not_found_during_module_execution_propagates(self): + code = textwrap.dedent(""" + lazy import test.test_lazy_import.data.self_named_module_not_found + + try: + _ = test.test_lazy_import.data.self_named_module_not_found + except ModuleNotFoundError as exc: + assert exc.name == "test.test_lazy_import.data.self_named_module_not_found", exc.name + assert str(exc) == "boom", exc + else: + raise AssertionError("ModuleNotFoundError was not raised") + """) + assert_python_ok("-c", code) + + def test_none_in_sys_modules_during_submodule_resolution_propagates(self): + code = textwrap.dedent(""" + import sys + + sys.modules["test.test_lazy_import.data.blocked_module"] = None + lazy import test.test_lazy_import.data.blocked_module + + try: + _ = test.test_lazy_import.data.blocked_module + except ModuleNotFoundError as exc: + assert exc.name == "test.test_lazy_import.data.blocked_module", exc.name + assert str(exc) == ( + "import of test.test_lazy_import.data.blocked_module " + "halted; None in sys.modules" + ), exc + else: + raise AssertionError("ModuleNotFoundError was not raised") + """) + assert_python_ok("-c", code) def test_circular_lazy_import_does_not_crash_for_gh_144727(self): with tempfile.TemporaryDirectory() as tmpdir: diff --git a/Lib/test/test_lazy_import/data/missing_dependency.py b/Lib/test/test_lazy_import/data/missing_dependency.py new file mode 100644 index 000000000000000..aa2f2d0f65f73fe --- /dev/null +++ b/Lib/test/test_lazy_import/data/missing_dependency.py @@ -0,0 +1 @@ +import missing_dependency_for_lazy_import_test diff --git a/Lib/test/test_lazy_import/data/self_named_module_not_found.py b/Lib/test/test_lazy_import/data/self_named_module_not_found.py new file mode 100644 index 000000000000000..941230d5da211d8 --- /dev/null +++ b/Lib/test/test_lazy_import/data/self_named_module_not_found.py @@ -0,0 +1 @@ +raise ModuleNotFoundError("boom", name=__name__) diff --git a/Lib/test/test_pathlib/test_join_windows.py b/Lib/test/test_pathlib/test_join_windows.py index f30c80605f7f910..5afefa61b585f3b 100644 --- a/Lib/test/test_pathlib/test_join_windows.py +++ b/Lib/test/test_pathlib/test_join_windows.py @@ -87,6 +87,11 @@ def test_vfspath(self): p = self.cls(r'\\a\b\c\d') self.assertEqual(vfspath(p), '\\\\a\\b\\c\\d') + def test_invalid_vspath(self): + msg = "expected JoinablePath object, not NoneType" + with self.assertRaisesRegex(TypeError, msg): + vfspath(None) + def test_parts(self): P = self.cls p = P(r'c:a\b') diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 17574c7f03d8e1e..c1f340230159db6 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -2801,8 +2801,14 @@ def test_args(self): self.assertEqual(Literal[1, 2, 3].__args__, (1, 2, 3)) self.assertEqual(Literal[1, 2, 3, 3].__args__, (1, 2, 3)) self.assertEqual(Literal[1, Literal[2], Literal[3, 4]].__args__, (1, 2, 3, 4)) - # Mutable arguments will not be deduplicated - self.assertEqual(Literal[[], []].__args__, ([], [])) + # Unhashable arguments will be deduplicated too + self.assertEqual(Literal[[], []].__args__, ([],)) + self.assertEqual(Literal[{"a": 1}, {"a": 1}].__args__, ({"a": 1},)) + self.assertEqual( + Literal[1, {'a': 'b'}, 2, {'a': 'b'}, 3].__args__, + (1, {'a': 'b'}, 2, 3), + ) + self.assertEqual(Literal[{1}, {1}, {2}, {2}].__args__, ({1}, {2})) def test_flatten(self): l1 = Literal[Literal[1], Literal[2], Literal[3]] diff --git a/Lib/typing.py b/Lib/typing.py index 933336ff4cf37e2..054420865d7fb50 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -775,13 +775,16 @@ def open_helper(file: str, mode: MODE) -> str: # There is no '_type_check' call because arguments to Literal[...] are # values, not types. parameters = _flatten_literal_params(parameters) + value_and_type_parameters = list(_value_and_type_iter(parameters)) + deduplicated_parameters = tuple( + p + for p, _ in _deduplicate( + value_and_type_parameters, + unhashable_fallback=True, + ) + ) - try: - parameters = tuple(p for p, _ in _deduplicate(list(_value_and_type_iter(parameters)))) - except TypeError: # unhashable parameters - pass - - return _LiteralGenericAlias(self, parameters) + return _LiteralGenericAlias(self, deduplicated_parameters) @_SpecialForm diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-06-16-30-00.gh-issue-153236.lazy-import-errors.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-06-16-30-00.gh-issue-153236.lazy-import-errors.rst new file mode 100644 index 000000000000000..c5043af0a4f80b7 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-06-16-30-00.gh-issue-153236.lazy-import-errors.rst @@ -0,0 +1,2 @@ +Propagate exceptions raised while importing lazy submodules instead of +reporting them as missing attributes. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-14-57-19.gh-issue-153568.memoflt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-14-57-19.gh-issue-153568.memoflt.rst new file mode 100644 index 000000000000000..9e78e5ad545a83a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-14-57-19.gh-issue-153568.memoflt.rst @@ -0,0 +1,2 @@ +Speed up the parser by letting memoization lookups that cannot match return +immediately. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-07-49.gh-issue-153568.identcache.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-07-49.gh-issue-153568.identcache.rst new file mode 100644 index 000000000000000..5d778139d06b50d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-07-49.gh-issue-153568.identcache.rst @@ -0,0 +1 @@ +Speed up the parser by caching repeated identifiers during a parse. diff --git a/Misc/NEWS.d/next/Library/2026-05-06-23-13-40.gh-issue-149319.PLM93t.rst b/Misc/NEWS.d/next/Library/2026-05-06-23-13-40.gh-issue-149319.PLM93t.rst new file mode 100644 index 000000000000000..4018ec43f5a47d9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-06-23-13-40.gh-issue-149319.PLM93t.rst @@ -0,0 +1 @@ +The :mod:`asyncio` REPL now ignores :envvar:`PYTHONSTARTUP` and :envvar:`PYTHON_BASIC_REPL` when :option:`-E` or :option:`-I` is used. Patch by Jonathan Dung. diff --git a/Misc/NEWS.d/next/Library/2026-07-18-06-16-55.gh-issue-153903.mJFrs8.rst b/Misc/NEWS.d/next/Library/2026-07-18-06-16-55.gh-issue-153903.mJFrs8.rst new file mode 100644 index 000000000000000..00f89b471175f45 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-18-06-16-55.gh-issue-153903.mJFrs8.rst @@ -0,0 +1,2 @@ +Add :func:`ctypes.util.wrap_dll_function` for creating +:class:`~ctypes._CFuncPtr` objects from a function signature. diff --git a/Misc/NEWS.d/next/Library/2026-07-18-10-52-10.gh-issue-153896.87oevp.rst b/Misc/NEWS.d/next/Library/2026-07-18-10-52-10.gh-issue-153896.87oevp.rst new file mode 100644 index 000000000000000..217a3d3d272ed07 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-18-10-52-10.gh-issue-153896.87oevp.rst @@ -0,0 +1 @@ +Deduplicate unhashable args in :data:`typing.Literal`. diff --git a/Misc/NEWS.d/next/Library/2026-07-18-11-16-41.gh-issue-153908.82FiGk.rst b/Misc/NEWS.d/next/Library/2026-07-18-11-16-41.gh-issue-153908.82FiGk.rst new file mode 100644 index 000000000000000..6ee943a5fb66724 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-18-11-16-41.gh-issue-153908.82FiGk.rst @@ -0,0 +1 @@ +Fix data race when calling :func:`repr` on :class:`itertools.count` under the :term:`free-threaded build`. diff --git a/Misc/NEWS.d/next/Windows/2026-05-06-21-36-53.gh-issue-124111.m4OBX8.rst b/Misc/NEWS.d/next/Windows/2026-05-06-21-36-53.gh-issue-124111.m4OBX8.rst deleted file mode 100644 index 9a57536f1dc96b6..000000000000000 --- a/Misc/NEWS.d/next/Windows/2026-05-06-21-36-53.gh-issue-124111.m4OBX8.rst +++ /dev/null @@ -1 +0,0 @@ -Updated Windows builds to use Tcl/Tk 9.0.3. diff --git a/Misc/NEWS.d/next/Windows/2026-06-28-12-37-08.gh-issue-152433.FTMgqF.rst b/Misc/NEWS.d/next/Windows/2026-06-28-12-37-08.gh-issue-152433.FTMgqF.rst new file mode 100644 index 000000000000000..bd295fb4dd603f5 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-06-28-12-37-08.gh-issue-152433.FTMgqF.rst @@ -0,0 +1 @@ +Implement :func:`os.cpu_count` for Universal Windows Platform. diff --git a/Misc/NEWS.d/next/Windows/2026-07-18-11-56-53.gh-issue-124111.m4OBX8.rst b/Misc/NEWS.d/next/Windows/2026-07-18-11-56-53.gh-issue-124111.m4OBX8.rst new file mode 100644 index 000000000000000..802afc6f7a6fc48 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-07-18-11-56-53.gh-issue-124111.m4OBX8.rst @@ -0,0 +1 @@ +Updated Windows builds to use Tcl/Tk 9.0.4. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 523d20259adaaa9..cdacaf6f5869531 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -112,42 +112,42 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "7a1d1f3a2b8f4484a9c2a027a157963c18f85a81785e85fcb5d1e3df6b6a4fd4" + "checksumValue": "3ac2acd65ddaaac0b2b8df321b558419d21da20bf67ab39149b8248c85f0d214" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-9.0.3.0.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-9.0.4.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:9.0.3.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:9.0.4.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "tcl", "primaryPackagePurpose": "SOURCE", - "versionInfo": "9.0.3.0" + "versionInfo": "9.0.4.0" }, { "SPDXID": "SPDXRef-PACKAGE-tk", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "54fb59df12c489c6264f5b7d3d7444b150d1e3d6561fd59cdb11483440cec000" + "checksumValue": "66048966cfa88989333ff1632f454e10e9248516bb45efda7b4d2ae5a61642cb" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-9.0.3.1.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-9.0.4.1.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:9.0.3.1:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:9.0.4.1:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "tk", "primaryPackagePurpose": "SOURCE", - "versionInfo": "9.0.3.1" + "versionInfo": "9.0.4.1" }, { "SPDXID": "SPDXRef-PACKAGE-xz", diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 72bfab1abaf9cae..c0023c839ca7fe3 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -3675,9 +3675,11 @@ static PyObject * count_repr(PyObject *op) { countobject *lz = countobject_CAST(op); - if (lz->long_cnt == NULL) + if (lz->long_cnt == NULL) { + Py_ssize_t cnt = FT_ATOMIC_LOAD_SSIZE_RELAXED(lz->cnt); return PyUnicode_FromFormat("%s(%zd)", - _PyType_Name(Py_TYPE(lz)), lz->cnt); + _PyType_Name(Py_TYPE(lz)), cnt); + } if (PyLong_Check(lz->long_step)) { long step = PyLong_AsLong(lz->long_step); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 36debd6fe7aa153..4c26ee52279abcd 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -16003,7 +16003,9 @@ os_cpu_count_impl(PyObject *module) # ifdef MS_WINDOWS_DESKTOP ncpu = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); # else - ncpu = 0; + SYSTEM_INFO sysinfo; + GetSystemInfo(&sysinfo); + ncpu = sysinfo.dwNumberOfProcessors; # endif #elif defined(__hpux) diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index f447403ef31b43a..b8cd6025c20ba56 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -1299,8 +1299,6 @@ _PyModule_IsPossiblyShadowing(PyObject *origin) return result; } -// Check if `name` is a lazily pending submodule of module `m`. -// Returns a new reference on success, or NULL with no error set. static PyObject * try_load_lazy_submodule(PyModuleObject *m, PyObject *name) { @@ -1313,10 +1311,13 @@ try_load_lazy_submodule(PyModuleObject *m, PyObject *name) Py_DECREF(mod_name); return NULL; } - PyObject *result = _PyImport_TryLoadLazySubmodule(mod_name, name); + PyObject *result = NULL; + _PyLazySubmoduleImportResult status = + _PyImport_TryLoadLazySubmodule(mod_name, name, &result); Py_DECREF(mod_name); - if (result == NULL) { - PyErr_Clear(); + if (status != _Py_LAZY_SUBMODULE_LOADED) { + assert(status == _Py_LAZY_SUBMODULE_ERROR || + status == _Py_LAZY_SUBMODULE_NOT_FOUND); return NULL; } if (PyDict_SetItem(m->md_dict, name, result) < 0) { diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 47399fe65d1e542..6bedd21299a73e9 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -57,8 +57,8 @@ if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.5.7 set libraries=%libraries% mpdecimal-4.0.0 set libraries=%libraries% sqlite-3.53.2.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-9.0.3.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-9.0.3.1 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-9.0.4.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-9.0.4.1 set libraries=%libraries% xz-5.8.1.1 set libraries=%libraries% zlib-ng-2.2.4 set libraries=%libraries% zstd-1.5.7 @@ -80,7 +80,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.5.7 -if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-9.0.3.0 +if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-9.0.4.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-21.1.4.0 diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 7c5eab2eb75a1de..7739264f56ccc27 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -247,7 +247,7 @@ _sqlite3 https://www.sqlite.org/ _tkinter - Wraps version 9.0.3 of the Tk windowing system, which is downloaded + Wraps version 9.0.4 of the Tk windowing system, which is downloaded from our binaries repository at https://github.com/python/cpython-bin-deps. diff --git a/PCbuild/tcltk.props b/PCbuild/tcltk.props index 28e8c0db4d1eafd..1d6b4f57f09f747 100644 --- a/PCbuild/tcltk.props +++ b/PCbuild/tcltk.props @@ -2,7 +2,7 @@ - 9.0.3.0 + 9.0.4.0 $(TclVersion) $([System.Version]::Parse($(TclVersion)).Major) $([System.Version]::Parse($(TclVersion)).Minor) diff --git a/Parser/pegen.c b/Parser/pegen.c index 165dcb9f9950955..fcec810037e98d4 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -12,6 +12,16 @@ #include "tokenizer/helpers.h" #include "pegen.h" +#define IDENTIFIER_CACHE_SIZE 2048 // Must be a power of two. +#define IDENTIFIER_CACHE_MAX_PROBES 8 + +struct _identifier_cache_entry { + const char *key; // Borrowed from arena-owned token bytes. + Py_ssize_t len; + Py_hash_t hash; + PyObject *value; // Borrowed from an arena-owned identifier. +}; + // Internal parser functions asdl_stmt_seq* @@ -90,6 +100,7 @@ _PyPegen_insert_memo(Parser *p, int mark, int type, void *node) m->mark = p->mark; m->next = p->tokens[mark]->memo; p->tokens[mark]->memo = m; + p->tokens[mark]->memo_mask |= 1ULL << (type & 63); return 0; } @@ -350,6 +361,10 @@ _PyPegen_is_memoized(Parser *p, int type, void *pres) Token *t = p->tokens[p->mark]; + if (!(t->memo_mask & (1ULL << (type & 63)))) { + return 0; + } + for (Memo *m = t->memo; m != NULL; m = m->next) { if (m->type == type) { #if defined(Py_DEBUG) @@ -572,11 +587,44 @@ _PyPegen_name_from_token(Parser *p, Token* t) p->error_indicator = 1; return NULL; } + // Identifiers repeat constantly; a small span-keyed cache skips the + // UTF-8 decode + intern for repeated occurrences. Keys point into + // arena-owned token bytes and values are arena-owned interned strings, + // so borrowed references are valid for the lifetime of the parse + // (including the second error pass, which reuses parser and arena). + Py_ssize_t len = PyBytes_GET_SIZE(t->bytes); + Py_hash_t hash = PyObject_Hash(t->bytes); + if (hash == -1) { + p->error_indicator = 1; + return NULL; + } + IdentifierCacheEntry *free_slot = NULL; + size_t idx = (size_t)hash & (IDENTIFIER_CACHE_SIZE - 1); + for (int probe = 0; probe < IDENTIFIER_CACHE_MAX_PROBES; probe++) { + IdentifierCacheEntry *entry = &p->identifier_cache[ + (idx + probe) & (IDENTIFIER_CACHE_SIZE - 1)]; + if (entry->key == NULL) { + free_slot = entry; + break; + } + if (entry->hash == hash && entry->len == len && + memcmp(entry->key, s, len) == 0) + { + return _PyAST_Name(entry->value, Load, t->lineno, t->col_offset, + t->end_lineno, t->end_col_offset, p->arena); + } + } PyObject *id = _PyPegen_new_identifier(p, s); if (id == NULL) { p->error_indicator = 1; return NULL; } + if (free_slot != NULL) { + free_slot->key = s; + free_slot->len = len; + free_slot->hash = hash; + free_slot->value = id; + } return _PyAST_Name(id, Load, t->lineno, t->col_offset, t->end_lineno, t->end_col_offset, p->arena); } @@ -844,6 +892,15 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags, p->flags = flags; p->feature_version = feature_version; p->known_err_token = NULL; + p->identifier_cache = PyMem_Calloc( + IDENTIFIER_CACHE_SIZE, sizeof(*p->identifier_cache)); + if (p->identifier_cache == NULL) { + growable_comment_array_deallocate(&p->type_ignore_comments); + PyMem_Free(p->tokens[0]); + PyMem_Free(p->tokens); + PyMem_Free(p); + return (Parser *) PyErr_NoMemory(); + } p->level = 0; p->call_invalid_rules = 0; p->last_stmt_location.lineno = 0; @@ -859,6 +916,7 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags, void _PyPegen_Parser_Free(Parser *p) { + PyMem_Free(p->identifier_cache); Py_XDECREF(p->normalize); for (int i = 0; i < p->size; i++) { PyMem_Free(p->tokens[i]); diff --git a/Parser/pegen.h b/Parser/pegen.h index cbf44ba474fa395..3cca698692bf6bf 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -42,6 +42,10 @@ typedef struct { int level; int lineno, col_offset, end_lineno, end_col_offset; Memo *memo; + // Filter over the rule types present in `memo` (bit `type & 63` is set + // for every entry): lets lookups skip walking the list on definite + // misses, which are the common case. + uint64_t memo_mask; PyObject *metadata; } Token; @@ -67,6 +71,8 @@ typedef struct { int end_col_offset; } location; +typedef struct _identifier_cache_entry IdentifierCacheEntry; + typedef struct { struct tok_state *tok; Token **tokens; @@ -91,6 +97,7 @@ typedef struct { int call_invalid_rules; int debug; location last_stmt_location; + IdentifierCacheEntry *identifier_cache; } Parser; typedef struct { diff --git a/Python/import.c b/Python/import.c index 63e23e21beb1266..5ca78a971fa54c6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -4103,7 +4103,9 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) } static PyObject * -import_find_and_load(PyThreadState *tstate, PyObject *abs_name) +import_find_and_load_with_name(PyThreadState *tstate, PyObject *abs_name, + PyObject *find_and_load, + PyObject *not_found) { PyObject *mod = NULL; PyInterpreterState *interp = tstate->interp; @@ -4130,12 +4132,14 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) if (PyDTrace_IMPORT_FIND_LOAD_START_ENABLED()) PyDTrace_IMPORT_FIND_LOAD_START(PyUnicode_AsUTF8(abs_name)); - mod = PyObject_CallMethodObjArgs(IMPORTLIB(interp), &_Py_ID(_find_and_load), + mod = PyObject_CallMethodObjArgs(IMPORTLIB(interp), find_and_load, abs_name, IMPORT_FUNC(interp), NULL); - if (PyDTrace_IMPORT_FIND_LOAD_DONE_ENABLED()) + if (PyDTrace_IMPORT_FIND_LOAD_DONE_ENABLED()) { + int found = mod != NULL && mod != not_found; PyDTrace_IMPORT_FIND_LOAD_DONE(PyUnicode_AsUTF8(abs_name), - mod != NULL); + found); + } if (import_time) { PyTime_t t2; @@ -4156,6 +4160,13 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) #undef accumulated } +static PyObject * +import_find_and_load(PyThreadState *tstate, PyObject *abs_name) +{ + return import_find_and_load_with_name( + tstate, abs_name, &_Py_ID(_find_and_load), NULL); +} + static PyObject * get_abs_name(PyThreadState *tstate, PyObject *name, PyObject *globals, int level) @@ -4439,52 +4450,70 @@ register_from_lazy_on_parent(PyThreadState *tstate, PyObject *abs_name, return res; } -PyObject * -_PyImport_TryLoadLazySubmodule(PyObject *mod_name, PyObject *attr_name) +_PyLazySubmoduleImportResult +_PyImport_TryLoadLazySubmodule(PyObject *mod_name, PyObject *attr_name, + PyObject **result) { - PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(result != NULL); + *result = NULL; + + PyThreadState *tstate = _PyThreadState_GET(); + PyInterpreterState *interp = tstate->interp; PyObject *lazy_pending = LAZY_PENDING_SUBMODULES(interp); if (lazy_pending == NULL) { - return NULL; + return _Py_LAZY_SUBMODULE_NOT_FOUND; } PyObject *pending_set; int rc = PyDict_GetItemRef(lazy_pending, mod_name, &pending_set); - if (rc <= 0) { - return NULL; + if (rc < 0) { + return _Py_LAZY_SUBMODULE_ERROR; + } + if (rc == 0) { + return _Py_LAZY_SUBMODULE_NOT_FOUND; } int contains = PySet_Contains(pending_set, attr_name); - if (contains <= 0) { + if (contains < 0) { Py_DECREF(pending_set); - return NULL; + return _Py_LAZY_SUBMODULE_ERROR; + } + if (contains == 0) { + Py_DECREF(pending_set); + return _Py_LAZY_SUBMODULE_NOT_FOUND; } PyObject *full_name = PyUnicode_FromFormat("%U.%U", mod_name, attr_name); if (full_name == NULL) { Py_DECREF(pending_set); - return NULL; + return _Py_LAZY_SUBMODULE_ERROR; } - PyObject *mod = PyImport_ImportModuleLevelObject( - full_name, NULL, NULL, NULL, 0); + PyObject *mod = import_find_and_load_with_name( + tstate, full_name, &_Py_ID(_find_and_load_lazy_submodule), Py_None); if (mod == NULL) { Py_DECREF(pending_set); Py_DECREF(full_name); - return NULL; + remove_importlib_frames(tstate); + return _Py_LAZY_SUBMODULE_ERROR; + } + if (mod == Py_None) { + Py_DECREF(mod); + Py_DECREF(pending_set); + Py_DECREF(full_name); + return _Py_LAZY_SUBMODULE_NOT_FOUND; } - Py_DECREF(mod); if (PySet_Discard(pending_set, attr_name) < 0) { + Py_DECREF(mod); Py_DECREF(pending_set); Py_DECREF(full_name); - return NULL; + return _Py_LAZY_SUBMODULE_ERROR; } Py_DECREF(pending_set); - - PyObject *submod = PyImport_GetModule(full_name); Py_DECREF(full_name); - return submod; + *result = mod; + return _Py_LAZY_SUBMODULE_LOADED; } PyObject * diff --git a/Tools/check-c-api-docs/ignored_c_api.txt b/Tools/check-c-api-docs/ignored_c_api.txt index fa53b205c4ff6af..aeae9e6553a3aa6 100644 --- a/Tools/check-c-api-docs/ignored_c_api.txt +++ b/Tools/check-c-api-docs/ignored_c_api.txt @@ -29,26 +29,12 @@ PY_DWORD_MAX PY_BIG_ENDIAN # cpython/methodobject.h PyCFunction_GET_CLASS -# cpython/compile.h -PyCF_ALLOW_INCOMPLETE_INPUT -PyCF_COMPILE_MASK -PyCF_DONT_IMPLY_DEDENT -PyCF_IGNORE_COOKIE -PyCF_MASK -PyCF_MASK_OBSOLETE -PyCF_SOURCE_IS_UTF8 # cpython/descrobject.h PyDescr_NAME PyDescr_TYPE PyWrapperFlag_KEYWORDS # cpython/fileobject.h Py_UniversalNewlineFgets -# cpython/pyframe.h -PyUnstable_EXECUTABLE_KINDS -PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION -PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR -PyUnstable_EXECUTABLE_KIND_PY_FUNCTION -PyUnstable_EXECUTABLE_KIND_SKIP # cpython/pylifecycle.h Py_FrozenMain # pythonrun.h