diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index cae51fad2c586c..40464695a281af 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -39,6 +39,8 @@ jobs: run: | sudo ./.github/workflows/posix-deps-apt.sh # On ubuntu-26.04 image, clang is clang-21 by default + # NOTE: when bumping to a new version of clang, + # please update the versions in `Tools/pixi-packages/variants.yaml` too. echo "CC=clang" >> "$GITHUB_ENV" echo "CXX=clang++" >> "$GITHUB_ENV" - name: TSan option setup diff --git a/Doc/library/curses.ascii.rst b/Doc/library/curses.ascii.rst index 3ac76edd38dcc8..fad35973dada18 100644 --- a/Doc/library/curses.ascii.rst +++ b/Doc/library/curses.ascii.rst @@ -176,15 +176,18 @@ C library: Checks for a non-ASCII character (ordinal values 0x80 and above). -These functions accept either integers or single-character strings; when the argument is a -string, it is first converted using the built-in function :func:`ord`. +These functions accept an integer, a single-character string, or a +:class:`curses.complexchar`. +A string is converted using the built-in function :func:`ord`, and a +complexchar by the code of its single character; a complexchar that holds +combining characters is not a single character and matches no class. Note that all these functions check ordinal bit values derived from the character of the string you pass in; they do not actually know anything about the host machine's character encoding. -The following two functions take either a single-character string or integer -byte value; they return a value of the same type. +The following three functions take either a single-character string or an +integer byte value; they return a value of the same type. .. function:: ascii(c) @@ -194,8 +197,13 @@ byte value; they return a value of the same type. .. function:: ctrl(c) - Return the control character corresponding to the given character (the character - bit value is bitwise-anded with 0x1f). + Return the control character corresponding to the given ASCII character (the + character bit value is bitwise-anded with 0x1f). A non-ASCII character has no + control character and is returned unchanged. + + .. versionchanged:: next + A non-ASCII argument is now returned unchanged instead of masked to a + control character. .. function:: alt(c) @@ -203,7 +211,8 @@ byte value; they return a value of the same type. Return the 8-bit character corresponding to the given ASCII character (the character bit value is bitwise-ored with 0x80). -The following function takes either a single-character string or integer value; +The following function takes a single-character string, an integer value, or a +:class:`curses.complexchar`; it returns a string. diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 7b79d81790b667..910b0f00c0e7de 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -188,9 +188,10 @@ enclosed with either parentheses or double quotes) each string is quoted. However, the *password* argument to the ``LOGIN`` command is always quoted. If you want to avoid having an argument string quoted (eg: the *flags* argument to ``STORE``) then enclose the string in parentheses (eg: ``r'(\Deleted)'``). -In general, pass arguments unquoted and let the module quote them as needed. -An argument that is already enclosed in double quotes is left unchanged, -so that code which quotes arguments itself keeps working. +Or you can quote the string yourself; +an argument that is already enclosed in double quotes is left unchanged. +In general, however, it is better to pass arguments unquoted +and let the module quote them as needed. Mailbox names are encoded as modified UTF-7 (:rfc:`3501`, section 5.1.3), so a mailbox name containing non-ASCII characters can be passed as an @@ -211,11 +212,70 @@ or mandated results from the command. Each *data* is either a ``bytes``, or a tuple. If a tuple, then the first part is the header of the response, and the second part contains the data (ie: 'literal' value). -The *message_set* options to commands below is a string specifying one or more -messages to be acted upon. It may be a simple message number (``'1'``), a range -of message numbers (``'2:4'``), or a group of non-contiguous ranges separated by -commas (``'1:3,6:9'``). A range can contain an asterisk to indicate an infinite -upper bound (``'3:*'``). +The *message_set* options to the commands below can be a string specifying +one or more messages to be acted upon. +It may be a simple message number (``'1'``), +a range of message numbers (``'2:4'``), +or a group of non-contiguous ranges separated by commas (``'1:3,6:9'``). +A range can contain an asterisk +to indicate an infinite upper bound (``'3:*'``). + +Alternatively it can be specified using integers and :class:`range` objects. +It may be a single message number or a sequence. +The sequence items may be integers, ``(start, stop)`` tuples +(where ``None`` or ``'*'`` stands for the last message), +or :class:`range` objects. +For example, ``[1, (3, 5), 8]`` and ``[range(1, 6), 8]`` +are both equivalent to ``'1,3:5,8'``. + +.. versionchanged:: next + Added support for the structured *message_set*. + +Command arguments that are parenthesized lists of atoms --- +such as the *flag_list* argument of :meth:`~IMAP4.store` and the *flags* +argument of :meth:`~IMAP4.append`, +the *names* argument of :meth:`~IMAP4.status`, +the *sort_criteria* argument of :meth:`~IMAP4.sort`, +or the *message_parts* argument of :meth:`~IMAP4.fetch` --- +can be passed as a sequence of strings instead of a single preformatted string. +For example, ``[r'\Seen', r'\Answered']`` +is equivalent to ``(\Seen \Answered)``. + +.. versionchanged:: next + Added support for passing these arguments as a sequence. + +.. _imap4-params: + +The value-bearing arguments of the search and fetch commands +can be quoted by hand, but this is error prone. +Instead, they may contain ``?`` placeholders that are substituted, and quoted +as required, from a *params* keyword argument, +in the manner of :mod:`sqlite3` parameter substitution:: + + # SEARCH FROM me@example.com SUBJECT "trip report" + M.search(None, 'FROM ? SUBJECT ?', params=['me@example.com', 'trip report']) + + # FETCH 1:5 (FLAGS BODY[HEADER.FIELDS (DATE FROM)]) + M.fetch('1:5', 'FLAGS BODY[HEADER.FIELDS ?]', params=[['DATE', 'FROM']]) + +The placeholders are: + +* ``?`` --- an ``astring``: a string (which will be quoted if necessary), + an integer, or a list of integers and/or strings + (which will be sent as a parenthesized list); +* ``?f`` --- a flag or a list of flags, sent verbatim without quoting; +* ``?s`` --- a *message_set* in the structured form described above. + +``??`` stands for a literal ``?``. + +Substitution is only performed when *params* is given; +if no *params* are given, an argument containing a literal ``?`` is unchanged. +The *params* keyword is accepted by :meth:`~IMAP4.search`, +:meth:`~IMAP4.fetch`, :meth:`~IMAP4.sort`, :meth:`~IMAP4.thread` and +:meth:`~IMAP4.uid`. + +.. versionadded:: next + The *params* keyword argument. An :class:`IMAP4` instance has the following methods: @@ -324,7 +384,7 @@ An :class:`IMAP4` instance has the following methods: Added the *message_set* and *uid* parameters. -.. method:: IMAP4.fetch(message_set, message_parts, *, uid=False) +.. method:: IMAP4.fetch(message_set, message_parts, *, uid=False, params=None) Fetch (parts of) messages. *message_parts* should be a string of message part names enclosed within parentheses, eg: ``"(UID BODY[TEXT])"``. Returned data @@ -333,8 +393,11 @@ An :class:`IMAP4` instance has the following methods: If *uid* is true, *message_set* is a set of UIDs and the message numbers in the response are UIDs (``UID FETCH``). + If *params* is given, ``?`` placeholders in *message_parts* are substituted + with the quoted parameters (see :ref:`the placeholders `). + .. versionchanged:: next - Added the *uid* parameter. + Added the *params* and *uid* parameters. .. method:: IMAP4.getacl(mailbox) @@ -598,7 +661,7 @@ An :class:`IMAP4` instance has the following methods: code, instead of the usual type. -.. method:: IMAP4.search(charset, criterion[, ...], *, uid=False) +.. method:: IMAP4.search(charset, criterion[, ...], *, uid=False, params=None) Search mailbox for matching messages. *charset* may be ``None``, in which case no ``CHARSET`` will be specified in the request to the server. The IMAP @@ -617,18 +680,22 @@ An :class:`IMAP4` instance has the following methods: When *charset* is ``None`` (as it must be under ``UTF8=ACCEPT``), the criterion is sent using the connection's encoding instead. + If *params* is given, ``?`` placeholders in the criteria are substituted + with the quoted parameters (see :ref:`the placeholders `). + Example:: # M is a connected IMAP4 instance... - typ, msgnums = M.search(None, 'FROM', '"LDJ"') + typ, msgnums = M.search(None, 'FROM', '"John Smith"') # or: - typ, msgnums = M.search(None, '(FROM "LDJ")') + typ, msgnums = M.search(None, '(FROM "John Smith")') - .. versionchanged:: next - Added the *uid* parameter. + # or, letting the module quote the value (this is recommended): + typ, msgnums = M.search(None, 'FROM ?', params=['John Smith']) .. versionchanged:: next + Added the *params* and *uid* parameters. ``str`` search criteria are encoded to *charset*. @@ -675,7 +742,7 @@ An :class:`IMAP4` instance has the following methods: Returns socket instance used to connect to server. -.. method:: IMAP4.sort(sort_criteria, charset, search_criterion[, ...], *, uid=False) +.. method:: IMAP4.sort(sort_criteria, charset, search_criterion[, ...], *, uid=False, params=None) The ``sort`` command is a variant of ``search`` with sorting semantics for the results. Returned data contains a space separated list of matching message @@ -696,12 +763,13 @@ An :class:`IMAP4` instance has the following methods: a *search_criterion* passed as :class:`str` is encoded to *charset*; pass :class:`bytes` to send one already encoded. - This is an ``IMAP4rev1`` extension command. + If *params* is given, ``?`` placeholders in the search criteria are + substituted with the quoted parameters (see :ref:`the placeholders `). - .. versionchanged:: next - Added the *uid* parameter. + This is an ``IMAP4rev1`` extension command. .. versionchanged:: next + Added the *params* and *uid* parameters. ``str`` search criteria are encoded to *charset*. @@ -745,7 +813,7 @@ An :class:`IMAP4` instance has the following methods: typ, data = M.search(None, 'ALL') for num in data[0].split(): - M.store(num, '+FLAGS', '\\Deleted') + M.store(num, '+FLAGS', r'\Deleted') M.expunge() .. note:: @@ -768,7 +836,7 @@ An :class:`IMAP4` instance has the following methods: Subscribe to new mailbox. -.. method:: IMAP4.thread(threading_algorithm, charset, search_criterion[, ...], *, uid=False) +.. method:: IMAP4.thread(threading_algorithm, charset, search_criterion[, ...], *, uid=False, params=None) The ``thread`` command is a variant of ``search`` with threading semantics for the results. Returned data contains a space separated list of thread members. @@ -793,22 +861,30 @@ An :class:`IMAP4` instance has the following methods: a *search_criterion* passed as :class:`str` is encoded to *charset*; pass :class:`bytes` to send one already encoded. - This is an ``IMAP4rev1`` extension command. + If *params* is given, ``?`` placeholders in the search criteria are + substituted with the quoted parameters (see :ref:`the placeholders `). - .. versionchanged:: next - Added the *uid* parameter. + This is an ``IMAP4rev1`` extension command. .. versionchanged:: next + Added the *params* and *uid* parameters. ``str`` search criteria are encoded to *charset*. -.. method:: IMAP4.uid(command, arg[, ...]) +.. method:: IMAP4.uid(command, arg[, ...], *, params=None) Execute command args with messages identified by UID, rather than message number. Returns response appropriate to command. At least one argument must be supplied; if none are provided, the server will return an error and an exception will be raised. + If *params* is given, ``?`` placeholders in the ``SEARCH``, ``SORT`` and + ``THREAD`` criteria or in the ``FETCH`` parts are substituted with the quoted + parameters (see :ref:`the placeholders `). + + .. versionchanged:: next + Added the *params* parameter. + .. method:: IMAP4.unsubscribe(mailbox) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index b6a5e4c7fbf1b8..ed6d303c1e28ad 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -290,6 +290,17 @@ imaplib the criteria are sent using the connection encoding instead. (Contributed by Serhiy Storchaka in :gh:`153494`.) +* Command methods now accept structured arguments, + so the module takes care of quoting instead of the caller. + A *message_set* and lists of flags or other atoms + can be passed as sequences instead of preformatted strings, + and the :meth:`~imaplib.IMAP4.search`, :meth:`~imaplib.IMAP4.fetch`, + :meth:`~imaplib.IMAP4.sort`, :meth:`~imaplib.IMAP4.thread` and + :meth:`~imaplib.IMAP4.uid` methods accept a *params* keyword argument + that substitutes and quotes ``?`` placeholders, + in the manner of :mod:`sqlite3` parameter substitution. + (Contributed by Serhiy Storchaka in :gh:`153521`.) + ipaddress --------- diff --git a/Lib/curses/ascii.py b/Lib/curses/ascii.py index 95acff33925ed7..cb55300e05dd4a 100644 --- a/Lib/curses/ascii.py +++ b/Lib/curses/ascii.py @@ -1,5 +1,8 @@ """Constants and membership tests for ASCII characters""" +# A character-cell type, present on wide and narrow builds. +from _curses import complexchar as _complexchar + NUL = 0x00 # ^@ SOH = 0x01 # ^A STX = 0x02 # ^B @@ -48,8 +51,12 @@ def _ctoi(c): if isinstance(c, str): return ord(c) - else: - return c + if isinstance(c, _complexchar): + # A character cell: its single character, or -1 (matches no class) + # for a cell with combining characters. + s = str(c) + return ord(s) if len(s) == 1 else -1 + return c def isalnum(c): return isalpha(c) or isdigit(c) def isalpha(c): return isupper(c) or islower(c) @@ -69,22 +76,23 @@ def isctrl(c): return 0 <= _ctoi(c) < 32 def ismeta(c): return _ctoi(c) > 127 def ascii(c): - if isinstance(c, str): - return chr(_ctoi(c) & 0x7f) - else: + if isinstance(c, int): return _ctoi(c) & 0x7f + else: + return chr(_ctoi(c) & 0x7f) def ctrl(c): - if isinstance(c, str): - return chr(_ctoi(c) & 0x1f) - else: - return _ctoi(c) & 0x1f + code = _ctoi(c) + if not 0 <= code < 128: + # No control character outside ASCII: return c unchanged. + return c if isinstance(c, int) else str(c) + return code & 0x1f if isinstance(c, int) else chr(code & 0x1f) def alt(c): - if isinstance(c, str): - return chr(_ctoi(c) | 0x80) - else: + if isinstance(c, int): return _ctoi(c) | 0x80 + else: + return chr(_ctoi(c) | 0x80) def unctrl(c): bits = _ctoi(c) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 538a858b33e8ae..139da1d3bb6fb8 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -133,6 +133,7 @@ # Only NUL, CR and LF are unsafe (they cannot be represented even in # a quoted string); other control characters are sent quoted. _control_chars = re.compile(b'[\x00\r\n]') +_control_chars_str = re.compile('[\x00\r\n]') _non_astring_char = re.compile(br'[(){ \x00-\x1f\x7f-\xff%*\\"]') _non_list_char = re.compile(br'[(){ \x00-\x1f\x7f-\xff\\"]') _quoted = re.compile(br'"(?:[^"\\]|\\.)*+"') @@ -144,6 +145,98 @@ def _paren_depth(data, depth=0): return depth + data.count(b'(') - data.count(b')') +def _seq_number(n): + # A single message number; None or '*' is the last message. + if n is None or n == '*': + return '*' + if not isinstance(n, int): + raise TypeError('message number must be an integer, not %s' + % type(n).__name__) + return str(n) + + +def _seq_range(item): + if isinstance(item, range): + if item.step == 1 and len(item): + item = (item.start, item[-1]) + else: + return ','.join(map(str, item)) + if isinstance(item, tuple): + start, stop = item + return '%s:%s' % (_seq_number(start), _seq_number(stop)) + return _seq_number(item) + + +def _format_sequence_set(arg): + if isinstance(arg, (int, str)): + return str(arg) + # A sequence of message numbers and ranges. + return ','.join(map(_seq_range, arg)) + + +# Characters that prevent a string from being sent as a bare atom. +_astring_special = re.compile(r'[(){ %*\\"\x00-\x1f\x7f-\U0010ffff]') +# A flag: an atom, optionally prefixed by a backslash. +_flag = re.compile(r'\\?[^(){ %*"\\\]\x00-\x1f\x7f-\U0010ffff]+') +# A placeholder in a format string: '?', '?f', '?s' or the escape '??'. +_placeholder = re.compile(r'\?[?fs]?') + + +def _format_astring(value): + if isinstance(value, (list, tuple)): + return '(' + ' '.join(map(_format_astring, value)) + ')' + if not isinstance(value, bool) and isinstance(value, int): + return str(value) + if isinstance(value, (bytes, bytearray)): + value = str(value, 'ascii') + elif not isinstance(value, str): + raise TypeError('expected a string, an integer or a list, not %s' + % type(value).__name__) + if value and _astring_special.search(value) is None: + return value # an atom, sent unquoted + if _control_chars_str.search(value): + raise ValueError('NUL, CR and LF cannot be represented inline: %r' + % value) + return '"' + value.replace('\\', r'\\').replace('"', r'\"') + '"' + + +def _format_flags(value): + if isinstance(value, (list, tuple)): + # A nested sequence is not part of the API; it produces invalid + # syntax that is rejected by the server. + return '(' + ' '.join(map(_format_flags, value)) + ')' + if isinstance(value, (bytes, bytearray)): + value = str(value, 'ascii') + elif not isinstance(value, str): + raise TypeError('expected a flag string or a list, not %s' + % type(value).__name__) + if not _flag.fullmatch(value): + raise ValueError('invalid flag: %r' % value) + return value + + +def _substitute(format, params): + params = iter(params) + def replace(match): + spec = match.group() + if spec == '??': # an escaped literal '?' + return '?' + try: + value = next(params) + except StopIteration: + raise TypeError('not enough parameters for the format string') \ + from None + if spec == '?f': # a flag or a list of flags + return _format_flags(value) + if spec == '?s': # a message sequence set + return _format_sequence_set(value) + return _format_astring(value) # an astring or a list of astrings + result = _placeholder.sub(replace, format) + for value in params: + raise TypeError('too many parameters for the format string') + return result + + class IMAP4: r"""IMAP4 client class. @@ -665,7 +758,7 @@ def expunge(self, message_set=None, *, uid=False): return self._untagged_response(typ, dat, name) - def fetch(self, message_set, message_parts, *, uid=False): + def fetch(self, message_set, message_parts, *, uid=False, params=None): """Fetch (parts of) messages. (typ, [data, ...]) = .fetch(message_set, message_parts) @@ -673,12 +766,17 @@ def fetch(self, message_set, message_parts, *, uid=False): 'message_parts' should be a string of selected parts enclosed in parentheses, eg: "(UID BODY[TEXT])". + If 'params' is given, '?' placeholders in 'message_parts' are + substituted with the quoted parameters. + 'data' are tuples of message part envelope and data. If 'uid' is true, 'message_set' is a set of UIDs and the message numbers in the response are UIDs (UID FETCH). """ name = 'FETCH' + if params is not None: + message_parts = _substitute(message_parts, params) args = (self._sequence_set(message_set), self._fetch_parts(message_parts)) if uid: @@ -941,7 +1039,7 @@ def rename(self, oldmailbox, newmailbox): self._mailbox(newmailbox)) - def search(self, charset, *criteria, uid=False): + def search(self, charset, *criteria, uid=False, params=None): """Search mailbox for matching messages. (typ, [data]) = .search(charset, criterion, ...) @@ -953,8 +1051,13 @@ def search(self, charset, *criteria, uid=False): A 'criteria' passed as str is encoded to 'charset'; pass bytes to send criteria that are already encoded. + + If 'params' is given, '?' placeholders in the criteria are + substituted with the quoted parameters. """ name = 'SEARCH' + if params is not None: + criteria = (_substitute(' '.join(criteria), params),) if charset is not None: if self.utf8_enabled: raise IMAP4.error("Non-None charset not valid in UTF8 mode") @@ -1028,18 +1131,24 @@ def setquota(self, root, limits): return self._untagged_response(typ, dat, 'QUOTA') - def sort(self, sort_criteria, charset, *search_criteria, uid=False): + def sort(self, sort_criteria, charset, *search_criteria, uid=False, + params=None): """IMAP4rev1 extension SORT command. (typ, [data]) = .sort(sort_criteria, charset, search_criteria, ...) If 'uid' is true, the message numbers in the response are UIDs (UID SORT). + + If 'params' is given, '?' placeholders in the search criteria are + substituted with the quoted parameters. """ name = 'SORT' #if not name in self.capabilities: # Let the server decide! # raise self.error('unimplemented extension command: %s' % name) sort_criteria = self._set_quote(sort_criteria) + if params is not None: + search_criteria = (_substitute(' '.join(search_criteria), params),) search_criteria = self._encode_criteria(charset, search_criteria) if charset is not None: charset = self._astring(charset) @@ -1113,15 +1222,21 @@ def subscribe(self, mailbox): return self._simple_command('SUBSCRIBE', self._mailbox(mailbox)) - def thread(self, threading_algorithm, charset, *search_criteria, uid=False): + def thread(self, threading_algorithm, charset, *search_criteria, uid=False, + params=None): """IMAPrev1 extension THREAD command. (type, [data]) = .thread(threading_algorithm, charset, search_criteria, ...) If 'uid' is true, the message numbers in the response are UIDs (UID THREAD). + + If 'params' is given, '?' placeholders in the search criteria are + substituted with the quoted parameters. """ name = 'THREAD' + if params is not None: + search_criteria = (_substitute(' '.join(search_criteria), params),) search_criteria = self._encode_criteria(charset, search_criteria) if charset is not None: charset = self._astring(charset) @@ -1133,13 +1248,17 @@ def thread(self, threading_algorithm, charset, *search_criteria, uid=False): return self._untagged_response(typ, dat, name) - def uid(self, command, *args): + def uid(self, command, *args, params=None): """Execute "command arg ..." with messages identified by UID, rather than message number. (typ, [data]) = .uid(command, arg1, arg2, ...) Returns response appropriate to 'command'. + + If 'params' is given, '?' placeholders in the SEARCH, SORT and + THREAD criteria or in the FETCH parts are substituted with the + quoted parameters. """ command = command.upper() if not command in Commands: @@ -1156,6 +1275,8 @@ def uid(self, command, *args): self._mailbox(new_mailbox)) elif command == 'FETCH': message_set, message_parts = args + if params is not None: + message_parts = _substitute(message_parts, params) args = (self._sequence_set(message_set), self._fetch_parts(message_parts)) elif command == 'STORE': @@ -1164,6 +1285,9 @@ def uid(self, command, *args): self._set_quote(flags)) elif command == 'SORT': sort_criteria, charset, *search_criteria = args + if params is not None: + search_criteria = (_substitute(' '.join(search_criteria), + params),) search_criteria = self._encode_criteria(charset, search_criteria) if charset is not None: charset = self._astring(charset) @@ -1171,11 +1295,16 @@ def uid(self, command, *args): *search_criteria) elif command == 'THREAD': threading_algorithm, charset, *search_criteria = args + if params is not None: + search_criteria = (_substitute(' '.join(search_criteria), + params),) search_criteria = self._encode_criteria(charset, search_criteria) if charset is not None: charset = self._astring(charset) args = (self._atom(threading_algorithm), charset, *search_criteria) + elif command == 'SEARCH' and params is not None: + args = (_substitute(' '.join(args), params),) typ, dat = self._simple_command(name, self._atom(command), *args) if command in ('SEARCH', 'SORT', 'THREAD'): name = command @@ -1570,9 +1699,13 @@ def _atom(self, arg): return arg def _sequence_set(self, arg): - return arg + return _format_sequence_set(arg) def _set_quote(self, arg): + if not isinstance(arg, str): + # A sequence of atoms (flags, criteria, item names, etc.); + # wrap them in parentheses as a single argument. + return '(' + ' '.join(arg) + ')' if arg and arg[0] == '(' and arg[-1] == ')': return arg return '(' + arg + ')' @@ -1580,7 +1713,7 @@ def _set_quote(self, arg): def _fetch_parts(self, arg): # "ALL", "FULL" and "FAST" are macros, not data item names; # they cannot be enclosed in parentheses. - if arg.upper() in ('ALL', 'FULL', 'FAST'): + if isinstance(arg, str) and arg.upper() in ('ALL', 'FULL', 'FAST'): return arg return self._set_quote(arg) diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index 870393be5209df..b99e4efbb83ce4 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -2761,6 +2761,9 @@ def test_ctrl(self): self.assertEqual(ctrl('\n'), '\n') self.assertEqual(ctrl('@'), '\0') self.assertEqual(ctrl(ord('J')), ord('\n')) + # A non-ASCII argument is returned unchanged (no control character). + self.assertEqual(ctrl('\xe9'), '\xe9') + self.assertEqual(ctrl(0xe9), 0xe9) def test_alt(self): alt = curses.ascii.alt @@ -2785,6 +2788,38 @@ def test_unctrl(self): self.assertEqual(unctrl(ord('\x8a')), '!^J') self.assertEqual(unctrl(ord('\xc1')), '!A') + @unittest.skipUnless(hasattr(curses, 'complexchar'), + 'requires the curses.complexchar type') + def test_complexchar(self): + # The predicates, ctrl() and unctrl() accept a complexchar too, using + # its single character. A narrow build just forms fewer cells. + cc = curses.complexchar + def storable(s): + try: + cc(s) + except ValueError: + return False + return True + + self.assertTrue(curses.ascii.isupper(cc('A'))) + self.assertTrue(curses.ascii.isalpha(cc('A', curses.A_BOLD))) + self.assertFalse(curses.ascii.isdigit(cc('A'))) + self.assertTrue(curses.ascii.isdigit(cc('7'))) + self.assertTrue(curses.ascii.iscntrl(cc('\n'))) + self.assertEqual(curses.ascii.ctrl(cc('J')), '\n') + self.assertEqual(curses.ascii.unctrl(cc('\n')), '^J') + self.assertEqual(curses.ascii.unctrl(cc('A')), 'A') + # A non-ASCII character: classified by code point, no control character. + if storable('\xe9'): + self.assertFalse(curses.ascii.isascii(cc('\xe9'))) + self.assertTrue(curses.ascii.ismeta(cc('\xe9'))) + self.assertEqual(curses.ascii.ctrl(cc('\xe9')), '\xe9') + # A cell with combining marks is not a single character, so no + # predicate matches it (needs a wide build to store). + if storable('e\u0301'): + self.assertFalse(curses.ascii.isalpha(cc('e\u0301'))) + self.assertFalse(curses.ascii.isascii(cc('e\u0301'))) + def lorem_ipsum(win): text = [ diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 3eaca67299e7c3..100791a9c3eb2c 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -184,8 +184,8 @@ def test_astring(self): self.assertEqual(m._astring(b'INBOX'), b'INBOX') # Names with protocol-sensitive characters are quoted. self.assertEqual(m._astring('New folder'), b'"New folder"') - self.assertEqual(m._astring('a"b'), b'"a\\"b"') - self.assertEqual(m._astring('a\\b'), b'"a\\\\b"') + self.assertEqual(m._astring('a"b'), rb'"a\"b"') + self.assertEqual(m._astring(r'a\b'), rb'"a\\b"') self.assertEqual(m._astring(''), b'""') self.assertEqual(m._astring('*'), b'"*"') # A well-formed quoted string is passed through unchanged. @@ -193,12 +193,12 @@ def test_astring(self): self.assertEqual(m._astring('""'), b'""') # Including a lenient (non-RFC) backslash escape, which the server # may accept. - self.assertEqual(m._astring('"a\\b"'), b'"a\\b"') + self.assertEqual(m._astring(r'"a\b"'), rb'"a\b"') # A string that only looks quoted but is not a single token is # quoted as data, closing the argument injection vector. self.assertEqual(m._astring('"a" SELECT evil "'), - b'"\\"a\\" SELECT evil \\""') - self.assertEqual(m._astring('"'), b'"\\""') + rb'"\"a\" SELECT evil \""') + self.assertEqual(m._astring('"'), rb'"\""') # Non-ASCII names are only allowed in a quoted string or a # literal, never in an atom (RFC 6855). m._encoding = 'utf-8' @@ -287,6 +287,69 @@ def test_mailbox(self): self.assertEqual(m._mailbox('Entwürfe'), '"Entwürfe"'.encode()) self.assertEqual(m._mailbox('Entw&APw-rfe'), b'Entw&APw-rfe') + def test_sequence_set(self): + m = imaplib.IMAP4.__new__(imaplib.IMAP4) + # A scalar is passed through as a string. + self.assertEqual(m._sequence_set(5), '5') + self.assertEqual(m._sequence_set('1:3,7'), '1:3,7') + # A sequence of numbers and ranges is formatted as a sequence set. + self.assertEqual(m._sequence_set([1, 2, 5]), '1,2,5') + self.assertEqual(m._sequence_set([1, (3, 5), (8, '*')]), '1,3:5,8:*') + self.assertEqual(m._sequence_set([(5, None)]), '5:*') + # A range is inclusive; a non-unit step falls back to explicit numbers. + self.assertEqual(m._sequence_set([range(1, 4), 7]), '1:3,7') + self.assertEqual(m._sequence_set([range(1, 10, 2)]), '1,3,5,7,9') + # Message numbers must be integers: a string is not coerced (the + # string form is the whole preformatted set), nor is a float. + self.assertRaises(TypeError, m._sequence_set, ['7']) + self.assertRaises(TypeError, m._sequence_set, [2.9]) + self.assertRaises(TypeError, m._sequence_set, [(1, 2.9)]) + + def test_set_quote(self): + m = imaplib.IMAP4.__new__(imaplib.IMAP4) + # A string is parenthesized unless it already is. + self.assertEqual(m._set_quote(r'\Seen'), r'(\Seen)') + self.assertEqual(m._set_quote(r'(\Seen)'), r'(\Seen)') + # A sequence of atoms is joined and parenthesized. + self.assertEqual(m._set_quote([r'\Seen', r'\Answered']), + r'(\Seen \Answered)') + self.assertEqual(m._set_quote(['MESSAGES', 'UNSEEN']), + '(MESSAGES UNSEEN)') + + def test_substitute(self): + sub = imaplib._substitute + # '?' quotes an astring; an atom-safe value is left unquoted. + self.assertEqual(sub('FROM ?', ['me@host']), 'FROM me@host') + self.assertEqual(sub('SUBJECT ?', ['hello world']), + 'SUBJECT "hello world"') + self.assertEqual(sub('SUBJECT ?', ['a"b']), r'SUBJECT "a\"b"') + # An integer becomes a number, a list a parenthesized list. + self.assertEqual(sub('LARGER ?', [1000]), 'LARGER 1000') + self.assertEqual(sub('HEADER.FIELDS ?', [['DATE', 'FROM']]), + 'HEADER.FIELDS (DATE FROM)') + # '?f' emits flags verbatim, never quoted. + self.assertEqual(sub('?f', [r'\Seen']), r'\Seen') + self.assertEqual(sub('?f', [[r'\Seen', r'\Answered']]), + r'(\Seen \Answered)') + # '?s' formats a message sequence set. + self.assertEqual(sub('?s', [[1, (3, 5), (8, '*')]]), '1,3:5,8:*') + # '??' is a literal '?'. + self.assertEqual(sub('a?? b', []), 'a? b') + + def test_substitute_errors(self): + sub = imaplib._substitute + self.assertRaises(TypeError, sub, '? ?', ['x']) # too few parameters + self.assertRaises(TypeError, sub, '?', ['x', 'y']) # too many parameters + self.assertRaises(ValueError, sub, '?f', ['a b']) # not a valid flag + self.assertRaises(ValueError, sub, '?', ['a\r\nb']) # CR/LF not inline + self.assertRaises(TypeError, sub, '?', [True]) # bool is not a string + self.assertRaises(TypeError, sub, '?', [1.5]) # float is not a string + self.assertRaises(TypeError, sub, '?s', [['a']]) # not a message number + self.assertRaises(TypeError, sub, '?s', [[1.5]]) # not a message number + self.assertRaises(TypeError, sub, '?s', [[(1, 'a')]]) # not a message number + self.assertRaises(ValueError, sub, '?s', [[(1,)]]) # not a range pair + self.assertRaises(ValueError, sub, '?s', [[(1, 2, 3)]]) # not a range pair + if ssl: class SecureTCPServer(socketserver.TCPServer): @@ -1342,6 +1405,11 @@ def test_copy(self): self.assertEqual(data, [b'COPY completed']) self.assertEqual(server.args, ['2:4', '"New folder"']) + # A structured message set is formatted into a sequence set. + typ, data = client.copy([2, (3, 5)], 'MEETING') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['2,3:5', 'MEETING']) + def test_uid_copy(self): client, server = self._setup(make_simple_handler('UID', completed='UID COPY completed')) @@ -1363,6 +1431,11 @@ def test_uid_copy(self): self.assertEqual(data, [b'UID COPY completed']) self.assertEqual(server.args, ['COPY', '4827313:4828442', 'MEETING']) + # A structured message set is formatted into a sequence set. + typ, data = client.uid('copy', [1, (3, 5)], 'MEETING') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['COPY', '1,3:5', 'MEETING']) + def test_move(self): client, server = self._setup(make_simple_handler('MOVE')) client.login('user', 'pass') @@ -1377,6 +1450,11 @@ def test_move(self): self.assertEqual(data, [b'MOVE completed']) self.assertEqual(server.args, ['2:4', '"New folder"']) + # A structured message set is formatted into a sequence set. + typ, data = client.move([2, (3, 5)], 'MEETING') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['2,3:5', 'MEETING']) + def test_uid_move(self): client, server = self._setup(make_simple_handler('UID', completed='UID MOVE completed')) @@ -1398,6 +1476,11 @@ def test_uid_move(self): self.assertEqual(data, [b'UID MOVE completed']) self.assertEqual(server.args, ['MOVE', '4827313:4828442', 'MEETING']) + # A structured message set is formatted into a sequence set. + typ, data = client.uid('move', [1, (3, 5)], 'MEETING') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['MOVE', '1,3:5', 'MEETING']) + def test_store(self): client, server = self._setup(make_simple_handler('STORE', [ r'* 2 FETCH (FLAGS (\Deleted \Seen))', @@ -1419,6 +1502,12 @@ def test_store(self): self.assertEqual(typ, 'OK') self.assertEqual(server.args, ['2:4', '+FLAGS', r'(\Deleted)']) + # The flags may be a sequence, and the message set may be structured. + typ, data = client.store([2, (3, 4)], '+FLAGS', + [r'\Deleted', r'\Seen']) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['2,3:4', '+FLAGS', r'(\Deleted \Seen)']) + def test_uid_store(self): client, server = self._setup(make_simple_handler('UID', [ r'* 23 FETCH (FLAGS (\Deleted \Seen) UID 4827313)', @@ -1451,6 +1540,13 @@ def test_uid_store(self): ]) self.assertEqual(server.args, ['STORE', '4827313:4828442', '+FLAGS', r'(\Deleted)']) + # The flags may be a sequence. + typ, data = client.uid('store', '4827313:4828442', '+FLAGS', + [r'\Deleted', r'\Seen']) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, + ['STORE', '4827313:4828442', '+FLAGS', r'(\Deleted \Seen)']) + def test_fetch(self): # The handler expands the requested sequence set and answers for # exactly those messages, so the test exercises the round trip of @@ -1508,6 +1604,23 @@ def cmd_FETCH(self, tag, args): self.assertEqual(typ, 'OK') self.assertEqual(server.args, ['2:4', 'fast']) + # A structured message set is formatted into a sequence set. + typ, data = client.fetch([2, (3, 4)], '(FLAGS)') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['2,3:4', '(FLAGS)']) + + # message_parts may be a sequence of items. + typ, data = client.fetch('1', ['UID', 'FLAGS']) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['1', '(UID FLAGS)']) + + # 'params' substitutes and quotes '?' placeholders. + typ, data = client.fetch('1', 'FLAGS BODY[HEADER.FIELDS ?]', + params=[['DATE', 'FROM']]) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, + ['1', '(FLAGS BODY[HEADER.FIELDS (DATE FROM)])']) + def test_uid_fetch(self): client, server = self._setup(make_simple_handler('UID', [ r'* 23 FETCH (FLAGS (\Seen) UID 4827313)', @@ -1543,6 +1656,18 @@ def test_uid_fetch(self): ]) self.assertEqual(server.args, ['FETCH', '4827313:4828442', '(FLAGS)']) + # message_parts may be a sequence, and 'params' substitutes '?'. + typ, data = client.uid('fetch', '4827313:4828442', ['UID', 'FLAGS']) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['FETCH', '4827313:4828442', '(UID FLAGS)']) + + typ, data = client.uid('fetch', '4827313:4828442', + 'BODY[HEADER.FIELDS ?]', params=[['DATE', 'FROM']]) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, + ['FETCH', '4827313:4828442', + '(BODY[HEADER.FIELDS (DATE FROM)])']) + def test_partial(self): client, server = self._setup(make_simple_handler('PARTIAL', ['* 1 FETCH (RFC822.TEXT<0.10> "0123456789")'])) @@ -1590,6 +1715,19 @@ def test_search(self): self.assertEqual(typ, 'OK') self.assertEqual(server.args, ['CHARSET', '"NF_Z_62-010_(1973)"', 'TEXT', 'XXXXXX']) + # 'params' substitutes and quotes '?' placeholders. + response[:] = ['* SEARCH 1'] + typ, data = client.search(None, 'FROM ? SUBJECT ?', + params=['me@host', 'trip report']) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, + ['FROM', 'me@host', 'SUBJECT', '"trip report"']) + + # Without 'params', a literal '?' is sent unchanged. + typ, data = client.search(None, 'SUBJECT', '"what?"') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['SUBJECT', '"what?"']) + def test_uid_search(self): response = [] client, server = self._setup(make_simple_handler('UID', response, @@ -1633,6 +1771,14 @@ def test_uid_search(self): self.assertEqual(data, [b'43']) self.assertEqual(server.args, ['SEARCH', 'CHARSET', 'UTF-8', 'TEXT', 'XXXXXX']) + # 'params' substitutes and quotes '?' placeholders. + response[:] = ['* SEARCH 1'] + typ, data = client.uid('SEARCH', 'FROM ? SUBJECT ?', + params=['me@host', 'trip report']) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, + ['SEARCH', 'FROM', 'me@host', 'SUBJECT', '"trip report"']) + def test_sort(self): response = [] client, server = self._setup(make_simple_handler('SORT', response)) @@ -1666,6 +1812,15 @@ def test_sort(self): self.assertEqual(typ, 'OK') self.assertIn('"Київ"'.encode('koi8-u'), server.line) + # sort_criteria may be a sequence, and 'params' substitutes and + # quotes '?' (a value with a space becomes a quoted string). + response[:] = ['* SORT 1'] + typ, data = client.sort(['REVERSE', 'DATE'], 'UTF-8', 'SUBJECT ?', + params=['trip report']) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, + ['(REVERSE DATE)', 'UTF-8', 'SUBJECT', '"trip report"']) + def test_uid_sort(self): response = [] client, server = self._setup(make_simple_handler('UID', response, @@ -1707,6 +1862,15 @@ def test_uid_sort(self): self.assertEqual(data, [br'2 84 882']) self.assertEqual(server.args, ['SORT', '(SUBJECT)', 'UTF-8', 'SINCE', '1-Feb-1994']) + # sort_criteria may be a sequence, and 'params' substitutes and + # quotes '?' (a value with a space becomes a quoted string). + response[:] = ['* SORT 1'] + typ, data = client.uid('sort', ['REVERSE', 'DATE'], 'UTF-8', 'SUBJECT ?', + params=['trip report']) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, + ['SORT', '(REVERSE DATE)', 'UTF-8', 'SUBJECT', '"trip report"']) + def test_thread(self): response = [] client, server = self._setup(make_simple_handler('THREAD', response)) @@ -1754,6 +1918,15 @@ def test_thread(self): self.assertEqual(typ, 'OK') self.assertIn('"Київ"'.encode('koi8-u'), server.line) + # 'params' substitutes and quotes '?' (a value with a space becomes + # a quoted string). + response[:] = ['* THREAD (1)'] + typ, data = client.thread('REFERENCES', 'UTF-8', 'SUBJECT ?', + params=['trip report']) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, + ['REFERENCES', 'UTF-8', 'SUBJECT', '"trip report"']) + def test_uid_thread(self): response = [] client, server = self._setup(make_simple_handler('UID', response, @@ -1810,6 +1983,15 @@ def test_uid_thread(self): self.assertEqual(data, [b'(166)(167)(168)']) self.assertEqual(server.args, ['THREAD', 'ORDEREDSUBJECT', 'UTF-8', 'SINCE', '5-MAR-2000']) + # 'params' substitutes and quotes '?' (a value with a space becomes + # a quoted string). + response[:] = ['* THREAD (1)'] + typ, data = client.uid('THREAD', 'REFERENCES', 'UTF-8', 'SUBJECT ?', + params=['trip report']) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, + ['THREAD', 'REFERENCES', 'UTF-8', 'SUBJECT', '"trip report"']) + def test_delete(self): client, server = self._setup(make_simple_handler('DELETE')) client.login('user', 'pass') @@ -1900,6 +2082,11 @@ def test_status(self): self.assertEqual(typ, 'OK') self.assertEqual(server.args, ['"New folder"', '(UIDNEXT MESSAGES)']) + # The names argument may be a sequence of item names. + typ, data = client.status('blurdybloop', ['UIDNEXT', 'MESSAGES']) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['blurdybloop', '(UIDNEXT MESSAGES)']) + def test_getacl(self): client, server = self._setup(make_simple_handler('GETACL', ['* ACL INBOX Fred rwipslxetad'])) diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 725c6e3b09e1d0..9693cfc259286d 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -188,6 +188,8 @@ def test_request_rate(self): parsed_request_rate.seconds, self.request_rate.seconds ) + else: + self.assertIsNone(parsed_request_rate) class EmptyFileTest(BaseRequestRateTest, unittest.TestCase): @@ -246,6 +248,32 @@ class InvalidCrawlDelayTest(BaseRobotTest, unittest.TestCase): bad = [] +class NonDecimalDigitsTest(BaseRequestRateTest, unittest.TestCase): + # Non-decimal Unicode digits pass str.isdigit() but int() rejects + # them, so the directive must be silently ignored, not raise. + robots_txt = """\ +User-Agent: * +Disallow: /tmp/ +Crawl-delay: ² +Request-rate: ²/5 + """ + good = ['/foo.html'] + bad = ['/tmp/'] + crawl_delay = None + request_rate = None + + +class NonDecimalDenominatorTest(BaseRequestRateTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /tmp/ +Request-rate: 5/² + """ + good = ['/foo.html'] + request_rate = None + bad = ['/tmp/'] + + class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase): # also test that Allow and Diasallow works well with each other robots_txt = """\ diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index f9803abb3db91f..3809f2d2643895 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -182,6 +182,16 @@ def test_calcsize(self): self.assertGreaterEqual(struct.calcsize('n'), struct.calcsize('i')) self.assertGreaterEqual(struct.calcsize('n'), struct.calcsize('P')) + def test_cache_bytes_vs_str_bb(self): + # Mixing str and bytes formats must not raise BytesWarning under -bb. + code = ( + 'import struct\n' + 'struct.calcsize(b"!d"); struct.calcsize("!d")\n' + 'struct.calcsize(">d"); struct.calcsize(b">d")\n' + 'struct.Struct(b"i"); struct.Struct("i")\n' + ) + assert_python_ok('-bb', '-c', code) + def test_integers(self): # Integer tests (bBhHiIlLqQnN). import binascii diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 61772d90e2d53b..8d0311d96f5e0b 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -148,15 +148,15 @@ def parse(self, lines): # before trying to convert to int we need to make # sure that robots.txt has valid syntax otherwise # it will crash - if line[1].strip().isdigit(): + if line[1].strip().isdecimal(): entry.delay = int(line[1]) state = 2 elif line[0] == "request-rate": if state != 0: numbers = line[1].split('/') # check if all values are sane - if (len(numbers) == 2 and numbers[0].strip().isdigit() - and numbers[1].strip().isdigit()): + if (len(numbers) == 2 and numbers[0].strip().isdecimal() + and numbers[1].strip().isdecimal()): entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1])) state = 2 elif line[0] == "sitemap": diff --git a/Misc/NEWS.d/next/Library/2026-07-06-14-30-00.gh-issue-153521.iMaP47.rst b/Misc/NEWS.d/next/Library/2026-07-06-14-30-00.gh-issue-153521.iMaP47.rst new file mode 100644 index 00000000000000..93edb1532d5d41 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-06-14-30-00.gh-issue-153521.iMaP47.rst @@ -0,0 +1,7 @@ +Add support for structured arguments in :mod:`imaplib` command methods. A +*message_set* and lists of flags or other atoms can now be passed as +sequences instead of preformatted strings, and the +:meth:`~imaplib.IMAP4.search`, :meth:`~imaplib.IMAP4.fetch`, +:meth:`~imaplib.IMAP4.sort`, :meth:`~imaplib.IMAP4.thread` and +:meth:`~imaplib.IMAP4.uid` methods accept a *params* keyword argument that +substitutes and quotes ``?`` placeholders. diff --git a/Misc/NEWS.d/next/Library/2026-07-09-10-14-08.gh-issue-153395.N8FuG3.rst b/Misc/NEWS.d/next/Library/2026-07-09-10-14-08.gh-issue-153395.N8FuG3.rst new file mode 100644 index 00000000000000..1c44219f710640 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-09-10-14-08.gh-issue-153395.N8FuG3.rst @@ -0,0 +1,4 @@ +:mod:`curses.ascii` predicates and the :func:`~curses.ascii.ctrl` and +:func:`~curses.ascii.unctrl` functions now accept a :class:`curses.complexchar`. +:func:`~curses.ascii.ctrl` now returns a non-ASCII argument unchanged instead +of masking it to a control character. diff --git a/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst b/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst new file mode 100644 index 00000000000000..0746d3f12149d5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst @@ -0,0 +1,4 @@ +:class:`urllib.robotparser.RobotFileParser` now silently ignores a +``Crawl-delay`` or ``Request-rate`` value written with non-decimal digits +(such as ``U+00B2 SUPERSCRIPT TWO``) instead of raising :exc:`ValueError` +and aborting the parse of the whole ``robots.txt`` file. diff --git a/Misc/NEWS.d/next/Library/2026-07-12-00-00-00.gh-issue-85943.8f906e.rst b/Misc/NEWS.d/next/Library/2026-07-12-00-00-00.gh-issue-85943.8f906e.rst new file mode 100644 index 00000000000000..0a93bf2e7b803e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-12-00-00-00.gh-issue-85943.8f906e.rst @@ -0,0 +1,4 @@ +Fix :mod:`struct` functions raising :exc:`BytesWarning` under the ``-bb`` +command line option when a :class:`str` format is used after an equal +:class:`bytes` format (or vice versa). The internal format cache no longer +mixes :class:`str` and :class:`bytes` keys. diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-07-10-14-38-31.gh-issue-152384.uaPADM.rst b/Misc/NEWS.d/next/Tools-Demos/2026-07-10-14-38-31.gh-issue-152384.uaPADM.rst new file mode 100644 index 00000000000000..08b94d5486f64d --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2026-07-10-14-38-31.gh-issue-152384.uaPADM.rst @@ -0,0 +1,8 @@ +The CPython Pixi packages are now all accessible at the same +``Tools/pixi-packages`` subdirectory, rather than at +``Tools/pixi-packages/{variant}`` as before. Variants are now selected not +via subdirectory but via ``flags``; see +https://pixi.prefix.dev/latest/concepts/package_specifications/#extras-and-flags +for usage instructions. The ``tsan-freethreading`` variant has been renamed +to ``tsan_freethreading``, while the ``default``, ``asan``, and +``freethreading`` variants retain their previous names. diff --git a/Modules/_struct.c b/Modules/_struct.c index 260dc85faf9fb9..352312fb0b4c19 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -2642,7 +2642,8 @@ static PyType_Spec PyStructType_spec = { static int cache_struct_converter(PyObject *module, PyObject *fmt, PyStructObject **ptr) { - PyObject * s_object; + PyObject *s_object; + PyObject *key; _structmodulestate *state = get_struct_state(module); if (fmt == NULL) { @@ -2650,24 +2651,41 @@ cache_struct_converter(PyObject *module, PyObject *fmt, PyStructObject **ptr) return 1; } - if (PyDict_GetItemRef(state->cache, fmt, &s_object) < 0) { + /* Use a str cache key: an equal str and bytes would collide and be + compared, raising BytesWarning under -bb. */ + if (PyBytes_Check(fmt)) { + key = PyUnicode_DecodeASCII(PyBytes_AS_STRING(fmt), + PyBytes_GET_SIZE(fmt), "surrogateescape"); + if (key == NULL) { + return 0; + } + } + else { + key = Py_NewRef(fmt); + } + + if (PyDict_GetItemRef(state->cache, key, &s_object) < 0) { + Py_DECREF(key); return 0; } if (s_object != NULL) { + Py_DECREF(key); *ptr = PyStructObject_CAST(s_object); return Py_CLEANUP_SUPPORTED; } - s_object = PyObject_CallOneArg(state->PyStructType, fmt); + s_object = PyObject_CallOneArg(state->PyStructType, key); if (s_object != NULL) { if (PyDict_GET_SIZE(state->cache) >= MAXCACHE) PyDict_Clear(state->cache); /* Attempt to cache the result */ - if (PyDict_SetItem(state->cache, fmt, s_object) == -1) + if (PyDict_SetItem(state->cache, key, s_object) == -1) PyErr_Clear(); + Py_DECREF(key); *ptr = (PyStructObject *)s_object; return Py_CLEANUP_SUPPORTED; } + Py_DECREF(key); return 0; } diff --git a/Tools/pixi-packages/README.md b/Tools/pixi-packages/README.md index d818fddaac6a1e..3e9df3f56f37fa 100644 --- a/Tools/pixi-packages/README.md +++ b/Tools/pixi-packages/README.md @@ -10,7 +10,8 @@ in a [Pixi workspace](https://pixi.sh/latest/first_workspace/), like: ```toml [dependencies] python.git = "https://github.com/python/cpython" -python.subdirectory = "Tools/pixi-packages/asan" +python.subdirectory = "Tools/pixi-packages" +python.flags = ["asan"] ``` This is particularly useful when developers need to build CPython from source @@ -18,17 +19,18 @@ This is particularly useful when developers need to build CPython from source clone or build steps. Instead, Pixi will automatically handle both the build and installation of the package. -Each package definition is contained in a subdirectory, but they share the build script -`build.sh` in this directory. Currently defined package variants: +Each package variant carries a 'flag' to enable selection of that variant — see +[the Pixi docs](https://pixi.prefix.dev/latest/concepts/package_specifications/#extras-and-flags) +for details of how to use this. Currently defined package variants: - `default` - `freethreading` - `asan`: ASan-instrumented build -- `tsan-freethreading`: TSan-instrumented free-threading build +- `tsan_freethreading`: TSan-instrumented free-threading build ## Maintenance -- Keep the `abi_tag` and `version` fields in each `variants.yaml` up to date with the +- Keep the `version` field in `variants.yaml` up to date with the Python version - Update `build.sh` for any breaking changes in the `configure` and `make` workflow @@ -36,8 +38,6 @@ Each package definition is contained in a subdirectory, but they share the build - More package variants (such as UBSan) - Support for Windows -- Using a single `pixi.toml` for all package variants is blocked on - [pixi#5248](https://github.com/prefix-dev/pixi/issues/5248) ## Troubleshooting diff --git a/Tools/pixi-packages/asan/variants.yaml b/Tools/pixi-packages/asan/variants.yaml deleted file mode 100644 index 2404948457e6bb..00000000000000 --- a/Tools/pixi-packages/asan/variants.yaml +++ /dev/null @@ -1,6 +0,0 @@ -variant: - - asan -abi_tag: - - asan_cp315 -version: - - 3.15 diff --git a/Tools/pixi-packages/build.sh b/Tools/pixi-packages/build.sh index 7e22e6243a5f77..b13e26d688e46a 100644 --- a/Tools/pixi-packages/build.sh +++ b/Tools/pixi-packages/build.sh @@ -7,7 +7,7 @@ if [[ "${PYTHON_VARIANT}" == "freethreading" ]]; then elif [[ "${PYTHON_VARIANT}" == "asan" ]]; then CONFIGURE_EXTRA="--with-address-sanitizer" export ASAN_OPTIONS="strict_init_order=true" -elif [[ "${PYTHON_VARIANT}" == "tsan-freethreading" ]]; then +elif [[ "${PYTHON_VARIANT}" == "tsan_freethreading" ]]; then CONFIGURE_EXTRA="--disable-gil --with-thread-sanitizer" export TSAN_OPTIONS="suppressions=${SRC_DIR}/Tools/tsan/suppressions_free_threading.txt" elif [[ "${PYTHON_VARIANT}" == "default" ]]; then @@ -17,6 +17,14 @@ else exit 1 fi +VER_REF=$(grep "\[PYTHON_VERSION\]\, \[" configure.ac | sed -n 's/.*\[\([0-9.]*\)\].*/\1/p') +VER=$(echo ${PKG_VERSION} | sed -E 's/^([0-9]+\.[0-9]+).*/\1/') + +if [[ "${VER_REF}" != "${VER}" ]]; then + echo "Unexpected version from conda package. Got ${VER}. Expected ${VER_REF}. Do you need to update 'version' in 'variants.yaml'?" + exit 1 +fi + # rattler-build by default set a target of 10.9 # override it to at least 10.12 case ${MACOSX_DEPLOYMENT_TARGET:-10.12} in diff --git a/Tools/pixi-packages/clone-recipe.sh b/Tools/pixi-packages/clone-recipe.sh deleted file mode 100755 index 25ceaf85c35f56..00000000000000 --- a/Tools/pixi-packages/clone-recipe.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -# Please always only modify default/recipe.yaml and default/pixi.toml and then run this -# script to propagate the changes to the other variants. -set -o errexit -cd "$(dirname "$0")" - -for variant in asan freethreading tsan-freethreading; do - cp -av default/pixi.toml ${variant}/ -done diff --git a/Tools/pixi-packages/default/pixi.toml b/Tools/pixi-packages/default/pixi.toml deleted file mode 100644 index bf9841e18677ca..00000000000000 --- a/Tools/pixi-packages/default/pixi.toml +++ /dev/null @@ -1,15 +0,0 @@ -# NOTE: Please always only modify default/pixi.toml and then run clone-recipe.sh to -# propagate the changes to the other variants. - -[workspace] -channels = ["https://prefix.dev/conda-forge"] -platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"] -preview = ["pixi-build"] -requires-pixi = ">=0.66.0" - -[package.build.backend] -name = "pixi-build-rattler-build" -version = "*" - -[package.build.config] -recipe = "../default/recipe.yaml" diff --git a/Tools/pixi-packages/default/variants.yaml b/Tools/pixi-packages/default/variants.yaml deleted file mode 100644 index f66e9e7a2226ba..00000000000000 --- a/Tools/pixi-packages/default/variants.yaml +++ /dev/null @@ -1,6 +0,0 @@ -variant: - - default -abi_tag: - - cp315 -version: - - 3.15 diff --git a/Tools/pixi-packages/freethreading/pixi.toml b/Tools/pixi-packages/freethreading/pixi.toml deleted file mode 100644 index bf9841e18677ca..00000000000000 --- a/Tools/pixi-packages/freethreading/pixi.toml +++ /dev/null @@ -1,15 +0,0 @@ -# NOTE: Please always only modify default/pixi.toml and then run clone-recipe.sh to -# propagate the changes to the other variants. - -[workspace] -channels = ["https://prefix.dev/conda-forge"] -platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"] -preview = ["pixi-build"] -requires-pixi = ">=0.66.0" - -[package.build.backend] -name = "pixi-build-rattler-build" -version = "*" - -[package.build.config] -recipe = "../default/recipe.yaml" diff --git a/Tools/pixi-packages/freethreading/variants.yaml b/Tools/pixi-packages/freethreading/variants.yaml deleted file mode 100644 index 022833d04c3821..00000000000000 --- a/Tools/pixi-packages/freethreading/variants.yaml +++ /dev/null @@ -1,6 +0,0 @@ -variant: - - freethreading -abi_tag: - - cp315t -version: - - 3.15 diff --git a/Tools/pixi-packages/asan/pixi.toml b/Tools/pixi-packages/pixi.toml similarity index 50% rename from Tools/pixi-packages/asan/pixi.toml rename to Tools/pixi-packages/pixi.toml index bf9841e18677ca..afbb5e1bcd56bf 100644 --- a/Tools/pixi-packages/asan/pixi.toml +++ b/Tools/pixi-packages/pixi.toml @@ -1,15 +1,9 @@ -# NOTE: Please always only modify default/pixi.toml and then run clone-recipe.sh to -# propagate the changes to the other variants. - [workspace] channels = ["https://prefix.dev/conda-forge"] platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"] preview = ["pixi-build"] -requires-pixi = ">=0.66.0" +requires-pixi = ">=0.72.2" [package.build.backend] name = "pixi-build-rattler-build" version = "*" - -[package.build.config] -recipe = "../default/recipe.yaml" diff --git a/Tools/pixi-packages/default/recipe.yaml b/Tools/pixi-packages/recipe.yaml similarity index 85% rename from Tools/pixi-packages/default/recipe.yaml rename to Tools/pixi-packages/recipe.yaml index 30d0d5a2ed2e04..c40fe2ad2e0c79 100644 --- a/Tools/pixi-packages/default/recipe.yaml +++ b/Tools/pixi-packages/recipe.yaml @@ -1,15 +1,13 @@ -# NOTE: Please always only modify default/recipe.yaml and then run clone-recipe.sh to -# propagate the changes to the other variants. - context: - # Keep up to date freethreading_tag: ${{ "t" if "freethreading" in variant else "" }} + abi_prefix: ${{ (variant | split("_"))[0] + "_" if "san" in variant else "" }} + abi_tag: ${{ abi_prefix }}cp${{ (version | split('.'))[:2] | join('') }}${{ freethreading_tag }} recipe: name: python source: - - path: ../../.. + - path: ../.. outputs: - package: @@ -25,12 +23,16 @@ outputs: name: python version: ${{ version }} build: + flags: + - ${{ variant }} + variant: + down_prioritize_variant: ${{ 0 if variant == "default" else 1 }} string: "0_${{ abi_tag }}" files: exclude: - "*.o" script: - file: ../build.sh + file: build.sh env: PYTHON_VARIANT: ${{ variant }} python: @@ -70,9 +72,6 @@ outputs: - libuuid - libmpdec-devel - expat - - if: linux and "san" in variant - then: - - libsanitizer - if: osx and "san" in variant then: - libcompiler-rt diff --git a/Tools/pixi-packages/tsan-freethreading/pixi.toml b/Tools/pixi-packages/tsan-freethreading/pixi.toml deleted file mode 100644 index bf9841e18677ca..00000000000000 --- a/Tools/pixi-packages/tsan-freethreading/pixi.toml +++ /dev/null @@ -1,15 +0,0 @@ -# NOTE: Please always only modify default/pixi.toml and then run clone-recipe.sh to -# propagate the changes to the other variants. - -[workspace] -channels = ["https://prefix.dev/conda-forge"] -platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"] -preview = ["pixi-build"] -requires-pixi = ">=0.66.0" - -[package.build.backend] -name = "pixi-build-rattler-build" -version = "*" - -[package.build.config] -recipe = "../default/recipe.yaml" diff --git a/Tools/pixi-packages/tsan-freethreading/variants.yaml b/Tools/pixi-packages/tsan-freethreading/variants.yaml deleted file mode 100644 index 6ed09fcc9b656b..00000000000000 --- a/Tools/pixi-packages/tsan-freethreading/variants.yaml +++ /dev/null @@ -1,6 +0,0 @@ -variant: - - tsan-freethreading -abi_tag: - - tsan_cp315t -version: - - 3.15 diff --git a/Tools/pixi-packages/variants.yaml b/Tools/pixi-packages/variants.yaml new file mode 100644 index 00000000000000..2519ec5ea0d6d6 --- /dev/null +++ b/Tools/pixi-packages/variants.yaml @@ -0,0 +1,15 @@ +version: ["3.16"] +variant: ["default", "asan", "freethreading", "tsan_freethreading"] + +openssl: + - '3.5' + +# match `.github/workflows/reusable-san.yml` +c_compiler: + - clang +c_compiler_version: + - 21 +cxx_compiler: + - clangxx +cxx_compiler_version: + - 21