From 597ec2d7e5f8bbd5a1eb9e6780444cae46cec3ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?tonghuaroot=20=28=E7=AB=A5=E8=AF=9D=29?= Date: Fri, 17 Jul 2026 13:54:46 +0800 Subject: [PATCH 1/5] gh-153404: Silently ignore non-decimal digits in robots.txt Crawl-delay and Request-rate (GH-153405) str.isdigit() returned True for non-decimal Unicode digits such as U+00B2 SUPERSCRIPT TWO, which int() then rejected with ValueError. --- Lib/test/test_robotparser.py | 28 +++++++++++++++++++ Lib/urllib/robotparser.py | 6 ++-- ...-07-09-16-35-47.gh-issue-153404.daRkes.rst | 4 +++ 3 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 725c6e3b09e1d0..9693cfc259286d 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -188,6 +188,8 @@ def test_request_rate(self): parsed_request_rate.seconds, self.request_rate.seconds ) + else: + self.assertIsNone(parsed_request_rate) class EmptyFileTest(BaseRequestRateTest, unittest.TestCase): @@ -246,6 +248,32 @@ class InvalidCrawlDelayTest(BaseRobotTest, unittest.TestCase): bad = [] +class NonDecimalDigitsTest(BaseRequestRateTest, unittest.TestCase): + # Non-decimal Unicode digits pass str.isdigit() but int() rejects + # them, so the directive must be silently ignored, not raise. + robots_txt = """\ +User-Agent: * +Disallow: /tmp/ +Crawl-delay: ² +Request-rate: ²/5 + """ + good = ['/foo.html'] + bad = ['/tmp/'] + crawl_delay = None + request_rate = None + + +class NonDecimalDenominatorTest(BaseRequestRateTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /tmp/ +Request-rate: 5/² + """ + good = ['/foo.html'] + request_rate = None + bad = ['/tmp/'] + + class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase): # also test that Allow and Diasallow works well with each other robots_txt = """\ diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 61772d90e2d53b..8d0311d96f5e0b 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -148,15 +148,15 @@ def parse(self, lines): # before trying to convert to int we need to make # sure that robots.txt has valid syntax otherwise # it will crash - if line[1].strip().isdigit(): + if line[1].strip().isdecimal(): entry.delay = int(line[1]) state = 2 elif line[0] == "request-rate": if state != 0: numbers = line[1].split('/') # check if all values are sane - if (len(numbers) == 2 and numbers[0].strip().isdigit() - and numbers[1].strip().isdigit()): + if (len(numbers) == 2 and numbers[0].strip().isdecimal() + and numbers[1].strip().isdecimal()): entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1])) state = 2 elif line[0] == "sitemap": diff --git a/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst b/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst new file mode 100644 index 00000000000000..0746d3f12149d5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst @@ -0,0 +1,4 @@ +:class:`urllib.robotparser.RobotFileParser` now silently ignores a +``Crawl-delay`` or ``Request-rate`` value written with non-decimal digits +(such as ``U+00B2 SUPERSCRIPT TWO``) instead of raising :exc:`ValueError` +and aborting the parse of the whole ``robots.txt`` file. From 2931f2067a6ecdc52cb44f694ce14ab8b50b9a32 Mon Sep 17 00:00:00 2001 From: Lucas Colley Date: Fri, 17 Jul 2026 08:32:29 +0200 Subject: [PATCH 2/5] gh-152384: pixi-packages: use `flags` to define variants (#152385) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: Isuru Fernando Co-authored-by: Nathan Goldbaum --- .github/workflows/reusable-san.yml | 2 ++ ...26-07-10-14-38-31.gh-issue-152384.uaPADM.rst | 8 ++++++++ Tools/pixi-packages/README.md | 14 +++++++------- Tools/pixi-packages/asan/variants.yaml | 6 ------ Tools/pixi-packages/build.sh | 10 +++++++++- Tools/pixi-packages/clone-recipe.sh | 10 ---------- Tools/pixi-packages/default/pixi.toml | 15 --------------- Tools/pixi-packages/default/variants.yaml | 6 ------ Tools/pixi-packages/freethreading/pixi.toml | 15 --------------- Tools/pixi-packages/freethreading/variants.yaml | 6 ------ Tools/pixi-packages/{asan => }/pixi.toml | 8 +------- Tools/pixi-packages/{default => }/recipe.yaml | 17 ++++++++--------- .../pixi-packages/tsan-freethreading/pixi.toml | 15 --------------- .../tsan-freethreading/variants.yaml | 6 ------ Tools/pixi-packages/variants.yaml | 15 +++++++++++++++ 15 files changed, 50 insertions(+), 103 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2026-07-10-14-38-31.gh-issue-152384.uaPADM.rst delete mode 100644 Tools/pixi-packages/asan/variants.yaml delete mode 100755 Tools/pixi-packages/clone-recipe.sh delete mode 100644 Tools/pixi-packages/default/pixi.toml delete mode 100644 Tools/pixi-packages/default/variants.yaml delete mode 100644 Tools/pixi-packages/freethreading/pixi.toml delete mode 100644 Tools/pixi-packages/freethreading/variants.yaml rename Tools/pixi-packages/{asan => }/pixi.toml (50%) rename Tools/pixi-packages/{default => }/recipe.yaml (85%) delete mode 100644 Tools/pixi-packages/tsan-freethreading/pixi.toml delete mode 100644 Tools/pixi-packages/tsan-freethreading/variants.yaml create mode 100644 Tools/pixi-packages/variants.yaml diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index cae51fad2c586c..40464695a281af 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -39,6 +39,8 @@ jobs: run: | sudo ./.github/workflows/posix-deps-apt.sh # On ubuntu-26.04 image, clang is clang-21 by default + # NOTE: when bumping to a new version of clang, + # please update the versions in `Tools/pixi-packages/variants.yaml` too. echo "CC=clang" >> "$GITHUB_ENV" echo "CXX=clang++" >> "$GITHUB_ENV" - name: TSan option setup diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-07-10-14-38-31.gh-issue-152384.uaPADM.rst b/Misc/NEWS.d/next/Tools-Demos/2026-07-10-14-38-31.gh-issue-152384.uaPADM.rst new file mode 100644 index 00000000000000..08b94d5486f64d --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2026-07-10-14-38-31.gh-issue-152384.uaPADM.rst @@ -0,0 +1,8 @@ +The CPython Pixi packages are now all accessible at the same +``Tools/pixi-packages`` subdirectory, rather than at +``Tools/pixi-packages/{variant}`` as before. Variants are now selected not +via subdirectory but via ``flags``; see +https://pixi.prefix.dev/latest/concepts/package_specifications/#extras-and-flags +for usage instructions. The ``tsan-freethreading`` variant has been renamed +to ``tsan_freethreading``, while the ``default``, ``asan``, and +``freethreading`` variants retain their previous names. diff --git a/Tools/pixi-packages/README.md b/Tools/pixi-packages/README.md index d818fddaac6a1e..3e9df3f56f37fa 100644 --- a/Tools/pixi-packages/README.md +++ b/Tools/pixi-packages/README.md @@ -10,7 +10,8 @@ in a [Pixi workspace](https://pixi.sh/latest/first_workspace/), like: ```toml [dependencies] python.git = "https://github.com/python/cpython" -python.subdirectory = "Tools/pixi-packages/asan" +python.subdirectory = "Tools/pixi-packages" +python.flags = ["asan"] ``` This is particularly useful when developers need to build CPython from source @@ -18,17 +19,18 @@ This is particularly useful when developers need to build CPython from source clone or build steps. Instead, Pixi will automatically handle both the build and installation of the package. -Each package definition is contained in a subdirectory, but they share the build script -`build.sh` in this directory. Currently defined package variants: +Each package variant carries a 'flag' to enable selection of that variant — see +[the Pixi docs](https://pixi.prefix.dev/latest/concepts/package_specifications/#extras-and-flags) +for details of how to use this. Currently defined package variants: - `default` - `freethreading` - `asan`: ASan-instrumented build -- `tsan-freethreading`: TSan-instrumented free-threading build +- `tsan_freethreading`: TSan-instrumented free-threading build ## Maintenance -- Keep the `abi_tag` and `version` fields in each `variants.yaml` up to date with the +- Keep the `version` field in `variants.yaml` up to date with the Python version - Update `build.sh` for any breaking changes in the `configure` and `make` workflow @@ -36,8 +38,6 @@ Each package definition is contained in a subdirectory, but they share the build - More package variants (such as UBSan) - Support for Windows -- Using a single `pixi.toml` for all package variants is blocked on - [pixi#5248](https://github.com/prefix-dev/pixi/issues/5248) ## Troubleshooting diff --git a/Tools/pixi-packages/asan/variants.yaml b/Tools/pixi-packages/asan/variants.yaml deleted file mode 100644 index 2404948457e6bb..00000000000000 --- a/Tools/pixi-packages/asan/variants.yaml +++ /dev/null @@ -1,6 +0,0 @@ -variant: - - asan -abi_tag: - - asan_cp315 -version: - - 3.15 diff --git a/Tools/pixi-packages/build.sh b/Tools/pixi-packages/build.sh index 7e22e6243a5f77..b13e26d688e46a 100644 --- a/Tools/pixi-packages/build.sh +++ b/Tools/pixi-packages/build.sh @@ -7,7 +7,7 @@ if [[ "${PYTHON_VARIANT}" == "freethreading" ]]; then elif [[ "${PYTHON_VARIANT}" == "asan" ]]; then CONFIGURE_EXTRA="--with-address-sanitizer" export ASAN_OPTIONS="strict_init_order=true" -elif [[ "${PYTHON_VARIANT}" == "tsan-freethreading" ]]; then +elif [[ "${PYTHON_VARIANT}" == "tsan_freethreading" ]]; then CONFIGURE_EXTRA="--disable-gil --with-thread-sanitizer" export TSAN_OPTIONS="suppressions=${SRC_DIR}/Tools/tsan/suppressions_free_threading.txt" elif [[ "${PYTHON_VARIANT}" == "default" ]]; then @@ -17,6 +17,14 @@ else exit 1 fi +VER_REF=$(grep "\[PYTHON_VERSION\]\, \[" configure.ac | sed -n 's/.*\[\([0-9.]*\)\].*/\1/p') +VER=$(echo ${PKG_VERSION} | sed -E 's/^([0-9]+\.[0-9]+).*/\1/') + +if [[ "${VER_REF}" != "${VER}" ]]; then + echo "Unexpected version from conda package. Got ${VER}. Expected ${VER_REF}. Do you need to update 'version' in 'variants.yaml'?" + exit 1 +fi + # rattler-build by default set a target of 10.9 # override it to at least 10.12 case ${MACOSX_DEPLOYMENT_TARGET:-10.12} in diff --git a/Tools/pixi-packages/clone-recipe.sh b/Tools/pixi-packages/clone-recipe.sh deleted file mode 100755 index 25ceaf85c35f56..00000000000000 --- a/Tools/pixi-packages/clone-recipe.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -# Please always only modify default/recipe.yaml and default/pixi.toml and then run this -# script to propagate the changes to the other variants. -set -o errexit -cd "$(dirname "$0")" - -for variant in asan freethreading tsan-freethreading; do - cp -av default/pixi.toml ${variant}/ -done diff --git a/Tools/pixi-packages/default/pixi.toml b/Tools/pixi-packages/default/pixi.toml deleted file mode 100644 index bf9841e18677ca..00000000000000 --- a/Tools/pixi-packages/default/pixi.toml +++ /dev/null @@ -1,15 +0,0 @@ -# NOTE: Please always only modify default/pixi.toml and then run clone-recipe.sh to -# propagate the changes to the other variants. - -[workspace] -channels = ["https://prefix.dev/conda-forge"] -platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"] -preview = ["pixi-build"] -requires-pixi = ">=0.66.0" - -[package.build.backend] -name = "pixi-build-rattler-build" -version = "*" - -[package.build.config] -recipe = "../default/recipe.yaml" diff --git a/Tools/pixi-packages/default/variants.yaml b/Tools/pixi-packages/default/variants.yaml deleted file mode 100644 index f66e9e7a2226ba..00000000000000 --- a/Tools/pixi-packages/default/variants.yaml +++ /dev/null @@ -1,6 +0,0 @@ -variant: - - default -abi_tag: - - cp315 -version: - - 3.15 diff --git a/Tools/pixi-packages/freethreading/pixi.toml b/Tools/pixi-packages/freethreading/pixi.toml deleted file mode 100644 index bf9841e18677ca..00000000000000 --- a/Tools/pixi-packages/freethreading/pixi.toml +++ /dev/null @@ -1,15 +0,0 @@ -# NOTE: Please always only modify default/pixi.toml and then run clone-recipe.sh to -# propagate the changes to the other variants. - -[workspace] -channels = ["https://prefix.dev/conda-forge"] -platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"] -preview = ["pixi-build"] -requires-pixi = ">=0.66.0" - -[package.build.backend] -name = "pixi-build-rattler-build" -version = "*" - -[package.build.config] -recipe = "../default/recipe.yaml" diff --git a/Tools/pixi-packages/freethreading/variants.yaml b/Tools/pixi-packages/freethreading/variants.yaml deleted file mode 100644 index 022833d04c3821..00000000000000 --- a/Tools/pixi-packages/freethreading/variants.yaml +++ /dev/null @@ -1,6 +0,0 @@ -variant: - - freethreading -abi_tag: - - cp315t -version: - - 3.15 diff --git a/Tools/pixi-packages/asan/pixi.toml b/Tools/pixi-packages/pixi.toml similarity index 50% rename from Tools/pixi-packages/asan/pixi.toml rename to Tools/pixi-packages/pixi.toml index bf9841e18677ca..afbb5e1bcd56bf 100644 --- a/Tools/pixi-packages/asan/pixi.toml +++ b/Tools/pixi-packages/pixi.toml @@ -1,15 +1,9 @@ -# NOTE: Please always only modify default/pixi.toml and then run clone-recipe.sh to -# propagate the changes to the other variants. - [workspace] channels = ["https://prefix.dev/conda-forge"] platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"] preview = ["pixi-build"] -requires-pixi = ">=0.66.0" +requires-pixi = ">=0.72.2" [package.build.backend] name = "pixi-build-rattler-build" version = "*" - -[package.build.config] -recipe = "../default/recipe.yaml" diff --git a/Tools/pixi-packages/default/recipe.yaml b/Tools/pixi-packages/recipe.yaml similarity index 85% rename from Tools/pixi-packages/default/recipe.yaml rename to Tools/pixi-packages/recipe.yaml index 30d0d5a2ed2e04..c40fe2ad2e0c79 100644 --- a/Tools/pixi-packages/default/recipe.yaml +++ b/Tools/pixi-packages/recipe.yaml @@ -1,15 +1,13 @@ -# NOTE: Please always only modify default/recipe.yaml and then run clone-recipe.sh to -# propagate the changes to the other variants. - context: - # Keep up to date freethreading_tag: ${{ "t" if "freethreading" in variant else "" }} + abi_prefix: ${{ (variant | split("_"))[0] + "_" if "san" in variant else "" }} + abi_tag: ${{ abi_prefix }}cp${{ (version | split('.'))[:2] | join('') }}${{ freethreading_tag }} recipe: name: python source: - - path: ../../.. + - path: ../.. outputs: - package: @@ -25,12 +23,16 @@ outputs: name: python version: ${{ version }} build: + flags: + - ${{ variant }} + variant: + down_prioritize_variant: ${{ 0 if variant == "default" else 1 }} string: "0_${{ abi_tag }}" files: exclude: - "*.o" script: - file: ../build.sh + file: build.sh env: PYTHON_VARIANT: ${{ variant }} python: @@ -70,9 +72,6 @@ outputs: - libuuid - libmpdec-devel - expat - - if: linux and "san" in variant - then: - - libsanitizer - if: osx and "san" in variant then: - libcompiler-rt diff --git a/Tools/pixi-packages/tsan-freethreading/pixi.toml b/Tools/pixi-packages/tsan-freethreading/pixi.toml deleted file mode 100644 index bf9841e18677ca..00000000000000 --- a/Tools/pixi-packages/tsan-freethreading/pixi.toml +++ /dev/null @@ -1,15 +0,0 @@ -# NOTE: Please always only modify default/pixi.toml and then run clone-recipe.sh to -# propagate the changes to the other variants. - -[workspace] -channels = ["https://prefix.dev/conda-forge"] -platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"] -preview = ["pixi-build"] -requires-pixi = ">=0.66.0" - -[package.build.backend] -name = "pixi-build-rattler-build" -version = "*" - -[package.build.config] -recipe = "../default/recipe.yaml" diff --git a/Tools/pixi-packages/tsan-freethreading/variants.yaml b/Tools/pixi-packages/tsan-freethreading/variants.yaml deleted file mode 100644 index 6ed09fcc9b656b..00000000000000 --- a/Tools/pixi-packages/tsan-freethreading/variants.yaml +++ /dev/null @@ -1,6 +0,0 @@ -variant: - - tsan-freethreading -abi_tag: - - tsan_cp315t -version: - - 3.15 diff --git a/Tools/pixi-packages/variants.yaml b/Tools/pixi-packages/variants.yaml new file mode 100644 index 00000000000000..2519ec5ea0d6d6 --- /dev/null +++ b/Tools/pixi-packages/variants.yaml @@ -0,0 +1,15 @@ +version: ["3.16"] +variant: ["default", "asan", "freethreading", "tsan_freethreading"] + +openssl: + - '3.5' + +# match `.github/workflows/reusable-san.yml` +c_compiler: + - clang +c_compiler_version: + - 21 +cxx_compiler: + - clangxx +cxx_compiler_version: + - 21 From 190d2ffbb969ad571fd12d4988b37519091194d5 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 17 Jul 2026 10:49:11 +0300 Subject: [PATCH 3/5] gh-85943: Fix BytesWarning in the struct format cache under -bb (GH-153627) Normalize bytes format strings to str before using them as the cache key, so that equal str and bytes formats no longer collide and get compared. Co-authored-by: Claude Opus 4.8 (1M context) --- Lib/test/test_struct.py | 10 +++++++ ...6-07-12-00-00-00.gh-issue-85943.8f906e.rst | 4 +++ Modules/_struct.c | 26 ++++++++++++++++--- 3 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-12-00-00-00.gh-issue-85943.8f906e.rst diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index f9803abb3db91f..3809f2d2643895 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -182,6 +182,16 @@ def test_calcsize(self): self.assertGreaterEqual(struct.calcsize('n'), struct.calcsize('i')) self.assertGreaterEqual(struct.calcsize('n'), struct.calcsize('P')) + def test_cache_bytes_vs_str_bb(self): + # Mixing str and bytes formats must not raise BytesWarning under -bb. + code = ( + 'import struct\n' + 'struct.calcsize(b"!d"); struct.calcsize("!d")\n' + 'struct.calcsize(">d"); struct.calcsize(b">d")\n' + 'struct.Struct(b"i"); struct.Struct("i")\n' + ) + assert_python_ok('-bb', '-c', code) + def test_integers(self): # Integer tests (bBhHiIlLqQnN). import binascii diff --git a/Misc/NEWS.d/next/Library/2026-07-12-00-00-00.gh-issue-85943.8f906e.rst b/Misc/NEWS.d/next/Library/2026-07-12-00-00-00.gh-issue-85943.8f906e.rst new file mode 100644 index 00000000000000..0a93bf2e7b803e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-12-00-00-00.gh-issue-85943.8f906e.rst @@ -0,0 +1,4 @@ +Fix :mod:`struct` functions raising :exc:`BytesWarning` under the ``-bb`` +command line option when a :class:`str` format is used after an equal +:class:`bytes` format (or vice versa). The internal format cache no longer +mixes :class:`str` and :class:`bytes` keys. diff --git a/Modules/_struct.c b/Modules/_struct.c index 260dc85faf9fb9..352312fb0b4c19 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -2642,7 +2642,8 @@ static PyType_Spec PyStructType_spec = { static int cache_struct_converter(PyObject *module, PyObject *fmt, PyStructObject **ptr) { - PyObject * s_object; + PyObject *s_object; + PyObject *key; _structmodulestate *state = get_struct_state(module); if (fmt == NULL) { @@ -2650,24 +2651,41 @@ cache_struct_converter(PyObject *module, PyObject *fmt, PyStructObject **ptr) return 1; } - if (PyDict_GetItemRef(state->cache, fmt, &s_object) < 0) { + /* Use a str cache key: an equal str and bytes would collide and be + compared, raising BytesWarning under -bb. */ + if (PyBytes_Check(fmt)) { + key = PyUnicode_DecodeASCII(PyBytes_AS_STRING(fmt), + PyBytes_GET_SIZE(fmt), "surrogateescape"); + if (key == NULL) { + return 0; + } + } + else { + key = Py_NewRef(fmt); + } + + if (PyDict_GetItemRef(state->cache, key, &s_object) < 0) { + Py_DECREF(key); return 0; } if (s_object != NULL) { + Py_DECREF(key); *ptr = PyStructObject_CAST(s_object); return Py_CLEANUP_SUPPORTED; } - s_object = PyObject_CallOneArg(state->PyStructType, fmt); + s_object = PyObject_CallOneArg(state->PyStructType, key); if (s_object != NULL) { if (PyDict_GET_SIZE(state->cache) >= MAXCACHE) PyDict_Clear(state->cache); /* Attempt to cache the result */ - if (PyDict_SetItem(state->cache, fmt, s_object) == -1) + if (PyDict_SetItem(state->cache, key, s_object) == -1) PyErr_Clear(); + Py_DECREF(key); *ptr = (PyStructObject *)s_object; return Py_CLEANUP_SUPPORTED; } + Py_DECREF(key); return 0; } From 456f3a713eb1fa62e1aa470b7ca8e23caa962c7d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 17 Jul 2026 11:06:45 +0300 Subject: [PATCH 4/5] gh-153521: Support structured arguments in imaplib commands (GH-153522) Command methods now accept a structured *message_set* (an integer, or a sequence of integers, (start, stop) ranges and range objects) and lists of flags or other atoms in place of preformatted parenthesized strings. The search, fetch, sort, thread and uid methods gain a keyword-only *params* argument that substitutes and quotes '?' placeholders in their value-bearing arguments, in the manner of sqlite3 parameter substitution. Co-Authored-By: Claude Opus 4.8 Co-authored-by: R. David Murray --- Doc/library/imaplib.rst | 126 ++++++++--- Doc/whatsnew/3.16.rst | 11 + Lib/imaplib.py | 147 ++++++++++++- Lib/test/test_imaplib.py | 197 +++++++++++++++++- ...-07-06-14-30-00.gh-issue-153521.iMaP47.rst | 7 + 5 files changed, 451 insertions(+), 37 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-06-14-30-00.gh-issue-153521.iMaP47.rst diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 7b79d81790b667..910b0f00c0e7de 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -188,9 +188,10 @@ enclosed with either parentheses or double quotes) each string is quoted. However, the *password* argument to the ``LOGIN`` command is always quoted. If you want to avoid having an argument string quoted (eg: the *flags* argument to ``STORE``) then enclose the string in parentheses (eg: ``r'(\Deleted)'``). -In general, pass arguments unquoted and let the module quote them as needed. -An argument that is already enclosed in double quotes is left unchanged, -so that code which quotes arguments itself keeps working. +Or you can quote the string yourself; +an argument that is already enclosed in double quotes is left unchanged. +In general, however, it is better to pass arguments unquoted +and let the module quote them as needed. Mailbox names are encoded as modified UTF-7 (:rfc:`3501`, section 5.1.3), so a mailbox name containing non-ASCII characters can be passed as an @@ -211,11 +212,70 @@ or mandated results from the command. Each *data* is either a ``bytes``, or a tuple. If a tuple, then the first part is the header of the response, and the second part contains the data (ie: 'literal' value). -The *message_set* options to commands below is a string specifying one or more -messages to be acted upon. It may be a simple message number (``'1'``), a range -of message numbers (``'2:4'``), or a group of non-contiguous ranges separated by -commas (``'1:3,6:9'``). A range can contain an asterisk to indicate an infinite -upper bound (``'3:*'``). +The *message_set* options to the commands below can be a string specifying +one or more messages to be acted upon. +It may be a simple message number (``'1'``), +a range of message numbers (``'2:4'``), +or a group of non-contiguous ranges separated by commas (``'1:3,6:9'``). +A range can contain an asterisk +to indicate an infinite upper bound (``'3:*'``). + +Alternatively it can be specified using integers and :class:`range` objects. +It may be a single message number or a sequence. +The sequence items may be integers, ``(start, stop)`` tuples +(where ``None`` or ``'*'`` stands for the last message), +or :class:`range` objects. +For example, ``[1, (3, 5), 8]`` and ``[range(1, 6), 8]`` +are both equivalent to ``'1,3:5,8'``. + +.. versionchanged:: next + Added support for the structured *message_set*. + +Command arguments that are parenthesized lists of atoms --- +such as the *flag_list* argument of :meth:`~IMAP4.store` and the *flags* +argument of :meth:`~IMAP4.append`, +the *names* argument of :meth:`~IMAP4.status`, +the *sort_criteria* argument of :meth:`~IMAP4.sort`, +or the *message_parts* argument of :meth:`~IMAP4.fetch` --- +can be passed as a sequence of strings instead of a single preformatted string. +For example, ``[r'\Seen', r'\Answered']`` +is equivalent to ``(\Seen \Answered)``. + +.. versionchanged:: next + Added support for passing these arguments as a sequence. + +.. _imap4-params: + +The value-bearing arguments of the search and fetch commands +can be quoted by hand, but this is error prone. +Instead, they may contain ``?`` placeholders that are substituted, and quoted +as required, from a *params* keyword argument, +in the manner of :mod:`sqlite3` parameter substitution:: + + # SEARCH FROM me@example.com SUBJECT "trip report" + M.search(None, 'FROM ? SUBJECT ?', params=['me@example.com', 'trip report']) + + # FETCH 1:5 (FLAGS BODY[HEADER.FIELDS (DATE FROM)]) + M.fetch('1:5', 'FLAGS BODY[HEADER.FIELDS ?]', params=[['DATE', 'FROM']]) + +The placeholders are: + +* ``?`` --- an ``astring``: a string (which will be quoted if necessary), + an integer, or a list of integers and/or strings + (which will be sent as a parenthesized list); +* ``?f`` --- a flag or a list of flags, sent verbatim without quoting; +* ``?s`` --- a *message_set* in the structured form described above. + +``??`` stands for a literal ``?``. + +Substitution is only performed when *params* is given; +if no *params* are given, an argument containing a literal ``?`` is unchanged. +The *params* keyword is accepted by :meth:`~IMAP4.search`, +:meth:`~IMAP4.fetch`, :meth:`~IMAP4.sort`, :meth:`~IMAP4.thread` and +:meth:`~IMAP4.uid`. + +.. versionadded:: next + The *params* keyword argument. An :class:`IMAP4` instance has the following methods: @@ -324,7 +384,7 @@ An :class:`IMAP4` instance has the following methods: Added the *message_set* and *uid* parameters. -.. method:: IMAP4.fetch(message_set, message_parts, *, uid=False) +.. method:: IMAP4.fetch(message_set, message_parts, *, uid=False, params=None) Fetch (parts of) messages. *message_parts* should be a string of message part names enclosed within parentheses, eg: ``"(UID BODY[TEXT])"``. Returned data @@ -333,8 +393,11 @@ An :class:`IMAP4` instance has the following methods: If *uid* is true, *message_set* is a set of UIDs and the message numbers in the response are UIDs (``UID FETCH``). + If *params* is given, ``?`` placeholders in *message_parts* are substituted + with the quoted parameters (see :ref:`the placeholders `). + .. versionchanged:: next - Added the *uid* parameter. + Added the *params* and *uid* parameters. .. method:: IMAP4.getacl(mailbox) @@ -598,7 +661,7 @@ An :class:`IMAP4` instance has the following methods: code, instead of the usual type. -.. method:: IMAP4.search(charset, criterion[, ...], *, uid=False) +.. method:: IMAP4.search(charset, criterion[, ...], *, uid=False, params=None) Search mailbox for matching messages. *charset* may be ``None``, in which case no ``CHARSET`` will be specified in the request to the server. The IMAP @@ -617,18 +680,22 @@ An :class:`IMAP4` instance has the following methods: When *charset* is ``None`` (as it must be under ``UTF8=ACCEPT``), the criterion is sent using the connection's encoding instead. + If *params* is given, ``?`` placeholders in the criteria are substituted + with the quoted parameters (see :ref:`the placeholders `). + Example:: # M is a connected IMAP4 instance... - typ, msgnums = M.search(None, 'FROM', '"LDJ"') + typ, msgnums = M.search(None, 'FROM', '"John Smith"') # or: - typ, msgnums = M.search(None, '(FROM "LDJ")') + typ, msgnums = M.search(None, '(FROM "John Smith")') - .. versionchanged:: next - Added the *uid* parameter. + # or, letting the module quote the value (this is recommended): + typ, msgnums = M.search(None, 'FROM ?', params=['John Smith']) .. versionchanged:: next + Added the *params* and *uid* parameters. ``str`` search criteria are encoded to *charset*. @@ -675,7 +742,7 @@ An :class:`IMAP4` instance has the following methods: Returns socket instance used to connect to server. -.. method:: IMAP4.sort(sort_criteria, charset, search_criterion[, ...], *, uid=False) +.. method:: IMAP4.sort(sort_criteria, charset, search_criterion[, ...], *, uid=False, params=None) The ``sort`` command is a variant of ``search`` with sorting semantics for the results. Returned data contains a space separated list of matching message @@ -696,12 +763,13 @@ An :class:`IMAP4` instance has the following methods: a *search_criterion* passed as :class:`str` is encoded to *charset*; pass :class:`bytes` to send one already encoded. - This is an ``IMAP4rev1`` extension command. + If *params* is given, ``?`` placeholders in the search criteria are + substituted with the quoted parameters (see :ref:`the placeholders `). - .. versionchanged:: next - Added the *uid* parameter. + This is an ``IMAP4rev1`` extension command. .. versionchanged:: next + Added the *params* and *uid* parameters. ``str`` search criteria are encoded to *charset*. @@ -745,7 +813,7 @@ An :class:`IMAP4` instance has the following methods: typ, data = M.search(None, 'ALL') for num in data[0].split(): - M.store(num, '+FLAGS', '\\Deleted') + M.store(num, '+FLAGS', r'\Deleted') M.expunge() .. note:: @@ -768,7 +836,7 @@ An :class:`IMAP4` instance has the following methods: Subscribe to new mailbox. -.. method:: IMAP4.thread(threading_algorithm, charset, search_criterion[, ...], *, uid=False) +.. method:: IMAP4.thread(threading_algorithm, charset, search_criterion[, ...], *, uid=False, params=None) The ``thread`` command is a variant of ``search`` with threading semantics for the results. Returned data contains a space separated list of thread members. @@ -793,22 +861,30 @@ An :class:`IMAP4` instance has the following methods: a *search_criterion* passed as :class:`str` is encoded to *charset*; pass :class:`bytes` to send one already encoded. - This is an ``IMAP4rev1`` extension command. + If *params* is given, ``?`` placeholders in the search criteria are + substituted with the quoted parameters (see :ref:`the placeholders `). - .. versionchanged:: next - Added the *uid* parameter. + This is an ``IMAP4rev1`` extension command. .. versionchanged:: next + Added the *params* and *uid* parameters. ``str`` search criteria are encoded to *charset*. -.. method:: IMAP4.uid(command, arg[, ...]) +.. method:: IMAP4.uid(command, arg[, ...], *, params=None) Execute command args with messages identified by UID, rather than message number. Returns response appropriate to command. At least one argument must be supplied; if none are provided, the server will return an error and an exception will be raised. + If *params* is given, ``?`` placeholders in the ``SEARCH``, ``SORT`` and + ``THREAD`` criteria or in the ``FETCH`` parts are substituted with the quoted + parameters (see :ref:`the placeholders `). + + .. versionchanged:: next + Added the *params* parameter. + .. method:: IMAP4.unsubscribe(mailbox) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index b6a5e4c7fbf1b8..ed6d303c1e28ad 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -290,6 +290,17 @@ imaplib the criteria are sent using the connection encoding instead. (Contributed by Serhiy Storchaka in :gh:`153494`.) +* Command methods now accept structured arguments, + so the module takes care of quoting instead of the caller. + A *message_set* and lists of flags or other atoms + can be passed as sequences instead of preformatted strings, + and the :meth:`~imaplib.IMAP4.search`, :meth:`~imaplib.IMAP4.fetch`, + :meth:`~imaplib.IMAP4.sort`, :meth:`~imaplib.IMAP4.thread` and + :meth:`~imaplib.IMAP4.uid` methods accept a *params* keyword argument + that substitutes and quotes ``?`` placeholders, + in the manner of :mod:`sqlite3` parameter substitution. + (Contributed by Serhiy Storchaka in :gh:`153521`.) + ipaddress --------- diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 538a858b33e8ae..139da1d3bb6fb8 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -133,6 +133,7 @@ # Only NUL, CR and LF are unsafe (they cannot be represented even in # a quoted string); other control characters are sent quoted. _control_chars = re.compile(b'[\x00\r\n]') +_control_chars_str = re.compile('[\x00\r\n]') _non_astring_char = re.compile(br'[(){ \x00-\x1f\x7f-\xff%*\\"]') _non_list_char = re.compile(br'[(){ \x00-\x1f\x7f-\xff\\"]') _quoted = re.compile(br'"(?:[^"\\]|\\.)*+"') @@ -144,6 +145,98 @@ def _paren_depth(data, depth=0): return depth + data.count(b'(') - data.count(b')') +def _seq_number(n): + # A single message number; None or '*' is the last message. + if n is None or n == '*': + return '*' + if not isinstance(n, int): + raise TypeError('message number must be an integer, not %s' + % type(n).__name__) + return str(n) + + +def _seq_range(item): + if isinstance(item, range): + if item.step == 1 and len(item): + item = (item.start, item[-1]) + else: + return ','.join(map(str, item)) + if isinstance(item, tuple): + start, stop = item + return '%s:%s' % (_seq_number(start), _seq_number(stop)) + return _seq_number(item) + + +def _format_sequence_set(arg): + if isinstance(arg, (int, str)): + return str(arg) + # A sequence of message numbers and ranges. + return ','.join(map(_seq_range, arg)) + + +# Characters that prevent a string from being sent as a bare atom. +_astring_special = re.compile(r'[(){ %*\\"\x00-\x1f\x7f-\U0010ffff]') +# A flag: an atom, optionally prefixed by a backslash. +_flag = re.compile(r'\\?[^(){ %*"\\\]\x00-\x1f\x7f-\U0010ffff]+') +# A placeholder in a format string: '?', '?f', '?s' or the escape '??'. +_placeholder = re.compile(r'\?[?fs]?') + + +def _format_astring(value): + if isinstance(value, (list, tuple)): + return '(' + ' '.join(map(_format_astring, value)) + ')' + if not isinstance(value, bool) and isinstance(value, int): + return str(value) + if isinstance(value, (bytes, bytearray)): + value = str(value, 'ascii') + elif not isinstance(value, str): + raise TypeError('expected a string, an integer or a list, not %s' + % type(value).__name__) + if value and _astring_special.search(value) is None: + return value # an atom, sent unquoted + if _control_chars_str.search(value): + raise ValueError('NUL, CR and LF cannot be represented inline: %r' + % value) + return '"' + value.replace('\\', r'\\').replace('"', r'\"') + '"' + + +def _format_flags(value): + if isinstance(value, (list, tuple)): + # A nested sequence is not part of the API; it produces invalid + # syntax that is rejected by the server. + return '(' + ' '.join(map(_format_flags, value)) + ')' + if isinstance(value, (bytes, bytearray)): + value = str(value, 'ascii') + elif not isinstance(value, str): + raise TypeError('expected a flag string or a list, not %s' + % type(value).__name__) + if not _flag.fullmatch(value): + raise ValueError('invalid flag: %r' % value) + return value + + +def _substitute(format, params): + params = iter(params) + def replace(match): + spec = match.group() + if spec == '??': # an escaped literal '?' + return '?' + try: + value = next(params) + except StopIteration: + raise TypeError('not enough parameters for the format string') \ + from None + if spec == '?f': # a flag or a list of flags + return _format_flags(value) + if spec == '?s': # a message sequence set + return _format_sequence_set(value) + return _format_astring(value) # an astring or a list of astrings + result = _placeholder.sub(replace, format) + for value in params: + raise TypeError('too many parameters for the format string') + return result + + class IMAP4: r"""IMAP4 client class. @@ -665,7 +758,7 @@ def expunge(self, message_set=None, *, uid=False): return self._untagged_response(typ, dat, name) - def fetch(self, message_set, message_parts, *, uid=False): + def fetch(self, message_set, message_parts, *, uid=False, params=None): """Fetch (parts of) messages. (typ, [data, ...]) = .fetch(message_set, message_parts) @@ -673,12 +766,17 @@ def fetch(self, message_set, message_parts, *, uid=False): 'message_parts' should be a string of selected parts enclosed in parentheses, eg: "(UID BODY[TEXT])". + If 'params' is given, '?' placeholders in 'message_parts' are + substituted with the quoted parameters. + 'data' are tuples of message part envelope and data. If 'uid' is true, 'message_set' is a set of UIDs and the message numbers in the response are UIDs (UID FETCH). """ name = 'FETCH' + if params is not None: + message_parts = _substitute(message_parts, params) args = (self._sequence_set(message_set), self._fetch_parts(message_parts)) if uid: @@ -941,7 +1039,7 @@ def rename(self, oldmailbox, newmailbox): self._mailbox(newmailbox)) - def search(self, charset, *criteria, uid=False): + def search(self, charset, *criteria, uid=False, params=None): """Search mailbox for matching messages. (typ, [data]) = .search(charset, criterion, ...) @@ -953,8 +1051,13 @@ def search(self, charset, *criteria, uid=False): A 'criteria' passed as str is encoded to 'charset'; pass bytes to send criteria that are already encoded. + + If 'params' is given, '?' placeholders in the criteria are + substituted with the quoted parameters. """ name = 'SEARCH' + if params is not None: + criteria = (_substitute(' '.join(criteria), params),) if charset is not None: if self.utf8_enabled: raise IMAP4.error("Non-None charset not valid in UTF8 mode") @@ -1028,18 +1131,24 @@ def setquota(self, root, limits): return self._untagged_response(typ, dat, 'QUOTA') - def sort(self, sort_criteria, charset, *search_criteria, uid=False): + def sort(self, sort_criteria, charset, *search_criteria, uid=False, + params=None): """IMAP4rev1 extension SORT command. (typ, [data]) = .sort(sort_criteria, charset, search_criteria, ...) If 'uid' is true, the message numbers in the response are UIDs (UID SORT). + + If 'params' is given, '?' placeholders in the search criteria are + substituted with the quoted parameters. """ name = 'SORT' #if not name in self.capabilities: # Let the server decide! # raise self.error('unimplemented extension command: %s' % name) sort_criteria = self._set_quote(sort_criteria) + if params is not None: + search_criteria = (_substitute(' '.join(search_criteria), params),) search_criteria = self._encode_criteria(charset, search_criteria) if charset is not None: charset = self._astring(charset) @@ -1113,15 +1222,21 @@ def subscribe(self, mailbox): return self._simple_command('SUBSCRIBE', self._mailbox(mailbox)) - def thread(self, threading_algorithm, charset, *search_criteria, uid=False): + def thread(self, threading_algorithm, charset, *search_criteria, uid=False, + params=None): """IMAPrev1 extension THREAD command. (type, [data]) = .thread(threading_algorithm, charset, search_criteria, ...) If 'uid' is true, the message numbers in the response are UIDs (UID THREAD). + + If 'params' is given, '?' placeholders in the search criteria are + substituted with the quoted parameters. """ name = 'THREAD' + if params is not None: + search_criteria = (_substitute(' '.join(search_criteria), params),) search_criteria = self._encode_criteria(charset, search_criteria) if charset is not None: charset = self._astring(charset) @@ -1133,13 +1248,17 @@ def thread(self, threading_algorithm, charset, *search_criteria, uid=False): return self._untagged_response(typ, dat, name) - def uid(self, command, *args): + def uid(self, command, *args, params=None): """Execute "command arg ..." with messages identified by UID, rather than message number. (typ, [data]) = .uid(command, arg1, arg2, ...) Returns response appropriate to 'command'. + + If 'params' is given, '?' placeholders in the SEARCH, SORT and + THREAD criteria or in the FETCH parts are substituted with the + quoted parameters. """ command = command.upper() if not command in Commands: @@ -1156,6 +1275,8 @@ def uid(self, command, *args): self._mailbox(new_mailbox)) elif command == 'FETCH': message_set, message_parts = args + if params is not None: + message_parts = _substitute(message_parts, params) args = (self._sequence_set(message_set), self._fetch_parts(message_parts)) elif command == 'STORE': @@ -1164,6 +1285,9 @@ def uid(self, command, *args): self._set_quote(flags)) elif command == 'SORT': sort_criteria, charset, *search_criteria = args + if params is not None: + search_criteria = (_substitute(' '.join(search_criteria), + params),) search_criteria = self._encode_criteria(charset, search_criteria) if charset is not None: charset = self._astring(charset) @@ -1171,11 +1295,16 @@ def uid(self, command, *args): *search_criteria) elif command == 'THREAD': threading_algorithm, charset, *search_criteria = args + if params is not None: + search_criteria = (_substitute(' '.join(search_criteria), + params),) search_criteria = self._encode_criteria(charset, search_criteria) if charset is not None: charset = self._astring(charset) args = (self._atom(threading_algorithm), charset, *search_criteria) + elif command == 'SEARCH' and params is not None: + args = (_substitute(' '.join(args), params),) typ, dat = self._simple_command(name, self._atom(command), *args) if command in ('SEARCH', 'SORT', 'THREAD'): name = command @@ -1570,9 +1699,13 @@ def _atom(self, arg): return arg def _sequence_set(self, arg): - return arg + return _format_sequence_set(arg) def _set_quote(self, arg): + if not isinstance(arg, str): + # A sequence of atoms (flags, criteria, item names, etc.); + # wrap them in parentheses as a single argument. + return '(' + ' '.join(arg) + ')' if arg and arg[0] == '(' and arg[-1] == ')': return arg return '(' + arg + ')' @@ -1580,7 +1713,7 @@ def _set_quote(self, arg): def _fetch_parts(self, arg): # "ALL", "FULL" and "FAST" are macros, not data item names; # they cannot be enclosed in parentheses. - if arg.upper() in ('ALL', 'FULL', 'FAST'): + if isinstance(arg, str) and arg.upper() in ('ALL', 'FULL', 'FAST'): return arg return self._set_quote(arg) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 3eaca67299e7c3..100791a9c3eb2c 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -184,8 +184,8 @@ def test_astring(self): self.assertEqual(m._astring(b'INBOX'), b'INBOX') # Names with protocol-sensitive characters are quoted. self.assertEqual(m._astring('New folder'), b'"New folder"') - self.assertEqual(m._astring('a"b'), b'"a\\"b"') - self.assertEqual(m._astring('a\\b'), b'"a\\\\b"') + self.assertEqual(m._astring('a"b'), rb'"a\"b"') + self.assertEqual(m._astring(r'a\b'), rb'"a\\b"') self.assertEqual(m._astring(''), b'""') self.assertEqual(m._astring('*'), b'"*"') # A well-formed quoted string is passed through unchanged. @@ -193,12 +193,12 @@ def test_astring(self): self.assertEqual(m._astring('""'), b'""') # Including a lenient (non-RFC) backslash escape, which the server # may accept. - self.assertEqual(m._astring('"a\\b"'), b'"a\\b"') + self.assertEqual(m._astring(r'"a\b"'), rb'"a\b"') # A string that only looks quoted but is not a single token is # quoted as data, closing the argument injection vector. self.assertEqual(m._astring('"a" SELECT evil "'), - b'"\\"a\\" SELECT evil \\""') - self.assertEqual(m._astring('"'), b'"\\""') + rb'"\"a\" SELECT evil \""') + self.assertEqual(m._astring('"'), rb'"\""') # Non-ASCII names are only allowed in a quoted string or a # literal, never in an atom (RFC 6855). m._encoding = 'utf-8' @@ -287,6 +287,69 @@ def test_mailbox(self): self.assertEqual(m._mailbox('Entwürfe'), '"Entwürfe"'.encode()) self.assertEqual(m._mailbox('Entw&APw-rfe'), b'Entw&APw-rfe') + def test_sequence_set(self): + m = imaplib.IMAP4.__new__(imaplib.IMAP4) + # A scalar is passed through as a string. + self.assertEqual(m._sequence_set(5), '5') + self.assertEqual(m._sequence_set('1:3,7'), '1:3,7') + # A sequence of numbers and ranges is formatted as a sequence set. + self.assertEqual(m._sequence_set([1, 2, 5]), '1,2,5') + self.assertEqual(m._sequence_set([1, (3, 5), (8, '*')]), '1,3:5,8:*') + self.assertEqual(m._sequence_set([(5, None)]), '5:*') + # A range is inclusive; a non-unit step falls back to explicit numbers. + self.assertEqual(m._sequence_set([range(1, 4), 7]), '1:3,7') + self.assertEqual(m._sequence_set([range(1, 10, 2)]), '1,3,5,7,9') + # Message numbers must be integers: a string is not coerced (the + # string form is the whole preformatted set), nor is a float. + self.assertRaises(TypeError, m._sequence_set, ['7']) + self.assertRaises(TypeError, m._sequence_set, [2.9]) + self.assertRaises(TypeError, m._sequence_set, [(1, 2.9)]) + + def test_set_quote(self): + m = imaplib.IMAP4.__new__(imaplib.IMAP4) + # A string is parenthesized unless it already is. + self.assertEqual(m._set_quote(r'\Seen'), r'(\Seen)') + self.assertEqual(m._set_quote(r'(\Seen)'), r'(\Seen)') + # A sequence of atoms is joined and parenthesized. + self.assertEqual(m._set_quote([r'\Seen', r'\Answered']), + r'(\Seen \Answered)') + self.assertEqual(m._set_quote(['MESSAGES', 'UNSEEN']), + '(MESSAGES UNSEEN)') + + def test_substitute(self): + sub = imaplib._substitute + # '?' quotes an astring; an atom-safe value is left unquoted. + self.assertEqual(sub('FROM ?', ['me@host']), 'FROM me@host') + self.assertEqual(sub('SUBJECT ?', ['hello world']), + 'SUBJECT "hello world"') + self.assertEqual(sub('SUBJECT ?', ['a"b']), r'SUBJECT "a\"b"') + # An integer becomes a number, a list a parenthesized list. + self.assertEqual(sub('LARGER ?', [1000]), 'LARGER 1000') + self.assertEqual(sub('HEADER.FIELDS ?', [['DATE', 'FROM']]), + 'HEADER.FIELDS (DATE FROM)') + # '?f' emits flags verbatim, never quoted. + self.assertEqual(sub('?f', [r'\Seen']), r'\Seen') + self.assertEqual(sub('?f', [[r'\Seen', r'\Answered']]), + r'(\Seen \Answered)') + # '?s' formats a message sequence set. + self.assertEqual(sub('?s', [[1, (3, 5), (8, '*')]]), '1,3:5,8:*') + # '??' is a literal '?'. + self.assertEqual(sub('a?? b', []), 'a? b') + + def test_substitute_errors(self): + sub = imaplib._substitute + self.assertRaises(TypeError, sub, '? ?', ['x']) # too few parameters + self.assertRaises(TypeError, sub, '?', ['x', 'y']) # too many parameters + self.assertRaises(ValueError, sub, '?f', ['a b']) # not a valid flag + self.assertRaises(ValueError, sub, '?', ['a\r\nb']) # CR/LF not inline + self.assertRaises(TypeError, sub, '?', [True]) # bool is not a string + self.assertRaises(TypeError, sub, '?', [1.5]) # float is not a string + self.assertRaises(TypeError, sub, '?s', [['a']]) # not a message number + self.assertRaises(TypeError, sub, '?s', [[1.5]]) # not a message number + self.assertRaises(TypeError, sub, '?s', [[(1, 'a')]]) # not a message number + self.assertRaises(ValueError, sub, '?s', [[(1,)]]) # not a range pair + self.assertRaises(ValueError, sub, '?s', [[(1, 2, 3)]]) # not a range pair + if ssl: class SecureTCPServer(socketserver.TCPServer): @@ -1342,6 +1405,11 @@ def test_copy(self): self.assertEqual(data, [b'COPY completed']) self.assertEqual(server.args, ['2:4', '"New folder"']) + # A structured message set is formatted into a sequence set. + typ, data = client.copy([2, (3, 5)], 'MEETING') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['2,3:5', 'MEETING']) + def test_uid_copy(self): client, server = self._setup(make_simple_handler('UID', completed='UID COPY completed')) @@ -1363,6 +1431,11 @@ def test_uid_copy(self): self.assertEqual(data, [b'UID COPY completed']) self.assertEqual(server.args, ['COPY', '4827313:4828442', 'MEETING']) + # A structured message set is formatted into a sequence set. + typ, data = client.uid('copy', [1, (3, 5)], 'MEETING') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['COPY', '1,3:5', 'MEETING']) + def test_move(self): client, server = self._setup(make_simple_handler('MOVE')) client.login('user', 'pass') @@ -1377,6 +1450,11 @@ def test_move(self): self.assertEqual(data, [b'MOVE completed']) self.assertEqual(server.args, ['2:4', '"New folder"']) + # A structured message set is formatted into a sequence set. + typ, data = client.move([2, (3, 5)], 'MEETING') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['2,3:5', 'MEETING']) + def test_uid_move(self): client, server = self._setup(make_simple_handler('UID', completed='UID MOVE completed')) @@ -1398,6 +1476,11 @@ def test_uid_move(self): self.assertEqual(data, [b'UID MOVE completed']) self.assertEqual(server.args, ['MOVE', '4827313:4828442', 'MEETING']) + # A structured message set is formatted into a sequence set. + typ, data = client.uid('move', [1, (3, 5)], 'MEETING') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['MOVE', '1,3:5', 'MEETING']) + def test_store(self): client, server = self._setup(make_simple_handler('STORE', [ r'* 2 FETCH (FLAGS (\Deleted \Seen))', @@ -1419,6 +1502,12 @@ def test_store(self): self.assertEqual(typ, 'OK') self.assertEqual(server.args, ['2:4', '+FLAGS', r'(\Deleted)']) + # The flags may be a sequence, and the message set may be structured. + typ, data = client.store([2, (3, 4)], '+FLAGS', + [r'\Deleted', r'\Seen']) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['2,3:4', '+FLAGS', r'(\Deleted \Seen)']) + def test_uid_store(self): client, server = self._setup(make_simple_handler('UID', [ r'* 23 FETCH (FLAGS (\Deleted \Seen) UID 4827313)', @@ -1451,6 +1540,13 @@ def test_uid_store(self): ]) self.assertEqual(server.args, ['STORE', '4827313:4828442', '+FLAGS', r'(\Deleted)']) + # The flags may be a sequence. + typ, data = client.uid('store', '4827313:4828442', '+FLAGS', + [r'\Deleted', r'\Seen']) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, + ['STORE', '4827313:4828442', '+FLAGS', r'(\Deleted \Seen)']) + def test_fetch(self): # The handler expands the requested sequence set and answers for # exactly those messages, so the test exercises the round trip of @@ -1508,6 +1604,23 @@ def cmd_FETCH(self, tag, args): self.assertEqual(typ, 'OK') self.assertEqual(server.args, ['2:4', 'fast']) + # A structured message set is formatted into a sequence set. + typ, data = client.fetch([2, (3, 4)], '(FLAGS)') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['2,3:4', '(FLAGS)']) + + # message_parts may be a sequence of items. + typ, data = client.fetch('1', ['UID', 'FLAGS']) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['1', '(UID FLAGS)']) + + # 'params' substitutes and quotes '?' placeholders. + typ, data = client.fetch('1', 'FLAGS BODY[HEADER.FIELDS ?]', + params=[['DATE', 'FROM']]) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, + ['1', '(FLAGS BODY[HEADER.FIELDS (DATE FROM)])']) + def test_uid_fetch(self): client, server = self._setup(make_simple_handler('UID', [ r'* 23 FETCH (FLAGS (\Seen) UID 4827313)', @@ -1543,6 +1656,18 @@ def test_uid_fetch(self): ]) self.assertEqual(server.args, ['FETCH', '4827313:4828442', '(FLAGS)']) + # message_parts may be a sequence, and 'params' substitutes '?'. + typ, data = client.uid('fetch', '4827313:4828442', ['UID', 'FLAGS']) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['FETCH', '4827313:4828442', '(UID FLAGS)']) + + typ, data = client.uid('fetch', '4827313:4828442', + 'BODY[HEADER.FIELDS ?]', params=[['DATE', 'FROM']]) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, + ['FETCH', '4827313:4828442', + '(BODY[HEADER.FIELDS (DATE FROM)])']) + def test_partial(self): client, server = self._setup(make_simple_handler('PARTIAL', ['* 1 FETCH (RFC822.TEXT<0.10> "0123456789")'])) @@ -1590,6 +1715,19 @@ def test_search(self): self.assertEqual(typ, 'OK') self.assertEqual(server.args, ['CHARSET', '"NF_Z_62-010_(1973)"', 'TEXT', 'XXXXXX']) + # 'params' substitutes and quotes '?' placeholders. + response[:] = ['* SEARCH 1'] + typ, data = client.search(None, 'FROM ? SUBJECT ?', + params=['me@host', 'trip report']) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, + ['FROM', 'me@host', 'SUBJECT', '"trip report"']) + + # Without 'params', a literal '?' is sent unchanged. + typ, data = client.search(None, 'SUBJECT', '"what?"') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['SUBJECT', '"what?"']) + def test_uid_search(self): response = [] client, server = self._setup(make_simple_handler('UID', response, @@ -1633,6 +1771,14 @@ def test_uid_search(self): self.assertEqual(data, [b'43']) self.assertEqual(server.args, ['SEARCH', 'CHARSET', 'UTF-8', 'TEXT', 'XXXXXX']) + # 'params' substitutes and quotes '?' placeholders. + response[:] = ['* SEARCH 1'] + typ, data = client.uid('SEARCH', 'FROM ? SUBJECT ?', + params=['me@host', 'trip report']) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, + ['SEARCH', 'FROM', 'me@host', 'SUBJECT', '"trip report"']) + def test_sort(self): response = [] client, server = self._setup(make_simple_handler('SORT', response)) @@ -1666,6 +1812,15 @@ def test_sort(self): self.assertEqual(typ, 'OK') self.assertIn('"Київ"'.encode('koi8-u'), server.line) + # sort_criteria may be a sequence, and 'params' substitutes and + # quotes '?' (a value with a space becomes a quoted string). + response[:] = ['* SORT 1'] + typ, data = client.sort(['REVERSE', 'DATE'], 'UTF-8', 'SUBJECT ?', + params=['trip report']) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, + ['(REVERSE DATE)', 'UTF-8', 'SUBJECT', '"trip report"']) + def test_uid_sort(self): response = [] client, server = self._setup(make_simple_handler('UID', response, @@ -1707,6 +1862,15 @@ def test_uid_sort(self): self.assertEqual(data, [br'2 84 882']) self.assertEqual(server.args, ['SORT', '(SUBJECT)', 'UTF-8', 'SINCE', '1-Feb-1994']) + # sort_criteria may be a sequence, and 'params' substitutes and + # quotes '?' (a value with a space becomes a quoted string). + response[:] = ['* SORT 1'] + typ, data = client.uid('sort', ['REVERSE', 'DATE'], 'UTF-8', 'SUBJECT ?', + params=['trip report']) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, + ['SORT', '(REVERSE DATE)', 'UTF-8', 'SUBJECT', '"trip report"']) + def test_thread(self): response = [] client, server = self._setup(make_simple_handler('THREAD', response)) @@ -1754,6 +1918,15 @@ def test_thread(self): self.assertEqual(typ, 'OK') self.assertIn('"Київ"'.encode('koi8-u'), server.line) + # 'params' substitutes and quotes '?' (a value with a space becomes + # a quoted string). + response[:] = ['* THREAD (1)'] + typ, data = client.thread('REFERENCES', 'UTF-8', 'SUBJECT ?', + params=['trip report']) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, + ['REFERENCES', 'UTF-8', 'SUBJECT', '"trip report"']) + def test_uid_thread(self): response = [] client, server = self._setup(make_simple_handler('UID', response, @@ -1810,6 +1983,15 @@ def test_uid_thread(self): self.assertEqual(data, [b'(166)(167)(168)']) self.assertEqual(server.args, ['THREAD', 'ORDEREDSUBJECT', 'UTF-8', 'SINCE', '5-MAR-2000']) + # 'params' substitutes and quotes '?' (a value with a space becomes + # a quoted string). + response[:] = ['* THREAD (1)'] + typ, data = client.uid('THREAD', 'REFERENCES', 'UTF-8', 'SUBJECT ?', + params=['trip report']) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, + ['THREAD', 'REFERENCES', 'UTF-8', 'SUBJECT', '"trip report"']) + def test_delete(self): client, server = self._setup(make_simple_handler('DELETE')) client.login('user', 'pass') @@ -1900,6 +2082,11 @@ def test_status(self): self.assertEqual(typ, 'OK') self.assertEqual(server.args, ['"New folder"', '(UIDNEXT MESSAGES)']) + # The names argument may be a sequence of item names. + typ, data = client.status('blurdybloop', ['UIDNEXT', 'MESSAGES']) + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['blurdybloop', '(UIDNEXT MESSAGES)']) + def test_getacl(self): client, server = self._setup(make_simple_handler('GETACL', ['* ACL INBOX Fred rwipslxetad'])) diff --git a/Misc/NEWS.d/next/Library/2026-07-06-14-30-00.gh-issue-153521.iMaP47.rst b/Misc/NEWS.d/next/Library/2026-07-06-14-30-00.gh-issue-153521.iMaP47.rst new file mode 100644 index 00000000000000..93edb1532d5d41 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-06-14-30-00.gh-issue-153521.iMaP47.rst @@ -0,0 +1,7 @@ +Add support for structured arguments in :mod:`imaplib` command methods. A +*message_set* and lists of flags or other atoms can now be passed as +sequences instead of preformatted strings, and the +:meth:`~imaplib.IMAP4.search`, :meth:`~imaplib.IMAP4.fetch`, +:meth:`~imaplib.IMAP4.sort`, :meth:`~imaplib.IMAP4.thread` and +:meth:`~imaplib.IMAP4.uid` methods accept a *params* keyword argument that +substitutes and quotes ``?`` placeholders. From abdd7aea18bde039fe35983b5c0d8036bc16f1a7 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 17 Jul 2026 11:10:21 +0300 Subject: [PATCH 5/5] gh-153395: Accept curses.complexchar in curses.ascii predicates and conversions (GH-153396) The curses.ascii predicates and the ctrl() and unctrl() functions now accept a curses.complexchar, classifying it by its single character. ctrl() now returns a non-ASCII argument unchanged instead of masking it to a control character. Co-authored-by: Claude Opus 4.8 --- Doc/library/curses.ascii.rst | 23 ++++++++---- Lib/curses/ascii.py | 32 ++++++++++------- Lib/test/test_curses.py | 35 +++++++++++++++++++ ...-07-09-10-14-08.gh-issue-153395.N8FuG3.rst | 4 +++ 4 files changed, 75 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-09-10-14-08.gh-issue-153395.N8FuG3.rst diff --git a/Doc/library/curses.ascii.rst b/Doc/library/curses.ascii.rst index 3ac76edd38dcc8..fad35973dada18 100644 --- a/Doc/library/curses.ascii.rst +++ b/Doc/library/curses.ascii.rst @@ -176,15 +176,18 @@ C library: Checks for a non-ASCII character (ordinal values 0x80 and above). -These functions accept either integers or single-character strings; when the argument is a -string, it is first converted using the built-in function :func:`ord`. +These functions accept an integer, a single-character string, or a +:class:`curses.complexchar`. +A string is converted using the built-in function :func:`ord`, and a +complexchar by the code of its single character; a complexchar that holds +combining characters is not a single character and matches no class. Note that all these functions check ordinal bit values derived from the character of the string you pass in; they do not actually know anything about the host machine's character encoding. -The following two functions take either a single-character string or integer -byte value; they return a value of the same type. +The following three functions take either a single-character string or an +integer byte value; they return a value of the same type. .. function:: ascii(c) @@ -194,8 +197,13 @@ byte value; they return a value of the same type. .. function:: ctrl(c) - Return the control character corresponding to the given character (the character - bit value is bitwise-anded with 0x1f). + Return the control character corresponding to the given ASCII character (the + character bit value is bitwise-anded with 0x1f). A non-ASCII character has no + control character and is returned unchanged. + + .. versionchanged:: next + A non-ASCII argument is now returned unchanged instead of masked to a + control character. .. function:: alt(c) @@ -203,7 +211,8 @@ byte value; they return a value of the same type. Return the 8-bit character corresponding to the given ASCII character (the character bit value is bitwise-ored with 0x80). -The following function takes either a single-character string or integer value; +The following function takes a single-character string, an integer value, or a +:class:`curses.complexchar`; it returns a string. diff --git a/Lib/curses/ascii.py b/Lib/curses/ascii.py index 95acff33925ed7..cb55300e05dd4a 100644 --- a/Lib/curses/ascii.py +++ b/Lib/curses/ascii.py @@ -1,5 +1,8 @@ """Constants and membership tests for ASCII characters""" +# A character-cell type, present on wide and narrow builds. +from _curses import complexchar as _complexchar + NUL = 0x00 # ^@ SOH = 0x01 # ^A STX = 0x02 # ^B @@ -48,8 +51,12 @@ def _ctoi(c): if isinstance(c, str): return ord(c) - else: - return c + if isinstance(c, _complexchar): + # A character cell: its single character, or -1 (matches no class) + # for a cell with combining characters. + s = str(c) + return ord(s) if len(s) == 1 else -1 + return c def isalnum(c): return isalpha(c) or isdigit(c) def isalpha(c): return isupper(c) or islower(c) @@ -69,22 +76,23 @@ def isctrl(c): return 0 <= _ctoi(c) < 32 def ismeta(c): return _ctoi(c) > 127 def ascii(c): - if isinstance(c, str): - return chr(_ctoi(c) & 0x7f) - else: + if isinstance(c, int): return _ctoi(c) & 0x7f + else: + return chr(_ctoi(c) & 0x7f) def ctrl(c): - if isinstance(c, str): - return chr(_ctoi(c) & 0x1f) - else: - return _ctoi(c) & 0x1f + code = _ctoi(c) + if not 0 <= code < 128: + # No control character outside ASCII: return c unchanged. + return c if isinstance(c, int) else str(c) + return code & 0x1f if isinstance(c, int) else chr(code & 0x1f) def alt(c): - if isinstance(c, str): - return chr(_ctoi(c) | 0x80) - else: + if isinstance(c, int): return _ctoi(c) | 0x80 + else: + return chr(_ctoi(c) | 0x80) def unctrl(c): bits = _ctoi(c) diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index 870393be5209df..b99e4efbb83ce4 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -2761,6 +2761,9 @@ def test_ctrl(self): self.assertEqual(ctrl('\n'), '\n') self.assertEqual(ctrl('@'), '\0') self.assertEqual(ctrl(ord('J')), ord('\n')) + # A non-ASCII argument is returned unchanged (no control character). + self.assertEqual(ctrl('\xe9'), '\xe9') + self.assertEqual(ctrl(0xe9), 0xe9) def test_alt(self): alt = curses.ascii.alt @@ -2785,6 +2788,38 @@ def test_unctrl(self): self.assertEqual(unctrl(ord('\x8a')), '!^J') self.assertEqual(unctrl(ord('\xc1')), '!A') + @unittest.skipUnless(hasattr(curses, 'complexchar'), + 'requires the curses.complexchar type') + def test_complexchar(self): + # The predicates, ctrl() and unctrl() accept a complexchar too, using + # its single character. A narrow build just forms fewer cells. + cc = curses.complexchar + def storable(s): + try: + cc(s) + except ValueError: + return False + return True + + self.assertTrue(curses.ascii.isupper(cc('A'))) + self.assertTrue(curses.ascii.isalpha(cc('A', curses.A_BOLD))) + self.assertFalse(curses.ascii.isdigit(cc('A'))) + self.assertTrue(curses.ascii.isdigit(cc('7'))) + self.assertTrue(curses.ascii.iscntrl(cc('\n'))) + self.assertEqual(curses.ascii.ctrl(cc('J')), '\n') + self.assertEqual(curses.ascii.unctrl(cc('\n')), '^J') + self.assertEqual(curses.ascii.unctrl(cc('A')), 'A') + # A non-ASCII character: classified by code point, no control character. + if storable('\xe9'): + self.assertFalse(curses.ascii.isascii(cc('\xe9'))) + self.assertTrue(curses.ascii.ismeta(cc('\xe9'))) + self.assertEqual(curses.ascii.ctrl(cc('\xe9')), '\xe9') + # A cell with combining marks is not a single character, so no + # predicate matches it (needs a wide build to store). + if storable('e\u0301'): + self.assertFalse(curses.ascii.isalpha(cc('e\u0301'))) + self.assertFalse(curses.ascii.isascii(cc('e\u0301'))) + def lorem_ipsum(win): text = [ diff --git a/Misc/NEWS.d/next/Library/2026-07-09-10-14-08.gh-issue-153395.N8FuG3.rst b/Misc/NEWS.d/next/Library/2026-07-09-10-14-08.gh-issue-153395.N8FuG3.rst new file mode 100644 index 00000000000000..1c44219f710640 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-09-10-14-08.gh-issue-153395.N8FuG3.rst @@ -0,0 +1,4 @@ +:mod:`curses.ascii` predicates and the :func:`~curses.ascii.ctrl` and +:func:`~curses.ascii.unctrl` functions now accept a :class:`curses.complexchar`. +:func:`~curses.ascii.ctrl` now returns a non-ASCII argument unchanged instead +of masking it to a control character.