From ff0cf0af10a775ea03dc2546b6a5506a708c2ab7 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Fri, 3 Oct 2025 09:58:32 -0700 Subject: [PATCH 001/112] gh-139525: Don't specialize functions which have a modified vectorcall (#139524) Don't specialize functions which have a modified vectorcall --- Lib/test/test_opcache.py | 8 ++++++++ Modules/_testinternalcapi.c | 20 ++++++++++++++++++++ Python/specialize.c | 9 +++++++++ 3 files changed, 37 insertions(+) diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index f1271fc540e..f23f8c053e8 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -567,6 +567,14 @@ def test(default=None): with self.assertRaises(RecursionError): test() + def test_dont_specialize_custom_vectorcall(self): + def f(): + raise Exception("no way") + + _testinternalcapi.set_vectorcall_nop(f) + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + f() + def make_deferred_ref_count_obj(): """Create an object that uses deferred reference counting. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index a4348e7e149..c2647d405e2 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2399,6 +2399,25 @@ simple_pending_call(PyObject *self, PyObject *callable) Py_RETURN_NONE; } +static PyObject * +vectorcall_nop(PyObject *callable, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + Py_RETURN_NONE; +} + +static PyObject * +set_vectorcall_nop(PyObject *self, PyObject *func) +{ + if (!PyFunction_Check(func)) { + PyErr_SetString(PyExc_TypeError, "expected function"); + return NULL; + } + + ((PyFunctionObject*)func)->vectorcall = vectorcall_nop; + Py_RETURN_NONE; +} + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -2507,6 +2526,7 @@ static PyMethodDef module_functions[] = { {"emscripten_set_up_async_input_device", emscripten_set_up_async_input_device, METH_NOARGS}, #endif {"simple_pending_call", simple_pending_call, METH_O}, + {"set_vectorcall_nop", set_vectorcall_nop, METH_O}, {NULL, NULL} /* sentinel */ }; diff --git a/Python/specialize.c b/Python/specialize.c index 47f7b27b490..a1c5dedd615 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -630,6 +630,7 @@ _PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, int enable_counters #define SPEC_FAIL_CALL_INIT_NOT_PYTHON 21 #define SPEC_FAIL_CALL_PEP_523 22 #define SPEC_FAIL_CALL_BOUND_METHOD 23 +#define SPEC_FAIL_CALL_VECTORCALL 24 #define SPEC_FAIL_CALL_CLASS_MUTABLE 26 #define SPEC_FAIL_CALL_METHOD_WRAPPER 28 #define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29 @@ -2071,6 +2072,10 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523); return -1; } + if (func->vectorcall != _PyFunction_Vectorcall) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_VECTORCALL); + return -1; + } int argcount = -1; if (kind == SPEC_FAIL_CODE_NOT_OPTIMIZED) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CODE_NOT_OPTIMIZED); @@ -2110,6 +2115,10 @@ specialize_py_call_kw(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523); return -1; } + if (func->vectorcall != _PyFunction_Vectorcall) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_VECTORCALL); + return -1; + } if (kind == SPEC_FAIL_CODE_NOT_OPTIMIZED) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CODE_NOT_OPTIMIZED); return -1; From 1ae92503647328544866f9586f57eec285e1949a Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Fri, 3 Oct 2025 15:49:03 -0400 Subject: [PATCH 002/112] gh-137638: Use macos-15-intel in GitHub Actions (#139154) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/actionlint.yaml | 3 ++- .github/workflows/build.yml | 6 +++--- .github/workflows/jit.yml | 7 +------ .github/workflows/reusable-macos.yml | 6 +++--- .github/workflows/tail-call.yml | 9 +-------- 5 files changed, 10 insertions(+), 21 deletions(-) diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 68aae196357..267ff6b42a8 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,6 +1,7 @@ self-hosted-runner: # Pending https://github.com/rhysd/actionlint/issues/533 - labels: ["windows-11-arm"] + # and https://github.com/rhysd/actionlint/issues/571 + labels: ["windows-11-arm", "macos-15-intel"] config-variables: null diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 83a668fc720..ebfaf32e193 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -202,13 +202,13 @@ jobs: strategy: fail-fast: false matrix: - # Cirrus and macos-14 are M1, macos-13 is default GHA Intel. + # Cirrus and macos-14 are M1, macos-15-intel is default GHA Intel. # macOS 13 only runs tests against the GIL-enabled CPython. # Cirrus used for upstream, macos-14 for forks. os: - ghcr.io/cirruslabs/macos-runner:sonoma - macos-14 - - macos-13 + - macos-15-intel is-fork: # only used for the exclusion trick - ${{ github.repository_owner != 'python' }} free-threading: @@ -219,7 +219,7 @@ jobs: is-fork: true - os: macos-14 is-fork: false - - os: macos-13 + - os: macos-15-intel free-threading: true uses: ./.github/workflows/reusable-macos.yml with: diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 80e4ae603a2..c32bf4fd63c 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -81,7 +81,7 @@ jobs: runner: windows-11-arm - target: x86_64-apple-darwin/clang architecture: x86_64 - runner: macos-13 + runner: macos-15-intel - target: aarch64-apple-darwin/clang architecture: aarch64 runner: macos-14 @@ -106,15 +106,10 @@ jobs: ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. - # This is a bug in the macOS runner image where the pre-installed Python is installed in the same - # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes - # the symlink to the pre-installed Python so that the Homebrew Python is used instead. - name: macOS if: runner.os == 'macOS' run: | brew update - find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete brew install llvm@${{ matrix.llvm }} export SDKROOT="$(xcrun --show-sdk-path)" # Set MACOSX_DEPLOYMENT_TARGET and -Werror=unguarded-availability to diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index de0c4022136..87bcd5786e7 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -60,15 +60,15 @@ jobs: --prefix=/opt/python-dev \ --with-openssl="$(brew --prefix openssl@3.0)" - name: Build CPython - if : ${{ inputs.free-threading || inputs.os != 'macos-13' }} + if : ${{ inputs.free-threading || inputs.os != 'macos-15-intel' }} run: gmake -j8 - name: Build CPython for compiler warning check - if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + if : ${{ !inputs.free-threading && inputs.os == 'macos-15-intel' }} run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt - name: Display build info run: make pythoninfo - name: Check compiler warnings - if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + if : ${{ !inputs.free-threading && inputs.os == 'macos-15-intel' }} run: >- python3 Tools/build/check_warnings.py --compiler-output-file-path=compiler_output_macos.txt diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index d4a84223b7b..e99e317182e 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -58,7 +58,7 @@ jobs: # runner: windows-2022 - target: x86_64-apple-darwin/clang architecture: x86_64 - runner: macos-13 + runner: macos-15-intel - target: aarch64-apple-darwin/clang architecture: aarch64 runner: macos-14 @@ -101,17 +101,10 @@ jobs: set LLVMInstallDir=C:\Program Files\LLVM ./PCbuild/build.bat --tail-call-interp -p ${{ matrix.architecture }} - # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. - # This is a bug in the macOS runner image where the pre-installed Python is installed in the same - # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes - # the symlink to the pre-installed Python so that the Homebrew Python is used instead. - # Note: when a new LLVM is released, the homebrew installation directory changes, so the builds will fail. - # We either need to upgrade LLVM or change the directory being pointed to. - name: Native macOS (release) if: runner.os == 'macOS' run: | brew update - find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete brew install llvm@${{ matrix.llvm }} export SDKROOT="$(xcrun --show-sdk-path)" export PATH="/usr/local/opt/llvm@${{ matrix.llvm }}/bin:$PATH" From 18d4e2ecd4e7414ef2285de457fcf81108f36a21 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sat, 4 Oct 2025 00:52:45 +0300 Subject: [PATCH 003/112] gh-133210: Fix `test_types` with `--without-doc-strings` (#139548) --- Lib/test/test_types.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 9b0ae709d79..4595e7e5d3e 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -714,7 +714,10 @@ def call(part): def test_frame_locals_proxy_type(self): self.assertIsInstance(types.FrameLocalsProxyType, type) - self.assertIsInstance(types.FrameLocalsProxyType.__doc__, str) + if MISSING_C_DOCSTRINGS: + self.assertIsNone(types.FrameLocalsProxyType.__doc__) + else: + self.assertIsInstance(types.FrameLocalsProxyType.__doc__, str) self.assertEqual(types.FrameLocalsProxyType.__module__, 'builtins') self.assertEqual(types.FrameLocalsProxyType.__name__, 'FrameLocalsProxy') From 37d16f7f620d79af7db1c00c8d638122af889bf9 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sat, 4 Oct 2025 06:09:37 +0100 Subject: [PATCH 004/112] Replace workflow with project automations (#136831) Remove unnecessary workflow --- .github/workflows/project-updater.yml | 31 --------------------------- 1 file changed, 31 deletions(-) delete mode 100644 .github/workflows/project-updater.yml diff --git a/.github/workflows/project-updater.yml b/.github/workflows/project-updater.yml deleted file mode 100644 index 1d9d637ec84..00000000000 --- a/.github/workflows/project-updater.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Update GH projects - -on: - issues: - types: - - opened - - labeled - -permissions: - contents: read - -jobs: - add-to-project: - name: Add issues to projects - runs-on: ubuntu-latest - timeout-minutes: 10 - strategy: - fail-fast: false - matrix: - include: - # if an issue has any of these labels, it will be added - # to the corresponding project - - { project: 2, label: "release-blocker, deferred-blocker" } - - { project: 32, label: sprint } - - steps: - - uses: actions/add-to-project@v1.0.0 - with: - project-url: https://github.com/orgs/python/projects/${{ matrix.project }} - github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} - labeled: ${{ matrix.label }} From c33dc154b4d219a696f3fbe8965a38907c371780 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 4 Oct 2025 13:58:51 +0100 Subject: [PATCH 005/112] GH-123299: Copyedit 3.14 What's New: New and Improved Modules (#139530) --- Doc/library/compression.rst | 2 + Doc/whatsnew/3.14.rst | 450 ++++++++++++++++++++++-------------- 2 files changed, 274 insertions(+), 178 deletions(-) diff --git a/Doc/library/compression.rst b/Doc/library/compression.rst index 618b4a3c2bd..98719be9992 100644 --- a/Doc/library/compression.rst +++ b/Doc/library/compression.rst @@ -1,6 +1,8 @@ The :mod:`!compression` package =============================== +.. module:: compression + .. versionadded:: 3.14 The :mod:`!compression` package contains the canonical compression modules diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 3e204eafac8..647b20bf6a9 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1212,10 +1212,30 @@ Default interactive shell New modules =========== -* :mod:`annotationlib`: For introspecting :term:`annotations `. - See :pep:`749` for more details. +* :mod:`annotationlib`: + For introspecting :term:`annotations `. + See :ref:`PEP 749 ` for more details. (Contributed by Jelle Zijlstra in :gh:`119180`.) +* :mod:`compression` (including :mod:`compression.zstd`): + A package for compression-related modules, + including a new module to support the Zstandard compression format. + See :ref:`PEP 784 ` for more details. + (Contributed by Emma Harper Smith, Adam Turner, Gregory P. Smith, Tomas Roun, + Victor Stinner, and Rogdham in :gh:`132983`.) + +* :mod:`concurrent.interpreters`: + Support for multiple interpreters in the standard library. + See :ref:`PEP 734 ` for more details. + (Contributed by Eric Snow in :gh:`134939`.) + +* :mod:`string.templatelib`: + Support for template string literals (t-strings). + See :ref:`PEP 750 ` for more details. + (Contributed by Jim Baker, Guido van Rossum, Paul Everitt, Koudai Aono, + Lysandros Nikolaou, Dave Peck, Adam Turner, Jelle Zijlstra, Bénédikt Tran, + and Pablo Galindo Salgado in :gh:`132661`.) + Improved modules ================ @@ -1233,7 +1253,7 @@ argparse and subparser names if mistyped by the user. (Contributed by Savannah Ostrowski in :gh:`124456`.) - .. _whatsnew314-color-argparse: +.. _whatsnew314-color-argparse: * Enable color for help text, which can be disabled with the optional *color* parameter to :class:`argparse.ArgumentParser`. @@ -1245,7 +1265,7 @@ argparse ast --- -* Add :func:`ast.compare` for comparing two ASTs. +* Add :func:`~ast.compare`, a function for comparing two ASTs. (Contributed by Batuhan Taskaya and Jeremy Hylton in :gh:`60191`.) * Add support for :func:`copy.replace` for AST nodes. @@ -1254,15 +1274,17 @@ ast * Docstrings are now removed from an optimized AST in optimization level 2. (Contributed by Irit Katriel in :gh:`123958`.) -* The ``repr()`` output for AST nodes now includes more information. +* The :func:`repr` output for AST nodes now includes more information. (Contributed by Tomas Roun in :gh:`116022`.) -* :func:`ast.parse`, when called with an AST as input, now always verifies - that the root node type is appropriate. +* When called with an AST as input, the :func:`~ast.parse` function + now always verifies that the root node type is appropriate. (Contributed by Irit Katriel in :gh:`130139`.) -* Add new ``--feature-version``, ``--optimize``, ``--show-empty`` options to - the command-line interface. +* Add new options to the command-line interface: + :option:`--feature-version `, + :option:`--optimize `, and + :option:`--show-empty `. (Contributed by Semyon Moroz in :gh:`133367`.) @@ -1281,20 +1303,23 @@ asyncio :meth:`asyncio.create_task`, :meth:`asyncio.loop.create_task`, :meth:`asyncio.TaskGroup.create_task`. + (Contributed by Thomas Grainger in :gh:`128307`.) * There are two new utility functions for introspecting and printing a program's call graph: :func:`~asyncio.capture_call_graph` and :func:`~asyncio.print_call_graph`. + See :ref:`Asyncio introspection capabilities + ` for more details. (Contributed by Yury Selivanov, Pablo Galindo Salgado, and Łukasz Langa in :gh:`91048`.) - .. _whatsnew314-color-calendar: - calendar -------- +.. _whatsnew314-color-calendar: + * By default, today's date is highlighted in color in :mod:`calendar`'s :ref:`command-line ` text output. This can be controlled by :ref:`environment variables @@ -1307,10 +1332,14 @@ concurrent.futures .. _whatsnew314-concurrent-futures-interp-pool: -* Add :class:`~concurrent.futures.InterpreterPoolExecutor`, - which exposes "subinterpreters" (multiple Python interpreters in the - same process) to Python code. This is separate from the proposed API - in :pep:`734`. +* Add a new executor class, :class:`~concurrent.futures.InterpreterPoolExecutor`, + which exposes multiple Python interpreters in the same process + ('subinterpreters') to Python code. + This uses a pool of independent Python interpreters to execute calls + asynchronously. + + This is separate from the new :mod:`~concurrent.interpreters` module + introduced by :ref:`PEP 734 `. (Contributed by Eric Snow in :gh:`124548`.) .. _whatsnew314-concurrent-futures-start-method: @@ -1334,31 +1363,36 @@ concurrent.futures (Contributed by Gregory P. Smith in :gh:`84559`.) -* Add :meth:`concurrent.futures.ProcessPoolExecutor.terminate_workers` and - :meth:`concurrent.futures.ProcessPoolExecutor.kill_workers` as - ways to terminate or kill all living worker processes in the given pool. +* Add two new methods to :class:`~concurrent.futures.ProcessPoolExecutor`, + :meth:`~concurrent.futures.ProcessPoolExecutor.terminate_workers` + and :meth:`~concurrent.futures.ProcessPoolExecutor.kill_workers`, + as ways to terminate or kill all living worker processes in the given pool. (Contributed by Charles Machalow in :gh:`130849`.) -* Add the optional ``buffersize`` parameter to - :meth:`concurrent.futures.Executor.map` to limit the number of submitted +* Add the optional *buffersize* parameter to :meth:`Executor.map + ` to limit the number of submitted tasks whose results have not yet been yielded. If the buffer is full, iteration over the *iterables* pauses until a result is yielded from the buffer. (Contributed by Enzo Bonnal and Josh Rosenberg in :gh:`74028`.) + configparser ------------ -* Security fix: will no longer write config files it cannot read. Attempting - to :meth:`configparser.ConfigParser.write` keys containing delimiters or - beginning with the section header pattern will raise a - :class:`configparser.InvalidWriteError`. +* :mod:`!configparser` will no longer write config files it cannot read, + to improve security. + Attempting to :meth:`~configparser.ConfigParser.write` keys containing + delimiters or beginning with the section header pattern will raise an + :class:`~configparser.InvalidWriteError`. (Contributed by Jacob Lincoln in :gh:`129270`.) + contextvars ----------- -* Support context manager protocol by :class:`contextvars.Token`. +* Support the :term:`context manager` protocol + for :class:`~contextvars.Token` objects. (Contributed by Andrew Svetlov in :gh:`129889`.) @@ -1366,8 +1400,8 @@ ctypes ------ * The layout of :ref:`bit fields ` - in :class:`~ctypes.Structure` and :class:`~ctypes.Union` - now matches platform defaults (GCC/Clang or MSVC) more closely. + in :class:`~ctypes.Structure` and :class:`~ctypes.Union` objects + is now a closer match to platform defaults (GCC/Clang or MSVC). In particular, fields no longer overlap. (Contributed by Matthias Görgens in :gh:`97702`.) @@ -1386,7 +1420,7 @@ ctypes * On Windows, the :func:`~ctypes.CopyComPointer` function is now public. (Contributed by Jun Komoda in :gh:`127275`.) -* :func:`ctypes.memoryview_at` now exists to create a +* Add :func:`~ctypes.memoryview_at`, a function to create a :class:`memoryview` object that refers to the supplied pointer and length. This works like :func:`ctypes.string_at` except it avoids a buffer copy, and is typically useful when implementing pure Python @@ -1394,7 +1428,7 @@ ctypes (Contributed by Rian Hunter in :gh:`112018`.) * Complex types, :class:`~ctypes.c_float_complex`, - :class:`~ctypes.c_double_complex` and :class:`~ctypes.c_longdouble_complex`, + :class:`~ctypes.c_double_complex`, and :class:`~ctypes.c_longdouble_complex`, are now available if both the compiler and the ``libffi`` library support complex C types. (Contributed by Sergey B Kirpichev in :gh:`61103`.) @@ -1404,50 +1438,57 @@ ctypes (Contributed by Brian Ward in :gh:`119349`.) * Move :func:`ctypes.POINTER` types cache from a global internal cache - (``_pointer_type_cache``) to the :attr:`ctypes._CData.__pointer_type__` - attribute of the corresponding :mod:`ctypes` types. + (``_pointer_type_cache``) to the :attr:`_CData.__pointer_type__ + ` attribute of the corresponding + :mod:`!ctypes` types. This will stop the cache from growing without limits in some situations. (Contributed by Sergey Miryanov in :gh:`100926`.) -* The :class:`ctypes.py_object` type now supports subscription, +* The :class:`~ctypes.py_object` type now supports subscription, making it a :term:`generic type`. (Contributed by Brian Schubert in :gh:`132168`.) -* :mod:`ctypes` now supports :term:`free-threading builds `. +* :mod:`!ctypes` now supports :term:`free-threading builds `. (Contributed by Kumar Aditya and Peter Bierma in :gh:`127945`.) + curses ------ * Add the :func:`~curses.assume_default_colors` function, a refinement of the :func:`~curses.use_default_colors` function which - allows to change the color pair ``0``. + allows changing the color pair ``0``. (Contributed by Serhiy Storchaka in :gh:`133139`.) + datetime -------- -* Add :meth:`datetime.time.strptime` and :meth:`datetime.date.strptime`. +* Add the :meth:`~datetime.date.strptime` method to the + :class:`datetime.date` and :class:`datetime.time` classes. (Contributed by Wannes Boeykens in :gh:`41431`.) + decimal ------- -* Add alternative :class:`~decimal.Decimal` constructor - :meth:`Decimal.from_number() `. +* Add :meth:`.Decimal.from_number` as an alternative constructor for + :class:`~decimal.Decimal`. (Contributed by Serhiy Storchaka in :gh:`121798`.) -* Expose :func:`decimal.IEEEContext` to support creation of contexts +* Expose :func:`~decimal.IEEEContext` to support creation of contexts corresponding to the IEEE 754 (2008) decimal interchange formats. (Contributed by Sergey B Kirpichev in :gh:`53032`.) + difflib ------- * Comparison pages with highlighted changes generated by the - :class:`difflib.HtmlDiff` class now support dark mode. + :class:`~difflib.HtmlDiff` class now support 'dark mode'. (Contributed by Jiahao Li in :gh:`129939`.) + dis --- @@ -1472,7 +1513,7 @@ dis errno ----- -* Add :data:`errno.EHWPOISON` error code. +* Add the :data:`~errno.EHWPOISON` error code constant. (Contributed by James Roy in :gh:`126585`.) @@ -1480,39 +1521,43 @@ faulthandler ------------ * Add support for printing the C stack trace on systems that - :ref:`support it ` via :func:`faulthandler.dump_c_stack` - or via the *c_stack* argument in :func:`faulthandler.enable`. + :ref:`support it ` via the new + :func:`~faulthandler.dump_c_stack` function or via the *c_stack* argument + in :func:`faulthandler.enable`. (Contributed by Peter Bierma in :gh:`127604`.) fnmatch ------- -* Added :func:`fnmatch.filterfalse` for excluding names matching a pattern. +* Add :func:`~fnmatch.filterfalse`, a function to reject names + matching a given pattern. (Contributed by Bénédikt Tran in :gh:`74598`.) fractions --------- -* Add support for converting any objects that have the - :meth:`!as_integer_ratio` method to a :class:`~fractions.Fraction`. +* A :class:`~fractions.Fraction` object may now be constructed from any + object with the :meth:`!as_integer_ratio` method. (Contributed by Serhiy Storchaka in :gh:`82017`.) -* Add alternative :class:`~fractions.Fraction` constructor - :meth:`Fraction.from_number() `. +* Add :meth:`.Fraction.from_number` as an alternative constructor for + :class:`~fractions.Fraction`. (Contributed by Serhiy Storchaka in :gh:`121797`.) functools --------- -* Add support to :func:`functools.partial` and - :func:`functools.partialmethod` for :data:`functools.Placeholder` sentinels - to reserve a place for positional arguments. +* Add the :data:`~functools.Placeholder` sentinel. + This may be used with the :func:`~functools.partial` + or :func:`~functools.partialmethod` functions to reserve a place + for positional arguments in the returned :ref:`partial object + `. (Contributed by Dominykas Grigonis in :gh:`119127`.) -* Allow the *initial* parameter of :func:`functools.reduce` to be passed +* Allow the *initial* parameter of :func:`~functools.reduce` to be passed as a keyword argument. (Contributed by Sayandip Dutta in :gh:`125916`.) @@ -1530,16 +1575,17 @@ getopt getpass ------- -* Support keyboard feedback by :func:`getpass.getpass` via the keyword-only - optional argument ``echo_char``. Placeholder characters are rendered whenever - a character is entered, and removed when a character is deleted. +* Support keyboard feedback in the :func:`~getpass.getpass` function via + the keyword-only optional argument *echo_char*. + Placeholder characters are rendered whenever a character is entered, + and removed when a character is deleted. (Contributed by Semyon Moroz in :gh:`77065`.) graphlib -------- -* Allow :meth:`graphlib.TopologicalSorter.prepare` to be called more than once +* Allow :meth:`.TopologicalSorter.prepare` to be called more than once as long as sorting has not started. (Contributed by Daniel Pope in :gh:`130914`.) @@ -1547,13 +1593,14 @@ graphlib heapq ----- -* Add functions for working with max-heaps: +* The :mod:`!heapq` module has improved support for working with max-heaps, + via the following new functions: - * :func:`heapq.heapify_max`, - * :func:`heapq.heappush_max`, - * :func:`heapq.heappop_max`, - * :func:`heapq.heapreplace_max` - * :func:`heapq.heappushpop_max` + * :func:`~heapq.heapify_max` + * :func:`~heapq.heappush_max` + * :func:`~heapq.heappop_max` + * :func:`~heapq.heapreplace_max` + * :func:`~heapq.heappushpop_max` hmac @@ -1578,9 +1625,12 @@ http the command-line interface (``python -m http.server``) through the following options: - * ``--tls-cert ``: Path to the TLS certificate file. - * ``--tls-key ``: Optional path to the private key file. - * ``--tls-password-file ``: Optional path to the password file for the private key. + * :option:`--tls-cert \ `: + Path to the TLS certificate file. + * :option:`--tls-key \ `: + Optional path to the private key file. + * :option:`--tls-password-file \ `: + Optional path to the password file for the private key. (Contributed by Semyon Moroz in :gh:`85162`.) @@ -1588,7 +1638,7 @@ http imaplib ------- -* Add :meth:`IMAP4.idle() `, implementing the IMAP4 +* Add :meth:`.IMAP4.idle`, implementing the IMAP4 ``IDLE`` command as defined in :rfc:`2177`. (Contributed by Forest in :gh:`55454`.) @@ -1596,15 +1646,16 @@ imaplib inspect ------- -* :func:`inspect.signature` takes a new argument *annotation_format* to control +* :func:`~inspect.signature` takes a new argument *annotation_format* to control the :class:`annotationlib.Format` used for representing annotations. (Contributed by Jelle Zijlstra in :gh:`101552`.) -* :meth:`inspect.Signature.format` takes a new argument *unquote_annotations*. - If true, string :term:`annotations ` are displayed without surrounding quotes. +* :meth:`.Signature.format` takes a new argument *unquote_annotations*. + If true, string :term:`annotations ` are displayed without + surrounding quotes. (Contributed by Jelle Zijlstra in :gh:`101552`.) -* Add function :func:`inspect.ispackage` to determine whether an object is a +* Add function :func:`~inspect.ispackage` to determine whether an object is a :term:`package` or not. (Contributed by Zhikang Yan in :gh:`125634`.) @@ -1616,7 +1667,7 @@ io :exc:`BlockingIOError` if the operation cannot immediately return bytes. (Contributed by Giovanni Siragusa in :gh:`109523`.) -* Add protocols :class:`io.Reader` and :class:`io.Writer` as simpler +* Add the :class:`~io.Reader` and :class:`~io.Writer` protocols as simpler alternatives to the pseudo-protocols :class:`typing.IO`, :class:`typing.TextIO`, and :class:`typing.BinaryIO`. (Contributed by Sebastian Rittau in :gh:`127648`.) @@ -1625,16 +1676,18 @@ io json ---- -* Add notes for JSON serialization errors that allow to identify the source - of the error. +* Add exception notes for JSON serialization errors that allow + identifying the source of the error. (Contributed by Serhiy Storchaka in :gh:`122163`.) -* Enable the :mod:`json` module to work as a script using the :option:`-m` - switch: :program:`python -m json`. +* Allow using the :mod:`json` module as a script using the :option:`-m` switch: + :program:`python -m json`. + This is now preferred to :program:`python -m json.tool`, + which is :term:`soft deprecated`. See the :ref:`JSON command-line interface ` documentation. (Contributed by Trey Hunner in :gh:`122873`.) - .. _whatsnew314-color-json: +.. _whatsnew314-color-json: * By default, the output of the :ref:`JSON command-line interface ` is highlighted in color. @@ -1642,18 +1695,19 @@ json `. (Contributed by Tomas Roun in :gh:`131952`.) + linecache --------- -* :func:`linecache.getline` can retrieve source code for frozen modules. +* :func:`~linecache.getline` can now retrieve source code for frozen modules. (Contributed by Tian Gao in :gh:`131638`.) logging.handlers ---------------- -* :class:`logging.handlers.QueueListener` now implements the context - manager protocol, allowing it to be used in a :keyword:`with` statement. +* :class:`~logging.handlers.QueueListener` objects now support the + :term:`context manager` protocol. (Contributed by Charles Machalow in :gh:`132106`.) * :meth:`QueueListener.start ` now @@ -1671,14 +1725,13 @@ math mimetypes --------- -* Document the command-line for :mod:`mimetypes`. - It now exits with ``1`` on failure instead of ``0`` - and ``2`` on incorrect command-line parameters instead of ``1``. - Also, errors are printed to stderr instead of stdout and their text is made - tighter. +* Add a public :ref:`command-line ` for the module, + invoked via :program:`python -m mimetypes`. (Contributed by Oleg Iarygin and Hugo van Kemenade in :gh:`93096`.) -* Add MS and :rfc:`8081` MIME types for fonts: +* Add several new MIME types based on RFCs and common usage: + + .. rubric:: Microsoft and :rfc:`8081` MIME types for fonts * Embedded OpenType: ``application/vnd.ms-fontobject`` * OpenType Layout (OTF) ``font/otf`` @@ -1686,18 +1739,14 @@ mimetypes * WOFF 1.0 ``font/woff`` * WOFF 2.0 ``font/woff2`` - (Contributed by Sahil Prajapati and Hugo van Kemenade in :gh:`84852`.) - -* Add :rfc:`9559` MIME types for Matroska audiovisual data container - structures, containing: + .. rubric:: :rfc:`9559` MIME types for Matroska audiovisual + data container structures * audio with no video: ``audio/matroska`` (``.mka``) * video: ``video/matroska`` (``.mkv``) * stereoscopic video: ``video/matroska-3d`` (``.mk3d``) - (Contributed by Hugo van Kemenade in :gh:`89416`.) - -* Add MIME types for images with RFCs: + .. rubric:: Images with RFCs * :rfc:`1494`: CCITT Group 3 (``.g3``) * :rfc:`3362`: Real-time Facsimile, T.38 (``.t38``) @@ -1706,9 +1755,7 @@ mimetypes * :rfc:`4047`: Flexible Image Transport System (``.fits``) * :rfc:`7903`: Enhanced Metafile (``.emf``) and Windows Metafile (``.wmf``) - (Contributed by Hugo van Kemenade in :gh:`85957`.) - -* More MIME type changes: + .. rubric:: Other MIME type additions and changes * :rfc:`2361`: Change type for ``.avi`` to ``video/vnd.avi`` and for ``.wav`` to ``audio/vnd.wave`` @@ -1716,6 +1763,8 @@ mimetypes * :rfc:`5334`: Add Ogg media (``.oga``, ``.ogg`` and ``.ogx``) * :rfc:`6713`: Add gzip ``application/gzip`` (``.gz``) * :rfc:`9639`: Add FLAC ``audio/flac`` (``.flac``) + * :rfc:`9512` ``application/yaml`` MIME type for YAML files (``.yaml`` + and ``.yml``) * Add 7z ``application/x-7z-compressed`` (``.7z``) * Add Android Package ``application/vnd.android.package-archive`` (``.apk``) when not strict @@ -1738,11 +1787,9 @@ mimetypes * `W3C `__: Add EPUB ``application/epub+zip`` (``.epub``) - (Contributed by Hugo van Kemenade in :gh:`129965`.) - -* Add :rfc:`9512` ``application/yaml`` MIME type for YAML files (``.yaml`` - and ``.yml``). (Contributed by Sasha "Nelie" Chernykh and Hugo van Kemenade - in :gh:`132056`.) + (Contributed by Sahil Prajapati and Hugo van Kemenade in :gh:`84852`, + by Sasha "Nelie" Chernykh and Hugo van Kemenade in :gh:`132056`, + and by Hugo van Kemenade in :gh:`89416`, :gh:`85957`, and :gh:`129965`.) multiprocessing @@ -1758,8 +1805,8 @@ multiprocessing ` remains the default start method. If the threading incompatible *fork* method is required, you must explicitly - request it via a context from :func:`multiprocessing.get_context` (preferred) - or change the default via :func:`multiprocessing.set_start_method`. + request it via a context from :func:`~multiprocessing.get_context` (preferred) + or change the default via :func:`~multiprocessing.set_start_method`. See :ref:`forkserver restrictions ` for information and differences with the *fork* method and how this change @@ -1768,7 +1815,7 @@ multiprocessing (Contributed by Gregory P. Smith in :gh:`84559`.) -* :mod:`multiprocessing`'s ``"forkserver"`` start method now authenticates +* :mod:`multiprocessing`'s ``'forkserver'`` start method now authenticates its control socket to avoid solely relying on filesystem permissions to restrict what other processes could cause the forkserver to spawn workers and run code. @@ -1784,20 +1831,22 @@ multiprocessing (Contributed by Roy Hyunjin Han for :gh:`103134`.) * Add support for shared :class:`set` objects via - :meth:`SyncManager.set() `. - The :func:`set` in :func:`multiprocessing.Manager` method is now available. + :meth:`.SyncManager.set`. + The :func:`set` in :func:`~multiprocessing.Manager` method is now available. (Contributed by Mingyu Park in :gh:`129949`.) -* Add :func:`multiprocessing.Process.interrupt` which terminates the child +* Add the :meth:`~multiprocessing.Process.interrupt` + to :class:`multiprocessing.Process` objects, which terminates the child process by sending :py:const:`~signal.SIGINT`. This enables :keyword:`finally` clauses to print a stack trace for the terminated process. (Contributed by Artem Pulkin in :gh:`131913`.) + operator -------- -* Two new functions :func:`operator.is_none` and :func:`operator.is_not_none` - have been added, such that ``operator.is_none(obj)`` is equivalent +* Add :func:`~operator.is_none` and :func:`~operator.is_not_none` as a pair + of functions, such that ``operator.is_none(obj)`` is equivalent to ``obj is None`` and ``operator.is_not_none(obj)`` is equivalent to ``obj is not None``. (Contributed by Raymond Hettinger and Nico Mexis in :gh:`115808`.) @@ -1806,17 +1855,17 @@ operator os -- -* Add the :func:`os.reload_environ` function to update :data:`os.environ` and +* Add the :func:`~os.reload_environ` function to update :data:`os.environ` and :data:`os.environb` with changes to the environment made by :func:`os.putenv`, by :func:`os.unsetenv`, or made outside Python in the same process. (Contributed by Victor Stinner in :gh:`120057`.) * Add the :data:`~os.SCHED_DEADLINE` and :data:`~os.SCHED_NORMAL` constants - to the :mod:`os` module. + to the :mod:`!os` module. (Contributed by James Roy in :gh:`127688`.) -* Add the :func:`os.readinto` function to read into a +* Add the :func:`~os.readinto` function to read into a :ref:`buffer object ` from a file descriptor. (Contributed by Cody Maloney in :gh:`129205`.) @@ -1824,8 +1873,8 @@ os os.path ------- -* The *strict* parameter to :func:`os.path.realpath` accepts a new value, - :data:`os.path.ALLOW_MISSING`. +* The *strict* parameter to :func:`~os.path.realpath` accepts a new value, + :data:`~os.path.ALLOW_MISSING`. If used, errors other than :exc:`FileNotFoundError` will be re-raised; the resulting path can be missing but it will be free of symlinks. (Contributed by Petr Viktorin for :cve:`2025-4517`.) @@ -1844,8 +1893,8 @@ pathlib (Contributed by Barney Gale in :gh:`73991`.) -* Add :attr:`pathlib.Path.info` attribute, which stores an object - implementing the :class:`pathlib.types.PathInfo` protocol (also new). The +* Add the :attr:`~pathlib.Path.info` attribute, which stores an object + implementing the new :class:`pathlib.types.PathInfo` protocol. The object supports querying the file type and internally caching :func:`~os.stat` results. Path objects generated by :meth:`~pathlib.Path.iterdir` are initialized with file type information @@ -1856,7 +1905,7 @@ pathlib pdb --- -* Hardcoded breakpoints (:func:`breakpoint` and :func:`pdb.set_trace`) now +* Hardcoded breakpoints (:func:`breakpoint` and :func:`~pdb.set_trace`) now reuse the most recent :class:`~pdb.Pdb` instance that calls :meth:`~pdb.Pdb.set_trace`, instead of creating a new one each time. As a result, all the instance specific data like :pdbcmd:`display` and @@ -1906,15 +1955,16 @@ pickle * Set the default protocol version on the :mod:`pickle` module to 5. For more details, see :ref:`pickle protocols `. -* Add notes for pickle serialization errors that allow to identify the source - of the error. +* Add exception notes for pickle serialization errors that allow + identifying the source of the error. (Contributed by Serhiy Storchaka in :gh:`122213`.) platform -------- -* Add :func:`platform.invalidate_caches` to invalidate the cached results. +* Add :func:`~platform.invalidate_caches`, a function to invalidate + cached results in the :mod:`!platform` module. (Contributed by Bénédikt Tran in :gh:`122549`.) @@ -1934,8 +1984,8 @@ re unlike ``\Z``, which has subtly different behavior. (Contributed by Serhiy Storchaka in :gh:`133306`.) -* ``\B`` in :mod:`regular expression ` now matches the empty input string. - Now it is always the opposite of ``\b``. +* ``\B`` in :mod:`regular expression ` now matches the empty input string, + meaning that it is now always the opposite of ``\b``. (Contributed by Serhiy Storchaka in :gh:`124130`.) @@ -1965,11 +2015,12 @@ socket * Add many new constants. (Contributed by Serhiy Storchaka in :gh:`132734`.) + ssl --- -* Indicate through :data:`ssl.HAS_PHA` whether the :mod:`ssl` module supports - TLSv1.3 post-handshake client authentication (PHA). +* Indicate through the :data:`~ssl.HAS_PHA` Boolean whether the :mod:`!ssl` + module supports TLSv1.3 post-handshake client authentication (PHA). (Contributed by Will Childs-Klein in :gh:`128036`.) @@ -1985,7 +2036,7 @@ struct symtable -------- -* Expose the following :class:`symtable.Symbol` methods: +* Expose the following :class:`~symtable.Symbol` methods: * :meth:`~symtable.Symbol.is_comp_cell` * :meth:`~symtable.Symbol.is_comp_iter` @@ -2000,28 +2051,41 @@ sys * The previously undocumented special function :func:`sys.getobjects`, which only exists in specialized builds of Python, may now return objects from other interpreters than the one it's called in. + (Contributed by Eric Snow in :gh:`125286`.) * Add :func:`sys._is_immortal` for determining if an object is :term:`immortal`. (Contributed by Peter Bierma in :gh:`128509`.) -* On FreeBSD, :data:`sys.platform` doesn't contain the major version anymore. +* On FreeBSD, :data:`sys.platform` no longer contains the major version number. It is always ``'freebsd'``, instead of ``'freebsd13'`` or ``'freebsd14'``. + (Contributed by Michael Osipov in :gh:`129393`.) * Raise :exc:`DeprecationWarning` for :func:`sys._clear_type_cache`. This function was deprecated in Python 3.13 but it didn't raise a runtime warning. +* Add :func:`sys.remote_exec` to implement the new external debugger interface. + See :ref:`PEP 768 ` for details. + (Contributed by Pablo Galindo Salgado, Matt Wozniski, and Ivona Stojanovic + in :gh:`131591`.) + +* Add the :data:`sys._jit` namespace, containing utilities for introspecting + just-in-time compilation. + (Contributed by Brandt Bucher in :gh:`133231`.) + sys.monitoring -------------- -* Two new events are added: :monitoring-event:`BRANCH_LEFT` and - :monitoring-event:`BRANCH_RIGHT`. The ``BRANCH`` event is deprecated. +* Add two new monitoring events, :monitoring-event:`BRANCH_LEFT` and + :monitoring-event:`BRANCH_RIGHT`. + These replace and deprecate the :monitoring-event:`!BRANCH` event. + (Contributed by Mark Shannon in :gh:`122548`.) sysconfig --------- -* Add ``ABIFLAGS`` key to :func:`sysconfig.get_config_vars` on Windows. +* Add ``ABIFLAGS`` key to :func:`~sysconfig.get_config_vars` on Windows. (Contributed by Xuehai Pan in :gh:`131799`.) @@ -2031,15 +2095,18 @@ tarfile * :func:`~tarfile.data_filter` now normalizes symbolic link targets in order to avoid path traversal attacks. (Contributed by Petr Viktorin in :gh:`127987` and :cve:`2025-4138`.) + * :func:`~tarfile.TarFile.extractall` now skips fixing up directory attributes when a directory was removed or replaced by another kind of file. (Contributed by Petr Viktorin in :gh:`127987` and :cve:`2024-12718`.) + * :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` now (re-)apply the extraction filter when substituting a link (hard or symbolic) with a copy of another archive member, and when fixing up directory attributes. The former raises a new exception, :exc:`~tarfile.LinkFallbackError`. (Contributed by Petr Viktorin for :cve:`2025-4330` and :cve:`2024-12718`.) + * :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` no longer extract rejected members when :func:`~tarfile.TarFile.errorlevel` is zero. @@ -2059,17 +2126,18 @@ tkinter ------- * Make :mod:`tkinter` widget methods :meth:`!after` and :meth:`!after_idle` - accept arguments passed by keyword. + accept keyword arguments. (Contributed by Zhikang Yan in :gh:`126899`.) -* Add ability to specify name for :class:`!tkinter.OptionMenu` and +* Add ability to specify a name for :class:`!tkinter.OptionMenu` and :class:`!tkinter.ttk.OptionMenu`. (Contributed by Zhikang Yan in :gh:`130482`.) + turtle ------ -* Add context managers for :func:`turtle.fill`, :func:`turtle.poly` +* Add context managers for :func:`turtle.fill`, :func:`turtle.poly`, and :func:`turtle.no_animation`. (Contributed by Marie Roald and Yngve Mardal Moe in :gh:`126350`.) @@ -2087,44 +2155,59 @@ typing .. _whatsnew314-typing-union: -* :class:`types.UnionType` and :class:`typing.Union` are now aliases for each other, - meaning that both old-style unions (created with ``Union[int, str]``) and new-style - unions (``int | str``) now create instances of the same runtime type. This unifies - the behavior between the two syntaxes, but leads to some differences in behavior that +* The :class:`types.UnionType` and :class:`typing.Union` types are now + aliases for each other, meaning that both old-style unions + (created with ``Union[int, str]``) and new-style unions (``int | str``) + now create instances of the same runtime type. This unifies the behavior + between the two syntaxes, but leads to some differences in behavior that may affect users who introspect types at runtime: - - Both syntaxes for creating a union now produce the same string representation in - ``repr()``. For example, ``repr(Union[int, str])`` - is now ``"int | str"`` instead of ``"typing.Union[int, str]"``. - - Unions created using the old syntax are no longer cached. Previously, running - ``Union[int, str]`` multiple times would return the same object - (``Union[int, str] is Union[int, str]`` would be ``True``), but now it will - return two different objects. Users should use ``==`` to compare unions for equality, not - ``is``. New-style unions have never been cached this way. - This change could increase memory usage for some programs that use a large number of - unions created by subscripting ``typing.Union``. However, several factors offset this cost: - unions used in annotations are no longer evaluated by default in Python 3.14 - because of :pep:`649`; an instance of :class:`types.UnionType` is - itself much smaller than the object returned by ``Union[]`` was on prior Python - versions; and removing the cache also saves some space. It is therefore - unlikely that this change will cause a significant increase in memory usage for most - users. + - Both syntaxes for creating a union now produce the same string + representation in :func:`repr`. + For example, ``repr(Union[int, str])`` is now ``"int | str"`` instead of + ``"typing.Union[int, str]"``. + + - Unions created using the old syntax are no longer cached. + Previously, running ``Union[int, str]`` multiple times would return + the same object (``Union[int, str] is Union[int, str]`` would be ``True``), + but now it will return two different objects. + Use ``==`` to compare unions for equality, not ``is``. + New-style unions have never been cached this way. + This change could increase memory usage for some programs that use + a large number of unions created by subscripting ``typing.Union``. + However, several factors offset this cost: + unions used in annotations are no longer evaluated by default in Python + 3.14 because of :pep:`649`; an instance of :class:`types.UnionType` is + itself much smaller than the object returned by ``Union[]`` was on prior + Python versions; and removing the cache also saves some space. + It is therefore unlikely that this change will cause a significant increase + in memory usage for most users. + - Previously, old-style unions were implemented using the private class - ``typing._UnionGenericAlias``. This class is no longer needed for the implementation, - but it has been retained for backward compatibility, with removal scheduled for Python - 3.17. Users should use documented introspection helpers like :func:`typing.get_origin` - and :func:`typing.get_args` instead of relying on private implementation details. - - It is now possible to use :class:`typing.Union` itself in :func:`isinstance` checks. - For example, ``isinstance(int | str, typing.Union)`` will return ``True``; previously - this raised :exc:`TypeError`. - - The ``__args__`` attribute of :class:`typing.Union` objects is no longer writable. - - It is no longer possible to set any attributes on :class:`typing.Union` objects. + ``typing._UnionGenericAlias``. + This class is no longer needed for the implementation, + but it has been retained for backward compatibility, + with removal scheduled for Python 3.17. + Users should use documented introspection helpers like + :func:`~typing.get_origin` and :func:`typing.get_args` instead of + relying on private implementation details. + + - It is now possible to use :class:`typing.Union` itself in + :func:`isinstance` checks. + For example, ``isinstance(int | str, typing.Union)`` will return ``True``; + previously this raised :exc:`TypeError`. + + - The :attr:`!__args__` attribute of :class:`typing.Union` objects is + no longer writable. + + - It is no longer possible to set any attributes on :class:`~typing.Union` + objects. This only ever worked for dunder attributes on previous versions, was never documented to work, and was subtly broken in many cases. (Contributed by Jelle Zijlstra in :gh:`105499`.) -* :class:`typing.TypeAliasType` now supports star unpacking. +* :class:`~typing.TypeAliasType` now supports star unpacking. unicodedata @@ -2133,11 +2216,11 @@ unicodedata * The Unicode database has been updated to Unicode 16.0.0. -.. _whatsnew314-color-unittest: - unittest -------- +.. _whatsnew314-color-unittest: + * :mod:`unittest` output is now colored by default. This can be controlled by :ref:`environment variables `. @@ -2175,7 +2258,7 @@ urllib * Improve ergonomics and standards compliance when parsing and emitting ``file:`` URLs. - In :func:`urllib.request.url2pathname`: + In :func:`~urllib.request.url2pathname`: - Accept a complete URL when the new *require_scheme* argument is set to true. @@ -2186,7 +2269,7 @@ urllib - Raise :exc:`~urllib.error.URLError` if a URL authority isn't local, except on Windows where we return a UNC path as before. - In :func:`urllib.request.pathname2url`: + In :func:`~urllib.request.pathname2url`: - Return a complete URL when the new *add_scheme* argument is set to true. - Include an empty URL authority when a path begins with a slash. For @@ -2202,16 +2285,17 @@ urllib uuid ---- -* Add support for UUID versions 6, 7, and 8 via :func:`uuid.uuid6`, - :func:`uuid.uuid7`, and :func:`uuid.uuid8` respectively, as specified +* Add support for UUID versions 6, 7, and 8 via :func:`~uuid.uuid6`, + :func:`~uuid.uuid7`, and :func:`~uuid.uuid8` respectively, as specified in :rfc:`9562`. (Contributed by Bénédikt Tran in :gh:`89083`.) -* :const:`uuid.NIL` and :const:`uuid.MAX` are now available to represent the +* :const:`~uuid.NIL` and :const:`~uuid.MAX` are now available to represent the Nil and Max UUID formats as defined by :rfc:`9562`. (Contributed by Nick Pope in :gh:`128427`.) -* Allow to generate multiple UUIDs at once via :option:`python -m uuid --count `. +* Allow generating multiple UUIDs simultaneously on the command-line via + :option:`python -m uuid --count `. (Contributed by Simon Legner in :gh:`131236`.) @@ -2229,14 +2313,13 @@ webbrowser zipfile ------- -* Added :func:`ZipInfo._for_archive ` +* Added :meth:`ZipInfo._for_archive `, a method to resolve suitable defaults for a :class:`~zipfile.ZipInfo` object as used by :func:`ZipFile.writestr `. (Contributed by Bénédikt Tran in :gh:`123424`.) -* :meth:`zipfile.ZipFile.writestr` now respects ``SOURCE_DATE_EPOCH`` that - distributions can set centrally and have build tools consume this in order - to produce reproducible output. +* :meth:`.ZipFile.writestr` now respects the :envvar:`SOURCE_DATE_EPOCH` + environment variable in order to better support reproducible builds. (Contributed by Jiahao Li in :gh:`91279`.) @@ -2256,17 +2339,18 @@ Optimizations (Contributed by Adam Turner, Bénédikt Tran, Chris Markiewicz, Eli Schwartz, Hugo van Kemenade, Jelle Zijlstra, and others in :gh:`118761`.) -* The interpreter avoids some reference count modifications internally when - it's safe to do so. This can lead to different values returned from - :func:`sys.getrefcount` and :c:func:`Py_REFCNT` compared to previous versions - of Python. See :ref:`below ` for details. +* The interpreter now avoids some reference count modifications internally + when it's safe to do so. + This can lead to different values being returned from :func:`sys.getrefcount` + and :c:func:`Py_REFCNT` compared to previous versions of Python. + See :ref:`below ` for details. asyncio ------- -* Standard benchmark results have improved by 10-20%, following the - implementation of a new per-thread double linked list +* Standard benchmark results have improved by 10-20% following the + implementation of a new per-thread doubly linked list for :class:`native tasks `, also reducing memory usage. This enables external introspection tools such as @@ -3267,6 +3351,16 @@ Changes in the Python API with annotations should continue to work, some undocumented details may behave differently. +* As part of making the :mod:`mimetypes` CLI public, + it now exits with ``1`` on failure instead of ``0`` + and ``2`` on incorrect command-line parameters instead of ``1``. + Error messages are now printed to stderr. + +* The ``\B`` pattern in regular expression now matches the empty string + when given as the entire pattern, which may cause behavioural changes. + +* On FreeBSD, :data:`sys.platform` no longer contains the major version number. + Changes in the C API -------------------- From db53ca30d761abba525bc8e47b16920b1fc43a83 Mon Sep 17 00:00:00 2001 From: Ho Kim Date: Sat, 4 Oct 2025 22:26:20 +0900 Subject: [PATCH 006/112] gh-138489: Add missing `build-details.json` step for building wasm (#139302) * fix: add missing `build-details.json` step for building wasm Signed-off-by: Ho Kim * gh-138489: Add missing build-details.json step for building wasm Signed-off-by: Ho Kim * Update Makefile.pre.in Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --------- Signed-off-by: Ho Kim Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Makefile.pre.in | 2 +- .../Build/2025-09-24-13-59-26.gh-issue-138489.1AcuZM.rst | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Build/2025-09-24-13-59-26.gh-issue-138489.1AcuZM.rst diff --git a/Makefile.pre.in b/Makefile.pre.in index 6651b093e20..37bde125166 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -801,7 +801,7 @@ build_all: check-clean-src check-app-store-compliance $(BUILDPYTHON) platform sh .PHONY: build_wasm build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \ - python-config checksharedmods + python-config checksharedmods build-details.json .PHONY: build_emscripten build_emscripten: build_wasm web_example web_example_pyrepl_jspi diff --git a/Misc/NEWS.d/next/Build/2025-09-24-13-59-26.gh-issue-138489.1AcuZM.rst b/Misc/NEWS.d/next/Build/2025-09-24-13-59-26.gh-issue-138489.1AcuZM.rst new file mode 100644 index 00000000000..b11098a3f87 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-09-24-13-59-26.gh-issue-138489.1AcuZM.rst @@ -0,0 +1,6 @@ +When cross-compiling for WASI by ``build_wasm`` or ``build_emscripten``, the +``build-details.json`` step is now included in the build process, just like +with native builds. + +This fixes the ``libinstall`` task which requires the ``build-details.json`` +file during the process. From 8d17d79299ba3aad9f8fd2aded5ee776dc71668d Mon Sep 17 00:00:00 2001 From: Semyon Moroz Date: Sat, 4 Oct 2025 17:53:43 +0400 Subject: [PATCH 007/112] gh-138044: Remove deprecated parameter alias for `importlib.resources.files` (#138059) --- Doc/library/importlib.resources.rst | 13 ++++--- Lib/importlib/resources/_common.py | 34 ------------------- .../test_importlib/resources/test_files.py | 8 ----- ...-08-22-12-48-14.gh-issue-138044.lEQULC.rst | 2 ++ 4 files changed, 8 insertions(+), 49 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-08-22-12-48-14.gh-issue-138044.lEQULC.rst diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index 7a11f4fe069..8cb43f0625f 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -72,13 +72,12 @@ for example, a package and its resources can be imported from a zip file using .. versionadded:: 3.9 - .. versionchanged:: 3.12 - *package* parameter was renamed to *anchor*. *anchor* can now - be a non-package module and if omitted will default to the caller's - module. *package* is still accepted for compatibility but will raise - a :exc:`DeprecationWarning`. Consider passing the anchor positionally or - using ``importlib_resources >= 5.10`` for a compatible interface - on older Pythons. + .. deprecated-removed:: 3.12 3.15 + *package* parameter was renamed to *anchor*. *anchor* can now be a + non-package module and if omitted will default to the caller's module. + *package* is no longer accepted since Python 3.15. Consider passing the + anchor positionally or using ``importlib_resources >= 5.10`` for a + compatible interface on older Pythons. .. function:: as_file(traversable) diff --git a/Lib/importlib/resources/_common.py b/Lib/importlib/resources/_common.py index 4e9014c45a0..d16ebe4520f 100644 --- a/Lib/importlib/resources/_common.py +++ b/Lib/importlib/resources/_common.py @@ -6,7 +6,6 @@ import types import importlib import inspect -import warnings import itertools from typing import Union, Optional, cast @@ -16,39 +15,6 @@ Anchor = Package -def package_to_anchor(func): - """ - Replace 'package' parameter as 'anchor' and warn about the change. - - Other errors should fall through. - - >>> files('a', 'b') - Traceback (most recent call last): - TypeError: files() takes from 0 to 1 positional arguments but 2 were given - - Remove this compatibility in Python 3.14. - """ - undefined = object() - - @functools.wraps(func) - def wrapper(anchor=undefined, package=undefined): - if package is not undefined: - if anchor is not undefined: - return func(anchor, package) - warnings.warn( - "First parameter to files is renamed to 'anchor'", - DeprecationWarning, - stacklevel=2, - ) - return func(package) - elif anchor is undefined: - return func() - return func(anchor) - - return wrapper - - -@package_to_anchor def files(anchor: Optional[Anchor] = None) -> Traversable: """ Get a Traversable resource for an anchor. diff --git a/Lib/test/test_importlib/resources/test_files.py b/Lib/test/test_importlib/resources/test_files.py index 3ce44999f98..c935b1e10ac 100644 --- a/Lib/test/test_importlib/resources/test_files.py +++ b/Lib/test/test_importlib/resources/test_files.py @@ -38,14 +38,6 @@ def test_joinpath_with_multiple_args(self): binfile = files.joinpath('subdirectory', 'binary.file') self.assertTrue(binfile.is_file()) - def test_old_parameter(self): - """ - Files used to take a 'package' parameter. Make sure anyone - passing by name is still supported. - """ - with suppress_known_deprecation(): - resources.files(package=self.data) - class OpenDiskTests(FilesTests, util.DiskSetup, unittest.TestCase): pass diff --git a/Misc/NEWS.d/next/Library/2025-08-22-12-48-14.gh-issue-138044.lEQULC.rst b/Misc/NEWS.d/next/Library/2025-08-22-12-48-14.gh-issue-138044.lEQULC.rst new file mode 100644 index 00000000000..99ed3adef91 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-22-12-48-14.gh-issue-138044.lEQULC.rst @@ -0,0 +1,2 @@ +Remove compatibility shim for deprecated parameter *package* in +:func:`importlib.resources.files`. Patch by Semyon Moroz. From a7a485558c05a59665d24f9bc048db8a41daf532 Mon Sep 17 00:00:00 2001 From: Cycloctane Date: Sat, 4 Oct 2025 21:55:17 +0800 Subject: [PATCH 008/112] gh-133951: Remove lib64->lib symlink in venv creation (#137139) * Remove lib64->lib symlink in venv directory * fix test * remove unused import * add news --- Lib/test/test_venv.py | 10 ++-------- Lib/venv/__init__.py | 9 +++------ .../2025-07-27-17-03-17.gh-issue-133951.7kwt78.rst | 2 ++ 3 files changed, 7 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-07-27-17-03-17.gh-issue-133951.7kwt78.rst diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 3c18c9c2900..d46b45e5437 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -12,7 +12,6 @@ import pathlib import re import shutil -import struct import subprocess import sys import sysconfig @@ -138,14 +137,9 @@ def _check_output_of_default_create(self): self.isdir(self.bindir) self.isdir(self.include) self.isdir(*self.lib) - # Issue 21197 p = self.get_env_file('lib64') - conditions = ((struct.calcsize('P') == 8) and (os.name == 'posix') and - (sys.platform != 'darwin')) - if conditions: - self.assertTrue(os.path.islink(p)) - else: - self.assertFalse(os.path.exists(p)) + if os.path.exists(p): + self.assertFalse(os.path.islink(p)) data = self.get_text_file_contents('pyvenv.cfg') executable = sys._base_executable path = os.path.dirname(executable) diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index dc9c5991df7..e5addcc393a 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -174,6 +174,7 @@ def create_if_needed(d): context.python_exe = exename binpath = self._venv_path(env_dir, 'scripts') libpath = self._venv_path(env_dir, 'purelib') + platlibpath = self._venv_path(env_dir, 'platlib') # PEP 405 says venvs should create a local include directory. # See https://peps.python.org/pep-0405/#include-files @@ -191,12 +192,8 @@ def create_if_needed(d): create_if_needed(incpath) context.lib_path = libpath create_if_needed(libpath) - # Issue 21197: create lib64 as a symlink to lib on 64-bit non-OS X POSIX - if ((sys.maxsize > 2**32) and (os.name == 'posix') and - (sys.platform != 'darwin')): - link_path = os.path.join(env_dir, 'lib64') - if not os.path.exists(link_path): # Issue #21643 - os.symlink('lib', link_path) + context.platlib_path = platlibpath + create_if_needed(platlibpath) context.bin_path = binpath context.bin_name = os.path.relpath(binpath, env_dir) context.env_exe = os.path.join(binpath, exename) diff --git a/Misc/NEWS.d/next/Library/2025-07-27-17-03-17.gh-issue-133951.7kwt78.rst b/Misc/NEWS.d/next/Library/2025-07-27-17-03-17.gh-issue-133951.7kwt78.rst new file mode 100644 index 00000000000..dfda8e8f10c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-27-17-03-17.gh-issue-133951.7kwt78.rst @@ -0,0 +1,2 @@ +Remove lib64-lib symlink creation when creating new virtual environments in +:mod:`venv` module From ae6e7f572c18d9183cb23e6e53c3324dd9bf0c64 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 4 Oct 2025 13:56:43 +0000 Subject: [PATCH 009/112] gh-139308: Skip test_special_chars_csh on NetBSD due to csh variable expansion issue (#139341) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Skip test_special_chars_csh on NetBSD due to csh variable expansion issue Co-authored-by: Filipe Laíns 🇵🇸 --- Lib/test/test_venv.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index d46b45e5437..68bcf535ead 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -516,6 +516,8 @@ def test_special_chars_bash(self): # gh-124651: test quoted strings @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows') + @unittest.skipIf(sys.platform.startswith('netbsd'), + "NetBSD csh fails with quoted special chars; see gh-139308") def test_special_chars_csh(self): """ Test that the template strings are quoted properly (csh) From 9e3542a6c26a849012216f5f754f16043d775c42 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 4 Oct 2025 16:57:12 +0300 Subject: [PATCH 010/112] gh-136097: Fix sysconfig._parse_makefile() (#136166) * Fix potential infinite recursion. * Fix a bug when reference can cross boundaries of substitutions, e.g. a=$( b=$(a)a) * Fix potential quadratic complexity. * Fix KeyError for undefined CFLAGS, LDFLAGS, or CPPFLAGS. * Fix infinite recursion when keep_unresolved=False. * Unify behavior with keep_unresolved=False for bogus $ occurred before and after variable references. --- Lib/sysconfig/__main__.py | 148 ++++++++---------- Lib/test/test_sysconfig.py | 71 ++++++++- ...-07-01-14-44-03.gh-issue-136097.bI1n14.rst | 2 + 3 files changed, 133 insertions(+), 88 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-07-01-14-44-03.gh-issue-136097.bI1n14.rst diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index bc2197cfe79..0cf0cf4dbb9 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -21,8 +21,9 @@ # Regexes needed for parsing Makefile (and similar syntaxes, # like old-style Setup files). _variable_rx = r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)" -_findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)" -_findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}" +_findvar_rx = (r"\$(\([A-Za-z][A-Za-z0-9_]*\)" + r"|\{[A-Za-z][A-Za-z0-9_]*\}" + r"|\$?)") def _parse_makefile(filename, vars=None, keep_unresolved=True): @@ -49,26 +50,7 @@ def _parse_makefile(filename, vars=None, keep_unresolved=True): m = re.match(_variable_rx, line) if m: n, v = m.group(1, 2) - v = v.strip() - # `$$' is a literal `$' in make - tmpv = v.replace('$$', '') - - if "$" in tmpv: - notdone[n] = v - else: - try: - if n in _ALWAYS_STR: - raise ValueError - - v = int(v) - except ValueError: - # insert literal `$' - done[n] = v.replace('$$', '$') - else: - done[n] = v - - # do variable interpolation here - variables = list(notdone.keys()) + notdone[n] = v.strip() # Variables with a 'PY_' prefix in the makefile. These need to # be made available without that prefix through sysconfig. @@ -76,72 +58,64 @@ def _parse_makefile(filename, vars=None, keep_unresolved=True): # if the expansion uses the name without a prefix. renamed_variables = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS') - while len(variables) > 0: - for name in tuple(variables): - value = notdone[name] - m1 = re.search(_findvar1_rx, value) - m2 = re.search(_findvar2_rx, value) - if m1 and m2: - m = m1 if m1.start() < m2.start() else m2 - else: - m = m1 if m1 else m2 - if m is not None: - n = m.group(1) - found = True - if n in done: - item = str(done[n]) - elif n in notdone: - # get it on a subsequent round - found = False - elif n in os.environ: - # do it like make: fall back to environment - item = os.environ[n] - - elif n in renamed_variables: - if (name.startswith('PY_') and - name[3:] in renamed_variables): - item = "" - - elif 'PY_' + n in notdone: - found = False - - else: - item = str(done['PY_' + n]) - - else: - done[n] = item = "" - - if found: - after = value[m.end():] - value = value[:m.start()] + item + after - if "$" in after: - notdone[name] = value - else: - try: - if name in _ALWAYS_STR: - raise ValueError - value = int(value) - except ValueError: - done[name] = value.strip() - else: - done[name] = value - variables.remove(name) - - if name.startswith('PY_') \ - and name[3:] in renamed_variables: - - name = name[3:] - if name not in done: - done[name] = value - - else: - # Adds unresolved variables to the done dict. - # This is disabled when called from distutils.sysconfig + def resolve_var(name): + def repl(m): + n = m[1] + if n == '$': + return '$' + elif n == '': + # bogus variable reference (e.g. "prefix=$/opt/python") if keep_unresolved: - done[name] = value - # bogus variable reference (e.g. "prefix=$/opt/python"); - # just drop it since we can't deal - variables.remove(name) + return m[0] + raise ValueError + elif n[0] == '(' and n[-1] == ')': + n = n[1:-1] + elif n[0] == '{' and n[-1] == '}': + n = n[1:-1] + + if n in done: + return str(done[n]) + elif n in notdone: + return str(resolve_var(n)) + elif n in os.environ: + # do it like make: fall back to environment + return os.environ[n] + elif n in renamed_variables: + if name.startswith('PY_') and name[3:] in renamed_variables: + return "" + n = 'PY_' + n + if n in notdone: + return str(resolve_var(n)) + else: + assert n not in done + return "" + else: + done[n] = "" + return "" + + assert name not in done + done[name] = "" + try: + value = re.sub(_findvar_rx, repl, notdone[name]) + except ValueError: + del done[name] + return "" + value = value.strip() + if name not in _ALWAYS_STR: + try: + value = int(value) + except ValueError: + pass + done[name] = value + if name.startswith('PY_') and name[3:] in renamed_variables: + name = name[3:] + if name not in done: + done[name] = value + return value + + for n in notdone: + if n not in done: + resolve_var(n) # strip spurious spaces for k, v in done.items(): diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 6a0681a4148..59ea623bb92 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -764,8 +764,12 @@ def test_parse_makefile(self): print("var3=42", file=makefile) print("var4=$/invalid", file=makefile) print("var5=dollar$$5", file=makefile) - print("var6=${var3}/lib/python3.5/config-$(VAR2)$(var5)" + print("var6=${var7}/lib/python3.5/config-$(VAR2)$(var5)" "-x86_64-linux-gnu", file=makefile) + print("var7=${var3}", file=makefile) + print("var8=$$(var3)", file=makefile) + print("var9=$(var10)(var3)", file=makefile) + print("var10=$$", file=makefile) vars = _parse_makefile(TESTFN) self.assertEqual(vars, { 'var1': 'ab42', @@ -774,6 +778,71 @@ def test_parse_makefile(self): 'var4': '$/invalid', 'var5': 'dollar$5', 'var6': '42/lib/python3.5/config-b42dollar$5-x86_64-linux-gnu', + 'var7': 42, + 'var8': '$(var3)', + 'var9': '$(var3)', + 'var10': '$', + }) + + def _test_parse_makefile_recursion(self): + self.addCleanup(unlink, TESTFN) + with open(TESTFN, "w") as makefile: + print("var1=var1=$(var1)", file=makefile) + print("var2=var3=$(var3)", file=makefile) + print("var3=var2=$(var2)", file=makefile) + vars = _parse_makefile(TESTFN) + self.assertEqual(vars, { + 'var1': 'var1=', + 'var2': 'var3=var2=', + 'var3': 'var2=', + }) + + def test_parse_makefile_renamed_vars(self): + self.addCleanup(unlink, TESTFN) + with open(TESTFN, "w") as makefile: + print("var1=$(CFLAGS)", file=makefile) + print("PY_CFLAGS=-Wall $(CPPFLAGS)", file=makefile) + print("PY_LDFLAGS=-lm", file=makefile) + print("var2=$(LDFLAGS)", file=makefile) + print("var3=$(CPPFLAGS)", file=makefile) + vars = _parse_makefile(TESTFN) + self.assertEqual(vars, { + 'var1': '-Wall', + 'CFLAGS': '-Wall', + 'PY_CFLAGS': '-Wall', + 'LDFLAGS': '-lm', + 'PY_LDFLAGS': '-lm', + 'var2': '-lm', + 'var3': '', + }) + + def test_parse_makefile_keep_unresolved(self): + self.addCleanup(unlink, TESTFN) + with open(TESTFN, "w") as makefile: + print("var1=value", file=makefile) + print("var2=$/", file=makefile) + print("var3=$/$(var1)", file=makefile) + print("var4=var5=$(var5)", file=makefile) + print("var5=$/$(var1)", file=makefile) + print("var6=$(var1)$/", file=makefile) + print("var7=var8=$(var8)", file=makefile) + print("var8=$(var1)$/", file=makefile) + vars = _parse_makefile(TESTFN) + self.assertEqual(vars, { + 'var1': 'value', + 'var2': '$/', + 'var3': '$/value', + 'var4': 'var5=$/value', + 'var5': '$/value', + 'var6': 'value$/', + 'var7': 'var8=value$/', + 'var8': 'value$/', + }) + vars = _parse_makefile(TESTFN, keep_unresolved=False) + self.assertEqual(vars, { + 'var1': 'value', + 'var4': 'var5=', + 'var7': 'var8=', }) diff --git a/Misc/NEWS.d/next/Library/2025-07-01-14-44-03.gh-issue-136097.bI1n14.rst b/Misc/NEWS.d/next/Library/2025-07-01-14-44-03.gh-issue-136097.bI1n14.rst new file mode 100644 index 00000000000..209ca745a14 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-01-14-44-03.gh-issue-136097.bI1n14.rst @@ -0,0 +1,2 @@ +Fix potential infinite recursion and KeyError in ``sysconfig +--generate-posix-vars``. From fc48a3c9cac5b9e8c6a56ec2294b01a8da9f8dee Mon Sep 17 00:00:00 2001 From: Sebastian Pipping Date: Sat, 4 Oct 2025 16:19:06 +0200 Subject: [PATCH 011/112] gh-90949: Fix an "unused function" compiler warning introduced in GH-139234 (#139558) Fix a compiler warning `-Wunused-function` after f04bea44c37793561d753dd4ca6e7cd658137553. The `set_invalid_arg` function in `Modules/pyexpat.c` may be unused if the underlying Expat version is less than 2.4.0. --- Modules/pyexpat.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 7f6d84ad864..9949e185dce 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -198,12 +198,14 @@ set_error(pyexpat_state *state, xmlparseobject *self, enum XML_Error code) return NULL; } +#if XML_COMBINED_VERSION >= 20400 static PyObject * set_invalid_arg(pyexpat_state *state, xmlparseobject *self, const char *errmsg) { SET_XML_ERROR(state, self, XML_ERROR_INVALID_ARGUMENT, errmsg); return NULL; } +#endif #undef SET_XML_ERROR From f191db2e0eed421c4d11b007458549002785d96e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns=20=F0=9F=87=B5=F0=9F=87=B8?= Date: Sat, 4 Oct 2025 15:25:07 +0100 Subject: [PATCH 012/112] Add FFY00 to CODEOWNERS for generate-build-details (#139561) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- .github/CODEOWNERS | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ec1670e8e0b..1086b426204 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -86,6 +86,10 @@ Modules/makesetup @erlend-aasland @AA-Turner @emmatyping Modules/Setup* @erlend-aasland @AA-Turner @emmatyping Tools/build/regen-configure.sh @AA-Turner +# generate-build-details +Tools/build/generate-build-details.py @FFY00 +Lib/test/test_build_details.py @FFY00 + # ---------------------------------------------------------------------------- # Documentation From 04a2f80a604ccb214902f35deebf0d075543737c Mon Sep 17 00:00:00 2001 From: rowanbudge Date: Sat, 4 Oct 2025 16:59:39 +0100 Subject: [PATCH 013/112] gh-101100: Fix some Sphinx reference warnings in ``whatsnew/2.6.rst`` (#139236) Co-authored-by: rowanvil Co-authored-by: Adam Turner <9087854+aa-turner@users.noreply.github.com> --- Doc/whatsnew/2.6.rst | 174 +++++++++++++++++++++++-------------------- 1 file changed, 92 insertions(+), 82 deletions(-) diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index cbab2b57cbb..f5e3a47037c 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -56,7 +56,7 @@ Python 2.6 incorporates new features and syntax from 3.0 while remaining compatible with existing code by not removing older features or syntax. When it's not possible to do that, Python 2.6 tries to do what it can, adding compatibility functions in a -:mod:`future_builtins` module and a :option:`!-3` switch to warn about +:mod:`!future_builtins` module and a :option:`!-3` switch to warn about usages that will become unsupported in 3.0. Some significant new packages have been added to the standard library, @@ -109,7 +109,7 @@ are: Python 3.0 adds several new built-in functions and changes the semantics of some existing builtins. Functions that are new in 3.0 such as :func:`bin` have simply been added to Python 2.6, but existing -builtins haven't been changed; instead, the :mod:`future_builtins` +builtins haven't been changed; instead, the :mod:`!future_builtins` module has versions with the new 3.0 semantics. Code written to be compatible with 3.0 can do ``from future_builtins import hex, map`` as necessary. @@ -118,7 +118,7 @@ A new command-line switch, :option:`!-3`, enables warnings about features that will be removed in Python 3.0. You can run code with this switch to see how much work will be necessary to port code to 3.0. The value of this switch is available -to Python code as the boolean variable :data:`sys.py3kwarning`, +to Python code as the boolean variable :data:`!sys.py3kwarning`, and to C extension code as :c:data:`!Py_Py3kWarningFlag`. .. seealso:: @@ -307,9 +307,9 @@ The :mod:`threading` module's locks and condition variables also support the The lock is acquired before the block is executed and always released once the block is complete. -The :func:`localcontext` function in the :mod:`decimal` module makes it easy -to save and restore the current decimal context, which encapsulates the desired -precision and rounding characteristics for computations:: +The :func:`~decimal.localcontext` function in the :mod:`decimal` module makes +it easy to save and restore the current decimal context, which encapsulates +the desired precision and rounding characteristics for computations:: from decimal import Decimal, Context, localcontext @@ -337,12 +337,12 @@ underlying implementation and should keep reading. A high-level explanation of the context management protocol is: * The expression is evaluated and should result in an object called a "context - manager". The context manager must have :meth:`~object.__enter__` and :meth:`~object.__exit__` - methods. + manager". The context manager must have :meth:`~object.__enter__` and + :meth:`~object.__exit__` methods. -* The context manager's :meth:`~object.__enter__` method is called. The value returned - is assigned to *VAR*. If no ``as VAR`` clause is present, the value is simply - discarded. +* The context manager's :meth:`~object.__enter__` method is called. The value + returned is assigned to *VAR*. If no ``as VAR`` clause is present, the + value is simply discarded. * The code in *BLOCK* is executed. @@ -378,7 +378,7 @@ be to let the user write code like this:: The transaction should be committed if the code in the block runs flawlessly or rolled back if there's an exception. Here's the basic interface for -:class:`DatabaseConnection` that I'll assume:: +:class:`!DatabaseConnection` that I'll assume:: class DatabaseConnection: # Database interface @@ -431,14 +431,15 @@ The contextlib module The :mod:`contextlib` module provides some functions and a decorator that are useful when writing objects for use with the ':keyword:`with`' statement. -The decorator is called :func:`contextmanager`, and lets you write a single -generator function instead of defining a new class. The generator should yield -exactly one value. The code up to the :keyword:`yield` will be executed as the -:meth:`~object.__enter__` method, and the value yielded will be the method's return -value that will get bound to the variable in the ':keyword:`with`' statement's -:keyword:`!as` clause, if any. The code after the :keyword:`!yield` will be -executed in the :meth:`~object.__exit__` method. Any exception raised in the block will -be raised by the :keyword:`!yield` statement. +The decorator is called :func:`~contextlib.contextmanager`, and lets you write +a single generator function instead of defining a new class. The generator +should yield exactly one value. The code up to the :keyword:`yield` will be +executed as the :meth:`~object.__enter__` method, and the value yielded will +be the method's return value that will get bound to the variable in the +':keyword:`with`' statement's :keyword:`!as` clause, if any. The code after +the :keyword:`!yield` will be executed in the :meth:`~object.__exit__` method. +Any exception raised in the block will be raised by the :keyword:`!yield` +statement. Using this decorator, our database example from the previous section could be written as:: @@ -469,7 +470,7 @@ statement both starts a database transaction and acquires a thread lock:: with nested (db_transaction(db), lock) as (cursor, locked): ... -Finally, the :func:`closing` function returns its argument so that it can be +Finally, the :func:`~contextlib.closing` function returns its argument so that it can be bound to a variable, and calls the argument's ``.close()`` method at the end of the block. :: @@ -538,7 +539,7 @@ If you don't like the default directory, it can be overridden by an environment variable. :envvar:`PYTHONUSERBASE` sets the root directory used for all Python versions supporting this feature. On Windows, the directory for application-specific data can be changed by -setting the :envvar:`APPDATA` environment variable. You can also +setting the :envvar:`!APPDATA` environment variable. You can also modify the :file:`site.py` file for your Python installation. The feature can be disabled entirely by running Python with the @@ -568,11 +569,12 @@ The :mod:`multiprocessing` module started out as an exact emulation of the :mod:`threading` module using processes instead of threads. That goal was discarded along the path to Python 2.6, but the general approach of the module is still similar. The fundamental class -is the :class:`Process`, which is passed a callable object and -a collection of arguments. The :meth:`start` method +is the :class:`~multiprocessing.Process`, which is passed a callable object and +a collection of arguments. The :meth:`~multiprocessing.Process.start` method sets the callable running in a subprocess, after which you can call -the :meth:`is_alive` method to check whether the subprocess is still running -and the :meth:`join` method to wait for the process to exit. +the :meth:`~multiprocessing.Process.is_alive` method to check whether the +subprocess is still running and the :meth:`~multiprocessing.Process.join` +method to wait for the process to exit. Here's a simple example where the subprocess will calculate a factorial. The function doing the calculation is written strangely so @@ -619,13 +621,16 @@ the object to communicate. (If the parent were to change the value of the global variable, the child's value would be unaffected, and vice versa.) -Two other classes, :class:`Pool` and :class:`Manager`, provide -higher-level interfaces. :class:`Pool` will create a fixed number of -worker processes, and requests can then be distributed to the workers -by calling :meth:`apply` or :meth:`apply_async` to add a single request, -and :meth:`map` or :meth:`map_async` to add a number of -requests. The following code uses a :class:`Pool` to spread requests -across 5 worker processes and retrieve a list of results:: +Two other classes, :class:`~multiprocessing.pool.Pool` and +:class:`~multiprocessing.Manager`, provide higher-level interfaces. +:class:`~multiprocessing.pool.Pool` will create a fixed number of worker +processes, and requests can then be distributed to the workers by calling +:meth:`~multiprocessing.pool.Pool.apply` or +:meth:`~multiprocessing.pool.Pool.apply_async` to add a single request, and +:meth:`~multiprocessing.pool.Pool.map` or +:meth:`~multiprocessing.pool.Pool.map_async` to add a number of +requests. The following code uses a :class:`~multiprocessing.pool.Pool` to +spread requests across 5 worker processes and retrieve a list of results:: from multiprocessing import Pool @@ -646,15 +651,18 @@ This produces the following output:: 33452526613163807108170062053440751665152000000000 ... -The other high-level interface, the :class:`Manager` class, creates a -separate server process that can hold master copies of Python data +The other high-level interface, the :class:`~multiprocessing.Manager` class, +creates a separate server process that can hold master copies of Python data structures. Other processes can then access and modify these data structures using proxy objects. The following example creates a shared dictionary by calling the :meth:`dict` method; the worker processes then insert values into the dictionary. (Locking is not done for you automatically, which doesn't matter in this example. -:class:`Manager`'s methods also include :meth:`Lock`, :meth:`RLock`, -and :meth:`Semaphore` to create shared locks.) +:class:`~multiprocessing.Manager`'s methods also include +:meth:`~multiprocessing.managers.SyncManager.Lock`, +:meth:`~multiprocessing.managers.SyncManager.RLock`, +and :meth:`~multiprocessing.managers.SyncManager.Semaphore` to create +shared locks.) :: @@ -824,7 +832,7 @@ documentation for a :ref:`complete list `; here's a sample: format, followed by a percent sign. ===== ======================================================================== -Classes and types can define a :meth:`__format__` method to control how they're +Classes and types can define a :meth:`~object.__format__` method to control how they're formatted. It receives a single argument, the format specifier:: def __format__(self, format_spec): @@ -834,7 +842,7 @@ formatted. It receives a single argument, the format specifier:: return str(self) There's also a :func:`format` builtin that will format a single -value. It calls the type's :meth:`__format__` method with the +value. It calls the type's :meth:`~object.__format__` method with the provided specifier:: >>> format(75.6564, '.2f') @@ -1029,56 +1037,58 @@ PEP 3116: New I/O Library Python's built-in file objects support a number of methods, but file-like objects don't necessarily support all of them. Objects that -imitate files usually support :meth:`read` and :meth:`write`, but they -may not support :meth:`readline`, for example. Python 3.0 introduces -a layered I/O library in the :mod:`io` module that separates buffering -and text-handling features from the fundamental read and write -operations. +imitate files usually support :meth:`!read` and +:meth:`!write`, but they may not support :meth:`!readline`, +for example. Python 3.0 introduces a layered I/O library in the :mod:`io` +module that separates buffering and text-handling features from the +fundamental read and write operations. There are three levels of abstract base classes provided by the :mod:`io` module: -* :class:`RawIOBase` defines raw I/O operations: :meth:`read`, - :meth:`readinto`, - :meth:`write`, :meth:`seek`, :meth:`tell`, :meth:`truncate`, - and :meth:`close`. +* :class:`~io.RawIOBase` defines raw I/O operations: :meth:`~io.RawIOBase.read`, + :meth:`~io.RawIOBase.readinto`, :meth:`~io.RawIOBase.write`, + :meth:`~io.IOBase.seek`, :meth:`~io.IOBase.tell`, :meth:`~io.IOBase.truncate`, + and :meth:`~io.IOBase.close`. Most of the methods of this class will often map to a single system call. - There are also :meth:`readable`, :meth:`writable`, and :meth:`seekable` - methods for determining what operations a given object will allow. + There are also :meth:`~io.IOBase.readable`, :meth:`~io.IOBase.writable`, + and :meth:`~io.IOBase.seekable` methods for determining what operations a + given object will allow. Python 3.0 has concrete implementations of this class for files and sockets, but Python 2.6 hasn't restructured its file and socket objects in this way. -* :class:`BufferedIOBase` is an abstract base class that +* :class:`~io.BufferedIOBase` is an abstract base class that buffers data in memory to reduce the number of system calls used, making I/O processing more efficient. - It supports all of the methods of :class:`RawIOBase`, - and adds a :attr:`raw` attribute holding the underlying raw object. + It supports all of the methods of :class:`~io.RawIOBase`, + and adds a :attr:`~io.BufferedIOBase.raw` attribute holding the underlying + raw object. There are five concrete classes implementing this ABC. - :class:`BufferedWriter` and :class:`BufferedReader` are for objects - that support write-only or read-only usage that have a :meth:`seek` - method for random access. :class:`BufferedRandom` objects support + :class:`~io.BufferedWriter` and :class:`~io.BufferedReader` are for objects + that support write-only or read-only usage that have a :meth:`~io.IOBase.seek` + method for random access. :class:`~io.BufferedRandom` objects support read and write access upon the same underlying stream, and - :class:`BufferedRWPair` is for objects such as TTYs that have both + :class:`~io.BufferedRWPair` is for objects such as TTYs that have both read and write operations acting upon unconnected streams of data. - The :class:`BytesIO` class supports reading, writing, and seeking + The :class:`~io.BytesIO` class supports reading, writing, and seeking over an in-memory buffer. .. index:: single: universal newlines; What's new -* :class:`TextIOBase`: Provides functions for reading and writing +* :class:`~io.TextIOBase`: Provides functions for reading and writing strings (remember, strings will be Unicode in Python 3.0), - and supporting :term:`universal newlines`. :class:`TextIOBase` defines + and supporting :term:`universal newlines`. :class:`~io.TextIOBase` defines the :meth:`readline` method and supports iteration upon objects. - There are two concrete implementations. :class:`TextIOWrapper` + There are two concrete implementations. :class:`~io.TextIOWrapper` wraps a buffered I/O object, supporting all of the methods for - text I/O and adding a :attr:`buffer` attribute for access - to the underlying object. :class:`StringIO` simply buffers + text I/O and adding a :attr:`~io.TextIOBase.buffer` attribute for access + to the underlying object. :class:`~io.StringIO` simply buffers everything in memory without ever writing anything to disk. (In Python 2.6, :class:`io.StringIO` is implemented in @@ -1162,7 +1172,7 @@ Some object-oriented languages such as Java support interfaces, declaring that a class has a given set of methods or supports a given access protocol. Abstract Base Classes (or ABCs) are an equivalent feature for Python. The ABC support consists of an :mod:`abc` module -containing a metaclass called :class:`ABCMeta`, special handling of +containing a metaclass called :class:`~abc.ABCMeta`, special handling of this metaclass by the :func:`isinstance` and :func:`issubclass` builtins, and a collection of basic ABCs that the Python developers think will be widely useful. Future versions of Python will probably @@ -1172,17 +1182,17 @@ Let's say you have a particular class and wish to know whether it supports dictionary-style access. The phrase "dictionary-style" is vague, however. It probably means that accessing items with ``obj[1]`` works. Does it imply that setting items with ``obj[2] = value`` works? -Or that the object will have :meth:`keys`, :meth:`values`, and :meth:`items` -methods? What about the iterative variants such as :meth:`iterkeys`? :meth:`copy` -and :meth:`update`? Iterating over the object with :func:`iter`? +Or that the object will have :meth:`!keys`, :meth:`!values`, and :meth:`!items` +methods? What about the iterative variants such as :meth:`!iterkeys`? +:meth:`!copy`and :meth:`!update`? Iterating over the object with :func:`!iter`? The Python 2.6 :mod:`collections` module includes a number of different ABCs that represent these distinctions. :class:`Iterable` -indicates that a class defines :meth:`__iter__`, and -:class:`Container` means the class defines a :meth:`__contains__` +indicates that a class defines :meth:`~object.__iter__`, and +:class:`Container` means the class defines a :meth:`~object.__contains__` method and therefore supports ``x in y`` expressions. The basic dictionary interface of getting items, setting items, and -:meth:`keys`, :meth:`values`, and :meth:`items`, is defined by the +:meth:`!keys`, :meth:`!values`, and :meth:`!items`, is defined by the :class:`MutableMapping` ABC. You can derive your own classes from a particular ABC @@ -1196,7 +1206,7 @@ to indicate they support that ABC's interface:: Alternatively, you could write the class without deriving from the desired ABC and instead register the class by -calling the ABC's :meth:`register` method:: +calling the ABC's :meth:`~abc.ABCMeta.register` method:: import collections @@ -1206,10 +1216,10 @@ calling the ABC's :meth:`register` method:: collections.MutableMapping.register(Storage) For classes that you write, deriving from the ABC is probably clearer. -The :meth:`register` method is useful when you've written a new +The :meth:`~abc.ABCMeta.register` method is useful when you've written a new ABC that can describe an existing type or class, or if you want to declare that some third-party class implements an ABC. -For example, if you defined a :class:`PrintableType` ABC, +For example, if you defined a :class:`!PrintableType` ABC, it's legal to do:: # Register Python's types @@ -1256,16 +1266,16 @@ metaclass in a class definition:: ... -In the :class:`Drawable` ABC above, the :meth:`draw_doubled` method +In the :class:`!Drawable` ABC above, the :meth:`!draw_doubled` method renders the object at twice its size and can be implemented in terms -of other methods described in :class:`Drawable`. Classes implementing +of other methods described in :class:`!Drawable`. Classes implementing this ABC therefore don't need to provide their own implementation -of :meth:`draw_doubled`, though they can do so. An implementation -of :meth:`draw` is necessary, though; the ABC can't provide +of :meth:`!draw_doubled`, though they can do so. An implementation +of :meth:`!draw` is necessary, though; the ABC can't provide a useful generic implementation. -You can apply the ``@abstractmethod`` decorator to methods such as -:meth:`draw` that must be implemented; Python will then raise an +You can apply the :deco:`~abc.abstractmethod` decorator to methods such as +:meth:`!draw` that must be implemented; Python will then raise an exception for classes that don't define the method. Note that the exception is only raised when you actually try to create an instance of a subclass lacking the method:: @@ -1289,7 +1299,7 @@ Abstract data attributes can be declared using the def readonly(self): return self._x -Subclasses must then define a :meth:`readonly` property. +Subclasses must then define a ``readonly`` property. .. seealso:: @@ -2739,13 +2749,13 @@ numbers. .. ====================================================================== -The :mod:`future_builtins` module +The :mod:`!future_builtins` module -------------------------------------- Python 3.0 makes many changes to the repertoire of built-in functions, and most of the changes can't be introduced in the Python 2.x series because they would break compatibility. -The :mod:`future_builtins` module provides versions +The :mod:`!future_builtins` module provides versions of these built-in functions that can be imported when writing 3.0-compatible code. From 0b2168275e8ec491fe7ea6f8c662e804437dfdab Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 4 Oct 2025 21:44:05 +0530 Subject: [PATCH 014/112] gh-138703: clarify data buffer requirement of `asyncio.StreamWriter.write` (#139564) --- Doc/library/asyncio-stream.rst | 5 +++-- Lib/asyncio/selector_events.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index e1568ae330b..05445219510 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -316,13 +316,14 @@ StreamWriter If that fails, the data is queued in an internal write buffer until it can be sent. + The *data* buffer should be a bytes, bytearray, or C-contiguous one-dimensional + memoryview object. + The method should be used along with the ``drain()`` method:: stream.write(data) await stream.drain() - .. note:: - The *data* buffer should be a C contiguous one-dimensional :term:`bytes-like object `. .. method:: writelines(data) diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index a7e27ccf0aa..ff7e16df3c6 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -1050,8 +1050,8 @@ def _read_ready__on_eof(self): def write(self, data): if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError(f'data argument must be a bytes-like object, ' - f'not {type(data).__name__!r}') + raise TypeError(f'data argument must be a bytes, bytearray, or memoryview ' + f'object, not {type(data).__name__!r}') if self._eof: raise RuntimeError('Cannot call write() after write_eof()') if self._empty_waiter is not None: From 880c9526f91960b9cba557a18b54e2c32d2f254e Mon Sep 17 00:00:00 2001 From: Dave Peck Date: Sat, 4 Oct 2025 17:06:56 -0400 Subject: [PATCH 015/112] gh-138558: Improve description of ``Interpolation.expression`` (#139187) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/library/string.templatelib.rst | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Doc/library/string.templatelib.rst b/Doc/library/string.templatelib.rst index 85d65fa9de1..a5b2d796aaf 100644 --- a/Doc/library/string.templatelib.rst +++ b/Doc/library/string.templatelib.rst @@ -24,7 +24,7 @@ Template strings Template strings are a mechanism for custom string processing. They have the full flexibility of Python's :ref:`f-strings`, but return a :class:`Template` instance that gives access -to the static and interpolated (in curly braces) parts of a string +to the static and interpolated (in curly brackets) parts of a string *before* they are combined. To write a t-string, use a ``'t'`` prefix instead of an ``'f'``, like so: @@ -258,13 +258,16 @@ Types .. attribute:: expression :type: str - The text of a valid Python expression, or an empty string. + For interpolations created by t-string literals, :attr:`!expression` + is the expression text found inside the curly brackets (``{`` & ``}``), + including any whitespace, excluding the curly brackets themselves, + and ending before the first ``!``, ``:``, or ``=`` if any is present. + For manually created interpolations, :attr:`!expression` is the arbitrary + string provided when constructing the interpolation instance. - The :attr:`.expression` is the original text of the - interpolation's Python expression, if the interpolation was created - from a t-string literal. Developers creating interpolations manually - should either set this to an empty string or choose a suitable valid - Python expression. + We recommend using valid Python expressions or the empty string for the + ``expression`` field of manually created :class:`!Interpolation` + instances, although this is not enforced at runtime. >>> t'{1 + 2}'.interpolations[0].expression '1 + 2' From 0f0fc5a16368ea45541137cff6b90d63bad5eb26 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sun, 5 Oct 2025 01:15:29 +0100 Subject: [PATCH 016/112] gh-139573: Update Android to OpenSSL 3.0.18 (#139562) Update Android to OpenSSL 3.0.18. --- Android/android.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Android/android.py b/Android/android.py index 15046b6fe1e..b810a6bfb3c 100755 --- a/Android/android.py +++ b/Android/android.py @@ -184,10 +184,16 @@ def make_build_python(context): run(["make", "-j", str(os.cpu_count())]) +# To create new builds of these dependencies, usually all that's necessary is to +# push a tag to the cpython-android-source-deps repository, and GitHub Actions +# will do the rest. +# +# If you're a member of the Python core team, and you'd like to be able to push +# these tags yourself, please contact Malcolm Smith or Russell Keith-Magee. def unpack_deps(host, prefix_dir): os.chdir(prefix_dir) deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" - for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.15-4", + for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.18-0", "sqlite-3.50.4-0", "xz-5.4.6-1", "zstd-1.5.7-1"]: filename = f"{name_ver}-{host}.tar.gz" download(f"{deps_url}/{name_ver}/{filename}") From 98e748b3a0d97bd2c785efc63693f971113b3b63 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Sat, 4 Oct 2025 19:43:17 -0500 Subject: [PATCH 017/112] gh-139573: Update OpenSSL in CI (GH-139577) --- .github/workflows/build.yml | 6 +++--- .github/workflows/reusable-ubuntu.yml | 2 +- Doc/using/configure.rst | 2 +- Modules/_ssl_data_35.h | 9 +++++++-- Tools/ssl/multissltests.py | 10 +++++----- 5 files changed, 17 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ebfaf32e193..47d38b75429 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -273,7 +273,7 @@ jobs: # Keep 1.1.1w in our list despite it being upstream EOL and otherwise # unsupported as it most resembles other 1.1.1-work-a-like ssl APIs # supported by important vendors such as AWS-LC. - openssl_ver: [1.1.1w, 3.0.17, 3.2.5, 3.3.4, 3.4.2, 3.5.2] + openssl_ver: [1.1.1w, 3.0.18, 3.2.6, 3.3.5, 3.4.3, 3.5.4] # See Tools/ssl/make_ssl_data.py for notes on adding a new version env: OPENSSL_VER: ${{ matrix.openssl_ver }} @@ -438,7 +438,7 @@ jobs: needs: build-context if: needs.build-context.outputs.run-tests == 'true' env: - OPENSSL_VER: 3.0.16 + OPENSSL_VER: 3.0.18 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v4 @@ -558,7 +558,7 @@ jobs: matrix: os: [ubuntu-24.04] env: - OPENSSL_VER: 3.0.16 + OPENSSL_VER: 3.0.18 PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 76b19fd5d1a..7f8b9fdf5d6 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -30,7 +30,7 @@ jobs: runs-on: ${{ inputs.os }} timeout-minutes: 60 env: - OPENSSL_VER: 3.0.15 + OPENSSL_VER: 3.0.18 PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index b3a9e081edc..b05e0600114 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -22,7 +22,7 @@ Features and minimum versions required to build CPython: * Support for threads. -* OpenSSL 1.1.1 is the minimum version and OpenSSL 3.0.16 is the recommended +* OpenSSL 1.1.1 is the minimum version and OpenSSL 3.0.18 is the recommended minimum version for the :mod:`ssl` and :mod:`hashlib` extension modules. * SQLite 3.15.2 for the :mod:`sqlite3` extension module. diff --git a/Modules/_ssl_data_35.h b/Modules/_ssl_data_35.h index 9e69eaa910f..e4919b550e3 100644 --- a/Modules/_ssl_data_35.h +++ b/Modules/_ssl_data_35.h @@ -1,6 +1,6 @@ /* File generated by Tools/ssl/make_ssl_data.py */ -/* Generated on 2025-08-13T16:42:33.155822+00:00 */ -/* Generated from Git commit openssl-3.5.2-0-g0893a6235 */ +/* Generated on 2025-10-04T17:49:19.148321+00:00 */ +/* Generated from Git commit openssl-3.5.4-0-gc1eeb9406 */ /* generated from args.lib2errnum */ static struct py_ssl_library_code library_codes[] = { @@ -5338,6 +5338,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"FIPS_MODULE_ENTERING_ERROR_STATE", 57, 224}, #endif + #ifdef PROV_R_FIPS_MODULE_IMPORT_PCT_ERROR + {"FIPS_MODULE_IMPORT_PCT_ERROR", ERR_LIB_PROV, PROV_R_FIPS_MODULE_IMPORT_PCT_ERROR}, + #else + {"FIPS_MODULE_IMPORT_PCT_ERROR", 57, 253}, + #endif #ifdef PROV_R_FIPS_MODULE_IN_ERROR_STATE {"FIPS_MODULE_IN_ERROR_STATE", ERR_LIB_PROV, PROV_R_FIPS_MODULE_IN_ERROR_STATE}, #else diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index c0559446982..56976de4998 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -48,11 +48,11 @@ ] OPENSSL_RECENT_VERSIONS = [ - "3.0.16", - "3.2.5", - "3.3.4", - "3.4.2", - "3.5.2", + "3.0.18", + "3.2.6", + "3.3.5", + "3.4.3", + "3.5.4", # See make_ssl_data.py for notes on adding a new version. ] From 063cef9999d7490e62cb2ad2711634a992c090fb Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Sat, 4 Oct 2025 19:56:59 -0500 Subject: [PATCH 018/112] gh-139573: Update Windows builds to use OpenSSL 3.0.18 (GH-139574) --- .../2025-10-04-12-18-45.gh-issue-139573.EO9kVB.rst | 1 + Misc/externals.spdx.json | 8 ++++---- PCbuild/get_externals.bat | 4 ++-- PCbuild/python.props | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2025-10-04-12-18-45.gh-issue-139573.EO9kVB.rst diff --git a/Misc/NEWS.d/next/Windows/2025-10-04-12-18-45.gh-issue-139573.EO9kVB.rst b/Misc/NEWS.d/next/Windows/2025-10-04-12-18-45.gh-issue-139573.EO9kVB.rst new file mode 100644 index 00000000000..02a3bfb41ce --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2025-10-04-12-18-45.gh-issue-139573.EO9kVB.rst @@ -0,0 +1 @@ +Updated bundled version of OpenSSL to 3.0.18. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index a87af7f9173..59aceedb94d 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -70,21 +70,21 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "6bb739ecddbd2cfb6d255eb5898437a9b5739277dee931338d3275bac5d96ba2" + "checksumValue": "9b07560b6c1afa666bd78b8d3aa5c83fdda02149afdf048596d5b0e0dac1ee55" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.0.16.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.0.18.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:openssl:openssl:3.0.16:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:openssl:openssl:3.0.18:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "openssl", "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.0.16" + "versionInfo": "3.0.18" }, { "SPDXID": "SPDXRef-PACKAGE-sqlite", diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index eff8d1ccd7f..50a227b563a 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -54,7 +54,7 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 -if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.16 +if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.18 set libraries=%libraries% mpdecimal-4.0.0 set libraries=%libraries% sqlite-3.50.4.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.15.0 @@ -79,7 +79,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 -if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.16.2 +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.18 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-19.1.7.0 diff --git a/PCbuild/python.props b/PCbuild/python.props index 06af15a269c..cc157252655 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -82,8 +82,8 @@ $(libffiDir)$(ArchName)\ $(libffiOutDir)include $(ExternalsDir)\mpdecimal-4.0.0\ - $(ExternalsDir)openssl-3.0.16\ - $(ExternalsDir)openssl-bin-3.0.16.2\$(ArchName)\ + $(ExternalsDir)openssl-3.0.18\ + $(ExternalsDir)openssl-bin-3.0.18\$(ArchName)\ $(opensslOutDir)include $(ExternalsDir)\nasm-2.11.06\ $(ExternalsDir)\zlib-1.3.1\ From 9964320d155907bcb3858e5ee99ea48b140d4772 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Sat, 4 Oct 2025 20:05:37 -0700 Subject: [PATCH 019/112] GH-137218: Fix unnecessary recompile of `Programs/_freeze_module` (#139241) --- Makefile.pre.in | 2 +- configure | 54 ++++++++++++++++++++++++++----------------------- configure.ac | 50 ++++++++++++++++++++++----------------------- 3 files changed, 55 insertions(+), 51 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 37bde125166..244e25c348f 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -3132,7 +3132,7 @@ JIT_DEPS = \ jit_stencils.h @JIT_STENCILS_H@: $(JIT_DEPS) @REGEN_JIT_COMMAND@ -Python/jit.o: $(srcdir)/Python/jit.c jit_stencils.h @JIT_STENCILS_H@ +Python/jit.o: $(srcdir)/Python/jit.c @JIT_STENCILS_H@ $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< .PHONY: regen-jit diff --git a/configure b/configure index 7624cbf0d2a..0d1f6a29e9b 100755 --- a/configure +++ b/configure @@ -644,6 +644,7 @@ ac_includes_default="\ ac_header_c_list= ac_subst_vars='LTLIBOBJS MODULE_BLOCK +JIT_STENCILS_H MODULE_XXLIMITED_35_FALSE MODULE_XXLIMITED_35_TRUE MODULE_XXLIMITED_FALSE @@ -904,7 +905,6 @@ LDSHARED SHLIB_SUFFIX DSYMUTIL_PATH DSYMUTIL -JIT_STENCILS_H REGEN_JIT_COMMAND UNIVERSAL_ARCH_FLAGS WASM_STDLIB @@ -10876,7 +10876,6 @@ then : else case e in #( e) as_fn_append CFLAGS_NODIST " $jit_flags" REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\"" - JIT_STENCILS_H="jit_stencils.h" if test "x$Py_DEBUG" = xtrue then : as_fn_append REGEN_JIT_COMMAND " --debug" @@ -10884,7 +10883,6 @@ fi ;; esac fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $tier2_flags $jit_flags" >&5 printf "%s\n" "$tier2_flags $jit_flags" >&6; } @@ -34196,29 +34194,35 @@ printf "%s\n" "$py_cv_module_xxlimited_35" >&6; } # Determine JIT stencils header files based on target platform JIT_STENCILS_H="" -case "$host" in - aarch64-apple-darwin*) - JIT_STENCILS_H="jit_stencils-aarch64-apple-darwin.h" - ;; - x86_64-apple-darwin*) - JIT_STENCILS_H="jit_stencils-x86_64-apple-darwin.h" - ;; - aarch64-pc-windows-msvc) - JIT_STENCILS_H="jit_stencils-aarch64-pc-windows-msvc.h" - ;; - i686-pc-windows-msvc) - JIT_STENCILS_H="jit_stencils-i686-pc-windows-msvc.h" - ;; - x86_64-pc-windows-msvc) - JIT_STENCILS_H="jit_stencils-x86_64-pc-windows-msvc.h" - ;; - aarch64-*-linux-gnu) - JIT_STENCILS_H="jit_stencils-$host.h" - ;; - x86_64-*-linux-gnu) - JIT_STENCILS_H="jit_stencils-$host.h" - ;; +if test "x$enable_experimental_jit" = xno +then : + +else case e in #( + e) case "$host" in + aarch64-apple-darwin*) + JIT_STENCILS_H="jit_stencils-aarch64-apple-darwin.h" + ;; + x86_64-apple-darwin*) + JIT_STENCILS_H="jit_stencils-x86_64-apple-darwin.h" + ;; + aarch64-pc-windows-msvc) + JIT_STENCILS_H="jit_stencils-aarch64-pc-windows-msvc.h" + ;; + i686-pc-windows-msvc) + JIT_STENCILS_H="jit_stencils-i686-pc-windows-msvc.h" + ;; + x86_64-pc-windows-msvc) + JIT_STENCILS_H="jit_stencils-x86_64-pc-windows-msvc.h" + ;; + aarch64-*-linux-gnu) + JIT_STENCILS_H="jit_stencils-$host.h" + ;; + x86_64-*-linux-gnu) + JIT_STENCILS_H="jit_stencils-$host.h" + ;; + esac ;; esac +fi diff --git a/configure.ac b/configure.ac index 7a7e32d4294..7b5da6e0d15 100644 --- a/configure.ac +++ b/configure.ac @@ -2787,13 +2787,11 @@ AS_VAR_IF([jit_flags], [AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"]) AS_VAR_SET([REGEN_JIT_COMMAND], ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\""]) - AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"]) AS_VAR_IF([Py_DEBUG], [true], [AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])], [])]) AC_SUBST([REGEN_JIT_COMMAND]) -AC_SUBST([JIT_STENCILS_H]) AC_MSG_RESULT([$tier2_flags $jit_flags]) if test "$disable_gil" = "yes" -a "$enable_experimental_jit" != "no"; then @@ -8175,29 +8173,31 @@ PY_STDLIB_MOD([xxlimited_35], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_d # Determine JIT stencils header files based on target platform JIT_STENCILS_H="" -case "$host" in - aarch64-apple-darwin*) - JIT_STENCILS_H="jit_stencils-aarch64-apple-darwin.h" - ;; - x86_64-apple-darwin*) - JIT_STENCILS_H="jit_stencils-x86_64-apple-darwin.h" - ;; - aarch64-pc-windows-msvc) - JIT_STENCILS_H="jit_stencils-aarch64-pc-windows-msvc.h" - ;; - i686-pc-windows-msvc) - JIT_STENCILS_H="jit_stencils-i686-pc-windows-msvc.h" - ;; - x86_64-pc-windows-msvc) - JIT_STENCILS_H="jit_stencils-x86_64-pc-windows-msvc.h" - ;; - aarch64-*-linux-gnu) - JIT_STENCILS_H="jit_stencils-$host.h" - ;; - x86_64-*-linux-gnu) - JIT_STENCILS_H="jit_stencils-$host.h" - ;; -esac +AS_VAR_IF([enable_experimental_jit], [no], + [], + [case "$host" in + aarch64-apple-darwin*) + JIT_STENCILS_H="jit_stencils-aarch64-apple-darwin.h" + ;; + x86_64-apple-darwin*) + JIT_STENCILS_H="jit_stencils-x86_64-apple-darwin.h" + ;; + aarch64-pc-windows-msvc) + JIT_STENCILS_H="jit_stencils-aarch64-pc-windows-msvc.h" + ;; + i686-pc-windows-msvc) + JIT_STENCILS_H="jit_stencils-i686-pc-windows-msvc.h" + ;; + x86_64-pc-windows-msvc) + JIT_STENCILS_H="jit_stencils-x86_64-pc-windows-msvc.h" + ;; + aarch64-*-linux-gnu) + JIT_STENCILS_H="jit_stencils-$host.h" + ;; + x86_64-*-linux-gnu) + JIT_STENCILS_H="jit_stencils-$host.h" + ;; + esac]) AC_SUBST([JIT_STENCILS_H]) From 20758f9bb1c3baa35c6cb44d0074ae488ca9f4d9 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Sun, 5 Oct 2025 13:03:41 +0800 Subject: [PATCH 020/112] gh-139573: Update OpenSSL version used in iOS builds (#139582) Update OpenSSL version used in iOS builds. --- Apple/__main__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Apple/__main__.py b/Apple/__main__.py index 88b54e91ac8..96c2d34fbe0 100644 --- a/Apple/__main__.py +++ b/Apple/__main__.py @@ -312,11 +312,18 @@ def unpack_deps( On iOS, as a safety mechanism, any dynamic libraries will be purged from the unpacked dependencies. """ + # To create new builds of these dependencies, usually all that's necessary + # is to push a tag to the cpython-apple-source-deps repository, and GitHub + # Actions will do the rest. + # + # If you're a member of the Python core team, and you'd like to be able to + # push these tags yourself, please contact Malcolm Smith or Russell + # Keith-Magee. deps_url = "https://github.com/beeware/cpython-apple-source-deps/releases/download" for name_ver in [ "BZip2-1.0.8-2", "libFFI-3.4.7-2", - "OpenSSL-3.0.17-1", + "OpenSSL-3.0.18-1", "XZ-5.6.4-2", "mpdecimal-4.0.0-2", "zstd-1.5.7-1", From 41712c4e095b2cc988febfe3887616c2779c6210 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 5 Oct 2025 10:27:16 +0200 Subject: [PATCH 021/112] gh-139310: skip `test_aead_aes_gcm` for Linux kernel between 6.16.0 and 6.17.x (#139552) Currently, Fedora 42 uses a custom Linux Kernel 6.16.9 that backported an upstream change from 6.17-rc7 [1,3] but not its subsequent fix [2]. Until the issue is resolved upstream, we skip the failing test `test_socket.test_aead_aes_gcm` for kernel versions between 6.16 and 6.17.x. [1] https://github.com/torvalds/linux/commit/1b34cbbf4f011a121ef7b2d7d6e6920a036d5285 [2] https://github.com/torvalds/linux/commit/d0ca0df179c4b21e2a6c4a4fb637aa8fa14575cb. [3] https://gitlab.com/cki-project/kernel-ark/-/commit/45bcf60fe49b37daab1acee57b27211ad1574042 --- Lib/test/support/__init__.py | 10 ++++++++++ Lib/test/test_socket.py | 8 +++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 8d614ab3d42..098bdcc0542 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -310,6 +310,16 @@ def requires(resource, msg=None): if resource == 'gui' and not _is_gui_available(): raise ResourceDenied(_is_gui_available.reason) +def _get_kernel_version(sysname="Linux"): + import platform + if platform.system() != sysname: + return None + version_txt = platform.release().split('-', 1)[0] + try: + return tuple(map(int, version_txt.split('.'))) + except ValueError: + return None + def _requires_unix_version(sysname, min_version): """Decorator raising SkipTest if the OS is `sysname` and the version is less than `min_version`. diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index e4e7fa20f8a..24ee0f2c280 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -7139,8 +7139,14 @@ def test_aes_cbc(self): self.assertEqual(len(dec), msglen * multiplier) self.assertEqual(dec, msg * multiplier) - @support.requires_linux_version(4, 9) # see issue29324 + @support.requires_linux_version(4, 9) # see gh-73510 def test_aead_aes_gcm(self): + kernel_version = support._get_kernel_version("Linux") + if kernel_version is not None: + if kernel_version >= (6, 16) and kernel_version < (6, 18): + # See https://github.com/python/cpython/issues/139310. + self.skipTest("upstream Linux kernel issue") + key = bytes.fromhex('c939cc13397c1d37de6ae0e1cb7c423c') iv = bytes.fromhex('b3d8cc017cbb89b39e0f67e2') plain = bytes.fromhex('c3b3c41f113a31b73d9a5cd432103069') From efd223da0c4be9384e07f9d43328bf9528d02145 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 5 Oct 2025 12:49:03 +0200 Subject: [PATCH 022/112] gh-64327: Remove skipped pydoc tests (#139512) Tests skipped since 2014: since commit a46ef70bdfa0273a9d2cc40014c0ab74967fe654. --- Lib/test/test_pydoc/test_pydoc.py | 41 ------------------------------- 1 file changed, 41 deletions(-) diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 1793ef14870..65ad7649b9e 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -1334,47 +1334,6 @@ def test_url_search_package_error(self): finally: sys.path[:] = saved_paths - @unittest.skip('causes undesirable side-effects (#20128)') - def test_modules(self): - # See Helper.listmodules(). - num_header_lines = 2 - num_module_lines_min = 5 # Playing it safe. - num_footer_lines = 3 - expected = num_header_lines + num_module_lines_min + num_footer_lines - - output = StringIO() - helper = pydoc.Helper(output=output) - helper('modules') - result = output.getvalue().strip() - num_lines = len(result.splitlines()) - - self.assertGreaterEqual(num_lines, expected) - - @unittest.skip('causes undesirable side-effects (#20128)') - def test_modules_search(self): - # See Helper.listmodules(). - expected = 'pydoc - ' - - output = StringIO() - helper = pydoc.Helper(output=output) - with captured_stdout() as help_io: - helper('modules pydoc') - result = help_io.getvalue() - - self.assertIn(expected, result) - - @unittest.skip('some buildbots are not cooperating (#20128)') - def test_modules_search_builtin(self): - expected = 'gc - ' - - output = StringIO() - helper = pydoc.Helper(output=output) - with captured_stdout() as help_io: - helper('modules garbage') - result = help_io.getvalue() - - self.assertStartsWith(result, expected) - def test_importfile(self): try: loaded_pydoc = pydoc.importfile(pydoc.__file__) From 29616f3d2c86ac7575177a9f2b30d51f3b2004d2 Mon Sep 17 00:00:00 2001 From: Jost Migenda Date: Sun, 5 Oct 2025 12:03:54 +0100 Subject: [PATCH 023/112] gh-118767: Remove ``bool(NotImplemented)`` from pending-removal document (#139526) --- Doc/deprecations/pending-removal-in-future.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/Doc/deprecations/pending-removal-in-future.rst b/Doc/deprecations/pending-removal-in-future.rst index edb672ed8ad..7ed430625f3 100644 --- a/Doc/deprecations/pending-removal-in-future.rst +++ b/Doc/deprecations/pending-removal-in-future.rst @@ -15,7 +15,6 @@ although there is currently no date scheduled for their removal. * :mod:`builtins`: - * ``bool(NotImplemented)``. * Generators: ``throw(type, exc, tb)`` and ``athrow(type, exc, tb)`` signature is deprecated: use ``throw(exc)`` and ``athrow(exc)`` instead, the single argument signature. From 13dc2fde8cec1e8aad04c7635b3da4ff3e3dcb00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 5 Oct 2025 14:03:25 +0200 Subject: [PATCH 024/112] gh-70765: avoid waiting for HTTP headers when parsing HTTP/0.9 requests (#139514) --- Lib/http/server.py | 7 ++++ Lib/test/test_httpservers.py | 37 +++++++++++++++++++ ...5-10-02-17-40-10.gh-issue-70765.zVlLZn.rst | 5 +++ 3 files changed, 49 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-10-02-17-40-10.gh-issue-70765.zVlLZn.rst diff --git a/Lib/http/server.py b/Lib/http/server.py index a2ffbe2e44d..160d3eefc7c 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -302,6 +302,7 @@ def parse_request(self): error response has already been sent back. """ + is_http_0_9 = False self.command = None # set in case of error on the first line self.request_version = version = self.default_request_version self.close_connection = True @@ -359,6 +360,7 @@ def parse_request(self): HTTPStatus.BAD_REQUEST, "Bad HTTP/0.9 request type (%r)" % command) return False + is_http_0_9 = True self.command, self.path = command, path # gh-87389: The purpose of replacing '//' with '/' is to protect @@ -368,6 +370,11 @@ def parse_request(self): if self.path.startswith('//'): self.path = '/' + self.path.lstrip('/') # Reduce to a single / + # For HTTP/0.9, headers are not expected at all. + if is_http_0_9: + self.headers = {} + return True + # Examine the headers and look for a Connection directive. try: self.headers = http.client.parse_headers(self.rfile, diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 2548a7c5f29..85d3a346439 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -362,6 +362,43 @@ def test_head_via_send_error(self): self.assertEqual(b'', data) +class HTTP09ServerTestCase(BaseTestCase): + + class request_handler(NoLogRequestHandler, BaseHTTPRequestHandler): + """Request handler for HTTP/0.9 server.""" + + def do_GET(self): + self.wfile.write(f'OK: here is {self.path}\r\n'.encode()) + + def setUp(self): + super().setUp() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock = self.enterContext(self.sock) + self.sock.connect((self.HOST, self.PORT)) + + def test_simple_get(self): + self.sock.send(b'GET /index.html\r\n') + res = self.sock.recv(1024) + self.assertEqual(res, b"OK: here is /index.html\r\n") + + def test_invalid_request(self): + self.sock.send(b'POST /index.html\r\n') + res = self.sock.recv(1024) + self.assertIn(b"Bad HTTP/0.9 request type ('POST')", res) + + def test_single_request(self): + self.sock.send(b'GET /foo.html\r\n') + res = self.sock.recv(1024) + self.assertEqual(res, b"OK: here is /foo.html\r\n") + + self.sock.send(b'GET /bar.html\r\n') + res = self.sock.recv(1024) + # The server will not parse more input as it closed the connection. + # Note that the socket connection itself is still opened since the + # client is responsible for also closing it on their side. + self.assertEqual(res, b'') + + def certdata_file(*path): return os.path.join(os.path.dirname(__file__), "certdata", *path) diff --git a/Misc/NEWS.d/next/Library/2025-10-02-17-40-10.gh-issue-70765.zVlLZn.rst b/Misc/NEWS.d/next/Library/2025-10-02-17-40-10.gh-issue-70765.zVlLZn.rst new file mode 100644 index 00000000000..e1a9bbe9afe --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-02-17-40-10.gh-issue-70765.zVlLZn.rst @@ -0,0 +1,5 @@ +:mod:`http.server`: fix default handling of HTTP/0.9 requests in +:class:`~http.server.BaseHTTPRequestHandler`. Previously, +:meth:`!BaseHTTPRequestHandler.parse_request`` incorrectly +waited for headers in the request although those are not +supported in HTTP/0.9. Patch by Bénédikt Tran. From 6edb2ddb5f3695cf4938979d645f31d7fba43ec8 Mon Sep 17 00:00:00 2001 From: Sebastian Pipping Date: Sun, 5 Oct 2025 17:37:42 +0200 Subject: [PATCH 025/112] gh-139400: Make sure that parent parsers outlive their subparsers in `pyexpat` (#139403) * Modules/pyexpat.c: Disallow collection of in-use parent parsers. Within libexpat, a parser created via `XML_ExternalEntityParserCreate` is relying on its parent parser throughout its entire lifetime. Prior to this fix, is was possible for the parent parser to be garbage-collected too early. --- Lib/test/test_pyexpat.py | 36 +++++++++++++++++++ ...-09-29-00-01-28.gh-issue-139400.X2T-jO.rst | 4 +++ Modules/pyexpat.c | 25 +++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-09-29-00-01-28.gh-issue-139400.X2T-jO.rst diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index 8e0f7374b26..b4ce72dfd51 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -771,6 +771,42 @@ def resolve_entity(context, base, system_id, public_id): self.assertEqual(handler_call_args, [("bar", "baz")]) +class ParentParserLifetimeTest(unittest.TestCase): + """ + Subparsers make use of their parent XML_Parser inside of Expat. + As a result, parent parsers need to outlive subparsers. + + See https://github.com/python/cpython/issues/139400. + """ + + def test_parent_parser_outlives_its_subparsers__single(self): + parser = expat.ParserCreate() + subparser = parser.ExternalEntityParserCreate(None) + + # Now try to cause garbage collection of the parent parser + # while it's still being referenced by a related subparser. + del parser + + def test_parent_parser_outlives_its_subparsers__multiple(self): + parser = expat.ParserCreate() + subparser_one = parser.ExternalEntityParserCreate(None) + subparser_two = parser.ExternalEntityParserCreate(None) + + # Now try to cause garbage collection of the parent parser + # while it's still being referenced by a related subparser. + del parser + + def test_parent_parser_outlives_its_subparsers__chain(self): + parser = expat.ParserCreate() + subparser = parser.ExternalEntityParserCreate(None) + subsubparser = subparser.ExternalEntityParserCreate(None) + + # Now try to cause garbage collection of the parent parsers + # while they are still being referenced by a related subparser. + del parser + del subparser + + class ReparseDeferralTest(unittest.TestCase): def test_getter_setter_round_trip(self): parser = expat.ParserCreate() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-29-00-01-28.gh-issue-139400.X2T-jO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-29-00-01-28.gh-issue-139400.X2T-jO.rst new file mode 100644 index 00000000000..a5dea3b5f81 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-29-00-01-28.gh-issue-139400.X2T-jO.rst @@ -0,0 +1,4 @@ +:mod:`xml.parsers.expat`: Make sure that parent Expat parsers are only +garbage-collected once they are no longer referenced by subparsers created +by :meth:`~xml.parsers.expat.xmlparser.ExternalEntityParserCreate`. +Patch by Sebastian Pipping. diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 9949e185dce..9c252be9cf2 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -76,6 +76,15 @@ typedef struct { PyObject_HEAD XML_Parser itself; + /* + * Strong reference to a parent `xmlparseobject` if this parser + * is a child parser. Set to NULL if this parser is a root parser. + * This is needed to keep the parent parser alive as long as it has + * at least one child parser. + * + * See https://github.com/python/cpython/issues/139400 for details. + */ + PyObject *parent; int ordered_attributes; /* Return attributes as a list. */ int specified_attributes; /* Report only specified attributes. */ int in_callback; /* Is a callback active? */ @@ -1067,6 +1076,11 @@ pyexpat_xmlparser_ExternalEntityParserCreate_impl(xmlparseobject *self, return NULL; } + // The new subparser will make use of the parent XML_Parser inside of Expat. + // So we need to take subparsers into account with the reference counting + // of their parent parser. + Py_INCREF(self); + new_parser->buffer_size = self->buffer_size; new_parser->buffer_used = 0; new_parser->buffer = NULL; @@ -1076,6 +1090,7 @@ pyexpat_xmlparser_ExternalEntityParserCreate_impl(xmlparseobject *self, new_parser->ns_prefixes = self->ns_prefixes; new_parser->itself = XML_ExternalEntityParserCreate(self->itself, context, encoding); + new_parser->parent = (PyObject *)self; new_parser->handlers = 0; new_parser->intern = Py_XNewRef(self->intern); @@ -1083,11 +1098,13 @@ pyexpat_xmlparser_ExternalEntityParserCreate_impl(xmlparseobject *self, new_parser->buffer = PyMem_Malloc(new_parser->buffer_size); if (new_parser->buffer == NULL) { Py_DECREF(new_parser); + Py_DECREF(self); return PyErr_NoMemory(); } } if (!new_parser->itself) { Py_DECREF(new_parser); + Py_DECREF(self); return PyErr_NoMemory(); } @@ -1101,6 +1118,7 @@ pyexpat_xmlparser_ExternalEntityParserCreate_impl(xmlparseobject *self, new_parser->handlers = PyMem_New(PyObject *, i); if (!new_parser->handlers) { Py_DECREF(new_parser); + Py_DECREF(self); return PyErr_NoMemory(); } clear_handlers(new_parser, 1); @@ -1481,6 +1499,7 @@ newxmlparseobject(pyexpat_state *state, const char *encoding, /* namespace_separator is either NULL or contains one char + \0 */ self->itself = XML_ParserCreate_MM(encoding, &ExpatMemoryHandler, namespace_separator); + self->parent = NULL; if (self->itself == NULL) { PyErr_SetString(PyExc_RuntimeError, "XML_ParserCreate failed"); @@ -1517,6 +1536,7 @@ xmlparse_traverse(PyObject *op, visitproc visit, void *arg) for (size_t i = 0; handler_info[i].name != NULL; i++) { Py_VISIT(self->handlers[i]); } + Py_VISIT(self->parent); Py_VISIT(Py_TYPE(op)); return 0; } @@ -1527,6 +1547,10 @@ xmlparse_clear(PyObject *op) xmlparseobject *self = xmlparseobject_CAST(op); clear_handlers(self, 0); Py_CLEAR(self->intern); + // NOTE: We cannot call Py_CLEAR(self->parent) prior to calling + // XML_ParserFree(self->itself), or a subparser could lose its parent + // XML_Parser while still making use of it internally. + // https://github.com/python/cpython/issues/139400 return 0; } @@ -1540,6 +1564,7 @@ xmlparse_dealloc(PyObject *op) XML_ParserFree(self->itself); } self->itself = NULL; + Py_CLEAR(self->parent); if (self->handlers != NULL) { PyMem_Free(self->handlers); From 1fe89d324e6b96dc44a7bd593c428a90d1f39d55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 5 Oct 2025 18:51:16 +0200 Subject: [PATCH 026/112] gh-70765: fix an HTTP/0.9 flaky test post GH-139514 (#139610) Fix a flaky test introduced in 13dc2fde8cec1e8aad04c7635b3da4ff3e3dcb00. After a single HTTP/0.9 request, both client and server are expected to close the connection on their side. In particular, if a client sends two requests with the same connection, only the first one should be handled. In the tests, it might happen that checking for the second request to be ignored did not take into account that the server may have already closed the connection. This flaky behavior was first observed on macOS CI workers but could not be reproduced locally on a Linux machine. --- Lib/test/test_httpservers.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 85d3a346439..7da5e3a1957 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -391,12 +391,13 @@ def test_single_request(self): res = self.sock.recv(1024) self.assertEqual(res, b"OK: here is /foo.html\r\n") - self.sock.send(b'GET /bar.html\r\n') - res = self.sock.recv(1024) - # The server will not parse more input as it closed the connection. - # Note that the socket connection itself is still opened since the - # client is responsible for also closing it on their side. - self.assertEqual(res, b'') + # Ignore errors if the connection is already closed, + # as this is the expected behavior of HTTP/0.9. + with contextlib.suppress(OSError): + self.sock.send(b'GET /bar.html\r\n') + res = self.sock.recv(1024) + # The server should not process our request. + self.assertEqual(res, b'') def certdata_file(*path): From 5389234fca91f474bf9df8bff38cea2d30091d80 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sun, 5 Oct 2025 18:05:29 +0100 Subject: [PATCH 027/112] GH-123299: Copyedit 3.14 What's New: New Features (#139543) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/whatsnew/3.14.rst | 1428 ++++++++++++++++++++--------------------- 1 file changed, 711 insertions(+), 717 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 647b20bf6a9..9179861ed0b 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -80,404 +80,71 @@ plus syntax highlighting in the REPL, as well as the usual deprecations and removals, and improvements in user-friendliness and correctness. +-------------- + .. PEP-sized items next. -* :ref:`PEP 779: Free-threaded Python is officially supported ` -* :ref:`PEP 649 and 749: deferred evaluation of annotations ` -* :ref:`PEP 734: Multiple interpreters in the stdlib ` -* :ref:`PEP 741: Python configuration C API ` -* :ref:`PEP 750: Template strings ` -* :ref:`PEP 758: Allow except and except* expressions without parentheses ` -* :ref:`PEP 761: Discontinuation of PGP signatures ` -* :ref:`PEP 765: Disallow return/break/continue that exit a finally block ` -* :ref:`Free-threaded mode improvements ` -* :ref:`PEP 768: Safe external debugger interface for CPython ` -* :ref:`PEP 784: Adding Zstandard to the standard library ` -* :ref:`A new type of interpreter ` -* :ref:`Syntax highlighting in the default interactive shell `, - and color output in :ref:`unittest `, - :ref:`argparse `, - :ref:`json ` and - :ref:`calendar ` CLIs -* :ref:`Binary releases for the experimental just-in-time compiler ` +Interpreter improvements: +* :pep:`649` and :pep:`749`: :ref:`Deferred evaluation of annotations + ` +* :pep:`734`: :ref:`Multiple interpreters in the standard library + ` +* :pep:`750`: :ref:`Template strings ` +* :pep:`758`: :ref:`Allow except and except* expressions without brackets + ` +* :pep:`765`: :ref:`Control flow in finally blocks + ` +* :pep:`768`: :ref:`Safe external debugger interface for CPython + ` +* :ref:`A new type of interpreter ` +* :ref:`Free-threaded mode improvements ` +* :ref:`Improved error messages ` + +Significant improvements in the standard library: + +* :pep:`784`: :ref:`Zstandard support in the standard library + ` +* :ref:`whatsnew314-asyncio-introspection` +* :ref:`whatsnew314-concurrent-warnings-control` +* :ref:`Syntax highlighting in the default interactive shell + `, and color output in several + standard library CLIs + +C API improvements: + +* :pep:`741`: :ref:`Python configuration C API ` + +Platform support: + +* :pep:`776`: Emscripten is now an :ref:`officially supported platform + `, at :pep:`tier 3 <11#tier-3>`. + +Release changes: + +* :pep:`779`: :ref:`Free-threaded Python is officially supported + ` +* :pep:`761`: :ref:`Discontinuation of PGP signatures ` +* :ref:`Windows and macOS binary releases now support the experimental + just-in-time compiler ` +* :ref:`Binary releases for Android are now provided ` New features ============ -.. _whatsnew314-pep779: +.. _whatsnew314-deferred-annotations: -PEP 779: Free-threaded Python is officially supported ------------------------------------------------------ - -The free-threaded build of Python is now supported and no longer experimental. -This is the start of phase II where free-threaded Python is officially supported -but still optional. - -We are confident that the project is on the right path, and we appreciate the -continued dedication from everyone working to make free-threading ready for -broader adoption across the Python community. - -With these recommendations and the acceptance of this PEP, we as the Python -developer community should broadly advertise that free-threading is a supported -Python build option now and into the future, and that it will not be removed -without a proper deprecation schedule. - -Any decision to transition to phase III, with free-threading as the default or -sole build of Python is still undecided, and dependent on many factors both -within CPython itself and the community. This decision is for the future. - -.. seealso:: - :pep:`779` and its `acceptance - `__. - -.. _whatsnew314-pep734: - -PEP 734: Multiple interpreters in the stdlib --------------------------------------------- - -The CPython runtime supports running multiple copies of Python in the -same process simultaneously and has done so for over 20 years. -Each of these separate copies is called an "interpreter". -However, the feature had been available only through the C-API. - -That limitation is removed in the 3.14 release, -with the new :mod:`concurrent.interpreters` module. - -There are at least two notable reasons why using multiple interpreters -is worth considering: - -* they support a new (to Python), human-friendly concurrency model -* true multi-core parallelism - -For some use cases, concurrency in software enables efficiency and -can simplify software, at a high level. At the same time, implementing -and maintaining all but the simplest concurrency is often a struggle -for the human brain. That especially applies to plain threads -(for example, :mod:`threading`), where all memory is shared between all threads. - -With multiple isolated interpreters, you can take advantage of a class -of concurrency models, like CSP or the actor model, that have found -success in other programming languages, like Smalltalk, Erlang, -Haskell, and Go. Think of multiple interpreters like threads -but with opt-in sharing. - -Regarding multi-core parallelism: as of the 3.12 release, interpreters -are now sufficiently isolated from one another to be used in parallel. -(See :pep:`684`.) This unlocks a variety of CPU-intensive use cases -for Python that were limited by the :term:`GIL`. - -Using multiple interpreters is similar in many ways to -:mod:`multiprocessing`, in that they both provide isolated logical -"processes" that can run in parallel, with no sharing by default. -However, when using multiple interpreters, an application will use -fewer system resources and will operate more efficiently (since it -stays within the same process). Think of multiple interpreters as -having the isolation of processes with the efficiency of threads. - -.. XXX Add an example or two. -.. XXX Link to the not-yet-added HOWTO doc. - -While the feature has been around for decades, multiple interpreters -have not been used widely, due to low awareness and the lack of a stdlib -module. Consequently, they currently have several notable limitations, -which will improve significantly now that the feature is finally -going mainstream. - -Current limitations: - -* starting each interpreter has not been optimized yet -* each interpreter uses more memory than necessary - (we will be working next on extensive internal sharing between - interpreters) -* there aren't many options *yet* for truly sharing objects or other - data between interpreters (other than :type:`memoryview`) -* many extension modules on PyPI are not compatible with multiple - interpreters yet (stdlib extension modules *are* compatible) -* the approach to writing applications that use multiple isolated - interpreters is mostly unfamiliar to Python users, for now - -The impact of these limitations will depend on future CPython -improvements, how interpreters are used, and what the community solves -through PyPI packages. Depending on the use case, the limitations may -not have much impact, so try it out! - -Furthermore, future CPython releases will reduce or eliminate overhead -and provide utilities that are less appropriate on PyPI. In the -meantime, most of the limitations can also be addressed through -extension modules, meaning PyPI packages can fill any gap for 3.14, and -even back to 3.12 where interpreters were finally properly isolated and -stopped sharing the :term:`GIL`. Likewise, we expect to slowly see -libraries on PyPI for high-level abstractions on top of interpreters. - -Regarding extension modules, work is in progress to update some PyPI -projects, as well as tools like Cython, pybind11, nanobind, and PyO3. -The steps for isolating an extension module are found at -:ref:`isolating-extensions-howto`. Isolating a module has a lot of -overlap with what is required to support -:ref:`free-threading `, -so the ongoing work in the community in that area will help accelerate -support for multiple interpreters. - -Also added in 3.14: :ref:`concurrent.futures.InterpreterPoolExecutor -`. - -.. seealso:: - :pep:`734`. - - -.. _whatsnew314-pep750: - -PEP 750: Template strings -------------------------- - -Template string literals (t-strings) are a generalization of f-strings, -using a ``t`` in place of the ``f`` prefix. Instead of evaluating -to :class:`str`, t-strings evaluate to a new :class:`!string.templatelib.Template` type: - -.. code-block:: python - - from string.templatelib import Template - - name = "World" - template: Template = t"Hello {name}" - -The template can then be combined with functions that operate on the template's -structure to produce a :class:`str` or a string-like result. -For example, sanitizing input: - -.. code-block:: python - - evil = "" - template = t"

{evil}

" - assert html(template) == "

<script>alert('evil')</script>

" - -As another example, generating HTML attributes from data: - -.. code-block:: python - - attributes = {"src": "shrubbery.jpg", "alt": "looks nice"} - template = t"" - assert html(template) == 'looks nice' - -Compared to using an f-string, the ``html`` function has access to template attributes -containing the original information: static strings, interpolations, and values -from the original scope. Unlike existing templating approaches, t-strings build -from the well-known f-string syntax and rules. Template systems thus benefit -from Python tooling as they are much closer to the Python language, syntax, -scoping, and more. - -Writing template handlers is straightforward: - -.. code-block:: python - - from string.templatelib import Template, Interpolation - - def lower_upper(template: Template) -> str: - """Render static parts lowercased and interpolations uppercased.""" - parts: list[str] = [] - for item in template: - if isinstance(item, Interpolation): - parts.append(str(item.value).upper()) - else: - parts.append(item.lower()) - return "".join(parts) - - name = "world" - assert lower_upper(t"HELLO {name}") == "hello WORLD" - -With this in place, developers can write template systems to sanitize SQL, make -safe shell operations, improve logging, tackle modern ideas in web development -(HTML, CSS, and so on), and implement lightweight, custom business DSLs. - -(Contributed by Jim Baker, Guido van Rossum, Paul Everitt, Koudai Aono, -Lysandros Nikolaou, Dave Peck, Adam Turner, Jelle Zijlstra, Bénédikt Tran, -and Pablo Galindo Salgado in :gh:`132661`.) - -.. seealso:: - :pep:`750`. - - -.. _whatsnew314-pep768: - -PEP 768: Safe external debugger interface for CPython ------------------------------------------------------ - -:pep:`768` introduces a zero-overhead debugging interface that allows debuggers and profilers -to safely attach to running Python processes. This is a significant enhancement to Python's -debugging capabilities allowing debuggers to forego unsafe alternatives. See -:ref:`below ` for how this feature is leveraged to -implement the new :mod:`pdb` module's remote attaching capabilities. - -The new interface provides safe execution points for attaching debugger code without modifying -the interpreter's normal execution path or adding runtime overhead. This enables tools to -inspect and interact with Python applications in real-time without stopping or restarting -them — a crucial capability for high-availability systems and production environments. - -For convenience, CPython implements this interface through the :mod:`sys` module with a -:func:`sys.remote_exec` function:: - - sys.remote_exec(pid, script_path) - -This function allows sending Python code to be executed in a target process at the next safe -execution point. However, tool authors can also implement the protocol directly as described -in the PEP, which details the underlying mechanisms used to safely attach to running processes. - -Here's a simple example that inspects object types in a running Python process: - - .. code-block:: python - - import os - import sys - import tempfile - - # Create a temporary script - with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: - script_path = f.name - f.write(f"import my_debugger; my_debugger.connect({os.getpid()})") - try: - # Execute in process with PID 1234 - print("Behold! An offering:") - sys.remote_exec(1234, script_path) - finally: - os.unlink(script_path) - -The debugging interface has been carefully designed with security in mind and includes several -mechanisms to control access: - -* A :envvar:`PYTHON_DISABLE_REMOTE_DEBUG` environment variable. -* A :option:`-X disable-remote-debug` command-line option. -* A :option:`--without-remote-debug` configure flag to completely disable the feature at build time. - -A key implementation detail is that the interface piggybacks on the interpreter's existing evaluation -loop and safe points, ensuring zero overhead during normal execution while providing a reliable way -for external processes to coordinate debugging operations. - -(Contributed by Pablo Galindo Salgado, Matt Wozniski, and Ivona Stojanovic in :gh:`131591`.) - -.. seealso:: - :pep:`768`. - - -.. _whatsnew314-pep784: - -PEP 784: Adding Zstandard to the standard library -------------------------------------------------- - -The new ``compression`` package contains modules :mod:`!compression.lzma`, -:mod:`!compression.bz2`, :mod:`!compression.gzip` and :mod:`!compression.zlib` -which re-export the :mod:`lzma`, :mod:`bz2`, :mod:`gzip` and :mod:`zlib` -modules respectively. The new import names under ``compression`` are the -canonical names for importing these compression modules going forward. However, -the existing modules names have not been deprecated. Any deprecation or removal -of the existing compression modules will occur no sooner than five years after -the release of 3.14. - -The new :mod:`!compression.zstd` module provides compression and decompression -APIs for the Zstandard format via bindings to `Meta's zstd library -`__. Zstandard is a widely adopted, highly -efficient, and fast compression format. In addition to the APIs introduced in -:mod:`!compression.zstd`, support for reading and writing Zstandard compressed -archives has been added to the :mod:`tarfile`, :mod:`zipfile`, and -:mod:`shutil` modules. - -Here's an example of using the new module to compress some data: - -.. code-block:: python - - from compression import zstd - import math - - data = str(math.pi).encode() * 20 - - compressed = zstd.compress(data) - - ratio = len(compressed) / len(data) - print(f"Achieved compression ratio of {ratio}") - -As can be seen, the API is similar to the APIs of the :mod:`!lzma` and -:mod:`!bz2` modules. - -(Contributed by Emma Harper Smith, Adam Turner, Gregory P. Smith, Tomas Roun, -Victor Stinner, and Rogdham in :gh:`132983`.) - -.. seealso:: - :pep:`784`. - - -.. _whatsnew314-remote-pdb: - -Remote attaching to a running Python process with PDB ------------------------------------------------------ - -The :mod:`pdb` module now supports remote attaching to a running Python process -using a new ``-p PID`` command-line option: - -.. code-block:: sh - - python -m pdb -p 1234 - -This will connect to the Python process with the given PID and allow you to -debug it interactively. Notice that due to how the Python interpreter works -attaching to a remote process that is blocked in a system call or waiting for -I/O will only work once the next bytecode instruction is executed or when the -process receives a signal. - -This feature uses :pep:`768` and the :func:`sys.remote_exec` function -to attach to the remote process and send the PDB commands to it. - - -(Contributed by Matt Wozniski and Pablo Galindo in :gh:`131591`.) - -.. seealso:: - :pep:`768`. - - -.. _whatsnew314-pep758: - -PEP 758 – Allow except and except* expressions without parentheses ------------------------------------------------------------------- - -The :keyword:`except` and :keyword:`except* ` expressions now allow -parentheses to be omitted when there are multiple exception types and the ``as`` clause is not used. -For example the following expressions are now valid: - -.. code-block:: python - - try: - connect_to_server() - except TimeoutError, ConnectionRefusedError: - print("Network issue encountered.") - - # The same applies to except* (for exception groups): - - try: - connect_to_server() - except* TimeoutError, ConnectionRefusedError: - print("Network issue encountered.") - -Check :pep:`758` for more details. - -(Contributed by Pablo Galindo and Brett Cannon in :gh:`131831`.) - -.. seealso:: - :pep:`758`. - - -.. _whatsnew314-pep649: - -PEP 649 and 749: deferred evaluation of annotations ---------------------------------------------------- +:pep:`649` & :pep:`749`: Deferred evaluation of annotations +------------------------------------------------------------ The :term:`annotations ` on functions, classes, and modules are no longer evaluated eagerly. Instead, annotations are stored in special-purpose :term:`annotate functions ` and evaluated only when necessary (except if ``from __future__ import annotations`` is used). -This is specified in :pep:`649` and :pep:`749`. -This change is designed to make annotations in Python more performant and more -usable in most circumstances. The runtime cost for defining annotations is +This change is designed to improve performance and usability of annotations +in Python in most circumstances. The runtime cost for defining annotations is minimized, but it remains possible to introspect annotations at runtime. It is no longer necessary to enclose annotations in strings if they contain forward references. @@ -505,70 +172,343 @@ This example shows how these formats behave: >>> get_annotations(func, format=Format.STRING) {'arg': 'Undefined'} -Implications for annotated code -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The :ref:`porting ` section contains guidance +on changes that may be needed due to these changes, though in the majority of +cases, code will continue working as-is. -If you define annotations in your code (for example, for use with a static type -checker), then this change probably does not affect you: you can keep -writing annotations the same way you did with previous versions of Python. - -You will likely be able to remove quoted strings in annotations, which are frequently -used for forward references. Similarly, if you use ``from __future__ import annotations`` -to avoid having to write strings in annotations, you may well be able to -remove that import once you support only Python 3.14 and newer. -However, if you rely on third-party libraries that read annotations, -those libraries may need changes to support unquoted annotations before they -work as expected. - -Implications for readers of ``__annotations__`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If your code reads the ``__annotations__`` attribute on objects, you may want -to make changes in order to support code that relies on deferred evaluation of -annotations. For example, you may want to use :func:`annotationlib.get_annotations` -with the :attr:`~annotationlib.Format.FORWARDREF` format, as the :mod:`dataclasses` -module now does. - -The external :pypi:`typing_extensions` package provides partial backports of some of the -functionality of the :mod:`annotationlib` module, such as the :class:`~annotationlib.Format` -enum and the :func:`~annotationlib.get_annotations` function. These can be used to -write cross-version code that takes advantage of the new behavior in Python 3.14. - -Related changes -^^^^^^^^^^^^^^^ - -The changes in Python 3.14 are designed to rework how ``__annotations__`` -works at runtime while minimizing breakage to code that contains -annotations in source code and to code that reads ``__annotations__``. However, -if you rely on undocumented details of the annotation behavior or on private -functions in the standard library, there are many ways in which your code may -not work in Python 3.14. To safeguard your code against future changes, -use only the documented functionality of the :mod:`annotationlib` module. - -In particular, do not read annotations directly from the namespace dictionary -attribute of type objects. Use :func:`annotationlib.get_annotate_from_class_namespace` -during class construction and :func:`annotationlib.get_annotations` afterwards. - -In previous releases, it was sometimes possible to access class annotations from -an instance of an annotated class. This behavior was undocumented and accidental, -and will no longer work in Python 3.14. - -``from __future__ import annotations`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In Python 3.7, :pep:`563` introduced the ``from __future__ import annotations`` -directive, which turns all annotations into strings. This directive is now -considered deprecated and it is expected to be removed in a future version of Python. -However, this removal will not happen until after Python 3.13, the last version of -Python without deferred evaluation of annotations, reaches its end of life in 2029. -In Python 3.14, the behavior of code using ``from __future__ import annotations`` -is unchanged. - -(Contributed by Jelle Zijlstra in :gh:`119180`; :pep:`649` was written by Larry Hastings.) +(Contributed by Jelle Zijlstra in :pep:`749` and :gh:`119180`; +:pep:`649` was written by Larry Hastings.) .. seealso:: - :pep:`649` and :pep:`749`. + :pep:`649` + Deferred Evaluation Of Annotations Using Descriptors + :pep:`749` + Implementing PEP 649 + + +.. _whatsnew314-multiple-interpreters: + +:pep:`734`: Multiple interpreters in the standard library +--------------------------------------------------------- + +The CPython runtime supports running multiple copies of Python in the +same process simultaneously and has done so for over 20 years. +Each of these separate copies is called an 'interpreter'. +However, the feature had been available only through +the :ref:`C-API `. + +That limitation is removed in Python 3.14, +with the new :mod:`concurrent.interpreters` module. + +There are at least two notable reasons why using multiple interpreters +has significant benefits: + +* they support a new (to Python), human-friendly concurrency model +* true multi-core parallelism + +For some use cases, concurrency in software enables efficiency and +can simplify design, at a high level. +At the same time, implementing and maintaining all but the simplest concurrency +is often a struggle for the human brain. +That especially applies to plain threads (for example, :mod:`threading`), +where all memory is shared between all threads. + +With multiple isolated interpreters, you can take advantage of a class +of concurrency models, like CSP or the actor model, that have found +success in other programming languages, like Smalltalk, Erlang, +Haskell, and Go. Think of multiple interpreters like threads +but with opt-in sharing. + +Regarding multi-core parallelism: as of Python 3.12, interpreters +are now sufficiently isolated from one another to be used in parallel +(see :pep:`684`). This unlocks a variety of CPU-intensive use cases +for Python that were limited by the :term:`GIL`. + +Using multiple interpreters is similar in many ways to +:mod:`multiprocessing`, in that they both provide isolated logical +"processes" that can run in parallel, with no sharing by default. +However, when using multiple interpreters, an application will use +fewer system resources and will operate more efficiently (since it +stays within the same process). Think of multiple interpreters as +having the isolation of processes with the efficiency of threads. + +.. XXX Add an example or two. +.. XXX Link to the not-yet-added HOWTO doc. + +While the feature has been around for decades, multiple interpreters +have not been used widely, due to low awareness and the lack of a +standard library module. Consequently, they currently have several +notable limitations, which will improve significantly now that the +feature is finally going mainstream. + +Current limitations: + +* starting each interpreter has not been optimized yet +* each interpreter uses more memory than necessary + (work continues on extensive internal sharing between interpreters) +* there aren't many options *yet* for truly sharing objects or other + data between interpreters (other than :type:`memoryview`) +* many third-party extension modules on PyPI are not yet compatible + with multiple interpreters + (all standard library extension modules *are* compatible) +* the approach to writing applications that use multiple isolated + interpreters is mostly unfamiliar to Python users, for now + +The impact of these limitations will depend on future CPython +improvements, how interpreters are used, and what the community solves +through PyPI packages. Depending on the use case, the limitations may +not have much impact, so try it out! + +Furthermore, future CPython releases will reduce or eliminate overhead +and provide utilities that are less appropriate on PyPI. In the +meantime, most of the limitations can also be addressed through +extension modules, meaning PyPI packages can fill any gap for 3.14, and +even back to 3.12 where interpreters were finally properly isolated and +stopped sharing the :term:`GIL`. Likewise, libraries on PyPI are expected +to emerge for high-level abstractions on top of interpreters. + +Regarding extension modules, work is in progress to update some PyPI +projects, as well as tools like Cython, pybind11, nanobind, and PyO3. +The steps for isolating an extension module are found at +:ref:`isolating-extensions-howto`. +Isolating a module has a lot of overlap with what is required to support +:ref:`free-threading `, so the ongoing +work in the community in that area will help accelerate support +for multiple interpreters. + +Also added in 3.14: :ref:`concurrent.futures.InterpreterPoolExecutor +`. + +(Contributed by Eric Snow in :gh:`134939`.) + +.. seealso:: :pep:`734` + + +.. _whatsnew314-template-string-literals: + +:pep:`750`: Template string literals +------------------------------------ + +Template strings are a new mechanism for custom string processing. +They share the familiar syntax of f-strings but, unlike f-strings, +return an object representing the static and interpolated parts of +the string, instead of a simple :class:`str`. + +To write a t-string, use a ``'t'`` prefix instead of an ``'f'``: + +.. doctest:: + + >>> variety = 'Stilton' + >>> template = t'Try some {variety} cheese!' + >>> type(template) + + +:class:`~string.templatelib.Template` objects provide access to the static +and interpolated (in curly braces) parts of a string *before* they are combined. +Iterate over :class:`!Template` instances to access their parts in order: + +.. testsetup:: + + variety = 'Stilton' + template = t'Try some {variety} cheese!' + +.. doctest:: + + >>> list(template) + ['Try some ', Interpolation('Stilton', 'variety', None, ''), ' cheese!'] + +It's easy to write (or call) code to process :class:`!Template` instances. +For example, here's a function that renders static parts lowercase and +:class:`~string.templatelib.Interpolation` instances uppercase: + +.. code-block:: python + + from string.templatelib import Interpolation + + def lower_upper(template): + """Render static parts lowercase and interpolations uppercase.""" + parts = [] + for part in template: + if isinstance(part, Interpolation): + parts.append(str(part.value).upper()) + else: + parts.append(part.lower()) + return ''.join(parts) + + name = 'Wenslydale' + template = t'Mister {name}' + assert lower_upper(template) == 'mister WENSLYDALE' + +Because :class:`!Template` instances distinguish between static strings and +interpolations at runtime, they can be useful for sanitising user input. +Writing a :func:`!html` function that escapes user input in HTML is an exercise +left to the reader! +Template processing code can provide improved flexibility. +For instance, a more advanced :func:`!html` function could accept +a :class:`!dict` of HTML attributes directly in the template: + +.. code-block:: python + + attributes = {'src': 'limburger.jpg', 'alt': 'lovely cheese'} + template = t'' + assert html(template) == 'lovely cheese' + +Of course, template processing code does not need to return a string-like result. +An even *more* advanced :func:`!html` could return a custom type representing +a DOM-like structure. + +With t-strings in place, developers can write systems that sanitise SQL, +make safe shell operations, improve logging, tackle modern ideas in web +development (HTML, CSS, and so on), and implement lightweight custom business DSLs. + +(Contributed by Jim Baker, Guido van Rossum, Paul Everitt, Koudai Aono, +Lysandros Nikolaou, Dave Peck, Adam Turner, Jelle Zijlstra, Bénédikt Tran, +and Pablo Galindo Salgado in :gh:`132661`.) + +.. seealso:: :pep:`750`. + + +.. _whatsnew314-remote-debugging: + +:pep:`768`: Safe external debugger interface +-------------------------------------------- + +Python 3.14 introduces a zero-overhead debugging interface that allows +debuggers and profilers to safely attach to running Python processes +without stopping or restarting them. +This is a significant enhancement to Python's debugging capabilities, +meaning that unsafe alternatives are no longer required. + +The new interface provides safe execution points for attaching debugger code +without modifying the interpreter's normal execution path +or adding any overhead at runtime. +Due to this, tools can now inspect and interact with Python applications +in real-time, which is a crucial capability for high-availability systems +and production environments. + +For convenience, this interface is implemented in the :func:`sys.remote_exec` +function. For example: + +.. code-block:: python + + import sys + from tempfile import NamedTemporaryFile + + with NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: + script_path = f.name + f.write(f'import my_debugger; my_debugger.connect({os.getpid()})') + + # Execute in process with PID 1234 + print('Behold! An offering:') + sys.remote_exec(1234, script_path) + + +This function allows sending Python code to be executed in a target process +at the next safe execution point. +However, tool authors can also implement the protocol directly as described +in the PEP, which details the underlying mechanisms used to safely attach to +running processes. + +The debugging interface has been carefully designed with security in mind +and includes several mechanisms to control access: + +* A :envvar:`PYTHON_DISABLE_REMOTE_DEBUG` environment variable. +* A :option:`-X disable-remote-debug` command-line option. +* A :option:`--without-remote-debug` configure flag to completely disable + the feature at build time. + +(Contributed by Pablo Galindo Salgado, Matt Wozniski, and Ivona Stojanovic +in :gh:`131591`.) + +.. seealso:: :pep:`768`. + + +.. _whatsnew314-tail-call-interpreter: + +A new type of interpreter +------------------------- + +A new type of interpreter has been added to CPython. +It uses tail calls between small C functions that implement individual +Python opcodes, rather than one large C ``case`` statement. +For certain newer compilers, this interpreter provides +significantly better performance. Preliminary benchmarks suggest a geometric +mean of 3-5% faster on the standard ``pyperformance`` benchmark suite, +depending on platform and architecture. +The baseline is Python 3.14 built with Clang 19, without this new interpreter. + +This interpreter currently only works with Clang 19 and newer +on x86-64 and AArch64 architectures. However, a future release +of GCC is expected will support this as well. + +This feature is opt-in for now. Enabling profile-guided optimization is highly +recommendeded when using the new interpreter as it is the only configuration +that has been tested and validated for improved performance. +For further information, see :option:`--with-tail-call-interp`. + +.. note:: + + This is not to be confused with `tail call optimization`__ of Python + functions, which is currently not implemented in CPython. + + This new interpreter type is an internal implementation detail of the CPython + interpreter. It doesn't change the visible behavior of Python programs at + all. It can improve their performance, but doesn't change anything else. + + __ https://en.wikipedia.org/wiki/Tail_call + +(Contributed by Ken Jin in :gh:`128563`, with ideas on how to implement this +in CPython by Mark Shannon, Garrett Gu, Haoran Xu, and Josh Haberman.) + + +.. _whatsnew314-free-threaded-cpython: + +Free-threaded mode improvements +------------------------------- + +CPython's free-threaded mode (:pep:`703`), initially added in 3.13, +has been significantly improved in Python 3.14. +The implementation described in PEP 703 has been finished, including C API +changes, and temporary workarounds in the interpreter were replaced with +more permanent solutions. +The specializing adaptive interpreter (:pep:`659`) is now enabled +in free-threaded mode, which along with many other optimizations +greatly improves its performance. +The performance penalty on single-threaded code in free-threaded mode +is now roughly 5-10%, depending on the platform and C compiler used. + +From Python 3.14, when compiling extension modules for the free-threaded build of +CPython on Windows, the preprocessor variable ``Py_GIL_DISABLED`` now needs to +be specified by the build backend, as it will no longer be determined +automatically by the C compiler. For a running interpreter, the setting that +was used at compile time can be found using :func:`sysconfig.get_config_var`. + +The new :option:`-X context_aware_warnings <-X>` flag controls if +:ref:`concurrent safe warnings control ` +is enabled. The flag defaults to true for the free-threaded build +and false for the GIL-enabled build. + +A new :data:`~sys.flags.thread_inherit_context` flag has been added, +which if enabled means that threads created with :class:`threading.Thread` +start with a copy of the :class:`~contextvars.Context()` of the caller of +:meth:`~threading.Thread.start`. Most significantly, this makes the warning +filtering context established by :class:`~warnings.catch_warnings` be +"inherited" by threads (or asyncio tasks) started within that context. It also +affects other modules that use context variables, such as the :mod:`decimal` +context manager. +This flag defaults to true for the free-threaded build and false for +the GIL-enabled build. + +(Contributed by Sam Gross, Matt Page, Neil Schemenauer, Thomas Wouters, +Donghee Na, Kirill Podoprigora, Ken Jin, Itamar Oren, Brett Simmers, +Dino Viehland, Nathan Goldbaum, Ralf Gommers, Lysandros Nikolaou, Kumar Aditya, +Edgar Margffoy, and many others. +Some of these contributors are employed by Meta, which has continued to provide +significant engineering resources to support this project.) + + +.. _whatsnew314-improved-error-messages: Improved error messages ----------------------- @@ -589,47 +529,12 @@ Improved error messages ^^^^^^ SyntaxError: invalid syntax. Did you mean 'while'? - >>> asynch def fetch_data(): - ... pass - Traceback (most recent call last): - File "", line 1 - asynch def fetch_data(): - ^^^^^^ - SyntaxError: invalid syntax. Did you mean 'async'? - - >>> async def foo(): - ... awaid fetch_data() - Traceback (most recent call last): - File "", line 2 - awaid fetch_data() - ^^^^^ - SyntaxError: invalid syntax. Did you mean 'await'? - - >>> raisee ValueError("Error") - Traceback (most recent call last): - File "", line 1 - raisee ValueError("Error") - ^^^^^^ - SyntaxError: invalid syntax. Did you mean 'raise'? - While the feature focuses on the most common cases, some variations of misspellings may still result in regular syntax errors. (Contributed by Pablo Galindo in :gh:`132449`.) -* When an unpacking assignment fails due to an incorrect number of variables, the - error message prints the received number of values in more cases than before. - (Contributed by Tushar Sadhwani in :gh:`122239`.) - - .. code-block:: pycon - - >>> x, y, z = 1, 2, 3, 4 - Traceback (most recent call last): - File "", line 1, in - x, y, z = 1, 2, 3, 4 - ^^^^^^^ - ValueError: too many values to unpack (expected 3, got 4) - -* :keyword:`elif` statements that follow an :keyword:`else` block now have a specific error message. +* :keyword:`elif` statements that follow an :keyword:`else` block now have + a specific error message. (Contributed by Steele Farnsworth in :gh:`129902`.) .. code-block:: pycon @@ -645,11 +550,9 @@ Improved error messages ^^^^ SyntaxError: 'elif' block follows an 'else' block -* If a statement (:keyword:`pass`, :keyword:`del`, :keyword:`return`, - :keyword:`yield`, :keyword:`raise`, :keyword:`break`, :keyword:`continue`, - :keyword:`assert`, :keyword:`import`, :keyword:`from`) is passed to the - :ref:`if_expr` after :keyword:`else`, or one of :keyword:`pass`, - :keyword:`break`, or :keyword:`continue` is passed before :keyword:`if`, then the +* If a statement is passed to the :ref:`if_expr` after :keyword:`else`, + or one of :keyword:`pass`, :keyword:`break`, or :keyword:`continue` + is passed before :keyword:`if`, then the error message highlights where the :token:`~python-grammar:expression` is required. (Contributed by Sergey Miryanov in :gh:`129515`.) @@ -669,10 +572,9 @@ Improved error messages ^^^^^^^^ SyntaxError: expected expression before 'if', but statement is given - * When incorrectly closed strings are detected, the error message suggests - that the string may be intended to be part of the string. (Contributed by - Pablo Galindo in :gh:`88535`.) + that the string may be intended to be part of the string. + (Contributed by Pablo Galindo in :gh:`88535`.) .. code-block:: pycon @@ -681,8 +583,8 @@ Improved error messages SyntaxError: invalid syntax. Is this intended to be part of the string? * When strings have incompatible prefixes, the error now shows - which prefixes are incompatible. (Contributed by - Nikita Sobolev in :gh:`133197`.) + which prefixes are incompatible. + (Contributed by Nikita Sobolev in :gh:`133197`.) .. code-block:: pycon @@ -699,20 +601,11 @@ Improved error messages - Except handlers: ``except ... as ...`` - Pattern-match cases: ``case ... as ...`` - (Contributed by Nikita Sobolev in :gh:`123539`, - :gh:`123562`, and :gh:`123440`.) - - .. code-block:: pycon - - >>> import ast as arr[0] - File "", line 1 - import ast as arr[0] - ^^^^^^ - SyntaxError: cannot use subscript as import target + (Contributed by Nikita Sobolev in :gh:`123539`, :gh:`123562`, and :gh:`123440`.) * Improved error message when trying to add an instance of an unhashable type to - a :class:`dict` or :class:`set`. (Contributed by CF Bolz-Tereick and Victor Stinner - in :gh:`132828`.) + a :class:`dict` or :class:`set`. + (Contributed by CF Bolz-Tereick and Victor Stinner in :gh:`132828`.) .. code-block:: pycon @@ -739,65 +632,70 @@ Improved error messages (Contributed by Bénédikt Tran in :gh:`128398`.) -.. _whatsnew314-pep741: +.. _whatsnew314-zstandard: -PEP 741: Python configuration C API ------------------------------------ +:pep:`784`: Zstandard support in the standard library +----------------------------------------------------- -Add a :ref:`PyInitConfig C API ` to configure the Python -initialization without relying on C structures and the ability to make -ABI-compatible changes in the future. +The new :mod:`!compression` package contains modules :mod:`!compression.lzma`, +:mod:`!compression.bz2`, :mod:`!compression.gzip` and :mod:`!compression.zlib` +which re-export the :mod:`lzma`, :mod:`bz2`, :mod:`gzip` and :mod:`zlib` +modules respectively. The new import names under :mod:`!compression` are the +preferred names for importing these compression modules from Python 3.14. However, +the existing modules names have not been deprecated. Any deprecation or removal +of the existing compression modules will occur no sooner than five years after +the release of 3.14. -Complete the :pep:`587` :ref:`PyConfig C API ` by adding -:c:func:`PyInitConfig_AddModule` which can be used to add a built-in extension -module; a feature previously referred to as the “inittab”. +The new :mod:`!compression.zstd` module provides compression and decompression +APIs for the Zstandard format via bindings to `Meta's zstd library +`__. Zstandard is a widely adopted, highly +efficient, and fast compression format. In addition to the APIs introduced in +:mod:`!compression.zstd`, support for reading and writing Zstandard compressed +archives has been added to the :mod:`tarfile`, :mod:`zipfile`, and +:mod:`shutil` modules. -Add :c:func:`PyConfig_Get` and :c:func:`PyConfig_Set` functions to get and set -the current runtime configuration. +Here's an example of using the new module to compress some data: -PEP 587 “Python Initialization Configuration” unified all the ways to configure -the Python initialization. This PEP unifies also the configuration of the -Python preinitialization and the Python initialization in a single API. -Moreover, this PEP only provides a single choice to embed Python, instead of -having two “Python” and “Isolated” choices (PEP 587), to simplify the API -further. +.. code-block:: python -The lower level PEP 587 PyConfig API remains available for use cases with an -intentionally higher level of coupling to CPython implementation details (such -as emulating the full functionality of CPython’s CLI, including its -configuration mechanisms). + from compression import zstd + import math -(Contributed by Victor Stinner in :gh:`107954`.) + data = str(math.pi).encode() * 20 + compressed = zstd.compress(data) + ratio = len(compressed) / len(data) + print(f"Achieved compression ratio of {ratio}") + +As can be seen, the API is similar to the APIs of the :mod:`!lzma` and +:mod:`!bz2` modules. + +(Contributed by Emma Harper Smith, Adam Turner, Gregory P. Smith, Tomas Roun, +Victor Stinner, and Rogdham in :gh:`132983`.) + +.. seealso:: :pep:`784`. -.. seealso:: - :pep:`741`. .. _whatsnew314-asyncio-introspection: Asyncio introspection capabilities ---------------------------------- -Added a new command-line interface to inspect running Python processes using -asynchronous tasks, available via: +Added a new command-line interface to inspect running Python processes +using asynchronous tasks, available via ``python -m asyncio ps PID`` +or ``python -m asyncio pstree PID``. -.. code-block:: bash +The ``ps`` subcommand inspects the given process ID (PID) and displays +information about currently running asyncio tasks. +It outputs a task table: a flat listing of all tasks, their names, +their coroutine stacks, and which tasks are awaiting them. - python -m asyncio ps PID -This tool inspects the given process ID (PID) and displays information about -currently running asyncio tasks. It outputs a task table: a flat -listing of all tasks, their names, their coroutine stacks, and which tasks are -awaiting them. - -.. code-block:: bash - - python -m asyncio pstree PID - -This tool fetches the same information, but renders a visual async call tree, -showing coroutine relationships in a hierarchical format. This command is -particularly useful for debugging long-running or stuck asynchronous programs. -It can help developers quickly identify where a program is blocked, what tasks -are pending, and how coroutines are chained together. +The ``pstree`` subcommand fetches the same information, but instead renders a +visual async call tree, showing coroutine relationships in a hierarchical format. +This command is particularly useful for debugging long-running or stuck +asynchronous programs. +It can help developers quickly identify where a program is blocked, +what tasks are pending, and how coroutines are chained together. For example given this code: @@ -805,23 +703,25 @@ For example given this code: import asyncio - async def play(track): + async def play_track(track): await asyncio.sleep(5) - print(f"🎵 Finished: {track}") + print(f'🎵 Finished: {track}') - async def album(name, tracks): + async def play_album(name, tracks): async with asyncio.TaskGroup() as tg: for track in tracks: - tg.create_task(play(track), name=track) + tg.create_task(play_track(track), name=track) async def main(): async with asyncio.TaskGroup() as tg: tg.create_task( - album("Sundowning", ["TNDNBTG", "Levitate"]), name="Sundowning") + play_album('Sundowning', ['TNDNBTG', 'Levitate']), + name='Sundowning') tg.create_task( - album("TMBTE", ["DYWTYLM", "Aqua Regia"]), name="TMBTE") + play_album('TMBTE', ['DYWTYLM', 'Aqua Regia']), + name='TMBTE') - if __name__ == "__main__": + if __name__ == '__main__': asyncio.run(main()) Executing the new tool on the running process will yield a table like this: @@ -886,139 +786,16 @@ prevent tree construction: (Contributed by Pablo Galindo, Łukasz Langa, Yury Selivanov, and Marta Gomez Macias in :gh:`91048`.) -.. _whatsnew314-tail-call: -A new type of interpreter -------------------------- - -A new type of interpreter has been added to CPython. -It uses tail calls between small C functions that implement individual -Python opcodes, rather than one large C case statement. -For certain newer compilers, this interpreter provides -significantly better performance. Preliminary numbers on our machines suggest -anywhere up to 30% faster Python code, and a geometric mean of 3-5% -faster on ``pyperformance`` depending on platform and architecture. The -baseline is Python 3.14 built with Clang 19 without this new interpreter. - -This interpreter currently only works with Clang 19 and newer -on x86-64 and AArch64 architectures. However, we expect -that a future release of GCC will support this as well. - -This feature is opt-in for now. We highly recommend enabling profile-guided -optimization with the new interpreter as it is the only configuration we have -tested and can validate its improved performance. -For further information on how to build Python, see -:option:`--with-tail-call-interp`. - -.. note:: - - This is not to be confused with `tail call optimization`__ of Python - functions, which is currently not implemented in CPython. - - This new interpreter type is an internal implementation detail of the CPython - interpreter. It doesn't change the visible behavior of Python programs at - all. It can improve their performance, but doesn't change anything else. - - __ https://en.wikipedia.org/wiki/Tail_call - -.. attention:: - - This section previously reported a 9-15% geometric mean speedup. This number has since been - cautiously revised down to 3-5%. While we expect performance results to be better - than what we report, our estimates are more conservative due to a - `compiler bug `_ found in - Clang/LLVM 19, which causes the normal interpreter to be slower. We were unaware of this bug, - resulting in inaccurate results. We sincerely apologize for - communicating results that were only accurate for LLVM v19.1.x and v20.1.0. In the meantime, - the bug has been fixed in LLVM v20.1.1 and for the upcoming v21.1, but it will remain - unfixed for LLVM v19.1.x and v20.1.0. Thus - any benchmarks with those versions of LLVM may produce inaccurate numbers. - (Thanks to Nelson Elhage for bringing this to light.) - -(Contributed by Ken Jin in :gh:`128563`, with ideas on how to implement this -in CPython by Mark Shannon, Garrett Gu, Haoran Xu, and Josh Haberman.) - -.. _whatsnew314-free-threaded-cpython: - -Free-threaded mode ------------------- - -Free-threaded mode (:pep:`703`), initially added in 3.13, has been significantly improved. -The implementation described in PEP 703 was finished, including C API changes, -and temporary workarounds in the interpreter were replaced with more permanent solutions. -The specializing adaptive interpreter (:pep:`659`) is now enabled in free-threaded mode, -which along with many other optimizations greatly improves its performance. -The performance penalty on single-threaded code in free-threaded mode is now roughly 5-10%, -depending on platform and C compiler used. - -This work was done by many contributors: Sam Gross, Matt Page, Neil Schemenauer, -Thomas Wouters, Donghee Na, Kirill Podoprigora, Ken Jin, Itamar Oren, -Brett Simmers, Dino Viehland, Nathan Goldbaum, Ralf Gommers, Lysandros Nikolaou, -Kumar Aditya, Edgar Margffoy, and many others. - -Some of these contributors are employed by Meta, which has continued to provide -significant engineering resources to support this project. - -From 3.14, when compiling extension modules for the free-threaded build of -CPython on Windows, the preprocessor variable ``Py_GIL_DISABLED`` now needs to -be specified by the build backend, as it will no longer be determined -automatically by the C compiler. For a running interpreter, the setting that -was used at compile time can be found using :func:`sysconfig.get_config_var`. - -A new flag has been added, :data:`~sys.flags.context_aware_warnings`. This -flag defaults to true for the free-threaded build and false for the GIL-enabled -build. If the flag is true then the :class:`warnings.catch_warnings` context -manager uses a context variable for warning filters. This makes the context -manager behave predictably when used with multiple threads or asynchronous -tasks. - -A new flag has been added, :data:`~sys.flags.thread_inherit_context`. This flag -defaults to true for the free-threaded build and false for the GIL-enabled -build. If the flag is true then threads created with :class:`threading.Thread` -start with a copy of the :class:`~contextvars.Context()` of the caller of -:meth:`~threading.Thread.start`. Most significantly, this makes the warning -filtering context established by :class:`~warnings.catch_warnings` be -"inherited" by threads (or asyncio tasks) started within that context. It also -affects other modules that use context variables, such as the :mod:`decimal` -context manager. - - -.. _whatsnew314-jit-compiler: - -Binary releases for the experimental just-in-time compiler ----------------------------------------------------------- - -The official macOS and Windows release binaries now include an *experimental* -just-in-time (JIT) compiler. Although it is **not** recommended for production -use, it can be tested by setting :envvar:`PYTHON_JIT=1 ` as an -environment variable. Downstream source builds and redistributors can use the -:option:`--enable-experimental-jit=yes-off` configuration option for similar -behavior. - -The JIT is at an early stage and still in active development. As such, the -typical performance impact of enabling it can range from 10% slower to 20% -faster, depending on workload. To aid in testing and evaluation, a set of -introspection functions has been provided in the :data:`sys._jit` namespace. -:func:`sys._jit.is_available` can be used to determine if the current executable -supports JIT compilation, while :func:`sys._jit.is_enabled` can be used to tell -if JIT compilation has been enabled for the current process. - -Currently, the most significant missing functionality is that native debuggers -and profilers like ``gdb`` and ``perf`` are unable to unwind through JIT frames -(Python debuggers and profilers, like :mod:`pdb` or :mod:`profile`, continue to -work without modification). Free-threaded builds do not support JIT compilation. - -Please report any bugs or major performance regressions that you encounter! - -.. seealso:: :pep:`744` +.. _whatsnew314-concurrent-warnings-control: Concurrent safe warnings control -------------------------------- The :class:`warnings.catch_warnings` context manager will now optionally -use a context variable for warning filters. This is enabled by setting +use a context variable for warning filters. This is enabled by setting the :data:`~sys.flags.context_aware_warnings` flag, either with the ``-X`` -command-line option or an environment variable. This gives predictable +command-line option or an environment variable. This gives predictable warnings control when using :class:`~warnings.catch_warnings` combined with multiple threads or asynchronous tasks. The flag defaults to true for the free-threaded build and false for the GIL-enabled build. @@ -1026,30 +803,6 @@ free-threaded build and false for the GIL-enabled build. (Contributed by Neil Schemenauer and Kumar Aditya in :gh:`130010`.) -Platform support -================ - -* :pep:`776`: Emscripten is now an officially supported platform at - :pep:`tier 3 <11#tier-3>`. As a part of this effort, more than 25 bugs in - `Emscripten libc`__ were fixed. Emscripten now includes support - for :mod:`ctypes`, :mod:`termios`, and :mod:`fcntl`, as well as - experimental support for the new :ref:`default interactive shell - `. - - (Contributed by R. Hood Chatham in :gh:`127146`, :gh:`127683`, and :gh:`136931`.) - - __ https://emscripten.org/docs/porting/emscripten-runtime-environment.html - -* iOS and macOS apps can now be configured to redirect ``stdout`` and - ``stderr`` content to the system log. - (Contributed by Russell Keith-Magee in :gh:`127592`.) - -* The iOS testbed is now able to stream test output while the test is running. - The testbed can also be used to run the test suite of projects other than - CPython itself. - (Contributed by Russell Keith-Magee in :gh:`127592`.) - - Other language changes ====================== @@ -1144,6 +897,26 @@ Command line and environment .. _Jython: https://www.jython.org/ +.. _whatsnew314-bracketless-except: + +PEP 758: Allow ``except`` and ``except*`` expressions without brackets +---------------------------------------------------------------------- + +The :keyword:`except` and :keyword:`except* ` expressions +now allow brackets to be omitted when there are multiple exception types +and the ``as`` clause is not used. +For example: + +.. code-block:: python + + try: + connect_to_server() + except TimeoutError, ConnectionRefusedError: + print('The network has ceased to be!') + +(Contributed by Pablo Galindo and Brett Cannon in :pep:`758` and :gh:`131831`.) + + .. _whatsnew314-finally-syntaxwarning: PEP 765: Control flow in :keyword:`finally` blocks @@ -1214,24 +987,24 @@ New modules * :mod:`annotationlib`: For introspecting :term:`annotations `. - See :ref:`PEP 749 ` for more details. + See :ref:`PEP 749 ` for more details. (Contributed by Jelle Zijlstra in :gh:`119180`.) * :mod:`compression` (including :mod:`compression.zstd`): A package for compression-related modules, including a new module to support the Zstandard compression format. - See :ref:`PEP 784 ` for more details. + See :ref:`PEP 784 ` for more details. (Contributed by Emma Harper Smith, Adam Turner, Gregory P. Smith, Tomas Roun, Victor Stinner, and Rogdham in :gh:`132983`.) * :mod:`concurrent.interpreters`: Support for multiple interpreters in the standard library. - See :ref:`PEP 734 ` for more details. + See :ref:`PEP 734 ` for more details. (Contributed by Eric Snow in :gh:`134939`.) * :mod:`string.templatelib`: Support for template string literals (t-strings). - See :ref:`PEP 750 ` for more details. + See :ref:`PEP 750 ` for more details. (Contributed by Jim Baker, Guido van Rossum, Paul Everitt, Koudai Aono, Lysandros Nikolaou, Dave Peck, Adam Turner, Jelle Zijlstra, Bénédikt Tran, and Pablo Galindo Salgado in :gh:`132661`.) @@ -1253,8 +1026,6 @@ argparse and subparser names if mistyped by the user. (Contributed by Savannah Ostrowski in :gh:`124456`.) -.. _whatsnew314-color-argparse: - * Enable color for help text, which can be disabled with the optional *color* parameter to :class:`argparse.ArgumentParser`. This can also be controlled by :ref:`environment variables @@ -1339,7 +1110,7 @@ concurrent.futures asynchronously. This is separate from the new :mod:`~concurrent.interpreters` module - introduced by :ref:`PEP 734 `. + introduced by :ref:`PEP 734 `. (Contributed by Eric Snow in :gh:`124548`.) .. _whatsnew314-concurrent-futures-start-method: @@ -1687,8 +1458,6 @@ json See the :ref:`JSON command-line interface ` documentation. (Contributed by Trey Hunner in :gh:`122873`.) -.. _whatsnew314-color-json: - * By default, the output of the :ref:`JSON command-line interface ` is highlighted in color. This can be controlled by :ref:`environment variables @@ -1905,6 +1674,25 @@ pathlib pdb --- +* The :mod:`pdb` module now supports remote attaching to a running Python process + using a new :option:`-p PID ` command-line option: + + .. code-block:: sh + + python -m pdb -p 1234 + + This will connect to the Python process with the given PID and allow you to + debug it interactively. Notice that due to how the Python interpreter works + attaching to a remote process that is blocked in a system call or waiting for + I/O will only work once the next bytecode instruction is executed or when the + process receives a signal. + + This feature uses :ref:`PEP 768 ` + and the new :func:`sys.remote_exec` function to attach to the remote process + and send the PDB commands to it. + + (Contributed by Matt Wozniski and Pablo Galindo in :gh:`131591`.) + * Hardcoded breakpoints (:func:`breakpoint` and :func:`~pdb.set_trace`) now reuse the most recent :class:`~pdb.Pdb` instance that calls :meth:`~pdb.Pdb.set_trace`, instead of creating a new one each time. @@ -2064,7 +1852,7 @@ sys function was deprecated in Python 3.13 but it didn't raise a runtime warning. * Add :func:`sys.remote_exec` to implement the new external debugger interface. - See :ref:`PEP 768 ` for details. + See :ref:`PEP 768 ` for details. (Contributed by Pablo Galindo Salgado, Matt Wozniski, and Ivona Stojanovic in :gh:`131591`.) @@ -2914,7 +2702,7 @@ CPython bytecode changes opcodes to construct new :class:`~string.templatelib.Interpolation` and :class:`~string.templatelib.Template` instances, respectively. (Contributed by Lysandros Nikolaou and others in :gh:`132661`; - see also :ref:`PEP 750: Template strings `). + see also :ref:`PEP 750: Template strings `). * Remove the :opcode:`!BUILD_CONST_KEY_MAP` opcode. Use :opcode:`BUILD_MAP` instead. @@ -2948,7 +2736,7 @@ Pseudo-instructions * Add the :opcode:`!ANNOTATIONS_PLACEHOLDER` pseudo instruction to support partially executed module-level annotations with - :ref:`deferred evaluation of annotations `. + :ref:`deferred evaluation of annotations `. (Contributed by Jelle Zijlstra in :gh:`130907`.) * Add the :opcode:`!BINARY_OP_EXTEND` pseudo instruction, @@ -2983,6 +2771,39 @@ Pseudo-instructions C API changes ============= +.. _whatsnew314-capi-config: + +Python configuration C API +-------------------------- + +Add a :ref:`PyInitConfig C API ` to configure the Python +initialization without relying on C structures and the ability to make +ABI-compatible changes in the future. + +Complete the :pep:`587` :ref:`PyConfig C API ` by adding +:c:func:`PyInitConfig_AddModule` which can be used to add a built-in extension +module; a feature previously referred to as the "inittab". + +Add :c:func:`PyConfig_Get` and :c:func:`PyConfig_Set` functions to get and set +the current runtime configuration. + +:pep:`587` 'Python Initialization Configuration' unified all the ways +to configure Python's initialization. This PEP also unifies the configuration +of Python's preinitialization and initialization in a single API. +Moreover, this PEP only provides a single choice to embed Python, +instead of having two 'Python' and 'Isolated' choices (PEP 587), +to further simplify the API. + +The lower level PEP 587 PyConfig API remains available for use cases +with an intentionally higher level of coupling to CPython implementation details +(such as emulating the full functionality of CPython's CLI, including its +configuration mechanisms). + +(Contributed by Victor Stinner in :gh:`107954`.) + +.. seealso:: :pep:`741` and :pep:`587` + + New features in the C API ------------------------- @@ -2998,7 +2819,7 @@ New features in the C API * Add functions to manipulate the configuration of the current runtime Python interpreter - (:ref:`PEP 741: Python configuration C API `): + (:ref:`PEP 741: Python configuration C API `): * :c:func:`PyConfig_Get` * :c:func:`PyConfig_GetInt` @@ -3008,7 +2829,7 @@ New features in the C API (Contributed by Victor Stinner in :gh:`107954`.) * Add functions to configure Python initialization - (:ref:`PEP 741: Python configuration C API `): + (:ref:`PEP 741: Python configuration C API `): * :c:func:`Py_InitializeFromInitConfig` * :c:func:`PyInitConfig_AddModule` @@ -3235,9 +3056,25 @@ Deprecated C APIs .. include:: ../deprecations/c-api-pending-removal-in-future.rst +.. _whatsnew314-build-changes: + Build Changes ============= +* :pep:`776`: Emscripten is now an officially supported platform at + :pep:`tier 3 <11#tier-3>`. As a part of this effort, more than 25 bugs in + `Emscripten libc`__ were fixed. Emscripten now includes support + for :mod:`ctypes`, :mod:`termios`, and :mod:`fcntl`, as well as + experimental support for the new :ref:`default interactive shell + `. + (Contributed by R. Hood Chatham in :gh:`127146`, :gh:`127683`, and :gh:`136931`.) + + __ https://emscripten.org/docs/porting/emscripten-runtime-environment.html + +* Official Android binary releases are now provided on python.org__. + + __ https://www.python.org/downloads/android/ + * GNU Autoconf 2.72 is now required to generate :file:`configure`. (Contributed by Erlend Aasland in :gh:`115765`.) @@ -3259,12 +3096,22 @@ Build Changes * The new :file:`configure` option :option:`--with-tail-call-interp` may be used to enable the experimental tail call interpreter. - See :ref:`whatsnew314-tail-call` for further details. + See :ref:`whatsnew314-tail-call-interpreter` for further details. * To disable the new remote debugging support, use the :option:`--without-remote-debug` :file:`configure` option. This may be useful for security reasons. +* iOS and macOS apps can now be configured to redirect ``stdout`` and + ``stderr`` content to the system log. + (Contributed by Russell Keith-Magee in :gh:`127592`.) + +* The iOS testbed is now able to stream test output while the test is running. + The testbed can also be used to run the test suite of projects other than + CPython itself. + (Contributed by Russell Keith-Magee in :gh:`127592`.) + + .. _whatsnew314-build_details: :file:`build-details.json` @@ -3285,6 +3132,7 @@ which can be found by running ``sysconfig.get_path('stdlib')``. :pep:`739` -- ``build-details.json`` 1.0 -- a static description file for Python build details + .. _whatsnew314-no-more-pgp: Discontinuation of PGP signatures @@ -3301,6 +3149,66 @@ This change in release process was specified in :pep:`761`. .. _Sigstore: https://www.sigstore.dev/ +.. _whatsnew314-free-threaded-now-supported: + +Free-threaded Python is officially supported +-------------------------------------------- + +The free-threaded build of Python is now supported and no longer experimental. +This is the start of `phase II `__ where +free-threaded Python is officially supported but still optional. + +The free-threading team are confident that the project is on the right path, +and appreciate the continued dedication from everyone working to make +free-threading ready for broader adoption across the Python community. + +With these recommendations and the acceptance of this PEP, the Python developer +community should broadly advertise that free-threading is a supported +Python build option now and into the future, and that it will not be removed +without a proper deprecation schedule. + +Any decision to transition to `phase III `__, +with free-threading as the default or sole build of Python is still undecided, +and dependent on many factors both within CPython itself and the community. +This decision is for the future. + +.. seealso:: + + :pep:`779` + + `PEP 779's acceptance `__ + + +.. _whatsnew314-jit-compiler: + +Binary releases for the experimental just-in-time compiler +---------------------------------------------------------- + +The official macOS and Windows release binaries now include an *experimental* +just-in-time (JIT) compiler. Although it is **not** recommended for production +use, it can be tested by setting :envvar:`PYTHON_JIT=1 ` as an +environment variable. Downstream source builds and redistributors can use the +:option:`--enable-experimental-jit=yes-off` configuration option for similar +behavior. + +The JIT is at an early stage and still in active development. As such, the +typical performance impact of enabling it can range from 10% slower to 20% +faster, depending on workload. To aid in testing and evaluation, a set of +introspection functions has been provided in the :data:`sys._jit` namespace. +:func:`sys._jit.is_available` can be used to determine if the current executable +supports JIT compilation, while :func:`sys._jit.is_enabled` can be used to tell +if JIT compilation has been enabled for the current process. + +Currently, the most significant missing functionality is that native debuggers +and profilers like ``gdb`` and ``perf`` are unable to unwind through JIT frames +(Python debuggers and profilers, like :mod:`pdb` or :mod:`profile`, continue to +work without modification). Free-threaded builds do not support JIT compilation. + +Please report any bugs or major performance regressions that you encounter! + +.. seealso:: :pep:`744` + + Porting to Python 3.14 ====================== @@ -3347,7 +3255,7 @@ Changes in the Python API (Contributed by Jelle Zijlstra in :gh:`105499`.) * The runtime behavior of annotations has changed in various ways; see - :ref:`above ` for details. While most code that interacts + :ref:`above ` for details. While most code that interacts with annotations should continue to work, some undocumented details may behave differently. @@ -3362,6 +3270,92 @@ Changes in the Python API * On FreeBSD, :data:`sys.platform` no longer contains the major version number. +.. _whatsnew314-porting-annotations: + +Changes in annotations (:pep:`649` and :pep:`749`) +-------------------------------------------------- + +This section contains guidance on changes that may be needed to annotations +or Python code that interacts with or introspects annotations, +due to the changes related to :ref:`deferred evaluation of annotations +`. + +In the majority of cases, working code from older versions of Python +will not require any changes. + + +Implications for annotated code +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you define annotations in your code (for example, for use with a static type +checker), then this change probably does not affect you: you can keep +writing annotations the same way you did with previous versions of Python. + +You will likely be able to remove quoted strings in annotations, which are frequently +used for forward references. Similarly, if you use ``from __future__ import annotations`` +to avoid having to write strings in annotations, you may well be able to +remove that import once you support only Python 3.14 and newer. +However, if you rely on third-party libraries that read annotations, +those libraries may need changes to support unquoted annotations before they +work as expected. + + +Implications for readers of ``__annotations__`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If your code reads the :attr:`~object.__annotations__` attribute on objects, +you may want to make changes in order to support code that relies on +deferred evaluation of annotations. +For example, you may want to use :func:`annotationlib.get_annotations` with +the :attr:`~annotationlib.Format.FORWARDREF` format, +as the :mod:`dataclasses` module now does. + +The external :pypi:`typing_extensions` package provides partial backports +of some of the functionality of the :mod:`annotationlib` module, +such as the :class:`~annotationlib.Format` enum and +the :func:`~annotationlib.get_annotations` function. +These can be used to write cross-version code that takes advantage of +the new behavior in Python 3.14. + + +Related changes +^^^^^^^^^^^^^^^ + +The changes in Python 3.14 are designed to rework how :attr:`!__annotations__` +works at runtime while minimizing breakage to code that contains +annotations in source code and to code that reads :attr:`!__annotations__`. +However, if you rely on undocumented details of the annotation behavior +or on private functions in the standard library, there are many ways in which +your code may not work in Python 3.14. +To safeguard your code against future changes, only use the documented +functionality of the :mod:`annotationlib` module. + +In particular, do not read annotations directly from the namespace dictionary +attribute of type objects. +Use :func:`annotationlib.get_annotate_from_class_namespace` during class +construction and :func:`annotationlib.get_annotations` afterwards. + +In previous releases, it was sometimes possible to access class annotations +from an instance of an annotated class. This behavior was undocumented +and accidental, and will no longer work in Python 3.14. + + +``from __future__ import annotations`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In Python 3.7, :pep:`563` introduced the ``from __future__ import annotations`` +:ref:`future statement `, which turns all annotations into strings. + +However, this statement is now deprecated and it is expected to be removed +in a future version of Python. +This removal will not happen until after Python 3.13 reaches its end of life +in 2029, being the last version of Python without support for deferred +evaluation of annotations. + +In Python 3.14, the behavior of code using ``from __future__ import annotations`` +is unchanged. + + Changes in the C API -------------------- From dadbb2662a405a34763db0298025f6af06342849 Mon Sep 17 00:00:00 2001 From: George Ogden <38294960+George-Ogden@users.noreply.github.com> Date: Sun, 5 Oct 2025 19:10:24 +0200 Subject: [PATCH 028/112] Replace ambiguous word "pound" by "hash" in `difflib` docs (#139601) --- Doc/library/difflib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index c55ecac3409..7a427078909 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -231,7 +231,7 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. *linejunk*: A function that accepts a single string argument, and returns true if the string is junk, or false if not. The default is ``None``. There is also a module-level function :func:`IS_LINE_JUNK`, which filters out lines - without visible characters, except for at most one pound character (``'#'``) + without visible characters, except for at most one hash character (``'#'``) -- however the underlying :class:`SequenceMatcher` class does a dynamic analysis of which lines are so frequent as to constitute noise, and this usually works better than using this function. From d1ca001d357400d3f1f64e7fa48ace99a59c558f Mon Sep 17 00:00:00 2001 From: Skip Montanaro Date: Sun, 5 Oct 2025 13:15:46 -0500 Subject: [PATCH 029/112] gh-127330: Comment correction in _ssl.c (#139603) --- Modules/_ssl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 5d075fe942b..e1bdc4033ba 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -148,7 +148,7 @@ static void _PySSLFixErrno(void) { #endif /* Include generated data (error codes) */ -/* See make_ssl_data.h for notes on adding a new version. */ +/* See Tools/ssl/make_ssl_data.py for notes on adding a new version. */ #if (OPENSSL_VERSION_NUMBER >= 0x30401000L) #include "_ssl_data_35.h" #elif (OPENSSL_VERSION_NUMBER >= 0x30100000L) From 46de475af7225e6ef4c3fd08ee9202115a22de10 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sun, 5 Oct 2025 20:45:58 +0100 Subject: [PATCH 030/112] GH-123299: Copyedit 3.14 What's New: Trivia (#139618) --- Doc/whatsnew/3.14.rst | 54 ++++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 9179861ed0b..c226f57e502 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1,8 +1,9 @@ + **************************** What's new in Python 3.14 **************************** -:Editor: Hugo van Kemenade +:Editors: Adam Turner and Hugo van Kemenade .. Rules for maintenance: @@ -45,41 +46,48 @@ when researching a change. This article explains the new features in Python 3.14, compared to 3.13. - +Python 3.14 will be released on 7 October 2025. For full details, see the :ref:`changelog `. .. seealso:: :pep:`745` -- Python 3.14 release schedule -.. note:: - Prerelease users should be aware that this document is currently in draft - form. It will be updated substantially as Python 3.14 moves towards release, - so it's worth checking back even after reading earlier versions. - - -Summary -- release highlights +Summary -- Release highlights ============================= .. This section singles out the most important changes in Python 3.14. Brevity is key. -Python 3.14 will be the latest stable release of the Python -programming language, with a mix of changes to the language, the -implementation and the standard library. +Python 3.14 will be the latest stable release of the Python programming +language, with a mix of changes to the language, the implementation, +and the standard library. +The biggest changes include :ref:`template string literals +`, +:ref:`deferred evaluation of annotations `, +and support for :ref:`subinterpreters ` in +the standard library. -The biggest changes to the implementation include template strings (:pep:`750`), -deferred evaluation of annotations (:pep:`649`), -and a new type of interpreter that uses tail calls. - -The library changes include the addition of a new :mod:`!annotationlib` module -for introspecting and wrapping annotations (:pep:`749`), -a new :mod:`!compression.zstd` module for Zstandard support (:pep:`784`), -plus syntax highlighting in the REPL, +The library changes include significantly improved capabilities for +:ref:`introspection in asyncio `, +:ref:`support for Zstandard ` via a new +:mod:`compression.zstd` module, syntax highlighting in the REPL, as well as the usual deprecations and removals, and improvements in user-friendliness and correctness. +This article doesn't attempt to provide a complete specification +of all new features, but instead gives a convenient overview. +For full details refer to the documentation, +such as the :ref:`Library Reference ` +and :ref:`Language Reference `. +To understand the complete implementation and design rationale for a change, +refer to the PEP for a particular new feature; +but note that PEPs usually are not kept up-to-date +once a feature has been fully implemented. +See `Porting to Python 3.14`_ for guidance on upgrading from +earlier versions of Python. + -------------- .. PEP-sized items next. @@ -100,6 +108,7 @@ Interpreter improvements: * :ref:`A new type of interpreter ` * :ref:`Free-threaded mode improvements ` * :ref:`Improved error messages ` +* :ref:`Incremental garbage collection ` Significant improvements in the standard library: @@ -124,7 +133,8 @@ Release changes: * :pep:`779`: :ref:`Free-threaded Python is officially supported ` -* :pep:`761`: :ref:`Discontinuation of PGP signatures ` +* :pep:`761`: :ref:`PGP signatures have been discontinued for official releases + ` * :ref:`Windows and macOS binary releases now support the experimental just-in-time compiler ` * :ref:`Binary releases for Android are now provided ` @@ -2440,6 +2450,7 @@ asyncio blocking_code() runner.run(operation_two()) + email ----- @@ -2731,6 +2742,7 @@ CPython bytecode changes * Add the :opcode:`POP_ITER` opcode to support 'virtual' iterators. (Contributed by Mark Shannon in :gh:`132554`.) + Pseudo-instructions ------------------- From 3195da0b1a5dc8a03faa5142d4ab4a1549797e53 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sun, 5 Oct 2025 21:15:36 +0100 Subject: [PATCH 031/112] gh-105812: Use the ``:deco:`` role in place of manual decorator markup (#139619) --- Doc/library/collections.abc.rst | 4 ++-- Doc/library/dataclasses.rst | 22 +++++++++---------- Doc/library/functools.rst | 5 ++--- Doc/library/typing.rst | 16 +++++++------- Doc/library/warnings.rst | 2 +- Doc/reference/datamodel.rst | 4 ++-- Doc/whatsnew/2.7.rst | 4 ++-- Doc/whatsnew/3.10.rst | 4 ++-- Misc/NEWS.d/3.10.0b1.rst | 6 ++--- Misc/NEWS.d/3.11.0a1.rst | 2 +- Misc/NEWS.d/3.11.0b1.rst | 2 +- Misc/NEWS.d/3.14.0a1.rst | 2 +- ...-08-27-17-05-36.gh-issue-138010.ZZJmPL.rst | 4 ++-- 13 files changed, 38 insertions(+), 39 deletions(-) diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 3d126bc83f5..e6daccb91f2 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -336,7 +336,7 @@ Collections Abstract Base Classes -- Detailed Descriptions .. note:: In CPython, generator-based coroutines (:term:`generators ` - decorated with :func:`@types.coroutine `) are + decorated with :deco:`types.coroutine`) are *awaitables*, even though they do not have an :meth:`~object.__await__` method. Using ``isinstance(gencoro, Awaitable)`` for them will return ``False``. Use :func:`inspect.isawaitable` to detect them. @@ -354,7 +354,7 @@ Collections Abstract Base Classes -- Detailed Descriptions .. note:: In CPython, generator-based coroutines (:term:`generators ` - decorated with :func:`@types.coroutine `) are + decorated with :deco:`types.coroutine`) are *awaitables*, even though they do not have an :meth:`~object.__await__` method. Using ``isinstance(gencoro, Coroutine)`` for them will return ``False``. Use :func:`inspect.isawaitable` to detect them. diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index ca432f2768a..cff36e25822 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -317,7 +317,7 @@ Module contents :func:`!field`, then the class attribute for this field will be replaced by the specified *default* value. If *default* is not provided, then the class attribute will be deleted. The intent is - that after the :func:`@dataclass ` decorator runs, the class + that after the :deco:`dataclass` decorator runs, the class attributes will all contain the default values for the fields, just as if the default value itself were specified. For example, after:: @@ -427,7 +427,7 @@ Module contents :data:`typing.Any` is used for ``type``. The values of *init*, *repr*, *eq*, *order*, *unsafe_hash*, *frozen*, *match_args*, *kw_only*, *slots*, and *weakref_slot* have - the same meaning as they do in :func:`@dataclass `. + the same meaning as they do in :deco:`dataclass`. If *module* is defined, the :attr:`!__module__` attribute of the dataclass is set to that value. @@ -435,12 +435,12 @@ Module contents The *decorator* parameter is a callable that will be used to create the dataclass. It should take the class object as a first argument and the same keyword arguments - as :func:`@dataclass `. By default, the :func:`@dataclass ` + as :deco:`dataclass`. By default, the :deco:`dataclass` function is used. This function is not strictly required, because any Python mechanism for creating a new class with :attr:`~object.__annotations__` can - then apply the :func:`@dataclass ` function to convert that class to + then apply the :deco:`dataclass` function to convert that class to a dataclass. This function is provided as a convenience. For example:: @@ -569,7 +569,7 @@ Post-init processing def __post_init__(self): self.c = self.a + self.b -The :meth:`~object.__init__` method generated by :func:`@dataclass ` does not call base +The :meth:`~object.__init__` method generated by :deco:`dataclass` does not call base class :meth:`!__init__` methods. If the base class has an :meth:`!__init__` method that has to be called, it is common to call this method in a :meth:`__post_init__` method:: @@ -599,7 +599,7 @@ parameters to :meth:`!__post_init__`. Also see the warning about how Class variables --------------- -One of the few places where :func:`@dataclass ` actually inspects the type +One of the few places where :deco:`dataclass` actually inspects the type of a field is to determine if a field is a class variable as defined in :pep:`526`. It does this by checking if the type of the field is :data:`typing.ClassVar`. If a field is a ``ClassVar``, it is excluded @@ -612,7 +612,7 @@ module-level :func:`fields` function. Init-only variables ------------------- -Another place where :func:`@dataclass ` inspects a type annotation is to +Another place where :deco:`dataclass` inspects a type annotation is to determine if a field is an init-only variable. It does this by seeing if the type of a field is of type :class:`InitVar`. If a field is an :class:`InitVar`, it is considered a pseudo-field called an init-only @@ -646,7 +646,7 @@ Frozen instances ---------------- It is not possible to create truly immutable Python objects. However, -by passing ``frozen=True`` to the :func:`@dataclass ` decorator you can +by passing ``frozen=True`` to the :deco:`dataclass` decorator you can emulate immutability. In that case, dataclasses will add :meth:`~object.__setattr__` and :meth:`~object.__delattr__` methods to the class. These methods will raise a :exc:`FrozenInstanceError` when invoked. @@ -662,7 +662,7 @@ must use :meth:`!object.__setattr__`. Inheritance ----------- -When the dataclass is being created by the :func:`@dataclass ` decorator, +When the dataclass is being created by the :deco:`dataclass` decorator, it looks through all of the class's base classes in reverse MRO (that is, starting at :class:`object`) and, for each dataclass that it finds, adds the fields from that base class to an ordered mapping of fields. @@ -786,7 +786,7 @@ for :attr:`!x` when creating a class instance will share the same copy of :attr:`!x`. Because dataclasses just use normal Python class creation they also share this behavior. There is no general way for Data Classes to detect this condition. Instead, the -:func:`@dataclass ` decorator will raise a :exc:`ValueError` if it +:deco:`dataclass` decorator will raise a :exc:`ValueError` if it detects an unhashable default parameter. The assumption is that if a value is unhashable, it is mutable. This is a partial solution, but it does protect against many common errors. @@ -820,7 +820,7 @@ default value have the following special behaviors: :meth:`~object.__get__` or :meth:`!__set__` method is called rather than returning or overwriting the descriptor object. -* To determine whether a field contains a default value, :func:`@dataclass ` +* To determine whether a field contains a default value, :deco:`dataclass` will call the descriptor's :meth:`!__get__` method using its class access form: ``descriptor.__get__(obj=None, type=cls)``. If the descriptor returns a value in this case, it will be used as the diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index beec9b942af..f8ffb3f41d1 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -690,7 +690,7 @@ The :mod:`functools` module defines the following functions: return not arg ``@singledispatchmethod`` supports nesting with other decorators such as - :func:`@classmethod`. Note that to allow for + :deco:`classmethod`. Note that to allow for ``dispatcher.register``, ``singledispatchmethod`` must be the *outer most* decorator. Here is the ``Negator`` class with the ``neg`` methods bound to the class, rather than an instance of the class:: @@ -712,8 +712,7 @@ The :mod:`functools` module defines the following functions: return not arg The same pattern can be used for other similar decorators: - :func:`@staticmethod`, - :func:`@abstractmethod`, and others. + :deco:`staticmethod`, :deco:`~abc.abstractmethod`, and others. .. versionadded:: 3.8 diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index e0122986e9b..cfeeb19efbd 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1390,7 +1390,7 @@ These can be used as types in annotations. They all support subscription using Using ``Annotated[T, x]`` as an annotation still allows for static typechecking of ``T``, as type checkers will simply ignore the metadata ``x``. In this way, ``Annotated`` differs from the - :func:`@no_type_check ` decorator, which can also be used for + :deco:`no_type_check` decorator, which can also be used for adding annotations outside the scope of the typing system, but completely disables typechecking for a function or class. @@ -2815,7 +2815,7 @@ Protocols --------- The following protocols are provided by the :mod:`!typing` module. All are decorated -with :func:`@runtime_checkable `. +with :deco:`runtime_checkable`. .. class:: SupportsAbs @@ -2996,7 +2996,7 @@ Functions and decorators The presence of ``@dataclass_transform()`` tells a static type checker that the decorated object performs runtime "magic" that transforms a class in a similar way to - :func:`@dataclasses.dataclass `. + :deco:`dataclasses.dataclass`. Example usage with a decorator function: @@ -3034,14 +3034,14 @@ Functions and decorators The ``CustomerModel`` classes defined above will be treated by type checkers similarly to classes created with - :func:`@dataclasses.dataclass `. + :deco:`dataclasses.dataclass`. For example, type checkers will assume these classes have ``__init__`` methods that accept ``id`` and ``name``. The decorated class, metaclass, or function may accept the following bool arguments which type checkers will assume have the same effect as they would have on the - :func:`@dataclasses.dataclass` decorator: ``init``, + :deco:`dataclasses.dataclass` decorator: ``init``, ``eq``, ``order``, ``unsafe_hash``, ``frozen``, ``match_args``, ``kw_only``, and ``slots``. It must be possible for the value of these arguments (``True`` or ``False``) to be statically evaluated. @@ -3169,12 +3169,12 @@ Functions and decorators .. function:: get_overloads(func) - Return a sequence of :func:`@overload `-decorated definitions for + Return a sequence of :deco:`overload`-decorated definitions for *func*. *func* is the function object for the implementation of the overloaded function. For example, given the definition of ``process`` in - the documentation for :func:`@overload `, + the documentation for :deco:`overload`, ``get_overloads(process)`` will return a sequence of three function objects for the three defined overloads. If called on a function with no overloads, ``get_overloads()`` returns an empty sequence. @@ -3331,7 +3331,7 @@ Introspection helpers If *globalns* or *localns* is not given, appropriate namespace dictionaries are inferred from *obj*. * ``None`` is replaced with :class:`types.NoneType`. - * If :func:`@no_type_check ` has been applied to *obj*, an + * If :deco:`no_type_check` has been applied to *obj*, an empty dictionary is returned. * If *obj* is a class ``C``, the function returns a dictionary that merges annotations from ``C``'s base classes with those on ``C`` directly. This diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 5b17bd009b3..f9c8c4fc3a8 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -590,7 +590,7 @@ Available Functions The deprecation message passed to the decorator is saved in the ``__deprecated__`` attribute on the decorated object. If applied to an overload, the decorator - must be after the :func:`@overload ` decorator + must be after the :deco:`~typing.overload` decorator for the attribute to exist on the overload as returned by :func:`typing.get_overloads`. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 7185d68c5cc..29f82fc12da 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2558,7 +2558,7 @@ instance dictionary. In contrast, non-data descriptors can be overridden by instances. Python methods (including those decorated with -:func:`@staticmethod ` and :func:`@classmethod `) are +:deco:`staticmethod` and :deco:`classmethod`) are implemented as non-data descriptors. Accordingly, instances can redefine and override methods. This allows individual instances to acquire behaviors that differ from other instances of the same class. @@ -2993,7 +2993,7 @@ class method ``__class_getitem__()``. When defined on a class, ``__class_getitem__()`` is automatically a class method. As such, there is no need for it to be decorated with - :func:`@classmethod` when it is defined. + :deco:`classmethod` when it is defined. The purpose of *__class_getitem__* diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index bcc5a3d5690..09feb185b82 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -858,8 +858,8 @@ Some smaller changes made to the core Python language are: .. XXX bytearray doesn't seem to be documented -* When using :class:`@classmethod ` and - :class:`@staticmethod ` to wrap +* When using :deco:`classmethod` and + :deco:`staticmethod` to wrap methods as class or static methods, the wrapper object now exposes the wrapped function as their :attr:`~method.__func__` attribute. diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 8db57f6f22f..d8251185fa7 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -847,8 +847,8 @@ Other Language Changes respectively. (Contributed by Joshua Bronson, Daniel Pope, and Justin Wang in :issue:`31861`.) -* Static methods (:func:`@staticmethod `) and class methods - (:func:`@classmethod `) now inherit the method attributes +* Static methods (:deco:`staticmethod`) and class methods + (:deco:`classmethod`) now inherit the method attributes (``__module__``, ``__name__``, ``__qualname__``, ``__doc__``, ``__annotations__``) and have a new ``__wrapped__`` attribute. Moreover, static methods are now callable as regular functions. diff --git a/Misc/NEWS.d/3.10.0b1.rst b/Misc/NEWS.d/3.10.0b1.rst index 406a5d7853e..5bc78b9007a 100644 --- a/Misc/NEWS.d/3.10.0b1.rst +++ b/Misc/NEWS.d/3.10.0b1.rst @@ -402,8 +402,8 @@ the heap. Should speed up dispatch in the interpreter. .. nonce: eUn4p5 .. section: Core and Builtins -Static methods (:func:`@staticmethod `) and class methods -(:func:`@classmethod `) now inherit the method attributes +Static methods (:deco:`staticmethod`) and class methods +(:deco:`classmethod`) now inherit the method attributes (``__module__``, ``__name__``, ``__qualname__``, ``__doc__``, ``__annotations__``) and have a new ``__wrapped__`` attribute. Patch by Victor Stinner. @@ -454,7 +454,7 @@ file locations. .. nonce: VSF3vg .. section: Core and Builtins -Static methods (:func:`@staticmethod `) are now callable as +Static methods (:deco:`staticmethod`) are now callable as regular functions. Patch by Victor Stinner. .. diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index 94e4868eb29..c7e57754ee3 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -2953,7 +2953,7 @@ support for Metadata 2.2. .. nonce: xTUyyX .. section: Library -Remove the :func:`@asyncio.coroutine ` :term:`decorator` +Remove the :deco:`asyncio.coroutine` :term:`decorator` enabling legacy generator-based coroutines to be compatible with async/await code; remove :class:`asyncio.coroutines.CoroWrapper` used for wrapping legacy coroutine objects in the debug mode. The decorator has been diff --git a/Misc/NEWS.d/3.11.0b1.rst b/Misc/NEWS.d/3.11.0b1.rst index c3a1942b881..7b8b983ebf9 100644 --- a/Misc/NEWS.d/3.11.0b1.rst +++ b/Misc/NEWS.d/3.11.0b1.rst @@ -664,7 +664,7 @@ for :func:`os.fcopyfile` available in macOs. .. nonce: l1p7CJ .. section: Library -For :func:`@dataclass `, add *weakref_slot*. +For :deco:`~dataclasses.dataclass`, add *weakref_slot*. The new parameter defaults to ``False``. If true, and if ``slots=True``, add a slot named ``"__weakref__"``, which will allow instances to be weakref'd. Contributed by Eric V. Smith diff --git a/Misc/NEWS.d/3.14.0a1.rst b/Misc/NEWS.d/3.14.0a1.rst index 07f2f521ece..305a0b65b98 100644 --- a/Misc/NEWS.d/3.14.0a1.rst +++ b/Misc/NEWS.d/3.14.0a1.rst @@ -1999,7 +1999,7 @@ with an escape character. .. nonce: vi2bP- .. section: Library -:func:`@warnings.deprecated ` now copies the coroutine +:deco:`warnings.deprecated` now copies the coroutine status of functions and methods so that :func:`inspect.iscoroutinefunction` returns the correct result. diff --git a/Misc/NEWS.d/next/Library/2025-08-27-17-05-36.gh-issue-138010.ZZJmPL.rst b/Misc/NEWS.d/next/Library/2025-08-27-17-05-36.gh-issue-138010.ZZJmPL.rst index 117f3ad6c67..1cc031a8bf5 100644 --- a/Misc/NEWS.d/next/Library/2025-08-27-17-05-36.gh-issue-138010.ZZJmPL.rst +++ b/Misc/NEWS.d/next/Library/2025-08-27-17-05-36.gh-issue-138010.ZZJmPL.rst @@ -1,4 +1,4 @@ -Fix an issue where defining a class with an :func:`@warnings.deprecated -`-decorated base class may not invoke the correct +Fix an issue where defining a class with a :deco:`warnings.deprecated`-decorated +base class may not invoke the correct :meth:`~object.__init_subclass__` method in cases involving multiple inheritance. Patch by Brian Schubert. From a9b0506d8db1aff8318759ed9324be56bf33eb31 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 6 Oct 2025 06:19:48 +0100 Subject: [PATCH 032/112] gh-137242: Allow Android testbed to take all Python command-line options (#138805) Modifies the Android test runner to ensure that all valid Python command line options are preserved when running the test suite. --- .github/workflows/build.yml | 2 +- Android/android.py | 75 +++++++++------- .../java/org/python/testbed/PythonSuite.kt | 2 +- .../testbed/app/src/main/c/main_activity.c | 88 +++++++++++++++---- .../java/org/python/testbed/MainActivity.kt | 40 ++++----- .../src/main/python/android_testbed_main.py | 48 ---------- Lib/test/libregrtest/main.py | 26 ++++-- 7 files changed, 152 insertions(+), 129 deletions(-) delete mode 100644 Android/testbed/app/src/main/python/android_testbed_main.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 47d38b75429..252ec831ed9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -421,7 +421,7 @@ jobs: with: persist-credentials: false - name: Build and test - run: ./Android/android.py ci ${{ matrix.arch }}-linux-android + run: ./Android/android.py ci --fast-ci ${{ matrix.arch }}-linux-android build-wasi: name: 'WASI' diff --git a/Android/android.py b/Android/android.py index b810a6bfb3c..25bb4ca70b5 100755 --- a/Android/android.py +++ b/Android/android.py @@ -2,6 +2,7 @@ import asyncio import argparse +import json import os import platform import re @@ -552,27 +553,33 @@ async def gradle_task(context): task_prefix = "connected" env["ANDROID_SERIAL"] = context.connected - if context.command: - mode = "-c" - module = context.command - else: - mode = "-m" - module = context.module or "test" + if context.ci_mode: + context.args[0:0] = [ + # See _add_ci_python_opts in libregrtest/main.py. + "-W", "error", "-bb", "-E", + + # Randomization is disabled because order-dependent failures are + # much less likely to pass on a rerun in single-process mode. + "-m", "test", + f"--{context.ci_mode}-ci", "--single-process", "--no-randomize" + ] + + if not any(arg in context.args for arg in ["-c", "-m"]): + context.args[0:0] = ["-m", "test"] args = [ gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest", ] + [ - # Build-time properties - f"-Ppython.{name}={value}" + f"-P{name}={value}" for name, value in [ - ("sitePackages", context.site_packages), ("cwd", context.cwd) - ] if value - ] + [ - # Runtime properties - f"-Pandroid.testInstrumentationRunnerArguments.python{name}={value}" - for name, value in [ - ("Mode", mode), ("Module", module), ("Args", join_command(context.args)) - ] if value + ("python.sitePackages", context.site_packages), + ("python.cwd", context.cwd), + ( + "android.testInstrumentationRunnerArguments.pythonArgs", + json.dumps(context.args), + ), + ] + if value ] if context.verbose >= 2: args.append("--info") @@ -740,15 +747,14 @@ def ci(context): else: with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir: print("::group::Tests") + # Prove the package is self-contained by using it to run the tests. shutil.unpack_archive(package_path, temp_dir) - - # Randomization is disabled because order-dependent failures are - # much less likely to pass on a rerun in single-process mode. - launcher_args = ["--managed", "maxVersion", "-v"] - test_args = ["--fast-ci", "--single-process", "--no-randomize"] + launcher_args = [ + "--managed", "maxVersion", "-v", f"--{context.ci_mode}-ci" + ] run( - ["./android.py", "test", *launcher_args, "--", *test_args], + ["./android.py", "test", *launcher_args], cwd=temp_dir ) print("::endgroup::") @@ -831,18 +837,11 @@ def add_parser(*args, **kwargs): test.add_argument( "--cwd", metavar="DIR", type=abspath, help="Directory to copy as the app's working directory.") - - mode_group = test.add_mutually_exclusive_group() - mode_group.add_argument( - "-c", dest="command", help="Execute the given Python code.") - mode_group.add_argument( - "-m", dest="module", help="Execute the module with the given name.") - test.epilog = ( - "If neither -c nor -m are passed, the default is '-m test', which will " - "run Python's own test suite.") test.add_argument( - "args", nargs="*", help=f"Arguments to add to sys.argv. " - f"Separate them from {SCRIPT_NAME}'s own arguments with `--`.") + "args", nargs="*", help=f"Python command-line arguments. " + f"Separate them from {SCRIPT_NAME}'s own arguments with `--`. " + f"If neither -c nor -m are included, `-m test` will be prepended, " + f"which will run Python's own test suite.") # Package arguments. for subcommand in [package, ci]: @@ -850,6 +849,16 @@ def add_parser(*args, **kwargs): "-g", action="store_true", default=False, dest="debug", help="Include debug information in package") + # CI arguments + for subcommand in [test, ci]: + group = subcommand.add_mutually_exclusive_group(required=subcommand is ci) + group.add_argument( + "--fast-ci", action="store_const", dest="ci_mode", const="fast", + help="Add test arguments for GitHub Actions") + group.add_argument( + "--slow-ci", action="store_const", dest="ci_mode", const="slow", + help="Add test arguments for buildbots") + return parser.parse_args() diff --git a/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt b/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt index 94be52dd2dc..e57243566f9 100644 --- a/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt +++ b/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt @@ -20,7 +20,7 @@ class PythonSuite { val status = PythonTestRunner( InstrumentationRegistry.getInstrumentation().targetContext ).run( - InstrumentationRegistry.getArguments() + InstrumentationRegistry.getArguments().getString("pythonArgs")!!, ) assertEquals(0, status) } finally { diff --git a/Android/testbed/app/src/main/c/main_activity.c b/Android/testbed/app/src/main/c/main_activity.c index ec7f93a3e5e..7f024f0a348 100644 --- a/Android/testbed/app/src/main/c/main_activity.c +++ b/Android/testbed/app/src/main/c/main_activity.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -15,6 +16,13 @@ static void throw_runtime_exception(JNIEnv *env, const char *message) { message); } +static void throw_errno(JNIEnv *env, const char *error_prefix) { + char error_message[1024]; + snprintf(error_message, sizeof(error_message), + "%s: %s", error_prefix, strerror(errno)); + throw_runtime_exception(env, error_message); +} + // --- Stdio redirection ------------------------------------------------------ @@ -95,10 +103,7 @@ JNIEXPORT void JNICALL Java_org_python_testbed_PythonTestRunner_redirectStdioToL for (StreamInfo *si = STREAMS; si->file; si++) { char *error_prefix; if ((error_prefix = redirect_stream(si))) { - char error_message[1024]; - snprintf(error_message, sizeof(error_message), - "%s: %s", error_prefix, strerror(errno)); - throw_runtime_exception(env, error_message); + throw_errno(env, error_prefix); return; } } @@ -107,13 +112,38 @@ JNIEXPORT void JNICALL Java_org_python_testbed_PythonTestRunner_redirectStdioToL // --- Python initialization --------------------------------------------------- -static PyStatus set_config_string( - JNIEnv *env, PyConfig *config, wchar_t **config_str, jstring value -) { - const char *value_utf8 = (*env)->GetStringUTFChars(env, value, NULL); - PyStatus status = PyConfig_SetBytesString(config, config_str, value_utf8); - (*env)->ReleaseStringUTFChars(env, value, value_utf8); - return status; +static char *init_signals() { + // Some tests use SIGUSR1, but that's blocked by default in an Android app in + // order to make it available to `sigwait` in the Signal Catcher thread. + // (https://cs.android.com/android/platform/superproject/+/android14-qpr3-release:art/runtime/signal_catcher.cc). + // That thread's functionality is only useful for debugging the JVM, so disabling + // it should not weaken the tests. + // + // There's no safe way of stopping the thread completely (#123982), but simply + // unblocking SIGUSR1 is enough to fix most tests. + // + // However, in tests that generate multiple different signals in quick + // succession, it's possible for SIGUSR1 to arrive while the main thread is busy + // running the C-level handler for a different signal. In that case, the SIGUSR1 + // may be sent to the Signal Catcher thread instead, which will generate a log + // message containing the text "reacting to signal". + // + // Such tests may need to be changed in one of the following ways: + // * Use a signal other than SIGUSR1 (e.g. test_stress_delivery_simultaneous in + // test_signal.py). + // * Send the signal to a specific thread rather than the whole process (e.g. + // test_signals in test_threadsignals.py. + sigset_t set; + if (sigemptyset(&set)) { + return "sigemptyset"; + } + if (sigaddset(&set, SIGUSR1)) { + return "sigaddset"; + } + if ((errno = pthread_sigmask(SIG_UNBLOCK, &set, NULL))) { + return "pthread_sigmask"; + } + return NULL; } static void throw_status(JNIEnv *env, PyStatus status) { @@ -121,27 +151,47 @@ static void throw_status(JNIEnv *env, PyStatus status) { } JNIEXPORT int JNICALL Java_org_python_testbed_PythonTestRunner_runPython( - JNIEnv *env, jobject obj, jstring home, jstring runModule + JNIEnv *env, jobject obj, jstring home, jarray args ) { + const char *home_utf8 = (*env)->GetStringUTFChars(env, home, NULL); + char cwd[PATH_MAX]; + snprintf(cwd, sizeof(cwd), "%s/%s", home_utf8, "cwd"); + if (chdir(cwd)) { + throw_errno(env, "chdir"); + return 1; + } + + char *error_prefix; + if ((error_prefix = init_signals())) { + throw_errno(env, error_prefix); + return 1; + } + PyConfig config; PyStatus status; - PyConfig_InitIsolatedConfig(&config); + PyConfig_InitPythonConfig(&config); - status = set_config_string(env, &config, &config.home, home); - if (PyStatus_Exception(status)) { + jsize argc = (*env)->GetArrayLength(env, args); + const char *argv[argc + 1]; + for (int i = 0; i < argc; i++) { + jobject arg = (*env)->GetObjectArrayElement(env, args, i); + argv[i] = (*env)->GetStringUTFChars(env, arg, NULL); + } + argv[argc] = NULL; + + // PyConfig_SetBytesArgv "must be called before other methods, since the + // preinitialization configuration depends on command line arguments" + if (PyStatus_Exception(status = PyConfig_SetBytesArgv(&config, argc, (char**)argv))) { throw_status(env, status); return 1; } - status = set_config_string(env, &config, &config.run_module, runModule); + status = PyConfig_SetBytesString(&config, &config.home, home_utf8); if (PyStatus_Exception(status)) { throw_status(env, status); return 1; } - // Some tests generate SIGPIPE and SIGXFSZ, which should be ignored. - config.install_signal_handlers = 1; - status = Py_InitializeFromConfig(&config); if (PyStatus_Exception(status)) { throw_status(env, status); diff --git a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt index ef28948486f..5727b0fe6c3 100644 --- a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt +++ b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt @@ -5,6 +5,7 @@ import android.os.* import android.system.Os import android.widget.TextView import androidx.appcompat.app.* +import org.json.JSONArray import java.io.* @@ -15,30 +16,25 @@ class MainActivity : AppCompatActivity() { override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) setContentView(R.layout.activity_main) - val status = PythonTestRunner(this).run("-m", "test", "-W -uall") + val status = PythonTestRunner(this).run("""["-m", "test", "-W", "-uall"]""") findViewById(R.id.tvHello).text = "Exit status $status" } } class PythonTestRunner(val context: Context) { - fun run(instrumentationArgs: Bundle) = run( - instrumentationArgs.getString("pythonMode")!!, - instrumentationArgs.getString("pythonModule")!!, - instrumentationArgs.getString("pythonArgs") ?: "", - ) - /** Run Python. * - * @param mode Either "-c" or "-m". - * @param module Python statements for "-c" mode, or a module name for - * "-m" mode. - * @param args Arguments to add to sys.argv. Will be parsed by `shlex.split`. + * @param args Python command-line, encoded as JSON. * @return The Python exit status: zero on success, nonzero on failure. */ - fun run(mode: String, module: String, args: String) : Int { - Os.setenv("PYTHON_MODE", mode, true) - Os.setenv("PYTHON_MODULE", module, true) - Os.setenv("PYTHON_ARGS", args, true) + fun run(args: String) : Int { + // We leave argument 0 as an empty string, which is a placeholder for the + // executable name in embedded mode. + val argsJsonArray = JSONArray(args) + val argsStringArray = Array(argsJsonArray.length() + 1) { it -> ""} + for (i in 0..) : Int } diff --git a/Android/testbed/app/src/main/python/android_testbed_main.py b/Android/testbed/app/src/main/python/android_testbed_main.py deleted file mode 100644 index 31b8e5343a8..00000000000 --- a/Android/testbed/app/src/main/python/android_testbed_main.py +++ /dev/null @@ -1,48 +0,0 @@ -import os -import runpy -import shlex -import signal -import sys - -# Some tests use SIGUSR1, but that's blocked by default in an Android app in -# order to make it available to `sigwait` in the Signal Catcher thread. -# (https://cs.android.com/android/platform/superproject/+/android14-qpr3-release:art/runtime/signal_catcher.cc). -# That thread's functionality is only useful for debugging the JVM, so disabling -# it should not weaken the tests. -# -# There's no safe way of stopping the thread completely (#123982), but simply -# unblocking SIGUSR1 is enough to fix most tests. -# -# However, in tests that generate multiple different signals in quick -# succession, it's possible for SIGUSR1 to arrive while the main thread is busy -# running the C-level handler for a different signal. In that case, the SIGUSR1 -# may be sent to the Signal Catcher thread instead, which will generate a log -# message containing the text "reacting to signal". -# -# Such tests may need to be changed in one of the following ways: -# * Use a signal other than SIGUSR1 (e.g. test_stress_delivery_simultaneous in -# test_signal.py). -# * Send the signal to a specific thread rather than the whole process (e.g. -# test_signals in test_threadsignals.py. -signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGUSR1]) - -mode = os.environ["PYTHON_MODE"] -module = os.environ["PYTHON_MODULE"] -sys.argv[1:] = shlex.split(os.environ["PYTHON_ARGS"]) - -cwd = f"{sys.prefix}/cwd" -if not os.path.exists(cwd): - # Empty directories are lost in the asset packing/unpacking process. - os.mkdir(cwd) -os.chdir(cwd) - -if mode == "-c": - # In -c mode, sys.path starts with an empty string, which means whatever the current - # working directory is at the moment of each import. - sys.path.insert(0, "") - exec(module, {}) -elif mode == "-m": - sys.path.insert(0, os.getcwd()) - runpy.run_module(module, run_name="__main__", alter_sys=True) -else: - raise ValueError(f"unknown mode: {mode}") diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index a2d01b157ac..0fc2548789e 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -646,15 +646,23 @@ def _add_cross_compile_opts(self, regrtest_opts): return (environ, keep_environ) def _add_ci_python_opts(self, python_opts, keep_environ): - # --fast-ci and --slow-ci add options to Python: - # "-u -W default -bb -E" + # --fast-ci and --slow-ci add options to Python. + # + # Some platforms run tests in embedded mode and cannot change options + # after startup, so if this function changes, consider also updating: + # * gradle_task in Android/android.py - # Unbuffered stdout and stderr - if not sys.stdout.write_through: + # Unbuffered stdout and stderr. This isn't helpful on Android, because + # it would cause lines to be split into multiple log messages. + if not sys.stdout.write_through and sys.platform != "android": python_opts.append('-u') - # Add warnings filter 'error' - if 'default' not in sys.warnoptions: + # Add warnings filter 'error', unless the user specified a different + # filter. Ignore BytesWarning since it's controlled by '-b' below. + if not [ + opt for opt in sys.warnoptions + if not opt.endswith("::BytesWarning") + ]: python_opts.extend(('-W', 'error')) # Error on bytes/str comparison @@ -673,8 +681,12 @@ def _execute_python(self, cmd, environ): cmd_text = shlex.join(cmd) try: - print(f"+ {cmd_text}", flush=True) + # Android and iOS run tests in embedded mode. To update their + # Python options, see the comment in _add_ci_python_opts. + if not cmd[0]: + raise ValueError("No Python executable is present") + print(f"+ {cmd_text}", flush=True) if hasattr(os, 'execv') and not MS_WINDOWS: os.execv(cmd[0], cmd) # On success, execv() do no return. From 6f3dae0dc5ccd47b2b8a6e052244353d8c37e59b Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 6 Oct 2025 09:38:01 +0000 Subject: [PATCH 033/112] gh-139624: Skip problematic locales on AIX in test_date_locale2 (GH-139625) --- Lib/test/test_strptime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py index d12816c9084..40e114aada6 100644 --- a/Lib/test/test_strptime.py +++ b/Lib/test/test_strptime.py @@ -570,7 +570,7 @@ def test_date_locale(self): def test_date_locale2(self): # Test %x directive loc = locale.getlocale(locale.LC_TIME)[0] - if sys.platform.startswith('sunos'): + if sys.platform.startswith(('sunos', 'aix')): if loc in ('en_US', 'de_DE', 'ar_AE'): self.skipTest(f'locale {loc!r} may not work on this platform') self.roundtrip('%x', slice(0, 3), (1900, 1, 1, 0, 0, 0, 0, 1, 0)) From 36a6c2cdfaddc4a82d14f2c84b04e05089802876 Mon Sep 17 00:00:00 2001 From: yihong Date: Mon, 6 Oct 2025 18:52:45 +0800 Subject: [PATCH 034/112] gh-139646: fix typo in `pickletools` error message (#139647) Signed-off-by: yihong0618 --- Lib/pickletools.py | 2 +- Lib/test/test_pickletools.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/pickletools.py b/Lib/pickletools.py index bcddfb722bd..254b6c7fcc9 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -348,7 +348,7 @@ def read_stringnl(f, decode=True, stripquotes=True, *, encoding='latin-1'): for q in (b'"', b"'"): if data.startswith(q): if not data.endswith(q): - raise ValueError("strinq quote %r not found at both " + raise ValueError("string quote %r not found at both " "ends of %r" % (q, data)) data = data[1:-1] break diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index a178d3353ee..cf990874621 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -384,13 +384,13 @@ def test_string_without_quotes(self): self.check_dis_error(b'Sabc"\n.', '', "no string quotes around b'abc\"'") self.check_dis_error(b"S'abc\n.", '', - '''strinq quote b"'" not found at both ends of b"'abc"''') + '''string quote b"'" not found at both ends of b"'abc"''') self.check_dis_error(b'S"abc\n.', '', - r"""strinq quote b'"' not found at both ends of b'"abc'""") + r"""string quote b'"' not found at both ends of b'"abc'""") self.check_dis_error(b"S'abc\"\n.", '', - r"""strinq quote b"'" not found at both ends of b'\\'abc"'""") + r"""string quote b"'" not found at both ends of b'\\'abc"'""") self.check_dis_error(b"S\"abc'\n.", '', - r"""strinq quote b'"' not found at both ends of b'"abc\\''""") + r"""string quote b'"' not found at both ends of b'"abc\\''""") def test_binstring(self): self.check_dis(b"T\x03\x00\x00\x00abc.", '''\ From b73aaffb3da0174ee8ae62767b6c363996cfe90c Mon Sep 17 00:00:00 2001 From: Cycloctane Date: Mon, 6 Oct 2025 20:40:48 +0800 Subject: [PATCH 035/112] gh-133951: Fix purelib packages not found in test_peg_generator TestCParser (GH-139607) also includes purelib in TestCParser import context --- Lib/test/test_peg_generator/test_c_parser.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index aa01a9b8f7e..395f15b9a62 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -100,11 +100,15 @@ def setUpClass(cls): with contextlib.ExitStack() as stack: python_exe = stack.enter_context(support.setup_venv_with_pip_setuptools("venv")) - sitepackages = subprocess.check_output( + platlib_path = subprocess.check_output( [python_exe, "-c", "import sysconfig; print(sysconfig.get_path('platlib'))"], text=True, ).strip() - stack.enter_context(import_helper.DirsOnSysPath(sitepackages)) + purelib_path = subprocess.check_output( + [python_exe, "-c", "import sysconfig; print(sysconfig.get_path('purelib'))"], + text=True, + ).strip() + stack.enter_context(import_helper.DirsOnSysPath(platlib_path, purelib_path)) cls.addClassCleanup(stack.pop_all().close) @support.requires_venv_with_pip() From 69cfad0b3e1e2805914cb647cce4c7c2bfed5910 Mon Sep 17 00:00:00 2001 From: Cycloctane Date: Mon, 6 Oct 2025 21:04:59 +0800 Subject: [PATCH 036/112] gh-116488: Mention `dict.get` in the data structures tutorial (GH-139643) --- Doc/tutorial/datastructures.rst | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index db8a066b369..1332c53f396 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -512,8 +512,12 @@ dictionary; this is also the way dictionaries are written on output. The main operations on a dictionary are storing a value with some key and extracting the value given the key. It is also possible to delete a key:value pair with ``del``. If you store using a key that is already in use, the old -value associated with that key is forgotten. It is an error to extract a value -using a non-existent key. +value associated with that key is forgotten. + +Extracting a value for a non-existent key by subscripting (``d[key]``) raises a +:exc:`KeyError`. To avoid getting this error when trying to access a possibly +non-existent key, use the :meth:`~dict.get` method instead, which returns +``None`` (or a specified default value) if the key is not in the dictionary. Performing ``list(d)`` on a dictionary returns a list of all the keys used in the dictionary, in insertion order (if you want it sorted, just use @@ -528,6 +532,12 @@ Here is a small example using a dictionary:: {'jack': 4098, 'sape': 4139, 'guido': 4127} >>> tel['jack'] 4098 + >>> tel['irv'] + Traceback (most recent call last): + File "", line 1, in + KeyError: 'irv' + >>> print(tel.get('irv')) + None >>> del tel['sape'] >>> tel['irv'] = 4127 >>> tel From 708de26e31b65cd3ae768882c8da59284917a80d Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Mon, 6 Oct 2025 17:51:10 +0300 Subject: [PATCH 037/112] gh-133210: Fix `test_pydoc` without docstrings (#139654) --- Lib/test/test_pydoc/test_pydoc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 65ad7649b9e..5aa8d92057e 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -1902,7 +1902,7 @@ def test_text_doc_routines_in_class(self, cls=pydocfodder.B): else: self.assertIn(' | get(...) method of builtins.dict instance', lines) self.assertIn(' | dict_get = get(...) method of builtins.dict instance', lines) - self.assertIn(' | sin(...)', lines) + self.assertIn(' | sin(object, /)', lines) lines = self.getsection(result, f' | Class methods {where}:', ' | ' + '-'*70) self.assertIn(' | B_classmethod(x)', lines) @@ -1992,7 +1992,7 @@ def test_text_doc_routines_in_module(self): if not support.MISSING_C_DOCSTRINGS: self.assertIn(' sin(x, /)', lines) else: - self.assertIn(' sin(...)', lines) + self.assertIn(' sin(object, /)', lines) def test_html_doc_routines_in_module(self): doc = pydoc.HTMLDoc() @@ -2037,7 +2037,7 @@ def test_html_doc_routines_in_module(self): if not support.MISSING_C_DOCSTRINGS: self.assertIn(' sin(x, /)', lines) else: - self.assertIn(' sin(...)', lines) + self.assertIn(' sin(object, /)', lines) @unittest.skipIf( From 55a44ccf5e121e76122a0fd25a56fce85cd325ab Mon Sep 17 00:00:00 2001 From: Sebastian Pipping Date: Mon, 6 Oct 2025 16:55:58 +0200 Subject: [PATCH 038/112] gh-139400: Move NEWS item from section "Core and Builtins" to section "Security" (GH-139606) (#139664) --- .../2025-09-29-00-01-28.gh-issue-139400.X2T-jO.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Misc/NEWS.d/next/{Core_and_Builtins => Security}/2025-09-29-00-01-28.gh-issue-139400.X2T-jO.rst (100%) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-29-00-01-28.gh-issue-139400.X2T-jO.rst b/Misc/NEWS.d/next/Security/2025-09-29-00-01-28.gh-issue-139400.X2T-jO.rst similarity index 100% rename from Misc/NEWS.d/next/Core_and_Builtins/2025-09-29-00-01-28.gh-issue-139400.X2T-jO.rst rename to Misc/NEWS.d/next/Security/2025-09-29-00-01-28.gh-issue-139400.X2T-jO.rst From 7c70cc5c23971ef448ea59827c6e6ae310157356 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Mon, 6 Oct 2025 19:48:50 +0300 Subject: [PATCH 039/112] gh-133210: Fix `test_inspect` without docstrings (#139651) --- Lib/test/test_inspect/test_inspect.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 555efb78dcc..e32e34c63b5 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -4261,8 +4261,14 @@ def __init__(self, a): self.assertEqual(self.signature(C, follow_wrapped=False), varargs_signature) - self.assertEqual(self.signature(C.__new__, follow_wrapped=False), - varargs_signature) + if support.MISSING_C_DOCSTRINGS: + self.assertRaisesRegex( + ValueError, "no signature found", + self.signature, C.__new__, follow_wrapped=False, + ) + else: + self.assertEqual(self.signature(C.__new__, follow_wrapped=False), + varargs_signature) def test_signature_on_class_with_wrapped_new(self): with self.subTest('FunctionType'): From 171f787a297ec4b02cfe8b3ebab8374018391f20 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 6 Oct 2025 17:42:26 +0000 Subject: [PATCH 040/112] gh-138854: Skip test_script_shadowing_stdlib_cwd_failure on AIX (GH-138855) --- Lib/test/test_import/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index abbd5f1ed5f..b71a36ec2f7 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1187,6 +1187,7 @@ class substr(str): @unittest.skipIf(sys.platform == 'win32', 'Cannot delete cwd on Windows') @unittest.skipIf(sys.platform == 'sunos5', 'Cannot delete cwd on Solaris/Illumos') + @unittest.skipIf(sys.platform.startswith('aix'), 'Cannot delete cwd on AIX') def test_script_shadowing_stdlib_cwd_failure(self): with os_helper.temp_dir() as tmp: subtmp = os.path.join(tmp, "subtmp") From 331158065b7426a791217157585e565157bb851c Mon Sep 17 00:00:00 2001 From: Alyssa Coghlan Date: Tue, 7 Oct 2025 05:41:08 +1000 Subject: [PATCH 041/112] Add warnings filter suggestions to PEP 765 entry in What's New (#139658) Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/whatsnew/3.14.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index c226f57e502..15fab9c6b3e 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -937,6 +937,19 @@ The compiler now emits a :exc:`SyntaxWarning` when a :keyword:`return`, leaving a :keyword:`finally` block. This change is specified in :pep:`765`. +In situations where this change is inconvenient (such as those where the +warnings are redundant due to code linting), the :ref:`warning filter +` can be used to turn off all syntax warnings by adding +``ignore::SyntaxWarning`` as a filter. This can be specified in combination +with a filter that converts other warnings to errors (for example, passing +``-Werror -Wignore::SyntaxWarning`` as CLI options, or setting +``PYTHONWARNINGS=error,ignore::SyntaxWarning``). + +Note that applying such a filter at runtime using the :mod:`warnings` module +will only suppress the warning in code that is compiled *after* the filter is +adjusted. Code that is compiled prior to the filter adjustment (for example, +when a module is imported) will still emit the syntax warning. + (Contributed by Irit Katriel in :gh:`130080`.) From 23410f0a9e2f691c8cd342cfd05cc3ca5f09c242 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Mon, 6 Oct 2025 21:17:09 -0500 Subject: [PATCH 042/112] gh-139573: Update macOS installer to use OpenSSL 3.0.18 (#139575) --- Mac/BuildScript/build-installer.py | 6 +++--- .../macOS/2025-10-04-12-29-31.gh-issue-139573.vVpHaP.rst | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/macOS/2025-10-04-12-29-31.gh-issue-139573.vVpHaP.rst diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 4da6d924848..29c552f5aa6 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 3.0.16", - url="https://github.com/openssl/openssl/releases/download/openssl-3.0.16/openssl-3.0.16.tar.gz", - checksum='57e03c50feab5d31b152af2b764f10379aecd8ee92f16c985983ce4a99f7ef86', + name="OpenSSL 3.0.18", + url="https://github.com/openssl/openssl/releases/download/openssl-3.0.18/openssl-3.0.18.tar.gz", + checksum='d80c34f5cf902dccf1f1b5df5ebb86d0392e37049e5d73df1b3abae72e4ffe8b', buildrecipe=build_universal_openssl, configure=None, install=None, diff --git a/Misc/NEWS.d/next/macOS/2025-10-04-12-29-31.gh-issue-139573.vVpHaP.rst b/Misc/NEWS.d/next/macOS/2025-10-04-12-29-31.gh-issue-139573.vVpHaP.rst new file mode 100644 index 00000000000..02a3bfb41ce --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2025-10-04-12-29-31.gh-issue-139573.vVpHaP.rst @@ -0,0 +1 @@ +Updated bundled version of OpenSSL to 3.0.18. From 6d804e4efb8ed2ebeb9cef9577945c46c0038178 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 7 Oct 2025 05:53:05 +0100 Subject: [PATCH 043/112] gh-124111: Update macOS installer to use Tcl/Tk 8.6.17. (#139682) --- Mac/BuildScript/build-installer.py | 6 +++--- .../macOS/2025-10-06-23-56-36.gh-issue-124111.KOlBvs.rst | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/macOS/2025-10-06-23-56-36.gh-issue-124111.KOlBvs.rst diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 29c552f5aa6..4fd8d55f35a 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -264,10 +264,10 @@ def library_recipes(): tk_patches = ['backport_gh71383_fix.patch', 'tk868_on_10_8_10_9.patch', 'backport_gh110950_fix.patch'] else: - tcl_tk_ver='8.6.16' - tcl_checksum='91cb8fa61771c63c262efb553059b7c7ad6757afa5857af6265e4b0bdc2a14a5' + tcl_tk_ver='8.6.17' + tcl_checksum='a3903371efcce8a405c5c245d029e9f6850258a60fa3761c4d58995610949b31' - tk_checksum='be9f94d3575d4b3099d84bc3c10de8994df2d7aa405208173c709cc404a7e5fe' + tk_checksum='e4982df6f969c08bf9dd858a6891059b4a3f50dc6c87c10abadbbe2fc4838946' tk_patches = [] diff --git a/Misc/NEWS.d/next/macOS/2025-10-06-23-56-36.gh-issue-124111.KOlBvs.rst b/Misc/NEWS.d/next/macOS/2025-10-06-23-56-36.gh-issue-124111.KOlBvs.rst new file mode 100644 index 00000000000..2787e5d37fa --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2025-10-06-23-56-36.gh-issue-124111.KOlBvs.rst @@ -0,0 +1 @@ +Update macOS installer to use Tcl/Tk 8.6.17. From 134ff810cfc479c56b770f769cf8d1b680e13009 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 7 Oct 2025 09:49:59 +0100 Subject: [PATCH 044/112] GH-123299: Announce final release in What's New in Python 3.14 (#139631) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/whatsnew/3.13.rst | 2 +- Doc/whatsnew/3.14.rst | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index bc5a21e172d..1548f128b5b 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -60,7 +60,7 @@ Summary -- Release Highlights .. This section singles out the most important changes in Python 3.13. Brevity is key. -Python 3.13 is the latest stable release of the Python programming +Python 3.13 is a stable release of the Python programming language, with a mix of changes to the language, the implementation and the standard library. The biggest changes include a new `interactive interpreter diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 15fab9c6b3e..b174333760e 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -46,7 +46,7 @@ when researching a change. This article explains the new features in Python 3.14, compared to 3.13. -Python 3.14 will be released on 7 October 2025. +Python 3.14 was released on 7 October 2025. For full details, see the :ref:`changelog `. .. seealso:: @@ -60,7 +60,7 @@ Summary -- Release highlights .. This section singles out the most important changes in Python 3.14. Brevity is key. -Python 3.14 will be the latest stable release of the Python programming +Python 3.14 is the latest stable release of the Python programming language, with a mix of changes to the language, the implementation, and the standard library. The biggest changes include :ref:`template string literals @@ -3083,7 +3083,7 @@ Deprecated C APIs .. _whatsnew314-build-changes: -Build Changes +Build changes ============= * :pep:`776`: Emscripten is now an officially supported platform at From 0e2cdd313ba5c67c5e2e21d993399b890e687c63 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 7 Oct 2025 13:29:18 +0100 Subject: [PATCH 045/112] gh-139436: Remove ``dist-pdf`` from the docs archives rebuild target (#139437) --- Doc/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/Makefile b/Doc/Makefile index 84578c5c57f..f6f4c721080 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -184,7 +184,7 @@ venv: fi .PHONY: dist-no-html -dist-no-html: dist-text dist-pdf dist-epub dist-texinfo +dist-no-html: dist-text dist-epub dist-texinfo .PHONY: dist dist: From 7094f09f547dc1a520c666dc6ce11b7b69a1b8da Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 7 Oct 2025 14:04:37 +0100 Subject: [PATCH 046/112] GH-139291: Fix C stack limits by factoring out finding hardware stack limits (GH-139294) --- Python/ceval.c | 60 +++++++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 0ccaacaf3ed..1b52128c858 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -438,31 +438,26 @@ int pthread_attr_destroy(pthread_attr_t *a) #endif - -void -_Py_InitializeRecursionLimits(PyThreadState *tstate) +static void +hardware_stack_limits(uintptr_t *top, uintptr_t *base) { - _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; #ifdef WIN32 ULONG_PTR low, high; GetCurrentThreadStackLimits(&low, &high); - _tstate->c_stack_top = (uintptr_t)high; + *top = (uintptr_t)high; ULONG guarantee = 0; SetThreadStackGuarantee(&guarantee); - _tstate->c_stack_hard_limit = ((uintptr_t)low) + guarantee + _PyOS_STACK_MARGIN_BYTES; - _tstate->c_stack_soft_limit = _tstate->c_stack_hard_limit + _PyOS_STACK_MARGIN_BYTES; + *base = (uintptr_t)low + guarantee; #elif defined(__APPLE__) pthread_t this_thread = pthread_self(); void *stack_addr = pthread_get_stackaddr_np(this_thread); // top of the stack size_t stack_size = pthread_get_stacksize_np(this_thread); - _tstate->c_stack_top = (uintptr_t)stack_addr; - _tstate->c_stack_hard_limit = _tstate->c_stack_top - stack_size; - _tstate->c_stack_soft_limit = _tstate->c_stack_hard_limit + _PyOS_STACK_MARGIN_BYTES; + *top = (uintptr_t)stack_addr; + *base = ((uintptr_t)stack_addr) - stack_size; #else - uintptr_t here_addr = _Py_get_machine_stack_pointer(); -/// XXX musl supports HAVE_PTHRED_GETATTR_NP, but the resulting stack size -/// (on alpine at least) is much smaller than expected and imposes undue limits -/// compared to the old stack size estimation. (We assume musl is not glibc.) + /// XXX musl supports HAVE_PTHRED_GETATTR_NP, but the resulting stack size + /// (on alpine at least) is much smaller than expected and imposes undue limits + /// compared to the old stack size estimation. (We assume musl is not glibc.) # if defined(HAVE_PTHREAD_GETATTR_NP) && !defined(_AIX) && \ !defined(__NetBSD__) && (defined(__GLIBC__) || !defined(__linux__)) size_t stack_size, guard_size; @@ -475,26 +470,35 @@ _Py_InitializeRecursionLimits(PyThreadState *tstate) err |= pthread_attr_destroy(&attr); } if (err == 0) { - uintptr_t base = ((uintptr_t)stack_addr) + guard_size; - _tstate->c_stack_top = base + stack_size; -#ifdef _Py_THREAD_SANITIZER - // Thread sanitizer crashes if we use a bit more than half the stack. - _tstate->c_stack_soft_limit = base + (stack_size / 2); -#else - _tstate->c_stack_soft_limit = base + _PyOS_STACK_MARGIN_BYTES * 2; -#endif - _tstate->c_stack_hard_limit = base + _PyOS_STACK_MARGIN_BYTES; - assert(_tstate->c_stack_soft_limit < here_addr); - assert(here_addr < _tstate->c_stack_top); + *base = ((uintptr_t)stack_addr) + guard_size; + *top = (uintptr_t)stack_addr + stack_size; return; } # endif - _tstate->c_stack_top = _Py_SIZE_ROUND_UP(here_addr, 4096); - _tstate->c_stack_soft_limit = _tstate->c_stack_top - Py_C_STACK_SIZE; - _tstate->c_stack_hard_limit = _tstate->c_stack_top - (Py_C_STACK_SIZE + _PyOS_STACK_MARGIN_BYTES); + uintptr_t here_addr = _Py_get_machine_stack_pointer(); + uintptr_t top_addr = _Py_SIZE_ROUND_UP(here_addr, 4096); + *top = top_addr; + *base = top_addr - Py_C_STACK_SIZE; #endif } +void +_Py_InitializeRecursionLimits(PyThreadState *tstate) +{ + uintptr_t top; + uintptr_t base; + hardware_stack_limits(&top, &base); +#ifdef _Py_THREAD_SANITIZER + // Thread sanitizer crashes if we use more than half the stack. + uintptr_t stacksize = top - base; + base += stacksize/2; +#endif + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + _tstate->c_stack_top = top; + _tstate->c_stack_hard_limit = base + _PyOS_STACK_MARGIN_BYTES; + _tstate->c_stack_soft_limit = base + _PyOS_STACK_MARGIN_BYTES * 2; +} + /* The function _Py_EnterRecursiveCallTstate() only calls _Py_CheckRecursiveCall() if the recursion_depth reaches recursion_limit. */ int From 96c59a6e427fab32d0bca89b77febca8cba8aada Mon Sep 17 00:00:00 2001 From: danigm Date: Tue, 7 Oct 2025 16:54:31 +0200 Subject: [PATCH 047/112] gh-138497: Support LLVM_VERSION configuration via env (#138498) Co-authored-by: Savannah Ostrowski --- ...-09-04-12-16-31.gh-issue-138497.Y_5YXh.rst | 4 ++ Tools/jit/README.md | 2 +- Tools/jit/_llvm.py | 49 ++++++++++++------- Tools/jit/_targets.py | 19 +++++-- Tools/jit/build.py | 3 ++ configure | 2 +- configure.ac | 2 +- 7 files changed, 56 insertions(+), 25 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2025-09-04-12-16-31.gh-issue-138497.Y_5YXh.rst diff --git a/Misc/NEWS.d/next/Build/2025-09-04-12-16-31.gh-issue-138497.Y_5YXh.rst b/Misc/NEWS.d/next/Build/2025-09-04-12-16-31.gh-issue-138497.Y_5YXh.rst new file mode 100644 index 00000000000..7eb07709968 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-09-04-12-16-31.gh-issue-138497.Y_5YXh.rst @@ -0,0 +1,4 @@ +The LLVM version used by the JIT at build time can now be modified using +the ``LLVM_VERSION`` environment variable. Use this at your own risk, as +there is only one officially supported LLVM version. For more information, +please check ``Tools/jit/README.md``. diff --git a/Tools/jit/README.md b/Tools/jit/README.md index ffc762d3828..35c7ffd7a28 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -9,7 +9,7 @@ ## Installing LLVM The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). -LLVM version 19 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. +LLVM version 19 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. It's easy to install all of the required tools: diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index f09a8404871..bc3b50ffe61 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -10,8 +10,8 @@ import _targets -_LLVM_VERSION = 19 -_LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+") + +_LLVM_VERSION = "19" _EXTERNALS_LLVM_TAG = "llvm-19.1.7.0" _P = typing.ParamSpec("_P") @@ -56,53 +56,66 @@ async def _run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str @_async_cache -async def _check_tool_version(name: str, *, echo: bool = False) -> bool: +async def _check_tool_version( + name: str, llvm_version: str, *, echo: bool = False +) -> bool: output = await _run(name, ["--version"], echo=echo) - return bool(output and _LLVM_VERSION_PATTERN.search(output)) + _llvm_version_pattern = re.compile(rf"version\s+{llvm_version}\.\d+\.\d+\S*\s+") + return bool(output and _llvm_version_pattern.search(output)) @_async_cache -async def _get_brew_llvm_prefix(*, echo: bool = False) -> str | None: - output = await _run("brew", ["--prefix", f"llvm@{_LLVM_VERSION}"], echo=echo) +async def _get_brew_llvm_prefix(llvm_version: str, *, echo: bool = False) -> str | None: + output = await _run("brew", ["--prefix", f"llvm@{llvm_version}"], echo=echo) return output and output.removesuffix("\n") @_async_cache -async def _find_tool(tool: str, *, echo: bool = False) -> str | None: +async def _find_tool(tool: str, llvm_version: str, *, echo: bool = False) -> str | None: # Unversioned executables: path = tool - if await _check_tool_version(path, echo=echo): + if await _check_tool_version(path, llvm_version, echo=echo): return path # Versioned executables: - path = f"{tool}-{_LLVM_VERSION}" - if await _check_tool_version(path, echo=echo): + path = f"{tool}-{llvm_version}" + if await _check_tool_version(path, llvm_version, echo=echo): return path # PCbuild externals: externals = os.environ.get("EXTERNALS_DIR", _targets.EXTERNALS) path = os.path.join(externals, _EXTERNALS_LLVM_TAG, "bin", tool) - if await _check_tool_version(path, echo=echo): + if await _check_tool_version(path, llvm_version, echo=echo): return path # Homebrew-installed executables: - prefix = await _get_brew_llvm_prefix(echo=echo) + prefix = await _get_brew_llvm_prefix(llvm_version, echo=echo) if prefix is not None: path = os.path.join(prefix, "bin", tool) - if await _check_tool_version(path, echo=echo): + if await _check_tool_version(path, llvm_version, echo=echo): return path # Nothing found: return None async def maybe_run( - tool: str, args: typing.Iterable[str], echo: bool = False + tool: str, + args: typing.Iterable[str], + echo: bool = False, + llvm_version: str = _LLVM_VERSION, ) -> str | None: """Run an LLVM tool if it can be found. Otherwise, return None.""" - path = await _find_tool(tool, echo=echo) + + path = await _find_tool(tool, llvm_version, echo=echo) return path and await _run(path, args, echo=echo) -async def run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str: +async def run( + tool: str, + args: typing.Iterable[str], + echo: bool = False, + llvm_version: str = _LLVM_VERSION, +) -> str: """Run an LLVM tool if it can be found. Otherwise, raise RuntimeError.""" - output = await maybe_run(tool, args, echo=echo) + + output = await maybe_run(tool, args, echo=echo, llvm_version=llvm_version) if output is None: - raise RuntimeError(f"Can't find {tool}-{_LLVM_VERSION}!") + raise RuntimeError(f"Can't find {tool}-{llvm_version}!") return output diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 2f3969e7d05..9fc3522d23d 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -50,6 +50,7 @@ class _Target(typing.Generic[_S, _R]): debug: bool = False verbose: bool = False cflags: str = "" + llvm_version: str = _llvm._LLVM_VERSION known_symbols: dict[str, int] = dataclasses.field(default_factory=dict) pyconfig_dir: pathlib.Path = pathlib.Path.cwd().resolve() @@ -81,7 +82,9 @@ def _compute_digest(self) -> str: async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: group = _stencils.StencilGroup() args = ["--disassemble", "--reloc", f"{path}"] - output = await _llvm.maybe_run("llvm-objdump", args, echo=self.verbose) + output = await _llvm.maybe_run( + "llvm-objdump", args, echo=self.verbose, llvm_version=self.llvm_version + ) if output is not None: # Make sure that full paths don't leak out (for reproducibility): long, short = str(path), str(path.name) @@ -99,7 +102,9 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: "--sections", f"{path}", ] - output = await _llvm.run("llvm-readobj", args, echo=self.verbose) + output = await _llvm.run( + "llvm-readobj", args, echo=self.verbose, llvm_version=self.llvm_version + ) # --elf-output-style=JSON is only *slightly* broken on Mach-O... output = output.replace("PrivateExtern\n", "\n") output = output.replace("Extern\n", "\n") @@ -175,12 +180,16 @@ async def _compile( # Allow user-provided CFLAGS to override any defaults *shlex.split(self.cflags), ] - await _llvm.run("clang", args_s, echo=self.verbose) + await _llvm.run( + "clang", args_s, echo=self.verbose, llvm_version=self.llvm_version + ) self.optimizer( s, label_prefix=self.label_prefix, symbol_prefix=self.symbol_prefix ).run() args_o = [f"--target={self.triple}", "-c", "-o", f"{o}", f"{s}"] - await _llvm.run("clang", args_o, echo=self.verbose) + await _llvm.run( + "clang", args_o, echo=self.verbose, llvm_version=self.llvm_version + ) return await self._parse(o) async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: @@ -224,6 +233,8 @@ def build( if not self.stable: warning = f"JIT support for {self.triple} is still experimental!" request = "Please report any issues you encounter.".center(len(warning)) + if self.llvm_version != _llvm._LLVM_VERSION: + request = f"Warning! Building with an LLVM version other than {_llvm._LLVM_VERSION} is not supported." outline = "=" * len(warning) print("\n".join(["", outline, warning, request, outline, ""])) digest = f"// {self._compute_digest()}\n" diff --git a/Tools/jit/build.py b/Tools/jit/build.py index a0733005929..127d93b317f 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -42,6 +42,7 @@ parser.add_argument( "--cflags", help="additional flags to pass to the compiler", default="" ) + parser.add_argument("--llvm-version", help="LLVM version to use") args = parser.parse_args() for target in args.target: target.debug = args.debug @@ -49,6 +50,8 @@ target.verbose = args.verbose target.cflags = args.cflags target.pyconfig_dir = args.pyconfig_dir + if args.llvm_version: + target.llvm_version = args.llvm_version target.build( comment=comment, force=args.force, diff --git a/configure b/configure index 0d1f6a29e9b..d80340e3015 100755 --- a/configure +++ b/configure @@ -10875,7 +10875,7 @@ then : else case e in #( e) as_fn_append CFLAGS_NODIST " $jit_flags" - REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\"" + REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\"" if test "x$Py_DEBUG" = xtrue then : as_fn_append REGEN_JIT_COMMAND " --debug" diff --git a/configure.ac b/configure.ac index 7b5da6e0d15..1e0c0f71b7c 100644 --- a/configure.ac +++ b/configure.ac @@ -2786,7 +2786,7 @@ AS_VAR_IF([jit_flags], [], [AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"]) AS_VAR_SET([REGEN_JIT_COMMAND], - ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\""]) + ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\""]) AS_VAR_IF([Py_DEBUG], [true], [AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])], From 539461d9ec8e5322ead638f7be733fd196aa6c79 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Tue, 7 Oct 2025 12:28:15 -0400 Subject: [PATCH 048/112] gh-139516: Fix lambda colon start format spec in f-string in tokenizer (#139657) --- Lib/test/test_fstring.py | 7 +++++++ Lib/test/test_tokenize.py | 17 +++++++++++++++++ ...25-10-06-13-15-26.gh-issue-139516.d9Pkur.rst | 1 + Parser/lexer/lexer.c | 2 +- Parser/lexer/state.h | 2 ++ 5 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-13-15-26.gh-issue-139516.d9Pkur.rst diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 41cefe0e286..05d0cbd2445 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1859,6 +1859,13 @@ def __format__(self, format): # Test multiple format specs in same raw f-string self.assertEqual(rf"{UnchangedFormat():\xFF} {UnchangedFormat():\n}", '\\xFF \\n') + def test_gh139516(self): + with temp_cwd(): + script = 'script.py' + with open(script, 'wb') as f: + f.write('''def f(a): pass\nf"{f(a=lambda: 'à'\n)}"'''.encode()) + assert_python_ok(script) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index d90a7659c42..8fdd03f347b 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1216,6 +1216,23 @@ def test_multiline_non_ascii_fstring_with_expr(self): FSTRING_END "\'\'\'" (3, 1) (3, 4) """) + # gh-139516, the '\n' is explicit to ensure no trailing whitespace which would invalidate the test + self.check_tokenize('''f"{f(a=lambda: 'à'\n)}"''', """\ + FSTRING_START \'f"\' (1, 0) (1, 2) + OP '{' (1, 2) (1, 3) + NAME 'f' (1, 3) (1, 4) + OP '(' (1, 4) (1, 5) + NAME 'a' (1, 5) (1, 6) + OP '=' (1, 6) (1, 7) + NAME 'lambda' (1, 7) (1, 13) + OP ':' (1, 13) (1, 14) + STRING "\'à\'" (1, 15) (1, 18) + NL '\\n' (1, 18) (1, 19) + OP ')' (2, 0) (2, 1) + OP '}' (2, 1) (2, 2) + FSTRING_END \'"\' (2, 2) (2, 3) + """) + class GenerateTokensTest(TokenizeTest): def check_tokenize(self, s, expected): # Format the tokens in s in a table format. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-13-15-26.gh-issue-139516.d9Pkur.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-13-15-26.gh-issue-139516.d9Pkur.rst new file mode 100644 index 00000000000..a7091123060 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-13-15-26.gh-issue-139516.d9Pkur.rst @@ -0,0 +1 @@ +Fix lambda colon erroneously start format spec in f-string in tokenizer. diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 81363cf8e81..a69994e9b3d 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -1376,7 +1376,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", c)); } - if( c == '=' && INSIDE_FSTRING_EXPR(current_tok)) { + if( c == '=' && INSIDE_FSTRING_EXPR_AT_TOP(current_tok)) { current_tok->in_debug = 1; } diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 5e8cac7249b..877127125a7 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -9,6 +9,8 @@ #define INSIDE_FSTRING(tok) (tok->tok_mode_stack_index > 0) #define INSIDE_FSTRING_EXPR(tok) (tok->curly_bracket_expr_start_depth >= 0) +#define INSIDE_FSTRING_EXPR_AT_TOP(tok) \ + (tok->curly_bracket_depth - tok->curly_bracket_expr_start_depth == 1) enum decoding_state { STATE_INIT, From 162997bb70e067668c039700141770687bc8f267 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 7 Oct 2025 20:15:26 +0300 Subject: [PATCH 049/112] gh-139700: Check consistency of the zip64 end of central directory record (GH-139702) Support records with "zip64 extensible data" if there are no bytes prepended to the ZIP file. --- Lib/test/test_zipfile/test_core.py | 82 ++++++++++++++++++- Lib/zipfile/__init__.py | 51 +++++++----- ...-10-07-19-31-34.gh-issue-139700.vNHU1O.rst | 3 + 3 files changed, 113 insertions(+), 23 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-10-07-19-31-34.gh-issue-139700.vNHU1O.rst diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index c033059a515..6acfefc74d6 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -898,6 +898,8 @@ def make_zip64_file( self, file_size_64_set=False, file_size_extra=False, compress_size_64_set=False, compress_size_extra=False, header_offset_64_set=False, header_offset_extra=False, + extensible_data=b'', + end_of_central_dir_size=None, offset_to_end_of_central_dir=None, ): """Generate bytes sequence for a zip with (incomplete) zip64 data. @@ -951,6 +953,12 @@ def make_zip64_file( central_dir_size = struct.pack(' 2: inferred = concat + offset_cd @@ -289,16 +286,15 @@ def _EndRecData64(fpin, offset, endrec): """ Read the ZIP64 end-of-archive records and use that to update endrec """ - try: - fpin.seek(offset - sizeEndCentDir64Locator, 2) - except OSError: - # If the seek fails, the file is not large enough to contain a ZIP64 + offset -= sizeEndCentDir64Locator + if offset < 0: + # The file is not large enough to contain a ZIP64 # end-of-archive record, so just return the end record we were given. return endrec - + fpin.seek(offset) data = fpin.read(sizeEndCentDir64Locator) if len(data) != sizeEndCentDir64Locator: - return endrec + raise OSError("Unknown I/O error") sig, diskno, reloff, disks = struct.unpack(structEndArchive64Locator, data) if sig != stringEndArchive64Locator: return endrec @@ -306,16 +302,33 @@ def _EndRecData64(fpin, offset, endrec): if diskno != 0 or disks > 1: raise BadZipFile("zipfiles that span multiple disks are not supported") - # Assume no 'zip64 extensible data' - fpin.seek(offset - sizeEndCentDir64Locator - sizeEndCentDir64, 2) + offset -= sizeEndCentDir64 + if reloff > offset: + raise BadZipFile("Corrupt zip64 end of central directory locator") + # First, check the assumption that there is no prepended data. + fpin.seek(reloff) + extrasz = offset - reloff data = fpin.read(sizeEndCentDir64) if len(data) != sizeEndCentDir64: - return endrec + raise OSError("Unknown I/O error") + if not data.startswith(stringEndArchive64) and reloff != offset: + # Since we already have seen the Zip64 EOCD Locator, it's + # possible we got here because there is prepended data. + # Assume no 'zip64 extensible data' + fpin.seek(offset) + extrasz = 0 + data = fpin.read(sizeEndCentDir64) + if len(data) != sizeEndCentDir64: + raise OSError("Unknown I/O error") + if not data.startswith(stringEndArchive64): + raise BadZipFile("Zip64 end of central directory record not found") + sig, sz, create_version, read_version, disk_num, disk_dir, \ dircount, dircount2, dirsize, diroffset = \ struct.unpack(structEndArchive64, data) - if sig != stringEndArchive64: - return endrec + if (diroffset + dirsize != reloff or + sz + 12 != sizeEndCentDir64 + extrasz): + raise BadZipFile("Corrupt zip64 end of central directory record") # Update the original endrec using data from the ZIP64 record endrec[_ECD_SIGNATURE] = sig @@ -325,6 +338,7 @@ def _EndRecData64(fpin, offset, endrec): endrec[_ECD_ENTRIES_TOTAL] = dircount2 endrec[_ECD_SIZE] = dirsize endrec[_ECD_OFFSET] = diroffset + endrec[_ECD_LOCATION] = offset - extrasz return endrec @@ -358,7 +372,7 @@ def _EndRecData(fpin): endrec.append(filesize - sizeEndCentDir) # Try to read the "Zip64 end of central directory" structure - return _EndRecData64(fpin, -sizeEndCentDir, endrec) + return _EndRecData64(fpin, filesize - sizeEndCentDir, endrec) # Either this is not a ZIP file, or it is a ZIP file with an archive # comment. Search the end of the file for the "end of central directory" @@ -382,8 +396,7 @@ def _EndRecData(fpin): endrec.append(maxCommentStart + start) # Try to read the "Zip64 end of central directory" structure - return _EndRecData64(fpin, maxCommentStart + start - filesize, - endrec) + return _EndRecData64(fpin, maxCommentStart + start, endrec) # Unable to find a valid end of central directory structure return None @@ -2142,7 +2155,7 @@ def _write_end_record(self): " would require ZIP64 extensions") zip64endrec = struct.pack( structEndArchive64, stringEndArchive64, - 44, 45, 45, 0, 0, centDirCount, centDirCount, + sizeEndCentDir64 - 12, 45, 45, 0, 0, centDirCount, centDirCount, centDirSize, centDirOffset) self.fp.write(zip64endrec) diff --git a/Misc/NEWS.d/next/Security/2025-10-07-19-31-34.gh-issue-139700.vNHU1O.rst b/Misc/NEWS.d/next/Security/2025-10-07-19-31-34.gh-issue-139700.vNHU1O.rst new file mode 100644 index 00000000000..a8e7a1f1878 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-10-07-19-31-34.gh-issue-139700.vNHU1O.rst @@ -0,0 +1,3 @@ +Check consistency of the zip64 end of central directory record. Support +records with "zip64 extensible data" if there are no bytes prepended to the +ZIP file. From d396a32b3d1a4b3a35bdfc8b44521d3021a108c7 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Tue, 7 Oct 2025 18:39:45 +0100 Subject: [PATCH 050/112] gh-139698: Fix typo in What's New 3.14 (#139699) --- Doc/whatsnew/3.14.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index b174333760e..e1a25c84813 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -449,8 +449,8 @@ depending on platform and architecture. The baseline is Python 3.14 built with Clang 19, without this new interpreter. This interpreter currently only works with Clang 19 and newer -on x86-64 and AArch64 architectures. However, a future release -of GCC is expected will support this as well. +on x86-64 and AArch64 architectures. +However, a future release of GCC is expected to support this as well. This feature is opt-in for now. Enabling profile-guided optimization is highly recommendeded when using the new interpreter as it is the only configuration From 25edfa7cf1c0ddeaae2dd76ca6c18807b339257a Mon Sep 17 00:00:00 2001 From: Cornelius Roemer Date: Tue, 7 Oct 2025 20:53:27 +0200 Subject: [PATCH 051/112] Doc: Improve clarity for subinterpreters in What's New in 3.14 (#139221) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/whatsnew/3.14.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index e1a25c84813..677365c2f59 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -217,7 +217,7 @@ has significant benefits: * they support a new (to Python), human-friendly concurrency model * true multi-core parallelism -For some use cases, concurrency in software enables efficiency and +For some use cases, concurrency in software improves efficiency and can simplify design, at a high level. At the same time, implementing and maintaining all but the simplest concurrency is often a struggle for the human brain. @@ -225,9 +225,10 @@ That especially applies to plain threads (for example, :mod:`threading`), where all memory is shared between all threads. With multiple isolated interpreters, you can take advantage of a class -of concurrency models, like CSP or the actor model, that have found +of concurrency models, like Communicating Sequential Processes (CSP) +or the actor model, that have found success in other programming languages, like Smalltalk, Erlang, -Haskell, and Go. Think of multiple interpreters like threads +Haskell, and Go. Think of multiple interpreters as threads but with opt-in sharing. Regarding multi-core parallelism: as of Python 3.12, interpreters @@ -249,8 +250,8 @@ having the isolation of processes with the efficiency of threads. While the feature has been around for decades, multiple interpreters have not been used widely, due to low awareness and the lack of a standard library module. Consequently, they currently have several -notable limitations, which will improve significantly now that the -feature is finally going mainstream. +notable limitations, which are expected to improve significantly now +that the feature is going mainstream. Current limitations: From f962e1eacf2141e6b14c96b58da62382a93b070b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 7 Oct 2025 23:49:08 +0200 Subject: [PATCH 052/112] gh-79315: Add Include/cpython/marshal.h header (#139725) --- Include/cpython/marshal.h | 17 +++++++++++++++++ Include/marshal.h | 23 +++++------------------ Makefile.pre.in | 1 + PCbuild/pythoncore.vcxproj | 1 + 4 files changed, 24 insertions(+), 18 deletions(-) create mode 100644 Include/cpython/marshal.h diff --git a/Include/cpython/marshal.h b/Include/cpython/marshal.h new file mode 100644 index 00000000000..6c1f7f96b6a --- /dev/null +++ b/Include/cpython/marshal.h @@ -0,0 +1,17 @@ +#ifndef _Py_CPYTHON_MARSHAL_H +# error "this header file must not be included directly" +#endif + +PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(const char *, + Py_ssize_t); +PyAPI_FUNC(PyObject *) PyMarshal_WriteObjectToString(PyObject *, int); + +#define Py_MARSHAL_VERSION 5 + +PyAPI_FUNC(long) PyMarshal_ReadLongFromFile(FILE *); +PyAPI_FUNC(int) PyMarshal_ReadShortFromFile(FILE *); +PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromFile(FILE *); +PyAPI_FUNC(PyObject *) PyMarshal_ReadLastObjectFromFile(FILE *); + +PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int); +PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int); diff --git a/Include/marshal.h b/Include/marshal.h index f773587bdd0..2ccb112b40c 100644 --- a/Include/marshal.h +++ b/Include/marshal.h @@ -1,31 +1,18 @@ - /* Interface for marshal.c */ #ifndef Py_MARSHAL_H #define Py_MARSHAL_H -#ifndef Py_LIMITED_API - #ifdef __cplusplus extern "C" { #endif -PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(const char *, - Py_ssize_t); -PyAPI_FUNC(PyObject *) PyMarshal_WriteObjectToString(PyObject *, int); - -#define Py_MARSHAL_VERSION 5 - -PyAPI_FUNC(long) PyMarshal_ReadLongFromFile(FILE *); -PyAPI_FUNC(int) PyMarshal_ReadShortFromFile(FILE *); -PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromFile(FILE *); -PyAPI_FUNC(PyObject *) PyMarshal_ReadLastObjectFromFile(FILE *); - -PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int); -PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int); +#ifndef Py_LIMITED_API +# define _Py_CPYTHON_MARSHAL_H +# include "cpython/marshal.h" +# undef _Py_CPYTHON_MARSHAL_H +#endif #ifdef __cplusplus } #endif - -#endif /* Py_LIMITED_API */ #endif /* !Py_MARSHAL_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index 244e25c348f..ff9a84e4c27 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1271,6 +1271,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/pylock.h \ $(srcdir)/Include/cpython/longintrepr.h \ $(srcdir)/Include/cpython/longobject.h \ + $(srcdir)/Include/cpython/marshal.h \ $(srcdir)/Include/cpython/memoryobject.h \ $(srcdir)/Include/cpython/methodobject.h \ $(srcdir)/Include/cpython/modsupport.h \ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index d7544d3a9fb..d043d23eed8 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -168,6 +168,7 @@ + From 1cf22600f10322d77ab2c2bb41acabf2360f84e9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 8 Oct 2025 00:48:18 +0200 Subject: [PATCH 053/112] gh-79315: Add Include/cpython/sliceobject.h header (#139729) --- Include/cpython/sliceobject.h | 20 ++++++++++++++++++++ Include/sliceobject.h | 25 ++++++------------------- Makefile.pre.in | 1 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 +++ 5 files changed, 31 insertions(+), 19 deletions(-) create mode 100644 Include/cpython/sliceobject.h diff --git a/Include/cpython/sliceobject.h b/Include/cpython/sliceobject.h new file mode 100644 index 00000000000..4c3ea1faceb --- /dev/null +++ b/Include/cpython/sliceobject.h @@ -0,0 +1,20 @@ +#ifndef Py_CPYTHON_SLICEOBJECT_H +# error "this header file must not be included directly" +#endif + +/* Slice object interface */ + +/* +A slice object containing start, stop, and step data members (the +names are from range). After much talk with Guido, it was decided to +let these be any arbitrary python type. Py_None stands for omitted values. +*/ +typedef struct { + PyObject_HEAD + PyObject *start, *stop, *step; /* not NULL */ +} PySliceObject; + +PyAPI_FUNC(PyObject *) _PySlice_FromIndices(Py_ssize_t start, Py_ssize_t stop); +PyAPI_FUNC(int) _PySlice_GetLongIndices(PySliceObject *self, PyObject *length, + PyObject **start_ptr, PyObject **stop_ptr, + PyObject **step_ptr); diff --git a/Include/sliceobject.h b/Include/sliceobject.h index 35e2ea254ca..00c70a6e911 100644 --- a/Include/sliceobject.h +++ b/Include/sliceobject.h @@ -16,19 +16,6 @@ PyAPI_DATA(PyObject) _Py_EllipsisObject; /* Don't use this directly */ /* Slice object interface */ -/* - -A slice object containing start, stop, and step data members (the -names are from range). After much talk with Guido, it was decided to -let these be any arbitrary python type. Py_None stands for omitted values. -*/ -#ifndef Py_LIMITED_API -typedef struct { - PyObject_HEAD - PyObject *start, *stop, *step; /* not NULL */ -} PySliceObject; -#endif - PyAPI_DATA(PyTypeObject) PySlice_Type; PyAPI_DATA(PyTypeObject) PyEllipsis_Type; @@ -36,12 +23,6 @@ PyAPI_DATA(PyTypeObject) PyEllipsis_Type; PyAPI_FUNC(PyObject *) PySlice_New(PyObject* start, PyObject* stop, PyObject* step); -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PySlice_FromIndices(Py_ssize_t start, Py_ssize_t stop); -PyAPI_FUNC(int) _PySlice_GetLongIndices(PySliceObject *self, PyObject *length, - PyObject **start_ptr, PyObject **stop_ptr, - PyObject **step_ptr); -#endif PyAPI_FUNC(int) PySlice_GetIndices(PyObject *r, Py_ssize_t length, Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step); Py_DEPRECATED(3.7) @@ -63,6 +44,12 @@ PyAPI_FUNC(Py_ssize_t) PySlice_AdjustIndices(Py_ssize_t length, Py_ssize_t step); #endif +#ifndef Py_LIMITED_API +# define Py_CPYTHON_SLICEOBJECT_H +# include "cpython/sliceobject.h" +# undef Py_CPYTHON_SLICEOBJECT_H +#endif + #ifdef __cplusplus } #endif diff --git a/Makefile.pre.in b/Makefile.pre.in index ff9a84e4c27..a356ac0ea94 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1297,6 +1297,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/pythonrun.h \ $(srcdir)/Include/cpython/pythread.h \ $(srcdir)/Include/cpython/setobject.h \ + $(srcdir)/Include/cpython/sliceobject.h \ $(srcdir)/Include/cpython/traceback.h \ $(srcdir)/Include/cpython/tracemalloc.h \ $(srcdir)/Include/cpython/tupleobject.h \ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index d043d23eed8..25a7313eb26 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -191,6 +191,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 1868b222f18..edbd88d930a 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -492,6 +492,9 @@ Include\cpython + + Include\cpython + Include\cpython From 5a77f02d72e0735877fe4a5d615559d1541bc232 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 8 Oct 2025 00:49:24 +0200 Subject: [PATCH 054/112] gh-79315: Remove Include/pylock.h and Include/monitoring.h (#139731) Keep Include/cpython/pylock.h and Include/cpython/monitoring.h. --- Include/Python.h | 4 ++-- Include/cpython/monitoring.h | 16 ++++++++++++++-- Include/cpython/pylock.h | 15 +++++++++++++-- Include/monitoring.h | 18 ------------------ Include/pylock.h | 16 ---------------- Makefile.pre.in | 2 -- Modules/_testcapi/monitoring.c | 2 -- PCbuild/pythoncore.vcxproj | 2 +- PCbuild/pythoncore.vcxproj.filters | 3 --- 9 files changed, 30 insertions(+), 48 deletions(-) delete mode 100644 Include/monitoring.h delete mode 100644 Include/pylock.h diff --git a/Include/Python.h b/Include/Python.h index 261b4d316bd..78083bbf31d 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -78,7 +78,7 @@ __pragma(warning(disable: 4201)) #include "pybuffer.h" #include "pystats.h" #include "pyatomic.h" -#include "pylock.h" +#include "cpython/pylock.h" #include "critical_section.h" #include "object.h" #include "refcount.h" @@ -105,7 +105,7 @@ __pragma(warning(disable: 4201)) #include "setobject.h" #include "methodobject.h" #include "moduleobject.h" -#include "monitoring.h" +#include "cpython/monitoring.h" #include "cpython/funcobject.h" #include "cpython/classobject.h" #include "fileobject.h" diff --git a/Include/cpython/monitoring.h b/Include/cpython/monitoring.h index ce92942404c..5094c8c23ae 100644 --- a/Include/cpython/monitoring.h +++ b/Include/cpython/monitoring.h @@ -1,7 +1,13 @@ -#ifndef Py_CPYTHON_MONITORING_H -# error "this header file must not be included directly" +#ifndef Py_MONITORING_H +#define Py_MONITORING_H +#ifndef Py_LIMITED_API +#ifdef __cplusplus +extern "C" { #endif +// There is currently no limited API for monitoring + + /* Local events. * These require bytecode instrumentation */ @@ -267,3 +273,9 @@ PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike } #undef _PYMONITORING_IF_ACTIVE + +#ifdef __cplusplus +} +#endif +#endif // !Py_LIMITED_API +#endif // !Py_MONITORING_H diff --git a/Include/cpython/pylock.h b/Include/cpython/pylock.h index 63886fca28e..460ac2c9f80 100644 --- a/Include/cpython/pylock.h +++ b/Include/cpython/pylock.h @@ -1,7 +1,11 @@ -#ifndef Py_CPYTHON_LOCK_H -# error "this header file must not be included directly" +#ifndef Py_LOCK_H +#define Py_LOCK_H +#ifndef Py_LIMITED_API +#ifdef __cplusplus +extern "C" { #endif + #define _Py_UNLOCKED 0 #define _Py_LOCKED 1 @@ -72,3 +76,10 @@ _PyMutex_IsLocked(PyMutex *m) return (_Py_atomic_load_uint8(&m->_bits) & _Py_LOCKED) != 0; } #define PyMutex_IsLocked _PyMutex_IsLocked + + +#ifdef __cplusplus +} +#endif +#endif // !Py_LIMITED_API +#endif // !Py_LOCK_H diff --git a/Include/monitoring.h b/Include/monitoring.h deleted file mode 100644 index 985f7f230e4..00000000000 --- a/Include/monitoring.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef Py_MONITORING_H -#define Py_MONITORING_H -#ifdef __cplusplus -extern "C" { -#endif - -// There is currently no limited API for monitoring - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_MONITORING_H -# include "cpython/monitoring.h" -# undef Py_CPYTHON_MONITORING_H -#endif - -#ifdef __cplusplus -} -#endif -#endif /* !Py_MONITORING_H */ diff --git a/Include/pylock.h b/Include/pylock.h deleted file mode 100644 index 1939ef269d3..00000000000 --- a/Include/pylock.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef Py_LOCK_H -#define Py_LOCK_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_LOCK_H -# include "cpython/pylock.h" -# undef Py_CPYTHON_LOCK_H -#endif - -#ifdef __cplusplus -} -#endif -#endif /* !Py_LOCK_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index a356ac0ea94..061305a9ed1 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1191,14 +1191,12 @@ PYTHON_HEADERS= \ $(srcdir)/Include/intrcheck.h \ $(srcdir)/Include/iterobject.h \ $(srcdir)/Include/listobject.h \ - $(srcdir)/Include/pylock.h \ $(srcdir)/Include/longobject.h \ $(srcdir)/Include/marshal.h \ $(srcdir)/Include/memoryobject.h \ $(srcdir)/Include/methodobject.h \ $(srcdir)/Include/modsupport.h \ $(srcdir)/Include/moduleobject.h \ - $(srcdir)/Include/monitoring.h \ $(srcdir)/Include/object.h \ $(srcdir)/Include/objimpl.h \ $(srcdir)/Include/opcode.h \ diff --git a/Modules/_testcapi/monitoring.c b/Modules/_testcapi/monitoring.c index e041943492d..3f99836c1eb 100644 --- a/Modules/_testcapi/monitoring.c +++ b/Modules/_testcapi/monitoring.c @@ -1,8 +1,6 @@ #include "parts.h" #include "util.h" -#include "monitoring.h" - #define Py_BUILD_CORE #include "internal/pycore_instruments.h" diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 25a7313eb26..248b63f2577 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -172,6 +172,7 @@ + @@ -334,7 +335,6 @@ - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index edbd88d930a..da07a139b7c 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -120,9 +120,6 @@ Include - - Include - Include From 85ec35d2ab8b764aefd6810efd59ff54c92554e9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 8 Oct 2025 01:19:50 +0200 Subject: [PATCH 055/112] gh-79315: Add Include/cpython/structseq.h header (#139730) --- Include/cpython/structseq.h | 12 ++++++++++++ Include/structseq.h | 12 +++--------- Makefile.pre.in | 1 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 +++ 5 files changed, 20 insertions(+), 9 deletions(-) create mode 100644 Include/cpython/structseq.h diff --git a/Include/cpython/structseq.h b/Include/cpython/structseq.h new file mode 100644 index 00000000000..328fbe86143 --- /dev/null +++ b/Include/cpython/structseq.h @@ -0,0 +1,12 @@ +#ifndef Py_CPYTHON_STRUCTSEQ_H +# error "this header file must not be included directly" +#endif + +PyAPI_FUNC(void) PyStructSequence_InitType(PyTypeObject *type, + PyStructSequence_Desc *desc); +PyAPI_FUNC(int) PyStructSequence_InitType2(PyTypeObject *type, + PyStructSequence_Desc *desc); + +typedef PyTupleObject PyStructSequence; +#define PyStructSequence_SET_ITEM PyStructSequence_SetItem +#define PyStructSequence_GET_ITEM PyStructSequence_GetItem diff --git a/Include/structseq.h b/Include/structseq.h index 29e24fee54e..e52d6188030 100644 --- a/Include/structseq.h +++ b/Include/structseq.h @@ -21,12 +21,6 @@ typedef struct PyStructSequence_Desc { PyAPI_DATA(const char * const) PyStructSequence_UnnamedField; -#ifndef Py_LIMITED_API -PyAPI_FUNC(void) PyStructSequence_InitType(PyTypeObject *type, - PyStructSequence_Desc *desc); -PyAPI_FUNC(int) PyStructSequence_InitType2(PyTypeObject *type, - PyStructSequence_Desc *desc); -#endif PyAPI_FUNC(PyTypeObject*) PyStructSequence_NewType(PyStructSequence_Desc *desc); PyAPI_FUNC(PyObject *) PyStructSequence_New(PyTypeObject* type); @@ -35,9 +29,9 @@ PyAPI_FUNC(void) PyStructSequence_SetItem(PyObject*, Py_ssize_t, PyObject*); PyAPI_FUNC(PyObject*) PyStructSequence_GetItem(PyObject*, Py_ssize_t); #ifndef Py_LIMITED_API -typedef PyTupleObject PyStructSequence; -#define PyStructSequence_SET_ITEM PyStructSequence_SetItem -#define PyStructSequence_GET_ITEM PyStructSequence_GetItem +# define Py_CPYTHON_STRUCTSEQ_H +# include "cpython/structseq.h" +# undef Py_CPYTHON_STRUCTSEQ_H #endif #ifdef __cplusplus diff --git a/Makefile.pre.in b/Makefile.pre.in index 061305a9ed1..987d55a9bdb 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1296,6 +1296,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/pythread.h \ $(srcdir)/Include/cpython/setobject.h \ $(srcdir)/Include/cpython/sliceobject.h \ + $(srcdir)/Include/cpython/structseq.h \ $(srcdir)/Include/cpython/traceback.h \ $(srcdir)/Include/cpython/tracemalloc.h \ $(srcdir)/Include/cpython/tupleobject.h \ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 248b63f2577..71f508a7e8b 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -193,6 +193,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index da07a139b7c..547e9ee1abf 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -486,6 +486,9 @@ Include\cpython + + Include\cpython + Include\cpython From c53c55b6ed9550dbffc5aad08aad7b920d095972 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Tue, 7 Oct 2025 19:29:43 -0500 Subject: [PATCH 056/112] Remove long-obsolete PCbuild/field3.py script (GH-139739) As far as I can tell, it has not actually been used since 2003. --- PCbuild/field3.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 PCbuild/field3.py diff --git a/PCbuild/field3.py b/PCbuild/field3.py deleted file mode 100644 index edcbe36ae08..00000000000 --- a/PCbuild/field3.py +++ /dev/null @@ -1,35 +0,0 @@ -# An absurd workaround for the lack of arithmetic in MS's resource compiler. -# After building Python, run this, then paste the output into the appropriate -# part of PC\python_nt.rc. -# Example output: -# -# * For 2.3a0, -# * PY_MICRO_VERSION = 0 -# * PY_RELEASE_LEVEL = 'alpha' = 0xA -# * PY_RELEASE_SERIAL = 1 -# * -# * and 0*1000 + 10*10 + 1 = 101. -# */ -# #define FIELD3 101 - -import sys - -major, minor, micro, level, serial = sys.version_info -levelnum = {'alpha': 0xA, - 'beta': 0xB, - 'candidate': 0xC, - 'final': 0xF, - }[level] -string = sys.version.split()[0] # like '2.3a0' - -print(" * For %s," % string) -print(" * PY_MICRO_VERSION = %d" % micro) -print(" * PY_RELEASE_LEVEL = %r = %s" % (level, hex(levelnum))) -print(" * PY_RELEASE_SERIAL = %d" % serial) -print(" *") - -field3 = micro * 1000 + levelnum * 10 + serial - -print(" * and %d*1000 + %d*10 + %d = %d" % (micro, levelnum, serial, field3)) -print(" */") -print("#define FIELD3", field3) From a15aeec29efa5b3d5d5568278c13bb3fc45f52ef Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Tue, 7 Oct 2025 19:25:06 -0700 Subject: [PATCH 057/112] GH-139590: Run `ruff format` on pre-commit for Tools/wasm (#139591) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .pre-commit-config.yaml | 4 + Tools/wasm/.ruff.toml | 28 ++ Tools/wasm/emscripten/__main__.py | 70 ++-- .../wasm/emscripten/prepare_external_wasm.py | 5 +- Tools/wasm/emscripten/wasm_assets.py | 4 +- Tools/wasm/emscripten/web_example/server.py | 10 +- Tools/wasm/wasi.py | 10 +- Tools/wasm/wasi/__main__.py | 322 +++++++++++------- 8 files changed, 305 insertions(+), 148 deletions(-) create mode 100644 Tools/wasm/.ruff.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 014dab1d2b5..0e00ffb3bd2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,6 +34,10 @@ repos: name: Run Ruff (format) on Tools/build/check_warnings.py args: [--check, --config=Tools/build/.ruff.toml] files: ^Tools/build/check_warnings.py + - id: ruff-format + name: Run Ruff (format) on Tools/wasm/ + args: [--check, --config=Tools/wasm/.ruff.toml] + files: ^Tools/wasm/ - repo: https://github.com/psf/black-pre-commit-mirror rev: 25.9.0 diff --git a/Tools/wasm/.ruff.toml b/Tools/wasm/.ruff.toml new file mode 100644 index 00000000000..aabcf8dc4f5 --- /dev/null +++ b/Tools/wasm/.ruff.toml @@ -0,0 +1,28 @@ +extend = "../../.ruff.toml" # Inherit the project-wide settings + +[format] +preview = true +docstring-code-format = true + +[lint] +select = [ + "C4", # flake8-comprehensions + "E", # pycodestyle + "F", # pyflakes + "I", # isort + "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "PYI", # flake8-pyi + "RUF100", # Ban unused `# noqa` comments + "UP", # pyupgrade + "W", # pycodestyle + "YTT", # flake8-2020 +] +ignore = [ + "E501", # Line too long + "F541", # f-string without any placeholders + "PYI024", # Use `typing.NamedTuple` instead of `collections.namedtuple` + "PYI025", # Use `from collections.abc import Set as AbstractSet` +] diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index 202dd298199..fdf3142c0a3 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -33,7 +33,9 @@ PREFIX_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "prefix" LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" -LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/emscripten.py\n".encode("utf-8") +LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/emscripten.py\n".encode( + "utf-8" +) def updated_env(updates={}): @@ -45,7 +47,9 @@ def updated_env(updates={}): # https://reproducible-builds.org/docs/source-date-epoch/ git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] try: - epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip() + epoch = subprocess.check_output( + git_epoch_cmd, encoding="utf-8" + ).strip() env_defaults["SOURCE_DATE_EPOCH"] = epoch except subprocess.CalledProcessError: pass # Might be building from a tarball. @@ -79,7 +83,11 @@ def wrapper(context): terminal_width = 80 print("⎯" * terminal_width) print("📁", working_dir) - if clean_ok and getattr(context, "clean", False) and working_dir.exists(): + if ( + clean_ok + and getattr(context, "clean", False) + and working_dir.exists() + ): print("🚮 Deleting directory (--clean)...") shutil.rmtree(working_dir) @@ -128,7 +136,9 @@ def build_python_path(): if not binary.is_file(): binary = binary.with_suffix(".exe") if not binary.is_file(): - raise FileNotFoundError("Unable to find `python(.exe)` in " f"{NATIVE_BUILD_DIR}") + raise FileNotFoundError( + f"Unable to find `python(.exe)` in {NATIVE_BUILD_DIR}" + ) return binary @@ -158,7 +168,8 @@ def make_build_python(context, working_dir): cmd = [ binary, "-c", - "import sys; " "print(f'{sys.version_info.major}.{sys.version_info.minor}')", + "import sys; " + "print(f'{sys.version_info.major}.{sys.version_info.minor}')", ] version = subprocess.check_output(cmd, encoding="utf-8").strip() @@ -173,7 +184,9 @@ def check_shasum(file: str, expected_shasum: str): def download_and_unpack(working_dir: Path, url: str, expected_shasum: str): - with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete_on_close=False) as tmp_file: + with tempfile.NamedTemporaryFile( + suffix=".tar.gz", delete_on_close=False + ) as tmp_file: with urlopen(url) as response: shutil.copyfileobj(response, tmp_file) tmp_file.close() @@ -186,7 +199,11 @@ def make_emscripten_libffi(context, working_dir): ver = "3.4.6" libffi_dir = working_dir / f"libffi-{ver}" shutil.rmtree(libffi_dir, ignore_errors=True) - download_and_unpack(working_dir, f"https://github.com/libffi/libffi/releases/download/v{ver}/libffi-{ver}.tar.gz", "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e") + download_and_unpack( + working_dir, + f"https://github.com/libffi/libffi/releases/download/v{ver}/libffi-{ver}.tar.gz", + "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e", + ) call( [EMSCRIPTEN_DIR / "make_libffi.sh"], env=updated_env({"PREFIX": PREFIX_DIR}), @@ -200,7 +217,11 @@ def make_mpdec(context, working_dir): ver = "4.0.1" mpdec_dir = working_dir / f"mpdecimal-{ver}" shutil.rmtree(mpdec_dir, ignore_errors=True) - download_and_unpack(working_dir, f"https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{ver}.tar.gz", "96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8") + download_and_unpack( + working_dir, + f"https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{ver}.tar.gz", + "96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8", + ) call( [ "emconfigure", @@ -214,10 +235,7 @@ def make_mpdec(context, working_dir): quiet=context.quiet, ) call( - [ - "make", - "install" - ], + ["make", "install"], cwd=mpdec_dir, quiet=context.quiet, ) @@ -226,17 +244,15 @@ def make_mpdec(context, working_dir): @subdir(HOST_DIR, clean_ok=True) def configure_emscripten_python(context, working_dir): """Configure the emscripten/host build.""" - config_site = os.fsdecode( - EMSCRIPTEN_DIR / "config.site-wasm32-emscripten" - ) + config_site = os.fsdecode(EMSCRIPTEN_DIR / "config.site-wasm32-emscripten") emscripten_build_dir = working_dir.relative_to(CHECKOUT) python_build_dir = NATIVE_BUILD_DIR / "build" lib_dirs = list(python_build_dir.glob("lib.*")) - assert ( - len(lib_dirs) == 1 - ), f"Expected a single lib.* directory in {python_build_dir}" + assert len(lib_dirs) == 1, ( + f"Expected a single lib.* directory in {python_build_dir}" + ) lib_dir = os.fsdecode(lib_dirs[0]) pydebug = lib_dir.endswith("-pydebug") python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] @@ -290,7 +306,9 @@ def configure_emscripten_python(context, working_dir): quiet=context.quiet, ) - shutil.copy(EMSCRIPTEN_DIR / "node_entry.mjs", working_dir / "node_entry.mjs") + shutil.copy( + EMSCRIPTEN_DIR / "node_entry.mjs", working_dir / "node_entry.mjs" + ) node_entry = working_dir / "node_entry.mjs" exec_script = working_dir / "python.sh" @@ -383,13 +401,15 @@ def main(): subcommands = parser.add_subparsers(dest="subcommand") build = subcommands.add_parser("build", help="Build everything") configure_build = subcommands.add_parser( - "configure-build-python", help="Run `configure` for the " "build Python" + "configure-build-python", help="Run `configure` for the build Python" ) make_mpdec_cmd = subcommands.add_parser( - "make-mpdec", help="Clone mpdec repo, configure and build it for emscripten" + "make-mpdec", + help="Clone mpdec repo, configure and build it for emscripten", ) make_libffi_cmd = subcommands.add_parser( - "make-libffi", help="Clone libffi repo, configure and build it for emscripten" + "make-libffi", + help="Clone libffi repo, configure and build it for emscripten", ) make_build = subcommands.add_parser( "make-build-python", help="Run `make` for the build Python" @@ -457,7 +477,11 @@ def main(): if not context.subcommand: # No command provided, display help and exit - print("Expected one of", ", ".join(sorted(dispatch.keys())), file=sys.stderr) + print( + "Expected one of", + ", ".join(sorted(dispatch.keys())), + file=sys.stderr, + ) parser.print_help(sys.stderr) sys.exit(1) dispatch[context.subcommand](context) diff --git a/Tools/wasm/emscripten/prepare_external_wasm.py b/Tools/wasm/emscripten/prepare_external_wasm.py index 960e5aefd24..1b0a9de4b1f 100644 --- a/Tools/wasm/emscripten/prepare_external_wasm.py +++ b/Tools/wasm/emscripten/prepare_external_wasm.py @@ -19,6 +19,7 @@ }}); """ + def prepare_wasm(input_file, output_file, function_name): # Read the compiled WASM as binary and convert to hex wasm_bytes = Path(input_file).read_bytes() @@ -31,9 +32,7 @@ def prepare_wasm(input_file, output_file, function_name): ) Path(output_file).write_text(js_content) - print( - f"Successfully compiled {input_file} and generated {output_file}" - ) + print(f"Successfully compiled {input_file} and generated {output_file}") return 0 diff --git a/Tools/wasm/emscripten/wasm_assets.py b/Tools/wasm/emscripten/wasm_assets.py index 78da913ff60..90f318f319a 100755 --- a/Tools/wasm/emscripten/wasm_assets.py +++ b/Tools/wasm/emscripten/wasm_assets.py @@ -27,7 +27,9 @@ WASM_STDLIB_ZIP = ( WASM_LIB / f"python{sys.version_info.major}{sys.version_info.minor}.zip" ) -WASM_STDLIB = WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}" +WASM_STDLIB = ( + WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}" +) WASM_DYNLOAD = WASM_STDLIB / "lib-dynload" diff --git a/Tools/wasm/emscripten/web_example/server.py b/Tools/wasm/emscripten/web_example/server.py index 768e6f84e07..f2e6ed56c6b 100755 --- a/Tools/wasm/emscripten/web_example/server.py +++ b/Tools/wasm/emscripten/web_example/server.py @@ -6,10 +6,16 @@ description="Start a local webserver with a Python terminal." ) parser.add_argument( - "--port", type=int, default=8000, help="port for the http server to listen on" + "--port", + type=int, + default=8000, + help="port for the http server to listen on", ) parser.add_argument( - "--bind", type=str, default="127.0.0.1", help="Bind address (empty for all)" + "--bind", + type=str, + default="127.0.0.1", + help="Bind address (empty for all)", ) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index b49b27cbbbe..af55e03d10f 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -1,10 +1,12 @@ -if __name__ == "__main__": +if __name__ == "__main__": import pathlib import runpy import sys - print("⚠️ WARNING: This script is deprecated and slated for removal in Python 3.20; " - "execute the `wasi/` directory instead (i.e. `python Tools/wasm/wasi`)\n", - file=sys.stderr) + print( + "⚠️ WARNING: This script is deprecated and slated for removal in Python 3.20; " + "execute the `wasi/` directory instead (i.e. `python Tools/wasm/wasi`)\n", + file=sys.stderr, + ) runpy.run_path(pathlib.Path(__file__).parent / "wasi", run_name="__main__") diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py index 973d78caa08..a0658cb351a 100644 --- a/Tools/wasm/wasi/__main__.py +++ b/Tools/wasm/wasi/__main__.py @@ -4,6 +4,7 @@ import contextlib import functools import os + try: from os import process_cpu_count as cpu_count except ImportError: @@ -17,15 +18,19 @@ CHECKOUT = pathlib.Path(__file__).parent.parent.parent.parent -assert (CHECKOUT / "configure").is_file(), "Please update the location of the file" +assert (CHECKOUT / "configure").is_file(), ( + "Please update the location of the file" +) CROSS_BUILD_DIR = CHECKOUT / "cross-build" # Build platform can also be found via `config.guess`. BUILD_DIR = CROSS_BUILD_DIR / sysconfig.get_config_var("BUILD_GNU_TYPE") LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" -LOCAL_SETUP_MARKER = ("# Generated by Tools/wasm/wasi .\n" - "# Required to statically build extension modules.").encode("utf-8") +LOCAL_SETUP_MARKER = ( + "# Generated by Tools/wasm/wasi .\n" + "# Required to statically build extension modules." +).encode("utf-8") WASI_SDK_VERSION = 24 @@ -42,7 +47,9 @@ def updated_env(updates={}): # https://reproducible-builds.org/docs/source-date-epoch/ git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] try: - epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip() + epoch = subprocess.check_output( + git_epoch_cmd, encoding="utf-8" + ).strip() env_defaults["SOURCE_DATE_EPOCH"] = epoch except subprocess.CalledProcessError: pass # Might be building from a tarball. @@ -63,6 +70,7 @@ def updated_env(updates={}): def subdir(working_dir, *, clean_ok=False): """Decorator to change to a working directory.""" + def decorator(func): @functools.wraps(func) def wrapper(context): @@ -71,16 +79,20 @@ def wrapper(context): if callable(working_dir): working_dir = working_dir(context) try: - tput_output = subprocess.check_output(["tput", "cols"], - encoding="utf-8") + tput_output = subprocess.check_output( + ["tput", "cols"], encoding="utf-8" + ) except subprocess.CalledProcessError: terminal_width = 80 else: terminal_width = int(tput_output.strip()) print("⎯" * terminal_width) print("📁", working_dir) - if (clean_ok and getattr(context, "clean", False) and - working_dir.exists()): + if ( + clean_ok + and getattr(context, "clean", False) + and working_dir.exists() + ): print("🚮 Deleting directory (--clean)...") shutil.rmtree(working_dir) @@ -110,11 +122,14 @@ def call(command, *, context=None, quiet=False, logdir=None, **kwargs): stdout = None stderr = None else: - stdout = tempfile.NamedTemporaryFile("w", encoding="utf-8", - delete=False, - dir=logdir, - prefix="cpython-wasi-", - suffix=".log") + stdout = tempfile.NamedTemporaryFile( + "w", + encoding="utf-8", + delete=False, + dir=logdir, + prefix="cpython-wasi-", + suffix=".log", + ) stderr = subprocess.STDOUT print(f"📝 Logging output to {stdout.name} (--quiet)...") @@ -127,8 +142,9 @@ def build_python_path(): if not binary.is_file(): binary = binary.with_suffix(".exe") if not binary.is_file(): - raise FileNotFoundError("Unable to find `python(.exe)` in " - f"{BUILD_DIR}") + raise FileNotFoundError( + f"Unable to find `python(.exe)` in {BUILD_DIR}" + ) return binary @@ -136,9 +152,11 @@ def build_python_path(): def build_python_is_pydebug(): """Find out if the build Python is a pydebug build.""" test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)" - result = subprocess.run([build_python_path(), "-c", test], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + result = subprocess.run( + [build_python_path(), "-c", test], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) return bool(result.returncode) @@ -154,7 +172,7 @@ def configure_build_python(context, working_dir): print(f"📝 Creating {LOCAL_SETUP} ...") LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) - configure = [os.path.relpath(CHECKOUT / 'configure', working_dir)] + configure = [os.path.relpath(CHECKOUT / "configure", working_dir)] if context.args: configure.extend(context.args) @@ -164,13 +182,15 @@ def configure_build_python(context, working_dir): @subdir(BUILD_DIR) def make_build_python(context, working_dir): """Make/build the build Python.""" - call(["make", "--jobs", str(cpu_count()), "all"], - context=context) + call(["make", "--jobs", str(cpu_count()), "all"], context=context) binary = build_python_path() - cmd = [binary, "-c", - "import sys; " - "print(f'{sys.version_info.major}.{sys.version_info.minor}')"] + cmd = [ + binary, + "-c", + "import sys; " + "print(f'{sys.version_info.major}.{sys.version_info.minor}')", + ] version = subprocess.check_output(cmd, encoding="utf-8").strip() print(f"🎉 {binary} {version}") @@ -188,8 +208,11 @@ def find_wasi_sdk(): # Starting with WASI SDK 23, the tarballs went from containing a directory named # ``wasi-sdk-{WASI_SDK_VERSION}.0`` to e.g. # ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``. - potential_sdks = [path for path in opt_path.glob(f"wasi-sdk-{WASI_SDK_VERSION}.0*") - if path.is_dir()] + potential_sdks = [ + path + for path in opt_path.glob(f"wasi-sdk-{WASI_SDK_VERSION}.0*") + if path.is_dir() + ] if len(potential_sdks) == 1: return potential_sdks[0] elif (default_path := opt_path / "wasi-sdk").is_dir(): @@ -200,8 +223,13 @@ def wasi_sdk_env(context): """Calculate environment variables for building with wasi-sdk.""" wasi_sdk_path = context.wasi_sdk_path sysroot = wasi_sdk_path / "share" / "wasi-sysroot" - env = {"CC": "clang", "CPP": "clang-cpp", "CXX": "clang++", - "AR": "llvm-ar", "RANLIB": "ranlib"} + env = { + "CC": "clang", + "CPP": "clang-cpp", + "CXX": "clang++", + "AR": "llvm-ar", + "RANLIB": "ranlib", + } for env_var, binary_name in list(env.items()): env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) @@ -212,16 +240,20 @@ def wasi_sdk_env(context): env["PKG_CONFIG_PATH"] = "" env["PKG_CONFIG_LIBDIR"] = os.pathsep.join( - map(os.fsdecode, - [sysroot / "lib" / "pkgconfig", - sysroot / "share" / "pkgconfig"])) + map( + os.fsdecode, + [sysroot / "lib" / "pkgconfig", sysroot / "share" / "pkgconfig"], + ) + ) env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot) env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path) env["WASI_SYSROOT"] = os.fsdecode(sysroot) - env["PATH"] = os.pathsep.join([os.fsdecode(wasi_sdk_path / "bin"), - os.environ["PATH"]]) + env["PATH"] = os.pathsep.join([ + os.fsdecode(wasi_sdk_path / "bin"), + os.environ["PATH"], + ]) return env @@ -230,54 +262,70 @@ def wasi_sdk_env(context): def configure_wasi_python(context, working_dir): """Configure the WASI/host build.""" if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): - raise ValueError("WASI-SDK not found; " - "download from " - "https://github.com/WebAssembly/wasi-sdk and/or " - "specify via $WASI_SDK_PATH or --wasi-sdk") + raise ValueError( + "WASI-SDK not found; " + "download from " + "https://github.com/WebAssembly/wasi-sdk and/or " + "specify via $WASI_SDK_PATH or --wasi-sdk" + ) - config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "wasi" / "config.site-wasm32-wasi") + config_site = os.fsdecode( + CHECKOUT / "Tools" / "wasm" / "wasi" / "config.site-wasm32-wasi" + ) wasi_build_dir = working_dir.relative_to(CHECKOUT) python_build_dir = BUILD_DIR / "build" lib_dirs = list(python_build_dir.glob("lib.*")) - assert len(lib_dirs) == 1, f"Expected a single lib.* directory in {python_build_dir}" + assert len(lib_dirs) == 1, ( + f"Expected a single lib.* directory in {python_build_dir}" + ) lib_dir = os.fsdecode(lib_dirs[0]) python_version = lib_dir.rpartition("-")[-1] - sysconfig_data_dir = f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}" + sysconfig_data_dir = ( + f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}" + ) # Use PYTHONPATH to include sysconfig data which must be anchored to the # WASI guest's `/` directory. - args = {"GUEST_DIR": "/", - "HOST_DIR": CHECKOUT, - "ENV_VAR_NAME": "PYTHONPATH", - "ENV_VAR_VALUE": f"/{sysconfig_data_dir}", - "PYTHON_WASM": working_dir / "python.wasm"} + args = { + "GUEST_DIR": "/", + "HOST_DIR": CHECKOUT, + "ENV_VAR_NAME": "PYTHONPATH", + "ENV_VAR_VALUE": f"/{sysconfig_data_dir}", + "PYTHON_WASM": working_dir / "python.wasm", + } # Check dynamically for wasmtime in case it was specified manually via # `--host-runner`. if WASMTIME_HOST_RUNNER_VAR in context.host_runner: if wasmtime := shutil.which("wasmtime"): args[WASMTIME_VAR_NAME] = wasmtime else: - raise FileNotFoundError("wasmtime not found; download from " - "https://github.com/bytecodealliance/wasmtime") + raise FileNotFoundError( + "wasmtime not found; download from " + "https://github.com/bytecodealliance/wasmtime" + ) host_runner = context.host_runner.format_map(args) env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} build_python = os.fsdecode(build_python_path()) # The path to `configure` MUST be relative, else `python.wasm` is unable # to find the stdlib due to Python not recognizing that it's being # executed from within a checkout. - configure = [os.path.relpath(CHECKOUT / 'configure', working_dir), - f"--host={context.host_triple}", - f"--build={BUILD_DIR.name}", - f"--with-build-python={build_python}"] + configure = [ + os.path.relpath(CHECKOUT / "configure", working_dir), + f"--host={context.host_triple}", + f"--build={BUILD_DIR.name}", + f"--with-build-python={build_python}", + ] if build_python_is_pydebug(): configure.append("--with-pydebug") if context.args: configure.extend(context.args) - call(configure, - env=updated_env(env_additions | wasi_sdk_env(context)), - context=context) + call( + configure, + env=updated_env(env_additions | wasi_sdk_env(context)), + context=context, + ) python_wasm = working_dir / "python.wasm" exec_script = working_dir / "python.sh" @@ -291,9 +339,11 @@ def configure_wasi_python(context, working_dir): @subdir(lambda context: CROSS_BUILD_DIR / context.host_triple) def make_wasi_python(context, working_dir): """Run `make` for the WASI/host build.""" - call(["make", "--jobs", str(cpu_count()), "all"], - env=updated_env(), - context=context) + call( + ["make", "--jobs", str(cpu_count()), "all"], + env=updated_env(), + context=context, + ) exec_script = working_dir / "python.sh" call([exec_script, "--version"], quiet=False) @@ -305,11 +355,16 @@ def make_wasi_python(context, working_dir): def build_all(context): """Build everything.""" - steps = [configure_build_python, make_build_python, configure_wasi_python, - make_wasi_python] + steps = [ + configure_build_python, + make_build_python, + configure_wasi_python, + make_wasi_python, + ] for step in steps: step(context) + def clean_contents(context): """Delete all files created by this script.""" if CROSS_BUILD_DIR.exists(): @@ -324,76 +379,113 @@ def clean_contents(context): def main(): default_host_triple = "wasm32-wasip1" default_wasi_sdk = find_wasi_sdk() - default_host_runner = (f"{WASMTIME_HOST_RUNNER_VAR} run " - # Make sure the stack size will work for a pydebug - # build. - # Use 16 MiB stack. - "--wasm max-wasm-stack=16777216 " - # Enable thread support; causes use of preview1. - #"--wasm threads=y --wasi threads=y " - # Map the checkout to / to load the stdlib from /Lib. - "--dir {HOST_DIR}::{GUEST_DIR} " - # Set PYTHONPATH to the sysconfig data. - "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}") + default_host_runner = ( + f"{WASMTIME_HOST_RUNNER_VAR} run " + # Make sure the stack size will work for a pydebug + # build. + # Use 16 MiB stack. + "--wasm max-wasm-stack=16777216 " + # Enable thread support; causes use of preview1. + # "--wasm threads=y --wasi threads=y " + # Map the checkout to / to load the stdlib from /Lib. + "--dir {HOST_DIR}::{GUEST_DIR} " + # Set PYTHONPATH to the sysconfig data. + "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}" + ) default_logdir = pathlib.Path(tempfile.gettempdir()) parser = argparse.ArgumentParser() subcommands = parser.add_subparsers(dest="subcommand") build = subcommands.add_parser("build", help="Build everything") - configure_build = subcommands.add_parser("configure-build-python", - help="Run `configure` for the " - "build Python") - make_build = subcommands.add_parser("make-build-python", - help="Run `make` for the build Python") - configure_host = subcommands.add_parser("configure-host", - help="Run `configure` for the " - "host/WASI (pydebug builds " - "are inferred from the build " - "Python)") - make_host = subcommands.add_parser("make-host", - help="Run `make` for the host/WASI") - subcommands.add_parser("clean", help="Delete files and directories " - "created by this script") - for subcommand in build, configure_build, make_build, configure_host, make_host: - subcommand.add_argument("--quiet", action="store_true", default=False, - dest="quiet", - help="Redirect output from subprocesses to a log file") - subcommand.add_argument("--logdir", type=pathlib.Path, default=default_logdir, - help="Directory to store log files; " - f"defaults to {default_logdir}") + configure_build = subcommands.add_parser( + "configure-build-python", help="Run `configure` for the build Python" + ) + make_build = subcommands.add_parser( + "make-build-python", help="Run `make` for the build Python" + ) + configure_host = subcommands.add_parser( + "configure-host", + help="Run `configure` for the " + "host/WASI (pydebug builds " + "are inferred from the build " + "Python)", + ) + make_host = subcommands.add_parser( + "make-host", help="Run `make` for the host/WASI" + ) + subcommands.add_parser( + "clean", help="Delete files and directories created by this script" + ) + for subcommand in ( + build, + configure_build, + make_build, + configure_host, + make_host, + ): + subcommand.add_argument( + "--quiet", + action="store_true", + default=False, + dest="quiet", + help="Redirect output from subprocesses to a log file", + ) + subcommand.add_argument( + "--logdir", + type=pathlib.Path, + default=default_logdir, + help=f"Directory to store log files; defaults to {default_logdir}", + ) for subcommand in configure_build, configure_host: - subcommand.add_argument("--clean", action="store_true", default=False, - dest="clean", - help="Delete any relevant directories before building") + subcommand.add_argument( + "--clean", + action="store_true", + default=False, + dest="clean", + help="Delete any relevant directories before building", + ) for subcommand in build, configure_build, configure_host: - subcommand.add_argument("args", nargs="*", - help="Extra arguments to pass to `configure`") + subcommand.add_argument( + "args", nargs="*", help="Extra arguments to pass to `configure`" + ) for subcommand in build, configure_host: - subcommand.add_argument("--wasi-sdk", type=pathlib.Path, - dest="wasi_sdk_path", - default=default_wasi_sdk, - help=f"Path to the WASI SDK; defaults to {default_wasi_sdk}") - subcommand.add_argument("--host-runner", action="store", - default=default_host_runner, dest="host_runner", - help="Command template for running the WASI host; defaults to " - f"`{default_host_runner}`") + subcommand.add_argument( + "--wasi-sdk", + type=pathlib.Path, + dest="wasi_sdk_path", + default=default_wasi_sdk, + help=f"Path to the WASI SDK; defaults to {default_wasi_sdk}", + ) + subcommand.add_argument( + "--host-runner", + action="store", + default=default_host_runner, + dest="host_runner", + help="Command template for running the WASI host; defaults to " + f"`{default_host_runner}`", + ) for subcommand in build, configure_host, make_host: - subcommand.add_argument("--host-triple", action="store", - default=default_host_triple, - help="The target triple for the WASI host build; " - f"defaults to {default_host_triple}") + subcommand.add_argument( + "--host-triple", + action="store", + default=default_host_triple, + help="The target triple for the WASI host build; " + f"defaults to {default_host_triple}", + ) context = parser.parse_args() context.init_dir = pathlib.Path().absolute() - dispatch = {"configure-build-python": configure_build_python, - "make-build-python": make_build_python, - "configure-host": configure_wasi_python, - "make-host": make_wasi_python, - "build": build_all, - "clean": clean_contents} + dispatch = { + "configure-build-python": configure_build_python, + "make-build-python": make_build_python, + "configure-host": configure_wasi_python, + "make-host": make_wasi_python, + "build": build_all, + "clean": clean_contents, + } dispatch[context.subcommand](context) -if __name__ == "__main__": +if __name__ == "__main__": main() From c4e7d245d61ac4547ecf3362b28f64fc00aa88c0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 8 Oct 2025 12:10:58 +0200 Subject: [PATCH 058/112] gh-138342: Move _PyObject_VisitType() to the internal C API (#139734) --- Include/cpython/object.h | 4 ---- Include/internal/pycore_object.h | 4 ++++ Modules/_dbmmodule.c | 1 + Modules/_decimal/_decimal.c | 2 +- Modules/_gdbmmodule.c | 5 +++-- Modules/_multiprocessing/semaphore.c | 1 + Modules/_sqlite/prepare_protocol.c | 7 +++++++ Modules/_sqlite/statement.c | 7 +++++++ Modules/blake2module.c | 5 +++-- Modules/md5module.c | 3 ++- Modules/sha1module.c | 1 + Modules/sha2module.c | 3 ++- Modules/sha3module.c | 1 + Modules/socketmodule.c | 1 + Modules/unicodedata.c | 1 + 15 files changed, 35 insertions(+), 11 deletions(-) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 4e6f86f29d8..e2f87524c21 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -492,7 +492,3 @@ PyAPI_FUNC(int) PyUnstable_TryIncRef(PyObject *); PyAPI_FUNC(void) PyUnstable_EnableTryIncRef(PyObject *); PyAPI_FUNC(int) PyUnstable_Object_IsUniquelyReferenced(PyObject *); - -/* Utility for the tp_traverse slot of mutable heap types that have no other - * references. */ -PyAPI_FUNC(int) _PyObject_VisitType(PyObject *op, visitproc visit, void *arg); diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 40f8ca68c00..77560e5da66 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -1047,6 +1047,10 @@ static inline Py_ALWAYS_INLINE void _Py_INCREF_MORTAL(PyObject *op) } #endif +/* Utility for the tp_traverse slot of mutable heap types that have no other + * references. */ +PyAPI_FUNC(int) _PyObject_VisitType(PyObject *op, visitproc visit, void *arg); + #ifdef __cplusplus } #endif diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index 3fdcf22ffd5..06712015418 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -8,6 +8,7 @@ #endif #include "Python.h" +#include "pycore_object.h" // _PyObject_VisitType() #include #include diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 0696c150cd4..04b6695f8af 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -30,9 +30,9 @@ #endif #include +#include "pycore_object.h" // _PyObject_VisitType() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_typeobject.h" -#include "complexobject.h" #include diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index a4bc0a0f675..87b84976f49 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -8,11 +8,12 @@ #endif #include "Python.h" -#include "pycore_pyerrors.h" // _PyErr_SetLocaleString() +#include "pycore_object.h" // _PyObject_VisitType() +#include "pycore_pyerrors.h" // _PyErr_SetLocaleString() #include "gdbm.h" #include -#include // free() +#include // free() #include #include diff --git a/Modules/_multiprocessing/semaphore.c b/Modules/_multiprocessing/semaphore.c index d5a1f27e9ff..85cc0ac70a6 100644 --- a/Modules/_multiprocessing/semaphore.c +++ b/Modules/_multiprocessing/semaphore.c @@ -8,6 +8,7 @@ */ #include "multiprocessing.h" +#include "pycore_object.h" // _PyObject_VisitType() #ifdef HAVE_SYS_TIME_H # include // gettimeofday() diff --git a/Modules/_sqlite/prepare_protocol.c b/Modules/_sqlite/prepare_protocol.c index d7ac09e3947..0e2812e1e4f 100644 --- a/Modules/_sqlite/prepare_protocol.c +++ b/Modules/_sqlite/prepare_protocol.c @@ -21,8 +21,15 @@ * 3. This notice may not be removed or altered from any source distribution. */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "prepare_protocol.h" +#include "pycore_object.h" // _PyObject_VisitType() + + static int pysqlite_prepare_protocol_init(PyObject *self, PyObject *args, PyObject *kwargs) { diff --git a/Modules/_sqlite/statement.c b/Modules/_sqlite/statement.c index 77181104eda..c551ca59b33 100644 --- a/Modules/_sqlite/statement.c +++ b/Modules/_sqlite/statement.c @@ -21,10 +21,17 @@ * 3. This notice may not be removed or altered from any source distribution. */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "connection.h" #include "statement.h" #include "util.h" +#include "pycore_object.h" // _PyObject_VisitType() + + #define _pysqlite_Statement_CAST(op) ((pysqlite_Statement *)(op)) /* prototypes */ diff --git a/Modules/blake2module.c b/Modules/blake2module.c index 4921e8f945e..89b0ebd516f 100644 --- a/Modules/blake2module.c +++ b/Modules/blake2module.c @@ -16,9 +16,10 @@ #include "Python.h" #include "hashlib.h" -#include "pycore_strhex.h" // _Py_strhex() -#include "pycore_typeobject.h" #include "pycore_moduleobject.h" +#include "pycore_object.h" // _PyObject_VisitType() +#include "pycore_strhex.h" // _Py_strhex() +#include "pycore_typeobject.h" // QUICK CPU AUTODETECTION // diff --git a/Modules/md5module.c b/Modules/md5module.c index 6b6457427b6..56e9faf4c62 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -22,7 +22,8 @@ #endif #include "Python.h" -#include "pycore_strhex.h" // _Py_strhex() +#include "pycore_object.h" // _PyObject_VisitType() +#include "pycore_strhex.h" // _Py_strhex() #include "hashlib.h" diff --git a/Modules/sha1module.c b/Modules/sha1module.c index d64eb91458c..89e66240d1d 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -21,6 +21,7 @@ #include "Python.h" #include "hashlib.h" +#include "pycore_object.h" // _PyObject_VisitType() #include "pycore_strhex.h" // _Py_strhex() #include "pycore_typeobject.h" // _PyType_GetModuleState() diff --git a/Modules/sha2module.c b/Modules/sha2module.c index 66259fe47a0..9453b0be512 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -22,8 +22,9 @@ #include "Python.h" #include "pycore_moduleobject.h" // _PyModule_GetState() -#include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "pycore_object.h" // _PyObject_VisitType() #include "pycore_strhex.h" // _Py_strhex() +#include "pycore_typeobject.h" // _PyType_GetModuleState() #include "hashlib.h" diff --git a/Modules/sha3module.c b/Modules/sha3module.c index 14c543b8641..38c9bc0405b 100644 --- a/Modules/sha3module.c +++ b/Modules/sha3module.c @@ -21,6 +21,7 @@ #endif #include "Python.h" +#include "pycore_object.h" // _PyObject_VisitType() #include "pycore_strhex.h" // _Py_strhex() #include "pycore_typeobject.h" // _PyType_GetModuleState() #include "hashlib.h" diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index ec8b53273bc..fa153efd8b7 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -109,6 +109,7 @@ Local naming conventions: #include "pycore_capsule.h" // _PyCapsule_SetTraverse() #include "pycore_fileutils.h" // _Py_set_inheritable() #include "pycore_moduleobject.h" // _PyModule_GetState +#include "pycore_object.h" // _PyObject_VisitType() #include "pycore_time.h" // _PyTime_AsMilliseconds() #include "pycore_pystate.h" // _Py_AssertHoldsTstate() diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index 3c9bc35fa35..a3699beff7d 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -17,6 +17,7 @@ #endif #include "Python.h" +#include "pycore_object.h" // _PyObject_VisitType() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI #include From 5cea8435943b4f9d22c89c80d86c2ba3b392c6f9 Mon Sep 17 00:00:00 2001 From: Rogdham <3994389+Rogdham@users.noreply.github.com> Date: Wed, 8 Oct 2025 14:29:37 +0200 Subject: [PATCH 059/112] gh-137589: Zipfile tests: close file objects (GH-138080) Zipfile tests: close file objects --- Lib/test/test_zipfile/_path/test_path.py | 11 +++++--- Lib/test/test_zipfile/test_core.py | 34 +++++++++++++----------- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/Lib/test/test_zipfile/_path/test_path.py b/Lib/test/test_zipfile/_path/test_path.py index 958a586b0dc..e7931b6f394 100644 --- a/Lib/test/test_zipfile/_path/test_path.py +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -274,7 +274,8 @@ def test_pathlike_construction(self, alpharep): """ zipfile_ondisk = self.zipfile_ondisk(alpharep) pathlike = FakePath(str(zipfile_ondisk)) - zipfile.Path(pathlike) + root = zipfile.Path(pathlike) + root.root.close() @pass_alpharep def test_traverse_pathlike(self, alpharep): @@ -373,6 +374,7 @@ def test_root_on_disk(self, alpharep): root = zipfile.Path(self.zipfile_ondisk(alpharep)) assert root.name == 'alpharep.zip' == root.filename.name assert root.stem == 'alpharep' == root.filename.stem + root.root.close() @pass_alpharep def test_suffix(self, alpharep): @@ -574,11 +576,13 @@ def test_inheritance(self, alpharep): ) def test_pickle(self, alpharep, path_type, subpath): zipfile_ondisk = path_type(str(self.zipfile_ondisk(alpharep))) - - saved_1 = pickle.dumps(zipfile.Path(zipfile_ondisk, at=subpath)) + root = zipfile.Path(zipfile_ondisk, at=subpath) + saved_1 = pickle.dumps(root) + root.root.close() restored_1 = pickle.loads(saved_1) first, *rest = restored_1.iterdir() assert first.read_text(encoding='utf-8').startswith('content of ') + restored_1.root.close() @pass_alpharep def test_extract_orig_with_implied_dirs(self, alpharep): @@ -590,6 +594,7 @@ def test_extract_orig_with_implied_dirs(self, alpharep): # wrap the zipfile for its side effect zipfile.Path(zf) zf.extractall(source_path.parent) + zf.close() @pass_alpharep def test_getinfo_missing(self, alpharep): diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 6acfefc74d6..1edb5dde998 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -312,26 +312,26 @@ def test_low_compression(self): self.assertEqual(openobj.read(1), b'2') def test_writestr_compression(self): - zipfp = zipfile.ZipFile(TESTFN2, "w") - zipfp.writestr("b.txt", "hello world", compress_type=self.compression) - info = zipfp.getinfo('b.txt') - self.assertEqual(info.compress_type, self.compression) + with zipfile.ZipFile(TESTFN2, "w") as zipfp: + zipfp.writestr("b.txt", "hello world", compress_type=self.compression) + info = zipfp.getinfo('b.txt') + self.assertEqual(info.compress_type, self.compression) def test_writestr_compresslevel(self): - zipfp = zipfile.ZipFile(TESTFN2, "w", compresslevel=1) - zipfp.writestr("a.txt", "hello world", compress_type=self.compression) - zipfp.writestr("b.txt", "hello world", compress_type=self.compression, - compresslevel=2) + with zipfile.ZipFile(TESTFN2, "w", compresslevel=1) as zipfp: + zipfp.writestr("a.txt", "hello world", compress_type=self.compression) + zipfp.writestr("b.txt", "hello world", compress_type=self.compression, + compresslevel=2) - # Compression level follows the constructor. - a_info = zipfp.getinfo('a.txt') - self.assertEqual(a_info.compress_type, self.compression) - self.assertEqual(a_info.compress_level, 1) + # Compression level follows the constructor. + a_info = zipfp.getinfo('a.txt') + self.assertEqual(a_info.compress_type, self.compression) + self.assertEqual(a_info.compress_level, 1) - # Compression level is overridden. - b_info = zipfp.getinfo('b.txt') - self.assertEqual(b_info.compress_type, self.compression) - self.assertEqual(b_info._compresslevel, 2) + # Compression level is overridden. + b_info = zipfp.getinfo('b.txt') + self.assertEqual(b_info.compress_type, self.compression) + self.assertEqual(b_info._compresslevel, 2) def test_read_return_size(self): # Issue #9837: ZipExtFile.read() shouldn't return more bytes @@ -2330,6 +2330,7 @@ def test_empty_zipfile(self): zipf = zipfile.ZipFile(TESTFN, mode="r") except zipfile.BadZipFile: self.fail("Unable to create empty ZIP file in 'w' mode") + zipf.close() zipf = zipfile.ZipFile(TESTFN, mode="a") zipf.close() @@ -2337,6 +2338,7 @@ def test_empty_zipfile(self): zipf = zipfile.ZipFile(TESTFN, mode="r") except: self.fail("Unable to create empty ZIP file in 'a' mode") + zipf.close() def test_open_empty_file(self): # Issue 1710703: Check that opening a file with less than 22 bytes From 3d3f126e86c653f9727c42081d9ce2e4a4d15de3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 8 Oct 2025 14:56:00 +0200 Subject: [PATCH 060/112] gh-139353: Rename formatter_unicode.c to unicode_formatter.c (#139723) * Move Python/formatter_unicode.c to Objects/unicode_formatter.c. * Move Objects/stringlib/localeutil.h content into unicode_formatter.c. Remove localeutil.h. * Move _PyUnicode_InsertThousandsGrouping() to unicode_formatter.c and mark the function as static. * Rename unicode_fill() to _PyUnicode_Fill() and export it in pycore_unicodeobject.h. * Move MAX_UNICODE to pycore_unicodeobject.h as _Py_MAX_UNICODE. --- Include/internal/pycore_unicodeobject.h | 53 ++-- Makefile.pre.in | 3 +- Objects/stringlib/localeutil.h | 97 -------- .../unicode_formatter.c | 235 ++++++++++++++++++ Objects/unicodeobject.c | 186 +------------- PCbuild/_freeze_module.vcxproj | 2 +- PCbuild/_freeze_module.vcxproj.filters | 6 +- PCbuild/pythoncore.vcxproj | 2 +- PCbuild/pythoncore.vcxproj.filters | 6 +- Python/fileutils.c | 8 +- 10 files changed, 293 insertions(+), 305 deletions(-) delete mode 100644 Objects/stringlib/localeutil.h rename Python/formatter_unicode.c => Objects/unicode_formatter.c (88%) diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index c85c01da89a..f1c9bcd4788 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -11,6 +11,44 @@ extern "C" { #include "pycore_fileutils.h" // _Py_error_handler #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI +// Maximum code point of Unicode 6.0: 0x10ffff (1,114,111). +#define _Py_MAX_UNICODE 0x10ffff + + +static inline void +_PyUnicode_Fill(int kind, void *data, Py_UCS4 value, + Py_ssize_t start, Py_ssize_t length) +{ + assert(0 <= start); + switch (kind) { + case PyUnicode_1BYTE_KIND: { + assert(value <= 0xff); + Py_UCS1 ch = (unsigned char)value; + Py_UCS1 *to = (Py_UCS1 *)data + start; + memset(to, ch, length); + break; + } + case PyUnicode_2BYTE_KIND: { + assert(value <= 0xffff); + Py_UCS2 ch = (Py_UCS2)value; + Py_UCS2 *to = (Py_UCS2 *)data + start; + const Py_UCS2 *end = to + length; + for (; to < end; ++to) *to = ch; + break; + } + case PyUnicode_4BYTE_KIND: { + assert(value <= _Py_MAX_UNICODE); + Py_UCS4 ch = value; + Py_UCS4 * to = (Py_UCS4 *)data + start; + const Py_UCS4 *end = to + length; + for (; to < end; ++to) *to = ch; + break; + } + default: Py_UNREACHABLE(); + } +} + + /* --- Characters Type APIs ----------------------------------------------- */ extern int _PyUnicode_IsXidStart(Py_UCS4 ch); @@ -240,21 +278,6 @@ extern PyObject* _PyUnicode_XStrip( ); -/* Using explicit passed-in values, insert the thousands grouping - into the string pointed to by buffer. For the argument descriptions, - see Objects/stringlib/localeutil.h */ -extern Py_ssize_t _PyUnicode_InsertThousandsGrouping( - _PyUnicodeWriter *writer, - Py_ssize_t n_buffer, - PyObject *digits, - Py_ssize_t d_pos, - Py_ssize_t n_digits, - Py_ssize_t min_width, - const char *grouping, - PyObject *thousands_sep, - Py_UCS4 *maxchar, - int forward); - /* Dedent a string. Behaviour is expected to be an exact match of `textwrap.dedent`. Return a new reference on success, NULL with exception set on error. diff --git a/Makefile.pre.in b/Makefile.pre.in index 987d55a9bdb..a5223246845 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -501,7 +501,6 @@ PYTHON_OBJS= \ Python/pystrtod.o \ Python/pystrhex.o \ Python/dtoa.o \ - Python/formatter_unicode.o \ Python/fileutils.o \ Python/suggestions.o \ Python/perf_trampoline.o \ @@ -558,6 +557,7 @@ OBJECT_OBJS= \ Objects/tupleobject.o \ Objects/typeobject.o \ Objects/typevarobject.o \ + Objects/unicode_formatter.o \ Objects/unicodeobject.o \ Objects/unicodectype.o \ Objects/unionobject.o \ @@ -2091,7 +2091,6 @@ UNICODE_DEPS = \ $(srcdir)/Objects/stringlib/fastsearch.h \ $(srcdir)/Objects/stringlib/find.h \ $(srcdir)/Objects/stringlib/find_max_char.h \ - $(srcdir)/Objects/stringlib/localeutil.h \ $(srcdir)/Objects/stringlib/partition.h \ $(srcdir)/Objects/stringlib/replace.h \ $(srcdir)/Objects/stringlib/repr.h \ diff --git a/Objects/stringlib/localeutil.h b/Objects/stringlib/localeutil.h deleted file mode 100644 index a4ab701de00..00000000000 --- a/Objects/stringlib/localeutil.h +++ /dev/null @@ -1,97 +0,0 @@ -/* _PyUnicode_InsertThousandsGrouping() helper functions */ - -typedef struct { - const char *grouping; - char previous; - Py_ssize_t i; /* Where we're currently pointing in grouping. */ -} GroupGenerator; - - -static void -GroupGenerator_init(GroupGenerator *self, const char *grouping) -{ - self->grouping = grouping; - self->i = 0; - self->previous = 0; -} - - -/* Returns the next grouping, or 0 to signify end. */ -static Py_ssize_t -GroupGenerator_next(GroupGenerator *self) -{ - /* Note that we don't really do much error checking here. If a - grouping string contains just CHAR_MAX, for example, then just - terminate the generator. That shouldn't happen, but at least we - fail gracefully. */ - switch (self->grouping[self->i]) { - case 0: - return self->previous; - case CHAR_MAX: - /* Stop the generator. */ - return 0; - default: { - char ch = self->grouping[self->i]; - self->previous = ch; - self->i++; - return (Py_ssize_t)ch; - } - } -} - - -/* Fill in some digits, leading zeros, and thousands separator. All - are optional, depending on when we're called. */ -static void -InsertThousandsGrouping_fill(_PyUnicodeWriter *writer, Py_ssize_t *buffer_pos, - PyObject *digits, Py_ssize_t *digits_pos, - Py_ssize_t n_chars, Py_ssize_t n_zeros, - PyObject *thousands_sep, Py_ssize_t thousands_sep_len, - Py_UCS4 *maxchar, int forward) -{ - if (!writer) { - /* if maxchar > 127, maxchar is already set */ - if (*maxchar == 127 && thousands_sep) { - Py_UCS4 maxchar2 = PyUnicode_MAX_CHAR_VALUE(thousands_sep); - *maxchar = Py_MAX(*maxchar, maxchar2); - } - return; - } - - if (thousands_sep) { - if (!forward) { - *buffer_pos -= thousands_sep_len; - } - /* Copy the thousands_sep chars into the buffer. */ - _PyUnicode_FastCopyCharacters(writer->buffer, *buffer_pos, - thousands_sep, 0, - thousands_sep_len); - if (forward) { - *buffer_pos += thousands_sep_len; - } - } - - if (!forward) { - *buffer_pos -= n_chars; - *digits_pos -= n_chars; - } - _PyUnicode_FastCopyCharacters(writer->buffer, *buffer_pos, - digits, *digits_pos, - n_chars); - if (forward) { - *buffer_pos += n_chars; - *digits_pos += n_chars; - } - - if (n_zeros) { - if (!forward) { - *buffer_pos -= n_zeros; - } - int kind = PyUnicode_KIND(writer->buffer); - void *data = PyUnicode_DATA(writer->buffer); - unicode_fill(kind, data, '0', *buffer_pos, n_zeros); - if (forward) { - *buffer_pos += n_zeros; - } - } -} diff --git a/Python/formatter_unicode.c b/Objects/unicode_formatter.c similarity index 88% rename from Python/formatter_unicode.c rename to Objects/unicode_formatter.c index 30807f428c7..b8604d13559 100644 --- a/Python/formatter_unicode.c +++ b/Objects/unicode_formatter.c @@ -8,6 +8,241 @@ #include "pycore_unicodeobject.h" // PyUnicode_MAX_CHAR_VALUE() #include + +/* _PyUnicode_InsertThousandsGrouping() helper functions */ + +typedef struct { + const char *grouping; + char previous; + Py_ssize_t i; /* Where we're currently pointing in grouping. */ +} GroupGenerator; + + +static void +GroupGenerator_init(GroupGenerator *self, const char *grouping) +{ + self->grouping = grouping; + self->i = 0; + self->previous = 0; +} + + +/* Returns the next grouping, or 0 to signify end. */ +static Py_ssize_t +GroupGenerator_next(GroupGenerator *self) +{ + /* Note that we don't really do much error checking here. If a + grouping string contains just CHAR_MAX, for example, then just + terminate the generator. That shouldn't happen, but at least we + fail gracefully. */ + switch (self->grouping[self->i]) { + case 0: + return self->previous; + case CHAR_MAX: + /* Stop the generator. */ + return 0; + default: { + char ch = self->grouping[self->i]; + self->previous = ch; + self->i++; + return (Py_ssize_t)ch; + } + } +} + + +/* Fill in some digits, leading zeros, and thousands separator. All + are optional, depending on when we're called. */ +static void +InsertThousandsGrouping_fill(_PyUnicodeWriter *writer, Py_ssize_t *buffer_pos, + PyObject *digits, Py_ssize_t *digits_pos, + Py_ssize_t n_chars, Py_ssize_t n_zeros, + PyObject *thousands_sep, Py_ssize_t thousands_sep_len, + Py_UCS4 *maxchar, int forward) +{ + if (!writer) { + /* if maxchar > 127, maxchar is already set */ + if (*maxchar == 127 && thousands_sep) { + Py_UCS4 maxchar2 = PyUnicode_MAX_CHAR_VALUE(thousands_sep); + *maxchar = Py_MAX(*maxchar, maxchar2); + } + return; + } + + if (thousands_sep) { + if (!forward) { + *buffer_pos -= thousands_sep_len; + } + /* Copy the thousands_sep chars into the buffer. */ + _PyUnicode_FastCopyCharacters(writer->buffer, *buffer_pos, + thousands_sep, 0, + thousands_sep_len); + if (forward) { + *buffer_pos += thousands_sep_len; + } + } + + if (!forward) { + *buffer_pos -= n_chars; + *digits_pos -= n_chars; + } + _PyUnicode_FastCopyCharacters(writer->buffer, *buffer_pos, + digits, *digits_pos, + n_chars); + if (forward) { + *buffer_pos += n_chars; + *digits_pos += n_chars; + } + + if (n_zeros) { + if (!forward) { + *buffer_pos -= n_zeros; + } + int kind = PyUnicode_KIND(writer->buffer); + void *data = PyUnicode_DATA(writer->buffer); + _PyUnicode_Fill(kind, data, '0', *buffer_pos, n_zeros); + if (forward) { + *buffer_pos += n_zeros; + } + } +} + + +/** + * InsertThousandsGrouping: + * @writer: Unicode writer. + * @n_buffer: Number of characters in @buffer. + * @digits: Digits we're reading from. If count is non-NULL, this is unused. + * @d_pos: Start of digits string. + * @n_digits: The number of digits in the string, in which we want + * to put the grouping chars. + * @min_width: The minimum width of the digits in the output string. + * Output will be zero-padded on the left to fill. + * @grouping: see definition in localeconv(). + * @thousands_sep: see definition in localeconv(). + * + * There are 2 modes: counting and filling. If @writer is NULL, + * we are in counting mode, else filling mode. + * If counting, the required buffer size is returned. + * If filling, we know the buffer will be large enough, so we don't + * need to pass in the buffer size. + * Inserts thousand grouping characters (as defined by grouping and + * thousands_sep) into @writer. + * + * Return value: -1 on error, number of characters otherwise. + **/ +static Py_ssize_t +_PyUnicode_InsertThousandsGrouping( + _PyUnicodeWriter *writer, + Py_ssize_t n_buffer, + PyObject *digits, + Py_ssize_t d_pos, + Py_ssize_t n_digits, + Py_ssize_t min_width, + const char *grouping, + PyObject *thousands_sep, + Py_UCS4 *maxchar, + int forward) +{ + min_width = Py_MAX(0, min_width); + if (writer) { + assert(digits != NULL); + assert(maxchar == NULL); + } + else { + assert(digits == NULL); + assert(maxchar != NULL); + } + assert(0 <= d_pos); + assert(0 <= n_digits); + assert(grouping != NULL); + + Py_ssize_t count = 0; + Py_ssize_t n_zeros; + int loop_broken = 0; + int use_separator = 0; /* First time through, don't append the + separator. They only go between + groups. */ + Py_ssize_t buffer_pos; + Py_ssize_t digits_pos; + Py_ssize_t len; + Py_ssize_t n_chars; + Py_ssize_t remaining = n_digits; /* Number of chars remaining to + be looked at */ + /* A generator that returns all of the grouping widths, until it + returns 0. */ + GroupGenerator groupgen; + GroupGenerator_init(&groupgen, grouping); + const Py_ssize_t thousands_sep_len = PyUnicode_GET_LENGTH(thousands_sep); + + /* if digits are not grouped, thousands separator + should be an empty string */ + assert(!(grouping[0] == CHAR_MAX && thousands_sep_len != 0)); + + digits_pos = d_pos + (forward ? 0 : n_digits); + if (writer) { + buffer_pos = writer->pos + (forward ? 0 : n_buffer); + assert(buffer_pos <= PyUnicode_GET_LENGTH(writer->buffer)); + assert(digits_pos <= PyUnicode_GET_LENGTH(digits)); + } + else { + buffer_pos = forward ? 0 : n_buffer; + } + + if (!writer) { + *maxchar = 127; + } + + while ((len = GroupGenerator_next(&groupgen)) > 0) { + len = Py_MIN(len, Py_MAX(Py_MAX(remaining, min_width), 1)); + n_zeros = Py_MAX(0, len - remaining); + n_chars = Py_MAX(0, Py_MIN(remaining, len)); + + /* Use n_zero zero's and n_chars chars */ + + /* Count only, don't do anything. */ + count += (use_separator ? thousands_sep_len : 0) + n_zeros + n_chars; + + /* Copy into the writer. */ + InsertThousandsGrouping_fill(writer, &buffer_pos, + digits, &digits_pos, + n_chars, n_zeros, + use_separator ? thousands_sep : NULL, + thousands_sep_len, maxchar, forward); + + /* Use a separator next time. */ + use_separator = 1; + + remaining -= n_chars; + min_width -= len; + + if (remaining <= 0 && min_width <= 0) { + loop_broken = 1; + break; + } + min_width -= thousands_sep_len; + } + if (!loop_broken) { + /* We left the loop without using a break statement. */ + + len = Py_MAX(Py_MAX(remaining, min_width), 1); + n_zeros = Py_MAX(0, len - remaining); + n_chars = Py_MAX(0, Py_MIN(remaining, len)); + + /* Use n_zero zero's and n_chars chars */ + count += (use_separator ? thousands_sep_len : 0) + n_zeros + n_chars; + + /* Copy into the writer. */ + InsertThousandsGrouping_fill(writer, &buffer_pos, + digits, &digits_pos, + n_chars, n_zeros, + use_separator ? thousands_sep : NULL, + thousands_sep_len, maxchar, forward); + } + return count; +} + + /* Raises an exception about an unknown presentation type for this * type. */ diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 11ba147a744..c71f9d3f71d 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -104,9 +104,7 @@ NOTE: In the interpreter's initialization phase, some globals are currently */ -// Maximum code point of Unicode 6.0: 0x10ffff (1,114,111). -// The value must be the same in fileutils.c. -#define MAX_UNICODE 0x10ffff +#define MAX_UNICODE _Py_MAX_UNICODE #ifdef Py_DEBUG # define _PyUnicode_CHECK(op) _PyUnicode_CheckConsistency(op, 0) @@ -420,39 +418,6 @@ static void clear_global_interned_strings(void) return unicode_get_empty(); \ } while (0) -static inline void -unicode_fill(int kind, void *data, Py_UCS4 value, - Py_ssize_t start, Py_ssize_t length) -{ - assert(0 <= start); - switch (kind) { - case PyUnicode_1BYTE_KIND: { - assert(value <= 0xff); - Py_UCS1 ch = (unsigned char)value; - Py_UCS1 *to = (Py_UCS1 *)data + start; - memset(to, ch, length); - break; - } - case PyUnicode_2BYTE_KIND: { - assert(value <= 0xffff); - Py_UCS2 ch = (Py_UCS2)value; - Py_UCS2 *to = (Py_UCS2 *)data + start; - const Py_UCS2 *end = to + length; - for (; to < end; ++to) *to = ch; - break; - } - case PyUnicode_4BYTE_KIND: { - assert(value <= MAX_UNICODE); - Py_UCS4 ch = value; - Py_UCS4 * to = (Py_UCS4 *)data + start; - const Py_UCS4 *end = to + length; - for (; to < end; ++to) *to = ch; - break; - } - default: Py_UNREACHABLE(); - } -} - /* Fast detection of the most frequent whitespace characters */ const unsigned char _Py_ascii_whitespace[] = { @@ -9735,142 +9700,6 @@ any_find_slice(PyObject* s1, PyObject* s2, return result; } -/* _PyUnicode_InsertThousandsGrouping() helper functions */ -#include "stringlib/localeutil.h" - -/** - * InsertThousandsGrouping: - * @writer: Unicode writer. - * @n_buffer: Number of characters in @buffer. - * @digits: Digits we're reading from. If count is non-NULL, this is unused. - * @d_pos: Start of digits string. - * @n_digits: The number of digits in the string, in which we want - * to put the grouping chars. - * @min_width: The minimum width of the digits in the output string. - * Output will be zero-padded on the left to fill. - * @grouping: see definition in localeconv(). - * @thousands_sep: see definition in localeconv(). - * - * There are 2 modes: counting and filling. If @writer is NULL, - * we are in counting mode, else filling mode. - * If counting, the required buffer size is returned. - * If filling, we know the buffer will be large enough, so we don't - * need to pass in the buffer size. - * Inserts thousand grouping characters (as defined by grouping and - * thousands_sep) into @writer. - * - * Return value: -1 on error, number of characters otherwise. - **/ -Py_ssize_t -_PyUnicode_InsertThousandsGrouping( - _PyUnicodeWriter *writer, - Py_ssize_t n_buffer, - PyObject *digits, - Py_ssize_t d_pos, - Py_ssize_t n_digits, - Py_ssize_t min_width, - const char *grouping, - PyObject *thousands_sep, - Py_UCS4 *maxchar, - int forward) -{ - min_width = Py_MAX(0, min_width); - if (writer) { - assert(digits != NULL); - assert(maxchar == NULL); - } - else { - assert(digits == NULL); - assert(maxchar != NULL); - } - assert(0 <= d_pos); - assert(0 <= n_digits); - assert(grouping != NULL); - - Py_ssize_t count = 0; - Py_ssize_t n_zeros; - int loop_broken = 0; - int use_separator = 0; /* First time through, don't append the - separator. They only go between - groups. */ - Py_ssize_t buffer_pos; - Py_ssize_t digits_pos; - Py_ssize_t len; - Py_ssize_t n_chars; - Py_ssize_t remaining = n_digits; /* Number of chars remaining to - be looked at */ - /* A generator that returns all of the grouping widths, until it - returns 0. */ - GroupGenerator groupgen; - GroupGenerator_init(&groupgen, grouping); - const Py_ssize_t thousands_sep_len = PyUnicode_GET_LENGTH(thousands_sep); - - /* if digits are not grouped, thousands separator - should be an empty string */ - assert(!(grouping[0] == CHAR_MAX && thousands_sep_len != 0)); - - digits_pos = d_pos + (forward ? 0 : n_digits); - if (writer) { - buffer_pos = writer->pos + (forward ? 0 : n_buffer); - assert(buffer_pos <= PyUnicode_GET_LENGTH(writer->buffer)); - assert(digits_pos <= PyUnicode_GET_LENGTH(digits)); - } - else { - buffer_pos = forward ? 0 : n_buffer; - } - - if (!writer) { - *maxchar = 127; - } - - while ((len = GroupGenerator_next(&groupgen)) > 0) { - len = Py_MIN(len, Py_MAX(Py_MAX(remaining, min_width), 1)); - n_zeros = Py_MAX(0, len - remaining); - n_chars = Py_MAX(0, Py_MIN(remaining, len)); - - /* Use n_zero zero's and n_chars chars */ - - /* Count only, don't do anything. */ - count += (use_separator ? thousands_sep_len : 0) + n_zeros + n_chars; - - /* Copy into the writer. */ - InsertThousandsGrouping_fill(writer, &buffer_pos, - digits, &digits_pos, - n_chars, n_zeros, - use_separator ? thousands_sep : NULL, - thousands_sep_len, maxchar, forward); - - /* Use a separator next time. */ - use_separator = 1; - - remaining -= n_chars; - min_width -= len; - - if (remaining <= 0 && min_width <= 0) { - loop_broken = 1; - break; - } - min_width -= thousands_sep_len; - } - if (!loop_broken) { - /* We left the loop without using a break statement. */ - - len = Py_MAX(Py_MAX(remaining, min_width), 1); - n_zeros = Py_MAX(0, len - remaining); - n_chars = Py_MAX(0, Py_MIN(remaining, len)); - - /* Use n_zero zero's and n_chars chars */ - count += (use_separator ? thousands_sep_len : 0) + n_zeros + n_chars; - - /* Copy into the writer. */ - InsertThousandsGrouping_fill(writer, &buffer_pos, - digits, &digits_pos, - n_chars, n_zeros, - use_separator ? thousands_sep : NULL, - thousands_sep_len, maxchar, forward); - } - return count; -} Py_ssize_t PyUnicode_Count(PyObject *str, @@ -10427,7 +10256,7 @@ _PyUnicode_FastFill(PyObject *unicode, Py_ssize_t start, Py_ssize_t length, assert(fill_char <= PyUnicode_MAX_CHAR_VALUE(unicode)); assert(start >= 0); assert(start + length <= PyUnicode_GET_LENGTH(unicode)); - unicode_fill(kind, data, fill_char, start, length); + _PyUnicode_Fill(kind, data, fill_char, start, length); } Py_ssize_t @@ -10496,9 +10325,10 @@ pad(PyObject *self, kind = PyUnicode_KIND(u); data = PyUnicode_DATA(u); if (left) - unicode_fill(kind, data, fill, 0, left); + _PyUnicode_Fill(kind, data, fill, 0, left); if (right) - unicode_fill(kind, data, fill, left + _PyUnicode_LENGTH(self), right); + _PyUnicode_Fill(kind, data, fill, + left + _PyUnicode_LENGTH(self), right); _PyUnicode_FastCopyCharacters(u, left, self, 0, _PyUnicode_LENGTH(self)); assert(_PyUnicode_CheckConsistency(u, 1)); return u; @@ -11910,7 +11740,7 @@ unicode_expandtabs_impl(PyObject *self, int tabsize) if (tabsize > 0) { incr = tabsize - (line_pos % tabsize); line_pos += incr; - unicode_fill(kind, dest_data, ' ', j, incr); + _PyUnicode_Fill(kind, dest_data, ' ', j, incr); j += incr; } } @@ -15405,7 +15235,7 @@ unicode_format_arg_output(struct unicode_formatter_t *ctx, /* Pad left with the fill character if needed */ if (arg->width > len && !(arg->flags & F_LJUST)) { sublen = arg->width - len; - unicode_fill(writer->kind, writer->data, fill, writer->pos, sublen); + _PyUnicode_Fill(writer->kind, writer->data, fill, writer->pos, sublen); writer->pos += sublen; arg->width = len; } @@ -15437,7 +15267,7 @@ unicode_format_arg_output(struct unicode_formatter_t *ctx, /* Pad right with the fill character if needed */ if (arg->width > len) { sublen = arg->width - len; - unicode_fill(writer->kind, writer->data, ' ', writer->pos, sublen); + _PyUnicode_Fill(writer->kind, writer->data, ' ', writer->pos, sublen); writer->pos += sublen; } return 0; diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 5ceddf759b8..c4a11fa9b24 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -165,6 +165,7 @@ + @@ -209,7 +210,6 @@ - diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 332d466b1f7..7bbbec2c988 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -160,9 +160,6 @@ Source Files - - Source Files - Source Files @@ -487,6 +484,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 71f508a7e8b..e2e1e415827 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -559,6 +559,7 @@ + @@ -605,7 +606,6 @@ - $(GeneratedFrozenModulesDir)Python;%(AdditionalIncludeDirectories) diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 547e9ee1abf..7e7ed9c2ae6 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1274,6 +1274,9 @@ Objects + + Objects + Objects @@ -1382,9 +1385,6 @@ Python - - Python - Python diff --git a/Python/fileutils.c b/Python/fileutils.c index 2a3f12d4e87..b808229716f 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2,6 +2,7 @@ #include "pycore_fileutils.h" // fileutils definitions #include "pycore_runtime.h" // _PyRuntime #include "pycore_pystate.h" // _Py_AssertHoldsTstate() +#include "pycore_unicodeobject.h" // _Py_MAX_UNICODE #include "osdefs.h" // SEP #include // mbstowcs() @@ -50,9 +51,6 @@ extern int winerror_to_errno(int); int _Py_open_cloexec_works = -1; #endif -// The value must be the same in unicodeobject.c. -#define MAX_UNICODE 0x10ffff - // mbstowcs() and mbrtowc() errors static const size_t DECODE_ERROR = ((size_t)-1); #ifdef HAVE_MBRTOWC @@ -123,7 +121,7 @@ is_valid_wide_char(wchar_t ch) { #ifdef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION /* Oracle Solaris doesn't use Unicode code points as wchar_t encoding - for non-Unicode locales, which makes values higher than MAX_UNICODE + for non-Unicode locales, which makes values higher than _Py_MAX_UNICODE possibly valid. */ return 1; #endif @@ -132,7 +130,7 @@ is_valid_wide_char(wchar_t ch) return 0; } #if SIZEOF_WCHAR_T > 2 - if (ch > MAX_UNICODE) { + if (ch > _Py_MAX_UNICODE) { // bpo-35883: Reject characters outside [U+0000; U+10ffff] range. // The glibc mbstowcs() UTF-8 decoder does not respect the RFC 3629, // it creates characters outside the [U+0000; U+10ffff] range: From 8b9606a2c577b0a0f7f425959a27a8554a9b474f Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 8 Oct 2025 06:28:29 -0700 Subject: [PATCH 061/112] gh-139452: Clarify redirect_stdout, stderr behavior (gh-139490) --- Doc/library/contextlib.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 176be4ff333..d0fa645093a 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -327,10 +327,10 @@ Functions and classes provided: .. function:: redirect_stdout(new_target) Context manager for temporarily redirecting :data:`sys.stdout` to - another file or file-like object. + another :term:`file object`. This tool adds flexibility to existing functions or classes whose output - is hardwired to stdout. + is hardwired to :data:`sys.stdout`. For example, the output of :func:`help` normally is sent to *sys.stdout*. You can capture that output in a string by redirecting the output to an @@ -366,8 +366,8 @@ Functions and classes provided: .. function:: redirect_stderr(new_target) - Similar to :func:`~contextlib.redirect_stdout` but redirecting - :data:`sys.stderr` to another file or file-like object. + Similar to :func:`~contextlib.redirect_stdout` but redirecting the global + :data:`sys.stderr` to another :term:`file object`. This context manager is :ref:`reentrant `. From 49fb46f555881c9f2d20ca87c8187c8718217c77 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 8 Oct 2025 19:49:54 +0530 Subject: [PATCH 062/112] gh-139774: use relaxed atomics for datetime hashes (#139775) --- Modules/_datetimemodule.c | 47 ++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 12d316985fc..46c4f57984b 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -15,6 +15,7 @@ #include "pycore_time.h" // _PyTime_ObjectToTime_t() #include "pycore_unicodeobject.h" // _PyUnicode_Copy() #include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_pyatomic_ft_wrappers.h" #include "datetime.h" @@ -2540,14 +2541,16 @@ static Py_hash_t delta_hash(PyObject *op) { PyDateTime_Delta *self = PyDelta_CAST(op); - if (self->hashcode == -1) { + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->hashcode); + if (hash == -1) { PyObject *temp = delta_getstate(self); if (temp != NULL) { - self->hashcode = PyObject_Hash(temp); + hash = PyObject_Hash(temp); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); Py_DECREF(temp); } } - return self->hashcode; + return hash; } static PyObject * @@ -3921,12 +3924,14 @@ static Py_hash_t date_hash(PyObject *op) { PyDateTime_Date *self = PyDate_CAST(op); - if (self->hashcode == -1) { - self->hashcode = generic_hash( + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->hashcode); + if (hash == -1) { + hash = generic_hash( (unsigned char *)self->data, _PyDateTime_DATE_DATASIZE); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); } - return self->hashcode; + return hash; } static PyObject * @@ -5043,7 +5048,8 @@ static Py_hash_t time_hash(PyObject *op) { PyDateTime_Time *self = PyTime_CAST(op); - if (self->hashcode == -1) { + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->hashcode); + if (hash == -1) { PyObject *offset, *self0; if (TIME_GET_FOLD(self)) { self0 = new_time_ex2(TIME_GET_HOUR(self), @@ -5065,10 +5071,11 @@ time_hash(PyObject *op) return -1; /* Reduce this to a hash of another object. */ - if (offset == Py_None) - self->hashcode = generic_hash( + if (offset == Py_None) { + hash = generic_hash( (unsigned char *)self->data, _PyDateTime_TIME_DATASIZE); - else { + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); + } else { PyObject *temp1, *temp2; int seconds, microseconds; assert(HASTZINFO(self)); @@ -5087,12 +5094,13 @@ time_hash(PyObject *op) Py_DECREF(offset); return -1; } - self->hashcode = PyObject_Hash(temp2); + hash = PyObject_Hash(temp2); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); Py_DECREF(temp2); } Py_DECREF(offset); } - return self->hashcode; + return hash; } /*[clinic input] @@ -6627,7 +6635,8 @@ static Py_hash_t datetime_hash(PyObject *op) { PyDateTime_DateTime *self = PyDateTime_CAST(op); - if (self->hashcode == -1) { + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->hashcode); + if (hash == -1) { PyObject *offset, *self0; if (DATE_GET_FOLD(self)) { self0 = new_datetime_ex2(GET_YEAR(self), @@ -6652,10 +6661,11 @@ datetime_hash(PyObject *op) return -1; /* Reduce this to a hash of another object. */ - if (offset == Py_None) - self->hashcode = generic_hash( + if (offset == Py_None) { + hash = generic_hash( (unsigned char *)self->data, _PyDateTime_DATETIME_DATASIZE); - else { + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); + } else { PyObject *temp1, *temp2; int days, seconds; @@ -6679,12 +6689,13 @@ datetime_hash(PyObject *op) Py_DECREF(offset); return -1; } - self->hashcode = PyObject_Hash(temp2); + hash = PyObject_Hash(temp2); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); Py_DECREF(temp2); } Py_DECREF(offset); } - return self->hashcode; + return hash; } /*[clinic input] From 59a6f9d8c52af40ccd31e5dca848f71808c24b06 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 8 Oct 2025 16:34:19 +0200 Subject: [PATCH 063/112] gh-135676: Add a summary of source characters (GH-138194) Co-authored-by: Carol Willing Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Co-authored-by: Blaise Pabon Co-authored-by: Micha Albert Co-authored-by: KeithTheEE --- Doc/reference/lexical_analysis.rst | 76 ++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 05ae410c168..0b0dba1a996 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -10,12 +10,76 @@ Lexical analysis A Python program is read by a *parser*. Input to the parser is a stream of :term:`tokens `, generated by the *lexical analyzer* (also known as the *tokenizer*). -This chapter describes how the lexical analyzer breaks a file into tokens. +This chapter describes how the lexical analyzer produces these tokens. -Python reads program text as Unicode code points; the encoding of a source file -can be given by an encoding declaration and defaults to UTF-8, see :pep:`3120` -for details. If the source file cannot be decoded, a :exc:`SyntaxError` is -raised. +The lexical analyzer determines the program text's :ref:`encoding ` +(UTF-8 by default), and decodes the text into +:ref:`source characters `. +If the text cannot be decoded, a :exc:`SyntaxError` is raised. + +Next, the lexical analyzer uses the source characters to generate a stream of tokens. +The type of a generated token generally depends on the next source character to +be processed. Similarly, other special behavior of the analyzer depends on +the first source character that hasn't yet been processed. +The following table gives a quick summary of these source characters, +with links to sections that contain more information. + +.. list-table:: + :header-rows: 1 + + * - Character + - Next token (or other relevant documentation) + + * - * space + * tab + * formfeed + - * :ref:`Whitespace ` + + * - * CR, LF + - * :ref:`New line ` + * :ref:`Indentation ` + + * - * backslash (``\``) + - * :ref:`Explicit line joining ` + * (Also significant in :ref:`string escape sequences `) + + * - * hash (``#``) + - * :ref:`Comment ` + + * - * quote (``'``, ``"``) + - * :ref:`String literal ` + + * - * ASCII letter (``a``-``z``, ``A``-``Z``) + * non-ASCII character + - * :ref:`Name ` + * Prefixed :ref:`string or bytes literal ` + + * - * underscore (``_``) + - * :ref:`Name ` + * (Can also be part of :ref:`numeric literals `) + + * - * number (``0``-``9``) + - * :ref:`Numeric literal ` + + * - * dot (``.``) + - * :ref:`Numeric literal ` + * :ref:`Operator ` + + * - * question mark (``?``) + * dollar (``$``) + * + .. (the following uses zero-width space characters to render + .. a literal backquote) + + backquote (``​`​``) + * control character + - * Error (outside string literals and comments) + + * - * other printing character + - * :ref:`Operator or delimiter ` + + * - * end of file + - * :ref:`End marker ` .. _line-structure: @@ -120,6 +184,8 @@ If an encoding is declared, the encoding name must be recognized by Python encoding is used for all lexical analysis, including string literals, comments and identifiers. +.. _lexical-source-character: + All lexical analysis, including string literals, comments and identifiers, works on Unicode text decoded using the source encoding. Any Unicode code point, except the NUL control character, can appear in From 570d17259f824302d20567a3a2f32c67ebdaefcd Mon Sep 17 00:00:00 2001 From: Wulian233 <1055917385@qq.com> Date: Wed, 8 Oct 2025 23:00:54 +0800 Subject: [PATCH 064/112] gh-139769: Update `PCBuild/find_python.bat` to allow discovery of Python 3.14 (GH-139770) Enable 3.14 py.exe can be use on PCBuild --- PCbuild/find_python.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat index d65d080ca71..841d83968c6 100644 --- a/PCbuild/find_python.bat +++ b/PCbuild/find_python.bat @@ -47,7 +47,7 @@ @rem If py.exe finds a recent enough version, use that one @rem It is fine to add new versions to this list when they have released, @rem but we do not use prerelease builds here. -@for %%p in (3.13 3.12 3.11 3.10) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found +@for %%p in (3.14 3.13 3.12 3.11 3.10) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found @if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%" @set _Py_NUGET=%NUGET% From b04a57deef66ce08233be57d1ab5873388df2cea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 8 Oct 2025 17:22:44 +0200 Subject: [PATCH 065/112] gh-139748: fix leaks in AC error paths when using unicode FS-based converters (#139765) --- Lib/test/test_clinic.py | 2 + Lib/test/test_compile.py | 15 +++++++ Lib/test/test_symtable.py | 7 ++++ ...-10-08-13-52-00.gh-issue-139748.jq0yFJ.rst | 2 + Modules/_ssl.c | 5 +-- Modules/clinic/_ssl.c.h | 7 +++- Modules/clinic/socketmodule.c.h | 7 +++- Modules/clinic/symtablemodule.c.h | 7 +++- Modules/posixmodule.c | 39 +++++++------------ Modules/socketmodule.c | 5 +-- Modules/symtablemodule.c | 6 +-- Python/bltinmodule.c | 5 +-- Python/clinic/bltinmodule.c.h | 7 +++- Tools/clinic/libclinic/converters.py | 20 ++++++++++ 14 files changed, 88 insertions(+), 46 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-08-13-52-00.gh-issue-139748.jq0yFJ.rst diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index d54dd546ea3..e0dbb062eb0 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -2980,6 +2980,8 @@ def test_cli_converters(self): "uint64", "uint8", "unicode", + "unicode_fs_decoded", + "unicode_fs_encoded", "unsigned_char", "unsigned_int", "unsigned_long", diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 277a2a18754..1660dabe681 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -651,6 +651,21 @@ def test_compile_filename(self): compile('pass', filename, 'exec') self.assertRaises(TypeError, compile, 'pass', list(b'file.py'), 'exec') + def test_compile_filename_refleak(self): + # Regression tests for reference leak in PyUnicode_FSDecoder. + # See https://github.com/python/cpython/issues/139748. + mortal_str = 'this is a mortal string' + # check error path when 'mode' AC conversion failed + self.assertRaises(TypeError, compile, b'', mortal_str, mode=1234) + # check error path when 'optimize' AC conversion failed + self.assertRaises(OverflowError, compile, b'', mortal_str, + 'exec', optimize=1 << 1000) + # check error path when 'dont_inherit' AC conversion failed + class EvilBool: + def __bool__(self): raise ValueError + self.assertRaises(ValueError, compile, b'', mortal_str, + 'exec', dont_inherit=EvilBool()) + @support.cpython_only def test_same_filename_used(self): s = """def f(): pass\ndef g(): pass""" diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 26c75fcbc2b..943e63fc13c 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -579,6 +579,13 @@ def test_nested_genexpr(self): self.assertEqual(sorted(st.get_identifiers()), [".0", "y"]) self.assertEqual(st.get_children(), []) + def test__symtable_refleak(self): + # Regression test for reference leak in PyUnicode_FSDecoder. + # See https://github.com/python/cpython/issues/139748. + mortal_str = 'this is a mortal string' + # check error path when 'compile_type' AC conversion failed + self.assertRaises(TypeError, symtable.symtable, '', mortal_str, 1) + class ComprehensionTests(unittest.TestCase): def get_identifiers_recursive(self, st, res): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-08-13-52-00.gh-issue-139748.jq0yFJ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-08-13-52-00.gh-issue-139748.jq0yFJ.rst new file mode 100644 index 00000000000..3b190f3f6df --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-08-13-52-00.gh-issue-139748.jq0yFJ.rst @@ -0,0 +1,2 @@ +Fix reference leaks in error branches of functions accepting path strings or +bytes such as :func:`compile` and :func:`os.system`. Patch by Bénédikt Tran. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index e1bdc4033ba..1fa44ef1de4 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -1858,14 +1858,14 @@ _certificate_to_der(_sslmodulestate *state, X509 *certificate) /*[clinic input] _ssl._test_decode_cert - path: object(converter="PyUnicode_FSConverter") + path: unicode_fs_encoded / [clinic start generated code]*/ static PyObject * _ssl__test_decode_cert_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=96becb9abb23c091 input=cdeaaf02d4346628]*/ +/*[clinic end generated code: output=96becb9abb23c091 input=cb4988d5e651a4f8]*/ { PyObject *retval = NULL; X509 *x=NULL; @@ -1895,7 +1895,6 @@ _ssl__test_decode_cert_impl(PyObject *module, PyObject *path) X509_free(x); fail0: - Py_DECREF(path); if (cert != NULL) BIO_free(cert); return retval; } diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 0e8e4a7c8a7..d1fb024903e 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -48,7 +48,7 @@ static PyObject * _ssl__test_decode_cert(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - PyObject *path; + PyObject *path = NULL; if (!PyUnicode_FSConverter(arg, &path)) { goto exit; @@ -56,6 +56,9 @@ _ssl__test_decode_cert(PyObject *module, PyObject *arg) return_value = _ssl__test_decode_cert_impl(module, path); exit: + /* Cleanup for path */ + Py_XDECREF(path); + return return_value; } @@ -3322,4 +3325,4 @@ exit: #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=5a630a1e83927d47 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3b6c9cbfc4660ecb input=a9049054013a1b77]*/ diff --git a/Modules/clinic/socketmodule.c.h b/Modules/clinic/socketmodule.c.h index 0cedab597db..e0cc1c50dcb 100644 --- a/Modules/clinic/socketmodule.c.h +++ b/Modules/clinic/socketmodule.c.h @@ -479,7 +479,7 @@ static PyObject * _socket_if_nametoindex(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - PyObject *oname; + PyObject *oname = NULL; if (!PyUnicode_FSConverter(arg, &oname)) { goto exit; @@ -487,6 +487,9 @@ _socket_if_nametoindex(PyObject *module, PyObject *arg) return_value = _socket_if_nametoindex_impl(module, oname); exit: + /* Cleanup for oname */ + Py_XDECREF(oname); + return return_value; } @@ -538,4 +541,4 @@ exit: #ifndef _SOCKET_IF_INDEXTONAME_METHODDEF #define _SOCKET_IF_INDEXTONAME_METHODDEF #endif /* !defined(_SOCKET_IF_INDEXTONAME_METHODDEF) */ -/*[clinic end generated code: output=0376c46b76ae2bce input=a9049054013a1b77]*/ +/*[clinic end generated code: output=36051ebf6ad1e6f8 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/symtablemodule.c.h b/Modules/clinic/symtablemodule.c.h index 2ecd3afc00d..bd55d77c540 100644 --- a/Modules/clinic/symtablemodule.c.h +++ b/Modules/clinic/symtablemodule.c.h @@ -22,7 +22,7 @@ _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *source; - PyObject *filename; + PyObject *filename = NULL; const char *startstr; if (!_PyArg_CheckPositional("symtable", nargs, 3, 3)) { @@ -48,6 +48,9 @@ _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return_value = _symtable_symtable_impl(module, source, filename, startstr); exit: + /* Cleanup for filename */ + Py_XDECREF(filename); + return return_value; } -/*[clinic end generated code: output=931964a76a72f850 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7a8545d9a1efe837 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f50167c223e..4189d300856 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -3137,17 +3137,6 @@ class dev_t_return_converter(unsigned_long_return_converter): conversion_fn = '_PyLong_FromDev' unsigned_cast = '(dev_t)' -class FSConverter_converter(CConverter): - type = 'PyObject *' - converter = 'PyUnicode_FSConverter' - def converter_init(self): - if self.default is not unspecified: - fail("FSConverter_converter does not support default values") - self.c_default = 'NULL' - - def cleanup(self): - return "Py_XDECREF(" + self.name + ");\n" - class pid_t_converter(CConverter): type = 'pid_t' format_unit = '" _Py_PARSE_PID "' @@ -3211,7 +3200,7 @@ class confname_converter(CConverter): """, argname=argname, converter=self.converter, table=self.table) [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=8189d5ae78244626]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=d2759f2332cd39b3]*/ /*[clinic input] @@ -6135,14 +6124,14 @@ os_system_impl(PyObject *module, const wchar_t *command) /*[clinic input] os.system -> long - command: FSConverter + command: unicode_fs_encoded Execute the command in a subshell. [clinic start generated code]*/ static long os_system_impl(PyObject *module, PyObject *command) -/*[clinic end generated code: output=290fc437dd4f33a0 input=86a58554ba6094af]*/ +/*[clinic end generated code: output=290fc437dd4f33a0 input=47c6f24b6dc92881]*/ { long result; const char *bytes = PyBytes_AsString(command); @@ -9328,7 +9317,7 @@ os_getgroups_impl(PyObject *module) /*[clinic input] os.initgroups - username as oname: FSConverter + username as oname: unicode_fs_encoded gid: int / @@ -9341,12 +9330,12 @@ group id. static PyObject * os_initgroups_impl(PyObject *module, PyObject *oname, int gid) -/*[clinic end generated code: output=7f074d30a425fd3a input=df3d54331b0af204]*/ +/*[clinic end generated code: output=7f074d30a425fd3a input=984e60c7fed88cb4]*/ #else /*[clinic input] os.initgroups - username as oname: FSConverter + username as oname: unicode_fs_encoded gid: gid_t / @@ -9359,7 +9348,7 @@ group id. static PyObject * os_initgroups_impl(PyObject *module, PyObject *oname, gid_t gid) -/*[clinic end generated code: output=59341244521a9e3f input=0cb91bdc59a4c564]*/ +/*[clinic end generated code: output=59341244521a9e3f input=17d8fbe2dea42ca4]*/ #endif { const char *username = PyBytes_AS_STRING(oname); @@ -13115,8 +13104,8 @@ os_putenv_impl(PyObject *module, PyObject *name, PyObject *value) /*[clinic input] os.putenv - name: FSConverter - value: FSConverter + name: unicode_fs_encoded + value: unicode_fs_encoded / Change or add an environment variable. @@ -13124,7 +13113,7 @@ Change or add an environment variable. static PyObject * os_putenv_impl(PyObject *module, PyObject *name, PyObject *value) -/*[clinic end generated code: output=d29a567d6b2327d2 input=a97bc6152f688d31]*/ +/*[clinic end generated code: output=d29a567d6b2327d2 input=84fcd30f873c8c45]*/ { const char *name_string = PyBytes_AS_STRING(name); const char *value_string = PyBytes_AS_STRING(value); @@ -13167,7 +13156,7 @@ os_unsetenv_impl(PyObject *module, PyObject *name) #else /*[clinic input] os.unsetenv - name: FSConverter + name: unicode_fs_encoded / Delete an environment variable. @@ -13175,7 +13164,7 @@ Delete an environment variable. static PyObject * os_unsetenv_impl(PyObject *module, PyObject *name) -/*[clinic end generated code: output=54c4137ab1834f02 input=2bb5288a599c7107]*/ +/*[clinic end generated code: output=54c4137ab1834f02 input=78ff12e505ade80a]*/ { if (PySys_Audit("os.unsetenv", "(O)", name) < 0) { return NULL; @@ -15229,14 +15218,14 @@ os_urandom_impl(PyObject *module, Py_ssize_t size) /*[clinic input] os.memfd_create - name: FSConverter + name: unicode_fs_encoded flags: unsigned_int(bitwise=True, c_default="MFD_CLOEXEC") = MFD_CLOEXEC [clinic start generated code]*/ static PyObject * os_memfd_create_impl(PyObject *module, PyObject *name, unsigned int flags) -/*[clinic end generated code: output=6681ede983bdb9a6 input=a42cfc199bcd56e9]*/ +/*[clinic end generated code: output=6681ede983bdb9a6 input=cd0eb092cfac474b]*/ { int fd; const char *bytes = PyBytes_AS_STRING(name); diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index fa153efd8b7..dd4b6892977 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -7278,7 +7278,7 @@ Returns a list of network interface information (index, name) tuples."); /*[clinic input] _socket.if_nametoindex - oname: object(converter="PyUnicode_FSConverter") + oname: unicode_fs_encoded / Returns the interface index corresponding to the interface name if_name. @@ -7286,7 +7286,7 @@ Returns the interface index corresponding to the interface name if_name. static PyObject * _socket_if_nametoindex_impl(PyObject *module, PyObject *oname) -/*[clinic end generated code: output=289a411614f30244 input=01e0f1205307fb77]*/ +/*[clinic end generated code: output=289a411614f30244 input=6125dc20683560cf]*/ { #ifdef MS_WINDOWS NET_IFINDEX index; @@ -7295,7 +7295,6 @@ _socket_if_nametoindex_impl(PyObject *module, PyObject *oname) #endif index = if_nametoindex(PyBytes_AS_STRING(oname)); - Py_DECREF(oname); if (index == 0) { /* if_nametoindex() doesn't set errno */ PyErr_SetString(PyExc_OSError, "no interface with this name"); diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index d0d5223e5ac..d353f406831 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -13,7 +13,7 @@ module _symtable _symtable.symtable source: object - filename: object(converter='PyUnicode_FSDecoder') + filename: unicode_fs_decoded startstr: str / @@ -23,7 +23,7 @@ Return symbol and scope dictionaries used internally by compiler. static PyObject * _symtable_symtable_impl(PyObject *module, PyObject *source, PyObject *filename, const char *startstr) -/*[clinic end generated code: output=59eb0d5fc7285ac4 input=9dd8a50c0c36a4d7]*/ +/*[clinic end generated code: output=59eb0d5fc7285ac4 input=436ffff90d02e4f6]*/ { struct symtable *st; PyObject *t; @@ -47,12 +47,10 @@ _symtable_symtable_impl(PyObject *module, PyObject *source, else { PyErr_SetString(PyExc_ValueError, "symtable() arg 3 must be 'exec' or 'eval' or 'single'"); - Py_DECREF(filename); Py_XDECREF(source_copy); return NULL; } st = _Py_SymtableStringObjectFlags(str, filename, start, &cf); - Py_DECREF(filename); Py_XDECREF(source_copy); if (st == NULL) { return NULL; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index b7e08ae54da..2551c2c961b 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -745,7 +745,7 @@ builtin_chr(PyObject *module, PyObject *i) compile as builtin_compile source: object - filename: object(converter="PyUnicode_FSDecoder") + filename: unicode_fs_decoded mode: str flags: int = 0 dont_inherit: bool = False @@ -771,7 +771,7 @@ static PyObject * builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, const char *mode, int flags, int dont_inherit, int optimize, int feature_version) -/*[clinic end generated code: output=b0c09c84f116d3d7 input=cc78e20e7c7682ba]*/ +/*[clinic end generated code: output=b0c09c84f116d3d7 input=8f0069edbdac381b]*/ { PyObject *source_copy; const char *str; @@ -889,7 +889,6 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, error: result = NULL; finally: - Py_DECREF(filename); return result; } diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 60e20a46af5..adb82f45c25 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -296,7 +296,7 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *argsbuf[7]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; PyObject *source; - PyObject *filename; + PyObject *filename = NULL; const char *mode; int flags = 0; int dont_inherit = 0; @@ -367,6 +367,9 @@ skip_optional_kwonly: return_value = builtin_compile_impl(module, source, filename, mode, flags, dont_inherit, optimize, feature_version); exit: + /* Cleanup for filename */ + Py_XDECREF(filename); + return return_value; } @@ -1274,4 +1277,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=c0b72519622c849e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7eada753dc2e046f input=a9049054013a1b77]*/ diff --git a/Tools/clinic/libclinic/converters.py b/Tools/clinic/libclinic/converters.py index 3154299e31b..201125b9165 100644 --- a/Tools/clinic/libclinic/converters.py +++ b/Tools/clinic/libclinic/converters.py @@ -916,6 +916,26 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st return super().parse_arg(argname, displayname, limited_capi=limited_capi) +class _unicode_fs_converter_base(CConverter): + type = 'PyObject *' + + def converter_init(self) -> None: + if self.default is not unspecified: + fail(f"{self.__class__.__name__} does not support default values") + self.c_default = 'NULL' + + def cleanup(self) -> str: + return f"Py_XDECREF({self.parser_name});" + + +class unicode_fs_encoded_converter(_unicode_fs_converter_base): + converter = 'PyUnicode_FSConverter' + + +class unicode_fs_decoded_converter(_unicode_fs_converter_base): + converter = 'PyUnicode_FSDecoder' + + @add_legacy_c_converter('u') @add_legacy_c_converter('u#', zeroes=True) @add_legacy_c_converter('Z', accept={str, NoneType}) From 99fd52563220f7dd09303fa7a2b232d8618da6ce Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Wed, 8 Oct 2025 16:26:22 +0100 Subject: [PATCH 066/112] gh-101100: Fix all Sphinx warnings in `Doc/library/subprocess.rst` (#139576) --- Doc/library/subprocess.rst | 4 ++-- Doc/tools/.nitignore | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 028a7861f36..1aade881745 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -649,7 +649,7 @@ functions. If specified, *env* must provide any variables required for the program to execute. On Windows, in order to run a `side-by-side assembly`_ the - specified *env* **must** include a valid :envvar:`SystemRoot`. + specified *env* **must** include a valid ``%SystemRoot%``. .. _side-by-side assembly: https://en.wikipedia.org/wiki/Side-by-Side_Assembly @@ -1473,7 +1473,7 @@ handling consistency are valid for these functions. Return ``(exitcode, output)`` of executing *cmd* in a shell. - Execute the string *cmd* in a shell with :meth:`Popen.check_output` and + Execute the string *cmd* in a shell with :func:`check_output` and return a 2-tuple ``(exitcode, output)``. *encoding* and *errors* are used to decode output; see the notes on :ref:`frequently-used-arguments` for more details. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 0ee92dce437..4ee09c6bbaa 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -35,7 +35,6 @@ Doc/library/smtplib.rst Doc/library/socket.rst Doc/library/ssl.rst Doc/library/stdtypes.rst -Doc/library/subprocess.rst Doc/library/termios.rst Doc/library/test.rst Doc/library/tkinter.rst From 72e370c910bd941806a36b01f2f57f4c53a78eed Mon Sep 17 00:00:00 2001 From: ed Date: Wed, 8 Oct 2025 15:36:53 +0000 Subject: [PATCH 067/112] gh-139736: Fix argparse indentation overshoot (#139738) Co-authored-by: Savannah Ostrowski --- Lib/argparse.py | 2 +- Lib/test/test_argparse.py | 12 ++++++------ .../2025-10-08-00-06-30.gh-issue-139736.baPeBd.rst | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-08-00-06-30.gh-issue-139736.baPeBd.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 863e951528b..d71e551401c 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -280,7 +280,7 @@ def add_argument(self, action): if action.help is not SUPPRESS: # find all invocations - get_invocation = self._format_action_invocation + get_invocation = lambda x: self._decolor(self._format_action_invocation(x)) invocation_lengths = [len(get_invocation(action)) + self._current_indent] for subaction in self._iter_indented_subactions(action): invocation_lengths.append(len(get_invocation(subaction)) + self._current_indent) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 90bfcd0ae3e..27e38040a98 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -7311,11 +7311,11 @@ def custom_formatter(prog): {heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] [{short}+f {label}FOO{reset}] {pos}spam{reset} {heading}positional arguments:{reset} - {pos_b}spam{reset} spam help + {pos_b}spam{reset} spam help {heading}options:{reset} - {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit - {short_b}+f{reset}, {long_b}++foo{reset} {label_b}FOO{reset} foo help + {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit + {short_b}+f{reset}, {long_b}++foo{reset} {label_b}FOO{reset} foo help ''')) def test_custom_formatter_class(self): @@ -7348,11 +7348,11 @@ def __init__(self, prog): {heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] [{short}+f {label}FOO{reset}] {pos}spam{reset} {heading}positional arguments:{reset} - {pos_b}spam{reset} spam help + {pos_b}spam{reset} spam help {heading}options:{reset} - {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit - {short_b}+f{reset}, {long_b}++foo{reset} {label_b}FOO{reset} foo help + {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit + {short_b}+f{reset}, {long_b}++foo{reset} {label_b}FOO{reset} foo help ''')) diff --git a/Misc/NEWS.d/next/Library/2025-10-08-00-06-30.gh-issue-139736.baPeBd.rst b/Misc/NEWS.d/next/Library/2025-10-08-00-06-30.gh-issue-139736.baPeBd.rst new file mode 100644 index 00000000000..820679632d1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-08-00-06-30.gh-issue-139736.baPeBd.rst @@ -0,0 +1,2 @@ +Fix excessive indentation in the default :mod:`argparse` +:class:`!HelpFormatter`. Patch by Alexander Edland. From d9cb191df804b73e90aa09a7e9ac2a8cdd9b90be Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Wed, 8 Oct 2025 17:14:09 +0100 Subject: [PATCH 068/112] gh-138843: Clean up downloads page (#138844) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/tools/templates/download.html | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index 26fc9571d17..523b505f596 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -42,22 +42,22 @@

{% trans %}Download Python {{ dl_version }} documentation{% endtrans %}

{% trans %}HTML{% endtrans %} - {% trans download_size="13" %}Download (ca. {{ download_size }} MiB){% endtrans %} - {% trans download_size="8" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans %}Download{% endtrans %} + {% trans %}Download{% endtrans %} {% trans %}Plain text{% endtrans %} - {% trans download_size="4" %}Download (ca. {{ download_size }} MiB){% endtrans %} - {% trans download_size="3" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans %}Download{% endtrans %} + {% trans %}Download{% endtrans %} {% trans %}Texinfo{% endtrans %} - {% trans download_size="9" %}Download (ca. {{ download_size }} MiB){% endtrans %} - {% trans download_size="7" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans %}Download{% endtrans %} + {% trans %}Download{% endtrans %} {% trans %}EPUB{% endtrans %} - {% trans download_size="6" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans %}Download{% endtrans %} @@ -71,6 +71,9 @@

{% trans %}Download Python {{ dl_version }} documentation{% endtrans %}

and run make dist-pdf in the Doc/ directory of a copy of the CPython repository. {% endtrans %}

+

{% trans %} +See the directory listing +for file sizes.{% endtrans %}

{% trans %}Unpacking{% endtrans %}

@@ -83,9 +86,8 @@

{% trans %}Unpacking{% endtrans %}

{% trans %}Windows users can use the ZIP archives since those are customary on that platform. These are created on Unix using the Info-ZIP zip program.{% endtrans %}

-

{% trans %}Problems{% endtrans %}

- -

{% trans %}If you have comments or suggestions for the Python documentation, please send -email to docs@python.org.{% endtrans %}

+{% set bugs = pathto('bugs') %} +

{% trans bugs = bugs %}Open an issue +if you have comments or suggestions for the Python documentation.{% endtrans %}

{% endblock %} From fa603542452ead916eda3702c1175d34486d080d Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 8 Oct 2025 11:23:27 -0700 Subject: [PATCH 069/112] gh-101100: Resolve some `os` sphinx reference warnings (#139636) --- Doc/library/os.rst | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 8c81e1dcd07..ba3d1894549 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -216,8 +216,8 @@ process and user. You can delete items in this mapping to unset environment variables. :func:`unsetenv` will be called automatically when an item is deleted from - :data:`os.environ`, and when one of the :meth:`pop` or :meth:`clear` methods is - called. + :data:`os.environ`, and when one of the :meth:`~dict.pop` or + :meth:`~dict.clear` methods is called. .. seealso:: @@ -430,8 +430,8 @@ process and user. associated with the effective user id of the process; the group access list may change over the lifetime of the process, it is not affected by calls to :func:`setgroups`, and its length is not limited to 16. The - deployment target value, :const:`MACOSX_DEPLOYMENT_TARGET`, can be - obtained with :func:`sysconfig.get_config_var`. + deployment target value can be obtained with + :func:`sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') `. .. function:: getlogin() @@ -2606,10 +2606,10 @@ features: Create a filesystem node (file, device special file or named pipe) named *path*. *mode* specifies both the permissions to use and the type of node - to be created, being combined (bitwise OR) with one of ``stat.S_IFREG``, - ``stat.S_IFCHR``, ``stat.S_IFBLK``, and ``stat.S_IFIFO`` (those constants are - available in :mod:`stat`). For ``stat.S_IFCHR`` and ``stat.S_IFBLK``, - *device* defines the newly created device special file (probably using + to be created, being combined (bitwise OR) with one of :const:`stat.S_IFREG`, + :const:`stat.S_IFCHR`, :const:`stat.S_IFBLK`, and :const:`stat.S_IFIFO`. + For :const:`stat.S_IFCHR` and :const:`stat.S_IFBLK`, *device* defines the + newly created device special file (probably using :func:`os.makedev`), otherwise it is ignored. This function can also support :ref:`paths relative to directory descriptors @@ -2627,13 +2627,13 @@ features: .. function:: major(device, /) Extract the device major number from a raw device number (usually the - :attr:`st_dev` or :attr:`st_rdev` field from :c:struct:`stat`). + :attr:`~stat_result.st_dev` or :attr:`~stat_result.st_rdev` field from :c:struct:`stat`). .. function:: minor(device, /) Extract the device minor number from a raw device number (usually the - :attr:`st_dev` or :attr:`st_rdev` field from :c:struct:`stat`). + :attr:`~stat_result.st_dev` or :attr:`~stat_result.st_rdev` field from :c:struct:`stat`). .. function:: makedev(major, minor, /) @@ -3364,8 +3364,8 @@ features: .. versionchanged:: 3.8 On Windows, the :attr:`st_mode` member now identifies special - files as :const:`S_IFCHR`, :const:`S_IFIFO` or :const:`S_IFBLK` - as appropriate. + files as :const:`~stat.S_IFCHR`, :const:`~stat.S_IFIFO` or + :const:`~stat.S_IFBLK` as appropriate. .. versionchanged:: 3.12 On Windows, :attr:`st_ctime` is deprecated. Eventually, it will @@ -4285,10 +4285,10 @@ to be ignored. .. function:: abort() - Generate a :const:`SIGABRT` signal to the current process. On Unix, the default + Generate a :const:`~signal.SIGABRT` signal to the current process. On Unix, the default behavior is to produce a core dump; on Windows, the process immediately returns an exit code of ``3``. Be aware that calling this function will not call the - Python signal handler registered for :const:`SIGABRT` with + Python signal handler registered for :const:`~signal.SIGABRT` with :func:`signal.signal`. From 6954077fde4aee07b55a666a610ef4bc4b9d76a8 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Wed, 8 Oct 2025 15:14:05 -0400 Subject: [PATCH 070/112] Remove Cirrus macOS runners from CI (#139799) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/build.yml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 252ec831ed9..625466151e3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -202,23 +202,15 @@ jobs: strategy: fail-fast: false matrix: - # Cirrus and macos-14 are M1, macos-15-intel is default GHA Intel. - # macOS 13 only runs tests against the GIL-enabled CPython. - # Cirrus used for upstream, macos-14 for forks. + # macos-14 is M1, macos-15-intel is Intel. + # macos-15-intel only runs tests against the GIL-enabled CPython. os: - - ghcr.io/cirruslabs/macos-runner:sonoma - macos-14 - macos-15-intel - is-fork: # only used for the exclusion trick - - ${{ github.repository_owner != 'python' }} free-threading: - false - true exclude: - - os: ghcr.io/cirruslabs/macos-runner:sonoma - is-fork: true - - os: macos-14 - is-fork: false - os: macos-15-intel free-threading: true uses: ./.github/workflows/reusable-macos.yml @@ -409,9 +401,8 @@ jobs: fail-fast: false matrix: include: - # Use the same runs-on configuration as build-macos and build-ubuntu. - arch: aarch64 - runs-on: ${{ github.repository_owner == 'python' && 'ghcr.io/cirruslabs/macos-runner:sonoma' || 'macos-14' }} + runs-on: macos-14 - arch: x86_64 runs-on: ubuntu-24.04 From a2850a3a91a1515cbdd9440463c78bdf2e0bd3cf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 8 Oct 2025 22:33:45 +0200 Subject: [PATCH 071/112] gh-70030: Remove _PyCode_ConstantKey() function (#139735) Move the function to the internal C API and no longer export it. --- Include/cpython/code.h | 9 --------- Include/internal/pycore_code.h | 9 +++++++++ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 3f0dce03455..84456a709a6 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -282,15 +282,6 @@ typedef struct _line_offsets { */ PyAPI_FUNC(int) _PyCode_CheckLineNumber(int lasti, PyCodeAddressRange *bounds); -/* Create a comparable key used to compare constants taking in account the - * object type. It is used to make sure types are not coerced (e.g., float and - * complex) _and_ to distinguish 0.0 from -0.0 e.g. on IEEE platforms - * - * Return (type(obj), obj, ...): a tuple with variable size (at least 2 items) - * depending on the type and the value. The type is the first item to not - * compare bytes and str which can raise a BytesWarning exception. */ -PyAPI_FUNC(PyObject*) _PyCode_ConstantKey(PyObject *obj); - PyAPI_FUNC(PyObject*) PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, PyObject *lnotab); diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 0ec47f0014b..2d7d81d491c 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -663,6 +663,15 @@ PyAPI_FUNC(int) _PyCode_VerifyStateless( PyAPI_FUNC(int) _PyCode_CheckPureFunction(PyCodeObject *, const char **); PyAPI_FUNC(int) _PyCode_ReturnsOnlyNone(PyCodeObject *); +/* Create a comparable key used to compare constants taking in account the + * object type. It is used to make sure types are not coerced (e.g., float and + * complex) _and_ to distinguish 0.0 from -0.0 e.g. on IEEE platforms + * + * Return (type(obj), obj, ...): a tuple with variable size (at least 2 items) + * depending on the type and the value. The type is the first item to not + * compare bytes and str which can raise a BytesWarning exception. */ +extern PyObject* _PyCode_ConstantKey(PyObject *obj); + #ifdef __cplusplus } From 678e0b818c0d6063907f55263bbc0e194b492c8e Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 9 Oct 2025 01:13:27 +0300 Subject: [PATCH 072/112] gh-139590: Stricter `ruff` rules for `Tools/wasm` (#139752) --- .pre-commit-config.yaml | 4 ++++ Tools/wasm/.ruff.toml | 3 --- Tools/wasm/emscripten/__main__.py | 9 ++++----- Tools/wasm/emscripten/wasm_assets.py | 5 ++--- Tools/wasm/wasi/__main__.py | 10 ++++------ 5 files changed, 14 insertions(+), 17 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0e00ffb3bd2..b0311f05279 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,6 +26,10 @@ repos: name: Run Ruff (lint) on Tools/peg_generator/ args: [--exit-non-zero-on-fix, --config=Tools/peg_generator/.ruff.toml] files: ^Tools/peg_generator/ + - id: ruff-check + name: Run Ruff (lint) on Tools/wasm/ + args: [--exit-non-zero-on-fix, --config=Tools/wasm/.ruff.toml] + files: ^Tools/wasm/ - id: ruff-format name: Run Ruff (format) on Doc/ args: [--check] diff --git a/Tools/wasm/.ruff.toml b/Tools/wasm/.ruff.toml index aabcf8dc4f5..3d8e59fa3f2 100644 --- a/Tools/wasm/.ruff.toml +++ b/Tools/wasm/.ruff.toml @@ -22,7 +22,4 @@ select = [ ] ignore = [ "E501", # Line too long - "F541", # f-string without any placeholders - "PYI024", # Use `typing.NamedTuple` instead of `collections.namedtuple` - "PYI025", # Use `from collections.abc import Set as AbstractSet` ] diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index fdf3142c0a3..c88e9edba6d 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -3,16 +3,16 @@ import argparse import contextlib import functools +import hashlib import os import shutil import subprocess import sys import sysconfig -import hashlib import tempfile -from urllib.request import urlopen from pathlib import Path from textwrap import dedent +from urllib.request import urlopen try: from os import process_cpu_count as cpu_count @@ -33,9 +33,7 @@ PREFIX_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "prefix" LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" -LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/emscripten.py\n".encode( - "utf-8" -) +LOCAL_SETUP_MARKER = b"# Generated by Tools/wasm/emscripten.py\n" def updated_env(updates={}): @@ -432,6 +430,7 @@ def main(): make_build, configure_host, make_host, + clean, ): subcommand.add_argument( "--quiet", diff --git a/Tools/wasm/emscripten/wasm_assets.py b/Tools/wasm/emscripten/wasm_assets.py index 90f318f319a..38479087235 100755 --- a/Tools/wasm/emscripten/wasm_assets.py +++ b/Tools/wasm/emscripten/wasm_assets.py @@ -15,7 +15,6 @@ import sys import sysconfig import zipfile -from typing import Dict # source directory SRCDIR = pathlib.Path(__file__).parents[3].absolute() @@ -134,7 +133,7 @@ def filterfunc(filename: str) -> bool: pzf.writepy(entry, filterfunc=filterfunc) -def detect_extension_modules(args: argparse.Namespace) -> Dict[str, bool]: +def detect_extension_modules(args: argparse.Namespace) -> dict[str, bool]: modules = {} # disabled by Modules/Setup.local ? @@ -149,7 +148,7 @@ def detect_extension_modules(args: argparse.Namespace) -> Dict[str, bool]: # disabled by configure? with open(args.sysconfig_data) as f: data = f.read() - loc: Dict[str, Dict[str, str]] = {} + loc: dict[str, dict[str, str]] = {} exec(data, globals(), loc) for key, value in loc["build_time_vars"].items(): diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py index a0658cb351a..b2f643ddbfc 100644 --- a/Tools/wasm/wasi/__main__.py +++ b/Tools/wasm/wasi/__main__.py @@ -16,7 +16,6 @@ import sysconfig import tempfile - CHECKOUT = pathlib.Path(__file__).parent.parent.parent.parent assert (CHECKOUT / "configure").is_file(), ( "Please update the location of the file" @@ -28,9 +27,9 @@ LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" LOCAL_SETUP_MARKER = ( - "# Generated by Tools/wasm/wasi .\n" - "# Required to statically build extension modules." -).encode("utf-8") + b"# Generated by Tools/wasm/wasi .\n" + b"# Required to statically build extension modules." +) WASI_SDK_VERSION = 24 @@ -154,8 +153,7 @@ def build_python_is_pydebug(): test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)" result = subprocess.run( [build_python_path(), "-c", test], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + capture_output=True, ) return bool(result.returncode) From e7e3d1d4a8dece01b1bbd0253684d5b46b2409d7 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Wed, 8 Oct 2025 23:34:40 +0100 Subject: [PATCH 073/112] gh-139805: Bump `test_repl_eio` timeout for slow builtbots (#139807) --- Lib/test/test_pyrepl/test_unix_console.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py index 6e422806cec..9ac2e9961df 100644 --- a/Lib/test/test_pyrepl/test_unix_console.py +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -361,7 +361,7 @@ def test_repl_eio(self): os.kill(proc.pid, signal.SIGUSR1) # sleep for pty to settle - _, err = proc.communicate(timeout=support.SHORT_TIMEOUT) + _, err = proc.communicate(timeout=support.LONG_TIMEOUT) self.assertEqual( proc.returncode, 1, From 7f155f9c46e2dd7dec5e12ebec3323dfd95fe1f0 Mon Sep 17 00:00:00 2001 From: Alper Date: Wed, 8 Oct 2025 23:30:47 -0700 Subject: [PATCH 074/112] gh-116738: make `mmap` module thread-safe (#139237) --- Lib/test/test_free_threading/test_mmap.py | 315 +++++++ Lib/test/test_mmap.py | 8 + ...-09-21-14-33-17.gh-issue-116738.vNaI4h.rst | 2 + Modules/clinic/mmapmodule.c.h | 799 ++++++++++++++++++ Modules/mmapmodule.c | 651 +++++++++----- 5 files changed, 1568 insertions(+), 207 deletions(-) create mode 100644 Lib/test/test_free_threading/test_mmap.py create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-09-21-14-33-17.gh-issue-116738.vNaI4h.rst create mode 100644 Modules/clinic/mmapmodule.c.h diff --git a/Lib/test/test_free_threading/test_mmap.py b/Lib/test/test_free_threading/test_mmap.py new file mode 100644 index 00000000000..ece13d33f3b --- /dev/null +++ b/Lib/test/test_free_threading/test_mmap.py @@ -0,0 +1,315 @@ +import unittest + +from test.support import import_helper, threading_helper +from test.support.threading_helper import run_concurrently + +import os +import string +import tempfile +import threading + +from collections import Counter + +mmap = import_helper.import_module("mmap") + +NTHREADS = 10 +ANONYMOUS_MEM = -1 + + +@threading_helper.requires_working_threading() +class MmapTests(unittest.TestCase): + def test_read_and_read_byte(self): + ascii_uppercase = string.ascii_uppercase.encode() + # Choose a total mmap size that evenly divides across threads and the + # read pattern (3 bytes per loop). + mmap_size = 3 * NTHREADS * len(ascii_uppercase) + num_bytes_to_read_per_thread = mmap_size // NTHREADS + bytes_read_from_mmap = [] + + def read(mm_obj): + nread = 0 + while nread < num_bytes_to_read_per_thread: + b = mm_obj.read_byte() + bytes_read_from_mmap.append(b) + b = mm_obj.read(2) + bytes_read_from_mmap.extend(b) + nread += 3 + + with mmap.mmap(ANONYMOUS_MEM, mmap_size) as mm_obj: + for i in range(mmap_size // len(ascii_uppercase)): + mm_obj.write(ascii_uppercase) + + mm_obj.seek(0) + run_concurrently( + worker_func=read, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + self.assertEqual(len(bytes_read_from_mmap), mmap_size) + # Count each letter/byte to verify read correctness + counter = Counter(bytes_read_from_mmap) + self.assertEqual(len(counter), len(ascii_uppercase)) + # Each letter/byte should be read (3 * NTHREADS) times + for letter in ascii_uppercase: + self.assertEqual(counter[letter], 3 * NTHREADS) + + def test_readline(self): + num_lines = 1000 + lines_read_from_mmap = [] + expected_lines = [] + + def readline(mm_obj): + for i in range(num_lines // NTHREADS): + line = mm_obj.readline() + lines_read_from_mmap.append(line) + + # Allocate mmap enough for num_lines (max line 5 bytes including NL) + with mmap.mmap(ANONYMOUS_MEM, num_lines * 5) as mm_obj: + for i in range(num_lines): + line = b"%d\n" % i + mm_obj.write(line) + expected_lines.append(line) + + mm_obj.seek(0) + run_concurrently( + worker_func=readline, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + self.assertEqual(len(lines_read_from_mmap), num_lines) + # Every line should be read once by threads; order is non-deterministic + # Sort numerically by integer value + lines_read_from_mmap.sort(key=lambda x: int(x)) + self.assertEqual(lines_read_from_mmap, expected_lines) + + def test_write_and_write_byte(self): + thread_letters = list(string.ascii_uppercase) + self.assertLessEqual(NTHREADS, len(thread_letters)) + per_thread_write_loop = 100 + + def write(mm_obj): + # Each thread picks a unique letter to write + thread_letter = thread_letters.pop(0) + thread_bytes = (thread_letter * 2).encode() + for _ in range(per_thread_write_loop): + mm_obj.write_byte(thread_bytes[0]) + mm_obj.write(thread_bytes) + + with mmap.mmap( + ANONYMOUS_MEM, per_thread_write_loop * 3 * NTHREADS + ) as mm_obj: + run_concurrently( + worker_func=write, + args=(mm_obj,), + nthreads=NTHREADS, + ) + mm_obj.seek(0) + data = mm_obj.read() + self.assertEqual(len(data), NTHREADS * per_thread_write_loop * 3) + counter = Counter(data) + self.assertEqual(len(counter), NTHREADS) + # Each thread letter should be written `per_thread_write_loop` * 3 + for letter in counter: + self.assertEqual(counter[letter], per_thread_write_loop * 3) + + def test_move(self): + ascii_uppercase = string.ascii_uppercase.encode() + num_letters = len(ascii_uppercase) + + def move(mm_obj): + for i in range(num_letters): + # Move 1 byte from the first half to the second half + mm_obj.move(0 + i, num_letters + i, 1) + + with mmap.mmap(ANONYMOUS_MEM, 2 * num_letters) as mm_obj: + mm_obj.write(ascii_uppercase) + run_concurrently( + worker_func=move, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + def test_seek_and_tell(self): + seek_per_thread = 10 + + def seek(mm_obj): + self.assertTrue(mm_obj.seekable()) + for _ in range(seek_per_thread): + before_seek = mm_obj.tell() + mm_obj.seek(1, os.SEEK_CUR) + self.assertLess(before_seek, mm_obj.tell()) + + with mmap.mmap(ANONYMOUS_MEM, 1024) as mm_obj: + run_concurrently( + worker_func=seek, + args=(mm_obj,), + nthreads=NTHREADS, + ) + # Each thread seeks from current position, the end position should + # be the sum of all seeks from all threads. + self.assertEqual(mm_obj.tell(), NTHREADS * seek_per_thread) + + def test_slice_update_and_slice_read(self): + thread_letters = list(string.ascii_uppercase) + self.assertLessEqual(NTHREADS, len(thread_letters)) + + def slice_update_and_slice_read(mm_obj): + # Each thread picks a unique letter to write + thread_letter = thread_letters.pop(0) + thread_bytes = (thread_letter * 1024).encode() + for _ in range(100): + mm_obj[:] = thread_bytes + read_bytes = mm_obj[:] + # Read bytes should be all the same letter, showing no + # interleaving + self.assertTrue(all_same(read_bytes)) + + with mmap.mmap(ANONYMOUS_MEM, 1024) as mm_obj: + run_concurrently( + worker_func=slice_update_and_slice_read, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + def test_item_update_and_item_read(self): + thread_indexes = [i for i in range(NTHREADS)] + + def item_update_and_item_read(mm_obj): + # Each thread picks a unique index to write + thread_index = thread_indexes.pop() + for i in range(100): + mm_obj[thread_index] = i + self.assertEqual(mm_obj[thread_index], i) + + # Read values set by other threads, all values + # should be less than '100' + for val in mm_obj: + self.assertLess(int.from_bytes(val), 100) + + with mmap.mmap(ANONYMOUS_MEM, NTHREADS + 1) as mm_obj: + run_concurrently( + worker_func=item_update_and_item_read, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + @unittest.skipUnless(os.name == "posix", "requires Posix") + @unittest.skipUnless(hasattr(mmap.mmap, "resize"), "requires mmap.resize") + def test_resize_and_size(self): + thread_indexes = [i for i in range(NTHREADS)] + + def resize_and_item_update(mm_obj): + # Each thread picks a unique index to write + thread_index = thread_indexes.pop() + mm_obj.resize(2048) + self.assertEqual(mm_obj.size(), 2048) + for i in range(100): + mm_obj[thread_index] = i + self.assertEqual(mm_obj[thread_index], i) + + with mmap.mmap(ANONYMOUS_MEM, 1024, flags=mmap.MAP_PRIVATE) as mm_obj: + run_concurrently( + worker_func=resize_and_item_update, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + def test_close_and_closed(self): + def close_mmap(mm_obj): + mm_obj.close() + self.assertTrue(mm_obj.closed) + + with mmap.mmap(ANONYMOUS_MEM, 1) as mm_obj: + run_concurrently( + worker_func=close_mmap, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + def test_find_and_rfind(self): + per_thread_loop = 10 + + def find_and_rfind(mm_obj): + pattern = b'Thread-Ident:"%d"' % threading.get_ident() + mm_obj.write(pattern) + for _ in range(per_thread_loop): + found_at = mm_obj.find(pattern, 0) + self.assertNotEqual(found_at, -1) + # Should not find it after the `found_at` + self.assertEqual(mm_obj.find(pattern, found_at + 1), -1) + found_at_rev = mm_obj.rfind(pattern, 0) + self.assertEqual(found_at, found_at_rev) + # Should not find it after the `found_at` + self.assertEqual(mm_obj.rfind(pattern, found_at + 1), -1) + + with mmap.mmap(ANONYMOUS_MEM, 1024) as mm_obj: + run_concurrently( + worker_func=find_and_rfind, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + @unittest.skipUnless(os.name == "posix", "requires Posix") + @unittest.skipUnless(hasattr(mmap.mmap, "resize"), "requires mmap.resize") + def test_flush(self): + mmap_filename = "test_mmap_file" + resize_to = 1024 + + def resize_and_flush(mm_obj): + mm_obj.resize(resize_to) + mm_obj.flush() + + with tempfile.TemporaryDirectory() as tmpdirname: + file_path = f"{tmpdirname}/{mmap_filename}" + with open(file_path, "wb+") as file: + file.write(b"CPython") + file.flush() + with mmap.mmap(file.fileno(), 1) as mm_obj: + run_concurrently( + worker_func=resize_and_flush, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + self.assertEqual(os.path.getsize(file_path), resize_to) + + def test_mmap_export_as_memoryview(self): + """ + Each thread creates a memoryview and updates the internal state of the + mmap object. + """ + buffer_size = 42 + + def create_memoryview_from_mmap(mm_obj): + memoryviews = [] + for _ in range(100): + mv = memoryview(mm_obj) + memoryviews.append(mv) + self.assertEqual(len(mv), buffer_size) + self.assertEqual(mv[:7], b"CPython") + + # Cannot close the mmap while it is exported as buffers + with self.assertRaisesRegex( + BufferError, "cannot close exported pointers exist" + ): + mm_obj.close() + + with mmap.mmap(ANONYMOUS_MEM, 42) as mm_obj: + mm_obj.write(b"CPython") + run_concurrently( + worker_func=create_memoryview_from_mmap, + args=(mm_obj,), + nthreads=NTHREADS, + ) + # Implicit mm_obj.close() verifies all exports (memoryviews) are + # properly freed. + + +def all_same(lst): + return all(item == lst[0] for item in lst) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 0571eed23f7..368af0cf89c 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -871,6 +871,10 @@ def test_madvise(self): size = 2 * PAGESIZE m = mmap.mmap(-1, size) + class Number: + def __index__(self): + return 2 + with self.assertRaisesRegex(ValueError, "madvise start out of bounds"): m.madvise(mmap.MADV_NORMAL, size) with self.assertRaisesRegex(ValueError, "madvise start out of bounds"): @@ -879,10 +883,14 @@ def test_madvise(self): m.madvise(mmap.MADV_NORMAL, 0, -1) with self.assertRaisesRegex(OverflowError, "madvise length too large"): m.madvise(mmap.MADV_NORMAL, PAGESIZE, sys.maxsize) + with self.assertRaisesRegex( + TypeError, "'str' object cannot be interpreted as an integer"): + m.madvise(mmap.MADV_NORMAL, PAGESIZE, "Not a Number") self.assertEqual(m.madvise(mmap.MADV_NORMAL), None) self.assertEqual(m.madvise(mmap.MADV_NORMAL, PAGESIZE), None) self.assertEqual(m.madvise(mmap.MADV_NORMAL, PAGESIZE, size), None) self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, 2), None) + self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, Number()), None) self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, size), None) @unittest.skipUnless(hasattr(mmap.mmap, 'resize'), 'requires mmap.resize') diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-21-14-33-17.gh-issue-116738.vNaI4h.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-21-14-33-17.gh-issue-116738.vNaI4h.rst new file mode 100644 index 00000000000..0668d57604b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-21-14-33-17.gh-issue-116738.vNaI4h.rst @@ -0,0 +1,2 @@ +Make :mod:`mmap` thread-safe on the :term:`free threaded ` +build. diff --git a/Modules/clinic/mmapmodule.c.h b/Modules/clinic/mmapmodule.c.h new file mode 100644 index 00000000000..f7fc172b3af --- /dev/null +++ b/Modules/clinic/mmapmodule.c.h @@ -0,0 +1,799 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + +PyDoc_STRVAR(mmap_mmap_close__doc__, +"close($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_CLOSE_METHODDEF \ + {"close", (PyCFunction)mmap_mmap_close, METH_NOARGS, mmap_mmap_close__doc__}, + +static PyObject * +mmap_mmap_close_impl(mmap_object *self); + +static PyObject * +mmap_mmap_close(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_close_impl((mmap_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_read_byte__doc__, +"read_byte($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_READ_BYTE_METHODDEF \ + {"read_byte", (PyCFunction)mmap_mmap_read_byte, METH_NOARGS, mmap_mmap_read_byte__doc__}, + +static PyObject * +mmap_mmap_read_byte_impl(mmap_object *self); + +static PyObject * +mmap_mmap_read_byte(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_read_byte_impl((mmap_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_readline__doc__, +"readline($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_READLINE_METHODDEF \ + {"readline", (PyCFunction)mmap_mmap_readline, METH_NOARGS, mmap_mmap_readline__doc__}, + +static PyObject * +mmap_mmap_readline_impl(mmap_object *self); + +static PyObject * +mmap_mmap_readline(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_readline_impl((mmap_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_read__doc__, +"read($self, n=None, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_READ_METHODDEF \ + {"read", _PyCFunction_CAST(mmap_mmap_read), METH_FASTCALL, mmap_mmap_read__doc__}, + +static PyObject * +mmap_mmap_read_impl(mmap_object *self, Py_ssize_t num_bytes); + +static PyObject * +mmap_mmap_read(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t num_bytes = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("read", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_Py_convert_optional_to_ssize_t(args[0], &num_bytes)) { + goto exit; + } +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_read_impl((mmap_object *)self, num_bytes); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_find__doc__, +"find($self, view, start=None, end=None, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_FIND_METHODDEF \ + {"find", _PyCFunction_CAST(mmap_mmap_find), METH_FASTCALL, mmap_mmap_find__doc__}, + +static PyObject * +mmap_mmap_find_impl(mmap_object *self, Py_buffer *view, PyObject *start, + PyObject *end); + +static PyObject * +mmap_mmap_find(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_buffer view = {NULL, NULL}; + PyObject *start = Py_None; + PyObject *end = Py_None; + + if (!_PyArg_CheckPositional("find", nargs, 1, 3)) { + goto exit; + } + if (PyObject_GetBuffer(args[0], &view, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + start = args[1]; + if (nargs < 3) { + goto skip_optional; + } + end = args[2]; +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_find_impl((mmap_object *)self, &view, start, end); + Py_END_CRITICAL_SECTION(); + +exit: + /* Cleanup for view */ + if (view.obj) { + PyBuffer_Release(&view); + } + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_rfind__doc__, +"rfind($self, view, start=None, end=None, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_RFIND_METHODDEF \ + {"rfind", _PyCFunction_CAST(mmap_mmap_rfind), METH_FASTCALL, mmap_mmap_rfind__doc__}, + +static PyObject * +mmap_mmap_rfind_impl(mmap_object *self, Py_buffer *view, PyObject *start, + PyObject *end); + +static PyObject * +mmap_mmap_rfind(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_buffer view = {NULL, NULL}; + PyObject *start = Py_None; + PyObject *end = Py_None; + + if (!_PyArg_CheckPositional("rfind", nargs, 1, 3)) { + goto exit; + } + if (PyObject_GetBuffer(args[0], &view, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + start = args[1]; + if (nargs < 3) { + goto skip_optional; + } + end = args[2]; +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_rfind_impl((mmap_object *)self, &view, start, end); + Py_END_CRITICAL_SECTION(); + +exit: + /* Cleanup for view */ + if (view.obj) { + PyBuffer_Release(&view); + } + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_write__doc__, +"write($self, bytes, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_WRITE_METHODDEF \ + {"write", (PyCFunction)mmap_mmap_write, METH_O, mmap_mmap_write__doc__}, + +static PyObject * +mmap_mmap_write_impl(mmap_object *self, Py_buffer *data); + +static PyObject * +mmap_mmap_write(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_buffer data = {NULL, NULL}; + + if (PyObject_GetBuffer(arg, &data, PyBUF_SIMPLE) != 0) { + goto exit; + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_write_impl((mmap_object *)self, &data); + Py_END_CRITICAL_SECTION(); + +exit: + /* Cleanup for data */ + if (data.obj) { + PyBuffer_Release(&data); + } + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_write_byte__doc__, +"write_byte($self, byte, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_WRITE_BYTE_METHODDEF \ + {"write_byte", (PyCFunction)mmap_mmap_write_byte, METH_O, mmap_mmap_write_byte__doc__}, + +static PyObject * +mmap_mmap_write_byte_impl(mmap_object *self, unsigned char value); + +static PyObject * +mmap_mmap_write_byte(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + unsigned char value; + + { + long ival = PyLong_AsLong(arg); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < 0) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is less than minimum"); + goto exit; + } + else if (ival > UCHAR_MAX) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is greater than maximum"); + goto exit; + } + else { + value = (unsigned char) ival; + } + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_write_byte_impl((mmap_object *)self, value); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_size__doc__, +"size($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_SIZE_METHODDEF \ + {"size", (PyCFunction)mmap_mmap_size, METH_NOARGS, mmap_mmap_size__doc__}, + +static PyObject * +mmap_mmap_size_impl(mmap_object *self); + +static PyObject * +mmap_mmap_size(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_size_impl((mmap_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if (defined(MS_WINDOWS) || defined(HAVE_MREMAP)) + +PyDoc_STRVAR(mmap_mmap_resize__doc__, +"resize($self, newsize, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_RESIZE_METHODDEF \ + {"resize", (PyCFunction)mmap_mmap_resize, METH_O, mmap_mmap_resize__doc__}, + +static PyObject * +mmap_mmap_resize_impl(mmap_object *self, Py_ssize_t new_size); + +static PyObject * +mmap_mmap_resize(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_ssize_t new_size; + + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(arg); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + new_size = ival; + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_resize_impl((mmap_object *)self, new_size); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +#endif /* (defined(MS_WINDOWS) || defined(HAVE_MREMAP)) */ + +PyDoc_STRVAR(mmap_mmap_tell__doc__, +"tell($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_TELL_METHODDEF \ + {"tell", (PyCFunction)mmap_mmap_tell, METH_NOARGS, mmap_mmap_tell__doc__}, + +static PyObject * +mmap_mmap_tell_impl(mmap_object *self); + +static PyObject * +mmap_mmap_tell(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_tell_impl((mmap_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_flush__doc__, +"flush($self, offset=0, size=-1, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_FLUSH_METHODDEF \ + {"flush", _PyCFunction_CAST(mmap_mmap_flush), METH_FASTCALL, mmap_mmap_flush__doc__}, + +static PyObject * +mmap_mmap_flush_impl(mmap_object *self, Py_ssize_t offset, Py_ssize_t size); + +static PyObject * +mmap_mmap_flush(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t offset = 0; + Py_ssize_t size = -1; + + if (!_PyArg_CheckPositional("flush", nargs, 0, 2)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + offset = ival; + } + if (nargs < 2) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + size = ival; + } +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_flush_impl((mmap_object *)self, offset, size); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_seek__doc__, +"seek($self, pos, whence=0, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_SEEK_METHODDEF \ + {"seek", _PyCFunction_CAST(mmap_mmap_seek), METH_FASTCALL, mmap_mmap_seek__doc__}, + +static PyObject * +mmap_mmap_seek_impl(mmap_object *self, Py_ssize_t dist, int how); + +static PyObject * +mmap_mmap_seek(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t dist; + int how = 0; + + if (!_PyArg_CheckPositional("seek", nargs, 1, 2)) { + goto exit; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + dist = ival; + } + if (nargs < 2) { + goto skip_optional; + } + how = PyLong_AsInt(args[1]); + if (how == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_seek_impl((mmap_object *)self, dist, how); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_seekable__doc__, +"seekable($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_SEEKABLE_METHODDEF \ + {"seekable", (PyCFunction)mmap_mmap_seekable, METH_NOARGS, mmap_mmap_seekable__doc__}, + +static PyObject * +mmap_mmap_seekable_impl(mmap_object *self); + +static PyObject * +mmap_mmap_seekable(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return mmap_mmap_seekable_impl((mmap_object *)self); +} + +PyDoc_STRVAR(mmap_mmap_move__doc__, +"move($self, dest, src, count, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_MOVE_METHODDEF \ + {"move", _PyCFunction_CAST(mmap_mmap_move), METH_FASTCALL, mmap_mmap_move__doc__}, + +static PyObject * +mmap_mmap_move_impl(mmap_object *self, Py_ssize_t dest, Py_ssize_t src, + Py_ssize_t cnt); + +static PyObject * +mmap_mmap_move(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t dest; + Py_ssize_t src; + Py_ssize_t cnt; + + if (!_PyArg_CheckPositional("move", nargs, 3, 3)) { + goto exit; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + dest = ival; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + src = ival; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[2]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + cnt = ival; + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_move_impl((mmap_object *)self, dest, src, cnt); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(mmap_mmap___enter____doc__, +"__enter__($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP___ENTER___METHODDEF \ + {"__enter__", (PyCFunction)mmap_mmap___enter__, METH_NOARGS, mmap_mmap___enter____doc__}, + +static PyObject * +mmap_mmap___enter___impl(mmap_object *self); + +static PyObject * +mmap_mmap___enter__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap___enter___impl((mmap_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap___exit____doc__, +"__exit__($self, exc_type, exc_value, traceback, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP___EXIT___METHODDEF \ + {"__exit__", _PyCFunction_CAST(mmap_mmap___exit__), METH_FASTCALL, mmap_mmap___exit____doc__}, + +static PyObject * +mmap_mmap___exit___impl(mmap_object *self, PyObject *exc_type, + PyObject *exc_value, PyObject *traceback); + +static PyObject * +mmap_mmap___exit__(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *exc_type; + PyObject *exc_value; + PyObject *traceback; + + if (!_PyArg_CheckPositional("__exit__", nargs, 3, 3)) { + goto exit; + } + exc_type = args[0]; + exc_value = args[1]; + traceback = args[2]; + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap___exit___impl((mmap_object *)self, exc_type, exc_value, traceback); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(mmap_mmap___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP___SIZEOF___METHODDEF \ + {"__sizeof__", (PyCFunction)mmap_mmap___sizeof__, METH_NOARGS, mmap_mmap___sizeof____doc__}, + +static PyObject * +mmap_mmap___sizeof___impl(mmap_object *self); + +static PyObject * +mmap_mmap___sizeof__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap___sizeof___impl((mmap_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if (defined(MS_WINDOWS) && defined(Py_DEBUG)) + +PyDoc_STRVAR(mmap_mmap__protect__doc__, +"_protect($self, flNewProtect, start, length, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP__PROTECT_METHODDEF \ + {"_protect", _PyCFunction_CAST(mmap_mmap__protect), METH_FASTCALL, mmap_mmap__protect__doc__}, + +static PyObject * +mmap_mmap__protect_impl(mmap_object *self, unsigned int flNewProtect, + Py_ssize_t start, Py_ssize_t length); + +static PyObject * +mmap_mmap__protect(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned int flNewProtect; + Py_ssize_t start; + Py_ssize_t length; + + if (!_PyArg_CheckPositional("_protect", nargs, 3, 3)) { + goto exit; + } + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[0], &flNewProtect, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + start = ival; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[2]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + length = ival; + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap__protect_impl((mmap_object *)self, flNewProtect, start, length); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +#endif /* (defined(MS_WINDOWS) && defined(Py_DEBUG)) */ + +#if defined(HAVE_MADVISE) + +PyDoc_STRVAR(mmap_mmap_madvise__doc__, +"madvise($self, option, start=0, length=None, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_MADVISE_METHODDEF \ + {"madvise", _PyCFunction_CAST(mmap_mmap_madvise), METH_FASTCALL, mmap_mmap_madvise__doc__}, + +static PyObject * +mmap_mmap_madvise_impl(mmap_object *self, int option, Py_ssize_t start, + PyObject *length_obj); + +static PyObject * +mmap_mmap_madvise(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int option; + Py_ssize_t start = 0; + PyObject *length_obj = Py_None; + + if (!_PyArg_CheckPositional("madvise", nargs, 1, 3)) { + goto exit; + } + option = PyLong_AsInt(args[0]); + if (option == -1 && PyErr_Occurred()) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + start = ival; + } + if (nargs < 3) { + goto skip_optional; + } + length_obj = args[2]; +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_madvise_impl((mmap_object *)self, option, start, length_obj); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +#endif /* defined(HAVE_MADVISE) */ + +#ifndef MMAP_MMAP_RESIZE_METHODDEF + #define MMAP_MMAP_RESIZE_METHODDEF +#endif /* !defined(MMAP_MMAP_RESIZE_METHODDEF) */ + +#ifndef MMAP_MMAP___SIZEOF___METHODDEF + #define MMAP_MMAP___SIZEOF___METHODDEF +#endif /* !defined(MMAP_MMAP___SIZEOF___METHODDEF) */ + +#ifndef MMAP_MMAP__PROTECT_METHODDEF + #define MMAP_MMAP__PROTECT_METHODDEF +#endif /* !defined(MMAP_MMAP__PROTECT_METHODDEF) */ + +#ifndef MMAP_MMAP_MADVISE_METHODDEF + #define MMAP_MMAP_MADVISE_METHODDEF +#endif /* !defined(MMAP_MMAP_MADVISE_METHODDEF) */ +/*[clinic end generated code: output=381f6cf4986ac867 input=a9049054013a1b77]*/ diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 41d11716288..ac8521f8aa9 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -91,6 +91,12 @@ my_getpagesize(void) # define MAP_ANONYMOUS MAP_ANON #endif +/*[clinic input] +module mmap +class mmap.mmap "mmap_object *" "" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=82a9f8a529905b9b]*/ + typedef enum { ACCESS_DEFAULT, @@ -129,6 +135,26 @@ typedef struct { #define mmap_object_CAST(op) ((mmap_object *)(op)) +#include "clinic/mmapmodule.c.h" + + +/* Return a Py_ssize_t from the object arg. This conversion logic is similar + to what AC uses for `Py_ssize_t` arguments. + + Returns -1 on error. Use PyErr_Occurred() to disambiguate. +*/ +static Py_ssize_t +_As_Py_ssize_t(PyObject *arg) { + assert(arg != NULL); + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(arg); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + return ival; +} + static void mmap_object_dealloc(PyObject *op) { @@ -165,10 +191,16 @@ mmap_object_dealloc(PyObject *op) Py_DECREF(tp); } +/*[clinic input] +@critical_section +mmap.mmap.close + +[clinic start generated code]*/ + static PyObject * -mmap_close_method(PyObject *op, PyObject *Py_UNUSED(ignored)) +mmap_mmap_close_impl(mmap_object *self) +/*[clinic end generated code: output=a1ae0c727546f78d input=25020035f047eae1]*/ { - mmap_object *self = mmap_object_CAST(op); if (self->exports > 0) { PyErr_SetString(PyExc_BufferError, "cannot close "\ "exported pointers exist"); @@ -472,10 +504,16 @@ _safe_PyBytes_FromStringAndSize(char *start, size_t num_bytes) } } +/*[clinic input] +@critical_section +mmap.mmap.read_byte + +[clinic start generated code]*/ + static PyObject * -mmap_read_byte_method(PyObject *op, PyObject *Py_UNUSED(ignored)) +mmap_mmap_read_byte_impl(mmap_object *self) +/*[clinic end generated code: output=d931da1319f3869b input=5b8c6a904bdddda9]*/ { - mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); if (self->pos >= self->size) { PyErr_SetString(PyExc_ValueError, "read byte out of range"); @@ -489,12 +527,18 @@ mmap_read_byte_method(PyObject *op, PyObject *Py_UNUSED(ignored)) return PyLong_FromLong((unsigned char) dest); } +/*[clinic input] +@critical_section +mmap.mmap.readline + +[clinic start generated code]*/ + static PyObject * -mmap_read_line_method(PyObject *op, PyObject *Py_UNUSED(ignored)) +mmap_mmap_readline_impl(mmap_object *self) +/*[clinic end generated code: output=b9d2bf9999283311 input=2c4efd1d06e1cdd1]*/ { Py_ssize_t remaining; char *start, *eol; - mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); @@ -519,15 +563,21 @@ mmap_read_line_method(PyObject *op, PyObject *Py_UNUSED(ignored)) return result; } -static PyObject * -mmap_read_method(PyObject *op, PyObject *args) -{ - Py_ssize_t num_bytes = PY_SSIZE_T_MAX, remaining; - mmap_object *self = mmap_object_CAST(op); +/*[clinic input] +@critical_section +mmap.mmap.read + + n as num_bytes: object(converter='_Py_convert_optional_to_ssize_t', type='Py_ssize_t', c_default='PY_SSIZE_T_MAX') = None + / + +[clinic start generated code]*/ + +static PyObject * +mmap_mmap_read_impl(mmap_object *self, Py_ssize_t num_bytes) +/*[clinic end generated code: output=3b4d4f3704ed0969 input=8f97f361d435e357]*/ +{ + Py_ssize_t remaining; - CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "|O&:read", _Py_convert_optional_to_ssize_t, &num_bytes)) - return NULL; CHECK_VALID(NULL); /* silently 'adjust' out-of-range requests */ @@ -544,81 +594,105 @@ mmap_read_method(PyObject *op, PyObject *args) } static PyObject * -mmap_gfind(mmap_object *self, - PyObject *args, - int reverse) +mmap_gfind_lock_held(mmap_object *self, Py_buffer *view, PyObject *start_obj, + PyObject *end_obj, int reverse) { Py_ssize_t start = self->pos; Py_ssize_t end = self->size; - Py_buffer view; CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, reverse ? "y*|nn:rfind" : "y*|nn:find", - &view, &start, &end)) { - return NULL; - } - else { - if (start < 0) - start += self->size; - if (start < 0) - start = 0; - else if (start > self->size) - start = self->size; - - if (end < 0) - end += self->size; - if (end < 0) - end = 0; - else if (end > self->size) - end = self->size; - - Py_ssize_t index; - PyObject *result; - CHECK_VALID_OR_RELEASE(NULL, view); - if (end < start) { - result = PyLong_FromSsize_t(-1); + if (start_obj != Py_None) { + start = _As_Py_ssize_t(start_obj); + if (start == -1 && PyErr_Occurred()) { + return NULL; } - else if (reverse) { - assert(0 <= start && start <= end && end <= self->size); - if (_safe_PyBytes_ReverseFind(&index, self, - self->data + start, end - start, - view.buf, view.len, start) < 0) - { - result = NULL; - } - else { - result = PyLong_FromSsize_t(index); + + if (end_obj != Py_None) { + end = _As_Py_ssize_t(end_obj); + if (end == -1 && PyErr_Occurred()) { + return NULL; } } + } + + if (start < 0) + start += self->size; + if (start < 0) + start = 0; + else if (start > self->size) + start = self->size; + + if (end < 0) + end += self->size; + if (end < 0) + end = 0; + else if (end > self->size) + end = self->size; + + Py_ssize_t index; + PyObject *result; + CHECK_VALID(NULL); + if (end < start) { + result = PyLong_FromSsize_t(-1); + } + else if (reverse) { + assert(0 <= start && start <= end && end <= self->size); + if (_safe_PyBytes_ReverseFind(&index, self, + self->data + start, end - start, + view->buf, view->len, start) < 0) + { + result = NULL; + } else { - assert(0 <= start && start <= end && end <= self->size); - if (_safe_PyBytes_Find(&index, self, - self->data + start, end - start, - view.buf, view.len, start) < 0) - { - result = NULL; - } - else { - result = PyLong_FromSsize_t(index); - } + result = PyLong_FromSsize_t(index); } - PyBuffer_Release(&view); - return result; } + else { + assert(0 <= start && start <= end && end <= self->size); + if (_safe_PyBytes_Find(&index, self, + self->data + start, end - start, + view->buf, view->len, start) < 0) + { + result = NULL; + } + else { + result = PyLong_FromSsize_t(index); + } + } + return result; } -static PyObject * -mmap_find_method(PyObject *op, PyObject *args) -{ - mmap_object *self = mmap_object_CAST(op); - return mmap_gfind(self, args, 0); -} +/*[clinic input] +@critical_section +mmap.mmap.find + + view: Py_buffer + start: object = None + end: object = None + / + +[clinic start generated code]*/ static PyObject * -mmap_rfind_method(PyObject *op, PyObject *args) +mmap_mmap_find_impl(mmap_object *self, Py_buffer *view, PyObject *start, + PyObject *end) +/*[clinic end generated code: output=ef8878a322f00192 input=0135504494b52c2b]*/ { - mmap_object *self = mmap_object_CAST(op); - return mmap_gfind(self, args, 1); + return mmap_gfind_lock_held(self, view, start, end, 0); +} + +/*[clinic input] +@critical_section +mmap.mmap.rfind = mmap.mmap.find + +[clinic start generated code]*/ + +static PyObject * +mmap_mmap_rfind_impl(mmap_object *self, Py_buffer *view, PyObject *start, + PyObject *end) +/*[clinic end generated code: output=73b918940d67c2b8 input=8aecdd1f70c06c62]*/ +{ + return mmap_gfind_lock_held(self, view, start, end, 1); } static int @@ -654,50 +728,55 @@ is_resizeable(mmap_object *self) #endif /* MS_WINDOWS || HAVE_MREMAP */ +/*[clinic input] +@critical_section +mmap.mmap.write + + bytes as data: Py_buffer + / + +[clinic start generated code]*/ + static PyObject * -mmap_write_method(PyObject *op, PyObject *args) +mmap_mmap_write_impl(mmap_object *self, Py_buffer *data) +/*[clinic end generated code: output=9e97063efb6fb27b input=3f16fa79aa89d6f7]*/ { - Py_buffer data; - mmap_object *self = mmap_object_CAST(op); - CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "y*:write", &data)) - return NULL; - if (!is_writable(self)) { - PyBuffer_Release(&data); return NULL; } - if (self->pos > self->size || self->size - self->pos < data.len) { - PyBuffer_Release(&data); + if (self->pos > self->size || self->size - self->pos < data->len) { PyErr_SetString(PyExc_ValueError, "data out of range"); return NULL; } - CHECK_VALID_OR_RELEASE(NULL, data); + CHECK_VALID(NULL); PyObject *result; - if (safe_memcpy(self->data + self->pos, data.buf, data.len) < 0) { + if (safe_memcpy(self->data + self->pos, data->buf, data->len) < 0) { result = NULL; } else { - self->pos += data.len; - result = PyLong_FromSsize_t(data.len); + self->pos += data->len; + result = PyLong_FromSsize_t(data->len); } - PyBuffer_Release(&data); return result; } +/*[clinic input] +@critical_section +mmap.mmap.write_byte + + byte as value: unsigned_char + / + +[clinic start generated code]*/ + static PyObject * -mmap_write_byte_method(PyObject *op, PyObject *args) +mmap_mmap_write_byte_impl(mmap_object *self, unsigned char value) +/*[clinic end generated code: output=aa11adada9b17510 input=32740bfa174f0991]*/ { - char value; - mmap_object *self = mmap_object_CAST(op); - CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "b:write_byte", &value)) - return(NULL); - if (!is_writable(self)) return NULL; @@ -707,17 +786,23 @@ mmap_write_byte_method(PyObject *op, PyObject *args) return NULL; } - if (safe_byte_copy(self->data + self->pos, &value) < 0) { + if (safe_byte_copy(self->data + self->pos, (const char*)&value) < 0) { return NULL; } self->pos++; Py_RETURN_NONE; } +/*[clinic input] +@critical_section +mmap.mmap.size + +[clinic start generated code]*/ + static PyObject * -mmap_size_method(PyObject *op, PyObject *Py_UNUSED(ignored)) +mmap_mmap_size_impl(mmap_object *self) +/*[clinic end generated code: output=c177e65e83a648ff input=f69c072efd2e1595]*/ { - mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); #ifdef MS_WINDOWS @@ -771,14 +856,21 @@ mmap_size_method(PyObject *op, PyObject *Py_UNUSED(ignored)) */ #if defined(MS_WINDOWS) || defined(HAVE_MREMAP) +/*[clinic input] +@critical_section +mmap.mmap.resize + + newsize as new_size: Py_ssize_t + / + +[clinic start generated code]*/ + static PyObject * -mmap_resize_method(PyObject *op, PyObject *args) +mmap_mmap_resize_impl(mmap_object *self, Py_ssize_t new_size) +/*[clinic end generated code: output=6f262537ce9c2dcc input=b6b5dee52a41b79f]*/ { - Py_ssize_t new_size; - mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "n:resize", &new_size) || - !is_resizeable(self)) { + if (!is_resizeable(self)) { return NULL; } if (new_size < 0 || PY_SSIZE_T_MAX - new_size < self->offset) { @@ -921,24 +1013,35 @@ mmap_resize_method(PyObject *op, PyObject *args) } #endif /* MS_WINDOWS || HAVE_MREMAP */ +/*[clinic input] +@critical_section +mmap.mmap.tell + +[clinic start generated code]*/ + static PyObject * -mmap_tell_method(PyObject *op, PyObject *Py_UNUSED(ignored)) +mmap_mmap_tell_impl(mmap_object *self) +/*[clinic end generated code: output=6034958630e1b1d1 input=fd163acacf45c3a5]*/ { - mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); return PyLong_FromSize_t(self->pos); } +/*[clinic input] +@critical_section +mmap.mmap.flush + + offset: Py_ssize_t = 0 + size: Py_ssize_t = -1 + / + +[clinic start generated code]*/ + static PyObject * -mmap_flush_method(PyObject *op, PyObject *args) +mmap_mmap_flush_impl(mmap_object *self, Py_ssize_t offset, Py_ssize_t size) +/*[clinic end generated code: output=956ced67466149cf input=c50b893bc69520ec]*/ { - Py_ssize_t offset = 0; - Py_ssize_t size = -1; - mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "|nn:flush", &offset, &size)) { - return NULL; - } if (size == -1) { size = self->size - offset; } @@ -969,60 +1072,80 @@ mmap_flush_method(PyObject *op, PyObject *args) #endif } +/*[clinic input] +@critical_section +mmap.mmap.seek + + pos as dist: Py_ssize_t + whence as how: int = 0 + / + +[clinic start generated code]*/ + static PyObject * -mmap_seek_method(PyObject *op, PyObject *args) +mmap_mmap_seek_impl(mmap_object *self, Py_ssize_t dist, int how) +/*[clinic end generated code: output=00310494e8b8c592 input=e2fda5d081c3db22]*/ { - Py_ssize_t dist; - mmap_object *self = mmap_object_CAST(op); - int how=0; CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "n|i:seek", &dist, &how)) - return NULL; - else { - Py_ssize_t where; - switch (how) { - case 0: /* relative to start */ - where = dist; - break; - case 1: /* relative to current position */ - if (PY_SSIZE_T_MAX - self->pos < dist) - goto onoutofrange; - where = self->pos + dist; - break; - case 2: /* relative to end */ - if (PY_SSIZE_T_MAX - self->size < dist) - goto onoutofrange; - where = self->size + dist; - break; - default: - PyErr_SetString(PyExc_ValueError, "unknown seek type"); - return NULL; - } - if (where > self->size || where < 0) + Py_ssize_t where; + switch (how) { + case 0: /* relative to start */ + where = dist; + break; + case 1: /* relative to current position */ + if (PY_SSIZE_T_MAX - self->pos < dist) goto onoutofrange; - self->pos = where; - return PyLong_FromSsize_t(self->pos); + where = self->pos + dist; + break; + case 2: /* relative to end */ + if (PY_SSIZE_T_MAX - self->size < dist) + goto onoutofrange; + where = self->size + dist; + break; + default: + PyErr_SetString(PyExc_ValueError, "unknown seek type"); + return NULL; } + if (where > self->size || where < 0) + goto onoutofrange; + self->pos = where; + return PyLong_FromSsize_t(self->pos); onoutofrange: PyErr_SetString(PyExc_ValueError, "seek out of range"); return NULL; } +/*[clinic input] +mmap.mmap.seekable + +[clinic start generated code]*/ + static PyObject * -mmap_seekable_method(PyObject *op, PyObject *Py_UNUSED(ignored)) +mmap_mmap_seekable_impl(mmap_object *self) +/*[clinic end generated code: output=6311dc3ea300fa38 input=5132505f6e259001]*/ { Py_RETURN_TRUE; } +/*[clinic input] +@critical_section +mmap.mmap.move + + dest: Py_ssize_t + src: Py_ssize_t + count as cnt: Py_ssize_t + / + +[clinic start generated code]*/ + static PyObject * -mmap_move_method(PyObject *op, PyObject *args) +mmap_mmap_move_impl(mmap_object *self, Py_ssize_t dest, Py_ssize_t src, + Py_ssize_t cnt) +/*[clinic end generated code: output=391f549a44181793 input=cf8cfe10d9f6b448]*/ { - Py_ssize_t dest, src, cnt; - mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "nnn:move", &dest, &src, &cnt) || - !is_writable(self)) { + if (!is_writable(self)) { return NULL; } else { /* bounds check the values */ @@ -1048,30 +1171,53 @@ static PyObject * mmap_closed_get(PyObject *op, void *Py_UNUSED(closure)) { mmap_object *self = mmap_object_CAST(op); + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); #ifdef MS_WINDOWS - return PyBool_FromLong(self->map_handle == NULL ? 1 : 0); + result = PyBool_FromLong(self->map_handle == NULL ? 1 : 0); #elif defined(UNIX) - return PyBool_FromLong(self->data == NULL ? 1 : 0); + result = PyBool_FromLong(self->data == NULL ? 1 : 0); #endif + Py_END_CRITICAL_SECTION(); + return result; } +/*[clinic input] +@critical_section +mmap.mmap.__enter__ + +[clinic start generated code]*/ + static PyObject * -mmap__enter__method(PyObject *op, PyObject *Py_UNUSED(ignored)) +mmap_mmap___enter___impl(mmap_object *self) +/*[clinic end generated code: output=92cfc59f4c4e2d26 input=a446541fbfe0b890]*/ { - mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); return Py_NewRef(self); } +/*[clinic input] +@critical_section +mmap.mmap.__exit__ + + exc_type: object + exc_value: object + traceback: object + / + +[clinic start generated code]*/ + static PyObject * -mmap__exit__method(PyObject *op, PyObject *Py_UNUSED(args)) +mmap_mmap___exit___impl(mmap_object *self, PyObject *exc_type, + PyObject *exc_value, PyObject *traceback) +/*[clinic end generated code: output=bec7e3e319c1f07e input=5f28e91cf752bc64]*/ { - return mmap_close_method(op, NULL); + return mmap_mmap_close_impl(self); } static PyObject * -mmap__repr__method(PyObject *op) +mmap__repr__method_lock_held(PyObject *op) { mmap_object *mobj = mmap_object_CAST(op); @@ -1115,11 +1261,27 @@ mmap__repr__method(PyObject *op) } } -#ifdef MS_WINDOWS static PyObject * -mmap__sizeof__method(PyObject *op, PyObject *Py_UNUSED(dummy)) +mmap__repr__method(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = mmap__repr__method_lock_held(op); + Py_END_CRITICAL_SECTION(); + return result; +} + +#ifdef MS_WINDOWS +/*[clinic input] +@critical_section +mmap.mmap.__sizeof__ + +[clinic start generated code]*/ + +static PyObject * +mmap_mmap___sizeof___impl(mmap_object *self) +/*[clinic end generated code: output=1aed30daff807d09 input=8a648868a089553c]*/ { - mmap_object *self = mmap_object_CAST(op); size_t res = _PyObject_SIZE(Py_TYPE(self)); if (self->tagname) { res += (wcslen(self->tagname) + 1) * sizeof(self->tagname[0]); @@ -1129,18 +1291,26 @@ mmap__sizeof__method(PyObject *op, PyObject *Py_UNUSED(dummy)) #endif #if defined(MS_WINDOWS) && defined(Py_DEBUG) +/*[clinic input] +@critical_section +mmap.mmap._protect + + flNewProtect: unsigned_int(bitwise=True) + start: Py_ssize_t + length: Py_ssize_t + / + +[clinic start generated code]*/ + static PyObject * -mmap_protect_method(PyObject *op, PyObject *args) { - DWORD flNewProtect, flOldProtect; - Py_ssize_t start, length; - mmap_object *self = mmap_object_CAST(op); +mmap_mmap__protect_impl(mmap_object *self, unsigned int flNewProtect, + Py_ssize_t start, Py_ssize_t length) +/*[clinic end generated code: output=a87271a34d1ad6cf input=9170498c5e1482da]*/ +{ + DWORD flOldProtect; CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "Inn:protect", &flNewProtect, &start, &length)) { - return NULL; - } - if (!VirtualProtect((void *) (self->data + start), length, flNewProtect, &flOldProtect)) { @@ -1153,18 +1323,32 @@ mmap_protect_method(PyObject *op, PyObject *args) { #endif #ifdef HAVE_MADVISE +/*[clinic input] +@critical_section +mmap.mmap.madvise + + option: int + start: Py_ssize_t = 0 + length as length_obj: object = None + / + +[clinic start generated code]*/ + static PyObject * -mmap_madvise_method(PyObject *op, PyObject *args) +mmap_mmap_madvise_impl(mmap_object *self, int option, Py_ssize_t start, + PyObject *length_obj) +/*[clinic end generated code: output=816be656f08c0e3c input=2d37f7a4c87f1053]*/ { - int option; - Py_ssize_t start = 0, length; - mmap_object *self = mmap_object_CAST(op); + Py_ssize_t length; CHECK_VALID(NULL); - length = self->size; - - if (!PyArg_ParseTuple(args, "i|nn:madvise", &option, &start, &length)) { - return NULL; + if (length_obj == Py_None) { + length = self->size; + } else { + length = _As_Py_ssize_t(length_obj); + if (length == -1 && PyErr_Occurred()) { + return NULL; + } } if (start < 0 || start >= self->size) { @@ -1200,34 +1384,26 @@ static struct PyMemberDef mmap_object_members[] = { }; static struct PyMethodDef mmap_object_methods[] = { - {"close", mmap_close_method, METH_NOARGS}, - {"find", mmap_find_method, METH_VARARGS}, - {"rfind", mmap_rfind_method, METH_VARARGS}, - {"flush", mmap_flush_method, METH_VARARGS}, -#ifdef HAVE_MADVISE - {"madvise", mmap_madvise_method, METH_VARARGS}, -#endif - {"move", mmap_move_method, METH_VARARGS}, - {"read", mmap_read_method, METH_VARARGS}, - {"read_byte", mmap_read_byte_method, METH_NOARGS}, - {"readline", mmap_read_line_method, METH_NOARGS}, -#if defined(MS_WINDOWS) || defined(HAVE_MREMAP) - {"resize", mmap_resize_method, METH_VARARGS}, -#endif - {"seek", mmap_seek_method, METH_VARARGS}, - {"seekable", mmap_seekable_method, METH_NOARGS}, - {"size", mmap_size_method, METH_NOARGS}, - {"tell", mmap_tell_method, METH_NOARGS}, - {"write", mmap_write_method, METH_VARARGS}, - {"write_byte", mmap_write_byte_method, METH_VARARGS}, - {"__enter__", mmap__enter__method, METH_NOARGS}, - {"__exit__", mmap__exit__method, METH_VARARGS}, -#ifdef MS_WINDOWS - {"__sizeof__", mmap__sizeof__method, METH_NOARGS}, -#ifdef Py_DEBUG - {"_protect", mmap_protect_method, METH_VARARGS}, -#endif // Py_DEBUG -#endif // MS_WINDOWS + MMAP_MMAP_CLOSE_METHODDEF + MMAP_MMAP_FIND_METHODDEF + MMAP_MMAP_RFIND_METHODDEF + MMAP_MMAP_FLUSH_METHODDEF + MMAP_MMAP_MADVISE_METHODDEF + MMAP_MMAP_MOVE_METHODDEF + MMAP_MMAP_READ_METHODDEF + MMAP_MMAP_READ_BYTE_METHODDEF + MMAP_MMAP_READLINE_METHODDEF + MMAP_MMAP_RESIZE_METHODDEF + MMAP_MMAP_SEEK_METHODDEF + MMAP_MMAP_SEEKABLE_METHODDEF + MMAP_MMAP_SIZE_METHODDEF + MMAP_MMAP_TELL_METHODDEF + MMAP_MMAP_WRITE_METHODDEF + MMAP_MMAP_WRITE_BYTE_METHODDEF + MMAP_MMAP___ENTER___METHODDEF + MMAP_MMAP___EXIT___METHODDEF + MMAP_MMAP___SIZEOF___METHODDEF + MMAP_MMAP__PROTECT_METHODDEF {NULL, NULL} /* sentinel */ }; @@ -1240,7 +1416,7 @@ static PyGetSetDef mmap_object_getset[] = { /* Functions for treating an mmap'ed file as a buffer */ static int -mmap_buffer_getbuf(PyObject *op, Py_buffer *view, int flags) +mmap_buffer_getbuf_lock_held(PyObject *op, Py_buffer *view, int flags) { mmap_object *self = mmap_object_CAST(op); CHECK_VALID(-1); @@ -1251,23 +1427,45 @@ mmap_buffer_getbuf(PyObject *op, Py_buffer *view, int flags) return 0; } +static int +mmap_buffer_getbuf(PyObject *op, Py_buffer *view, int flags) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(op); + result = mmap_buffer_getbuf_lock_held(op, view, flags); + Py_END_CRITICAL_SECTION(); + return result; +} + static void mmap_buffer_releasebuf(PyObject *op, Py_buffer *Py_UNUSED(view)) { mmap_object *self = mmap_object_CAST(op); + Py_BEGIN_CRITICAL_SECTION(self); self->exports--; + Py_END_CRITICAL_SECTION(); } static Py_ssize_t -mmap_length(PyObject *op) +mmap_length_lock_held(PyObject *op) { mmap_object *self = mmap_object_CAST(op); CHECK_VALID(-1); return self->size; } +static Py_ssize_t +mmap_length(PyObject *op) +{ + Py_ssize_t result; + Py_BEGIN_CRITICAL_SECTION(op); + result = mmap_length_lock_held(op); + Py_END_CRITICAL_SECTION(); + return result; +} + static PyObject * -mmap_item(PyObject *op, Py_ssize_t i) +mmap_item_lock_held(PyObject *op, Py_ssize_t i) { mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); @@ -1284,7 +1482,16 @@ mmap_item(PyObject *op, Py_ssize_t i) } static PyObject * -mmap_subscript(PyObject *op, PyObject *item) +mmap_item(PyObject *op, Py_ssize_t i) { + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = mmap_item_lock_held(op, i); + Py_END_CRITICAL_SECTION(); + return result; +} + +static PyObject * +mmap_subscript_lock_held(PyObject *op, PyObject *item) { mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); @@ -1346,8 +1553,18 @@ mmap_subscript(PyObject *op, PyObject *item) } } +static PyObject * +mmap_subscript(PyObject *op, PyObject *item) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = mmap_subscript_lock_held(op, item); + Py_END_CRITICAL_SECTION(); + return result; +} + static int -mmap_ass_item(PyObject *op, Py_ssize_t i, PyObject *v) +mmap_ass_item_lock_held(PyObject *op, Py_ssize_t i, PyObject *v) { const char *buf; mmap_object *self = mmap_object_CAST(op); @@ -1378,7 +1595,17 @@ mmap_ass_item(PyObject *op, Py_ssize_t i, PyObject *v) } static int -mmap_ass_subscript(PyObject *op, PyObject *item, PyObject *value) +mmap_ass_item(PyObject *op, Py_ssize_t i, PyObject *v) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(op); + result = mmap_ass_item_lock_held(op, i, v); + Py_END_CRITICAL_SECTION(); + return result; +} + +static int +mmap_ass_subscript_lock_held(PyObject *op, PyObject *item, PyObject *value) { mmap_object *self = mmap_object_CAST(op); CHECK_VALID(-1); @@ -1474,6 +1701,16 @@ mmap_ass_subscript(PyObject *op, PyObject *item, PyObject *value) } } +static int +mmap_ass_subscript(PyObject *op, PyObject *item, PyObject *value) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(op); + result = mmap_ass_subscript_lock_held(op, item, value); + Py_END_CRITICAL_SECTION(); + return result; +} + static PyObject * new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict); From 65089406a54df6d3fdc65f5f7e7691ec5de088d8 Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 9 Oct 2025 16:00:37 +0800 Subject: [PATCH 075/112] gh-139743: Avoid import-time print in test_sqlite3 (GH-139746) --- Lib/test/test_sqlite3/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_sqlite3/__init__.py b/Lib/test/test_sqlite3/__init__.py index d777fca82da..78a1e2078a5 100644 --- a/Lib/test/test_sqlite3/__init__.py +++ b/Lib/test/test_sqlite3/__init__.py @@ -8,8 +8,7 @@ # Implement the unittest "load tests" protocol. def load_tests(*args): + if verbose: + print(f"test_sqlite3: testing with SQLite version {sqlite3.sqlite_version}") pkg_dir = os.path.dirname(__file__) return load_package_tests(pkg_dir, *args) - -if verbose: - print(f"test_sqlite3: testing with SQLite version {sqlite3.sqlite_version}") From 197c610a1e8e396e354c10f6f5dd60ed6548be77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 9 Oct 2025 11:14:36 +0200 Subject: [PATCH 076/112] gh-88046: remove impossible conditional import for `_ssl.RAND_egd` (#139648) `_ssl.RAND_egd` was removed in b8d0fa035d74ae6ae00794c9af636b427c5dc650. --- Doc/library/ssl.rst | 5 ++--- Lib/ssl.py | 5 ----- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 0f2c2b89295..5b59a0698e4 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -354,9 +354,8 @@ Random generation .. function:: RAND_status() Return ``True`` if the SSL pseudo-random number generator has been seeded - with 'enough' randomness, and ``False`` otherwise. You can use - :func:`ssl.RAND_egd` and :func:`ssl.RAND_add` to increase the randomness of - the pseudo-random number generator. + with 'enough' randomness, and ``False`` otherwise. Use :func:`ssl.RAND_add` + to increase the randomness of the pseudo-random number generator. .. function:: RAND_add(bytes, entropy, /) diff --git a/Lib/ssl.py b/Lib/ssl.py index 493ea901d93..7ad7969a821 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -110,11 +110,6 @@ ) from _ssl import txt2obj as _txt2obj, nid2obj as _nid2obj from _ssl import RAND_status, RAND_add, RAND_bytes -try: - from _ssl import RAND_egd -except ImportError: - # RAND_egd is not supported on some platforms - pass from _ssl import get_sigalgs From 6fd141834193e3dc2423e785d97eb127c972a70e Mon Sep 17 00:00:00 2001 From: Anuradha Agrawal Date: Thu, 9 Oct 2025 14:54:53 +0530 Subject: [PATCH 077/112] gh-139742: Add support for Python 3.14 t-string prefixes in IDLE colorizer and tests (#139756) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add 't' prefix to colorizer.py stringprefix regex to support Python 3.14 template strings. Add t prefixes to test_colorizer.py source test text and adjust line numbers on test methods. --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Łukasz Langa Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Lib/idlelib/colorizer.py | 2 +- Lib/idlelib/idle_test/test_colorizer.py | 29 ++++++++++--------- ...-10-08-08-35-50.gh-issue-139742.B3fZLg.rst | 1 + 3 files changed, 17 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2025-10-08-08-35-50.gh-issue-139742.B3fZLg.rst diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index b4df353012b..bffa2ddd3cd 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -47,7 +47,7 @@ def make_pat(): name not in keyword.kwlist] builtin = r"([^.'\"\\#]\b|^)" + any("BUILTIN", builtinlist) + r"\b" comment = any("COMMENT", [r"#[^\n]*"]) - stringprefix = r"(?i:r|u|f|fr|rf|b|br|rb)?" + stringprefix = r"(?i:r|u|f|fr|rf|b|br|rb|t|rt|tr)?" sqstring = stringprefix + r"'[^'\\\n]*(\\.[^'\\\n]*)*'?" dqstring = stringprefix + r'"[^"\\\n]*(\\.[^"\\\n]*)*"?' sq3string = stringprefix + r"'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?" diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index 308bc389384..40800df97b0 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -36,6 +36,7 @@ async def f(): await g() # All valid prefixes for unicode and byte strings should be colored. r'x', u'x', R'x', U'x', f'x', F'x' fr'x', Fr'x', fR'x', FR'x', rf'x', rF'x', Rf'x', RF'x' + tr'x', Tr'x', tR'x', TR'x', rt'x', rT'x', Rt'x', RT'x' b'x',B'x', br'x',Br'x',bR'x',BR'x', rb'x', rB'x',Rb'x',RB'x' # Invalid combinations of legal characters should be half colored. ur'x', ru'x', uf'x', fu'x', UR'x', ufr'x', rfu'x', xf'x', fx'x' @@ -390,19 +391,19 @@ def test_recolorize_main(self, mock_notify): ('6.0', ('KEYWORD',)), ('6.10', ('DEFINITION',)), ('6.11', ()), ('8.0', ('STRING',)), ('8.4', ()), ('8.5', ('STRING',)), ('8.12', ()), ('8.14', ('STRING',)), - ('19.0', ('KEYWORD',)), - ('20.4', ('KEYWORD',)), ('20.16', ('KEYWORD',)),# ('20.19', ('KEYWORD',)), - #('22.4', ('KEYWORD',)), ('22.10', ('KEYWORD',)), ('22.14', ('KEYWORD',)), ('22.19', ('STRING',)), - #('23.12', ('KEYWORD',)), - ('24.8', ('KEYWORD',)), - ('25.4', ('KEYWORD',)), ('25.9', ('KEYWORD',)), - ('25.11', ('KEYWORD',)), ('25.15', ('STRING',)), - ('25.19', ('KEYWORD',)), ('25.22', ()), - ('25.24', ('KEYWORD',)), ('25.29', ('BUILTIN',)), ('25.37', ('KEYWORD',)), - ('26.4', ('KEYWORD',)), ('26.9', ('KEYWORD',)),# ('26.11', ('KEYWORD',)), ('26.14', (),), - ('27.25', ('STRING',)), ('27.38', ('STRING',)), - ('29.0', ('STRING',)), - ('30.1', ('STRING',)), + ('20.0', ('KEYWORD',)), + ('21.4', ('KEYWORD',)), ('21.16', ('KEYWORD',)),# ('21.19', ('KEYWORD',)), + #('23.4', ('KEYWORD',)), ('23.10', ('KEYWORD',)), ('23.14', ('KEYWORD',)), ('23.19', ('STRING',)), + #('24.12', ('KEYWORD',)), + ('25.8', ('KEYWORD',)), + ('26.4', ('KEYWORD',)), ('26.9', ('KEYWORD',)), + ('26.11', ('KEYWORD',)), ('26.15', ('STRING',)), + ('26.19', ('KEYWORD',)), ('26.22', ()), + ('26.24', ('KEYWORD',)), ('26.29', ('BUILTIN',)), ('26.37', ('KEYWORD',)), + ('27.4', ('KEYWORD',)), ('27.9', ('KEYWORD',)),# ('27.11', ('KEYWORD',)), ('27.14', (),), + ('28.25', ('STRING',)), ('28.38', ('STRING',)), + ('30.0', ('STRING',)), + ('31.1', ('STRING',)), # SYNC at the end of every line. ('1.55', ('SYNC',)), ('2.50', ('SYNC',)), ('3.34', ('SYNC',)), ) @@ -433,7 +434,7 @@ def test_recolorize_main(self, mock_notify): eq(text.tag_nextrange('STRING', '8.12'), ('8.14', '8.17')) eq(text.tag_nextrange('STRING', '8.17'), ('8.19', '8.26')) eq(text.tag_nextrange('SYNC', '8.0'), ('8.26', '9.0')) - eq(text.tag_nextrange('SYNC', '30.0'), ('30.10', '32.0')) + eq(text.tag_nextrange('SYNC', '31.0'), ('31.10', '33.0')) def _assert_highlighting(self, source, tag_ranges): """Check highlighting of a given piece of code. diff --git a/Misc/NEWS.d/next/IDLE/2025-10-08-08-35-50.gh-issue-139742.B3fZLg.rst b/Misc/NEWS.d/next/IDLE/2025-10-08-08-35-50.gh-issue-139742.B3fZLg.rst new file mode 100644 index 00000000000..c9e2f977a79 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2025-10-08-08-35-50.gh-issue-139742.B3fZLg.rst @@ -0,0 +1 @@ +Colorize t-string prefixes for template strings in IDLE, as done for f-string prefixes. From d2deb8fdef1ac5e54564448677cdb1522f1b776d Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Thu, 9 Oct 2025 10:34:35 +0100 Subject: [PATCH 078/112] gh-101100: Fix reference warnings in `c-api/init.rst` documenting `PyGILState_STATE` (#139572) --- Doc/c-api/init.rst | 25 +++++++++++++++++++------ Doc/tools/.nitignore | 1 - 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index ccf85e627f9..7a501ea22be 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1113,7 +1113,7 @@ code, or when embedding the Python interpreter: This function is safe to call without an :term:`attached thread state`; it will simply return ``NULL`` indicating that there was no prior thread state. - .. seealso: + .. seealso:: :c:func:`PyEval_ReleaseThread` .. note:: @@ -1124,6 +1124,19 @@ code, or when embedding the Python interpreter: The following functions use thread-local storage, and are not compatible with sub-interpreters: +.. c:type:: PyGILState_STATE + + The type of the value returned by :c:func:`PyGILState_Ensure` and passed to + :c:func:`PyGILState_Release`. + + .. c:enumerator:: PyGILState_LOCKED + + The GIL was already held when :c:func:`PyGILState_Ensure` was called. + + .. c:enumerator:: PyGILState_UNLOCKED + + The GIL was not held when :c:func:`PyGILState_Ensure` was called. + .. c:function:: PyGILState_STATE PyGILState_Ensure() Ensure that the current thread is ready to call the Python C API regardless @@ -1174,12 +1187,12 @@ with sub-interpreters: made on the main thread. This is mainly a helper/diagnostic function. .. note:: - This function does not account for :term:`thread states ` created - by something other than :c:func:`PyGILState_Ensure` (such as :c:func:`PyThreadState_New`). + This function may return non-``NULL`` even when the :term:`thread state` + is detached. Prefer :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` for most cases. - .. seealso: :c:func:`PyThreadState_Get`` + .. seealso:: :c:func:`PyThreadState_Get` .. c:function:: int PyGILState_Check() @@ -1278,11 +1291,11 @@ All of the following functions must be called after :c:func:`Py_Initialize`. must be :term:`attached ` .. versionchanged:: 3.9 - This function now calls the :c:member:`PyThreadState.on_delete` callback. + This function now calls the :c:member:`!PyThreadState.on_delete` callback. Previously, that happened in :c:func:`PyThreadState_Delete`. .. versionchanged:: 3.13 - The :c:member:`PyThreadState.on_delete` callback was removed. + The :c:member:`!PyThreadState.on_delete` callback was removed. .. c:function:: void PyThreadState_Delete(PyThreadState *tstate) diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 4ee09c6bbaa..827c5808fa5 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -4,7 +4,6 @@ Doc/c-api/descriptor.rst Doc/c-api/float.rst -Doc/c-api/init.rst Doc/c-api/init_config.rst Doc/c-api/intro.rst Doc/c-api/module.rst From 81959a0364f5bc15414d5bd61c5c0d019d486fe5 Mon Sep 17 00:00:00 2001 From: DeepWzh Date: Thu, 9 Oct 2025 22:16:48 +0800 Subject: [PATCH 079/112] gh-133400: Fixed Ctrl+D (^D) behavior in :mod:`_pyrepl` module (GH-133883) Co-authored-by: adam j hartz --- Lib/_pyrepl/commands.py | 19 +++-- Lib/test/test_pyrepl/test_pyrepl.py | 83 +++++++++++++++++++ ...-05-11-09-40-19.gh-issue-133400.zkWla8.rst | 1 + 3 files changed, 95 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-11-09-40-19.gh-issue-133400.zkWla8.rst diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 50c824995d8..10127e58897 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -420,14 +420,17 @@ class delete(EditCommand): def do(self) -> None: r = self.reader b = r.buffer - if ( - r.pos == 0 - and len(b) == 0 # this is something of a hack - and self.event[-1] == "\004" - ): - r.update_screen() - r.console.finish() - raise EOFError + if self.event[-1] == "\004": + if b and b[-1].endswith("\n"): + self.finish = True + elif ( + r.pos == 0 + and len(b) == 0 # this is something of a hack + ): + r.update_screen() + r.console.finish() + raise EOFError + for i in range(r.get_arg()): if r.pos != len(b): del b[r.pos] diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 47d384a209e..ad7464dc3d3 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -1825,3 +1825,86 @@ def test_detect_pip_usage_in_repl(self): " outside of the Python REPL" ) self.assertIn(hint, output) + +class TestPyReplCtrlD(TestCase): + """Test Ctrl+D behavior in _pyrepl to match old pre-3.13 REPL behavior. + + Ctrl+D should: + - Exit on empty buffer (raises EOFError) + - Delete character when cursor is in middle of line + - Perform no operation when cursor is at end of line without newline + - Exit multiline mode when cursor is at end with trailing newline + - Run code up to that point when pressed on blank line with preceding lines + """ + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_ctrl_d_empty_line(self): + """Test that pressing Ctrl+D on empty line exits the program""" + events = [ + Event(evt="key", data="\x04", raw=bytearray(b"\x04")), # Ctrl+D + ] + reader = self.prepare_reader(events) + with self.assertRaises(EOFError): + multiline_input(reader) + + def test_ctrl_d_multiline_with_new_line(self): + """Test that pressing Ctrl+D in multiline mode with trailing newline exits multiline mode""" + events = itertools.chain( + code_to_events("def f():\n pass\n"), # Enter multiline mode with trailing newline + [ + Event(evt="key", data="\x04", raw=bytearray(b"\x04")), # Ctrl+D + ], + ) + reader, _ = handle_all_events(events) + self.assertTrue(reader.finished) + self.assertEqual("def f():\n pass\n", "".join(reader.buffer)) + + def test_ctrl_d_multiline_middle_of_line(self): + """Test that pressing Ctrl+D in multiline mode with cursor in middle deletes character""" + events = itertools.chain( + code_to_events("def f():\n hello world"), # Enter multiline mode + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")) + ] * 5, # move cursor to 'w' in "world" + [ + Event(evt="key", data="\x04", raw=bytearray(b"\x04")) + ], # Ctrl+D should delete 'w' + ) + reader, _ = handle_all_events(events) + self.assertFalse(reader.finished) + self.assertEqual("def f():\n hello orld", "".join(reader.buffer)) + + def test_ctrl_d_multiline_end_of_line_no_newline(self): + """Test that pressing Ctrl+D at end of line without newline performs no operation""" + events = itertools.chain( + code_to_events("def f():\n hello"), # Enter multiline mode, no trailing newline + [ + Event(evt="key", data="\x04", raw=bytearray(b"\x04")) + ], # Ctrl+D should be no-op + ) + reader, _ = handle_all_events(events) + self.assertFalse(reader.finished) + self.assertEqual("def f():\n hello", "".join(reader.buffer)) + + def test_ctrl_d_single_line_middle_of_line(self): + """Test that pressing Ctrl+D in single line mode deletes current character""" + events = itertools.chain( + code_to_events("hello"), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], # move left + [Event(evt="key", data="\x04", raw=bytearray(b"\x04"))], # Ctrl+D + ) + reader, _ = handle_all_events(events) + self.assertEqual("hell", "".join(reader.buffer)) + + def test_ctrl_d_single_line_end_no_newline(self): + """Test that pressing Ctrl+D at end of single line without newline does nothing""" + events = itertools.chain( + code_to_events("hello"), # cursor at end of line + [Event(evt="key", data="\x04", raw=bytearray(b"\x04"))], # Ctrl+D + ) + reader, _ = handle_all_events(events) + self.assertEqual("hello", "".join(reader.buffer)) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-11-09-40-19.gh-issue-133400.zkWla8.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-11-09-40-19.gh-issue-133400.zkWla8.rst new file mode 100644 index 00000000000..2498d6ebaa5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-11-09-40-19.gh-issue-133400.zkWla8.rst @@ -0,0 +1 @@ +Fixed Ctrl+D (^D) behavior in _pyrepl module to match old pre-3.13 REPL behavior. From b8c8b8f1d30767e5358ffa93d41e6e27ca60570d Mon Sep 17 00:00:00 2001 From: yihong Date: Thu, 9 Oct 2025 22:58:01 +0800 Subject: [PATCH 080/112] gh-139391: properly handle `signal.signal()` in `UnixConsole` when called from a non-main thread (#139392) --- Lib/_pyrepl/unix_console.py | 7 ++++++- Lib/test/test_pyrepl/test_unix_console.py | 14 +++++++++++++- .../2025-09-28-16-34-11.gh-issue-139391.nRFnmx.rst | 3 +++ 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-09-28-16-34-11.gh-issue-139391.nRFnmx.rst diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index fe45b4eb384..09247de748e 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -390,7 +390,12 @@ def restore(self): os.write(self.output_fd, b"\033[?7h") if hasattr(self, "old_sigwinch"): - signal.signal(signal.SIGWINCH, self.old_sigwinch) + try: + signal.signal(signal.SIGWINCH, self.old_sigwinch) + except ValueError as e: + import threading + if threading.current_thread() is threading.main_thread(): + raise e del self.old_sigwinch def push_char(self, char: int | bytes) -> None: diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py index 9ac2e9961df..f4fb9237ffd 100644 --- a/Lib/test/test_pyrepl/test_unix_console.py +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -4,11 +4,12 @@ import signal import subprocess import sys +import threading import unittest from functools import partial from test import support from test.support import os_helper, force_not_colorized_test_class -from test.support import script_helper +from test.support import script_helper, threading_helper from unittest import TestCase from unittest.mock import MagicMock, call, patch, ANY, Mock @@ -318,6 +319,17 @@ def test_restore_with_invalid_environ_on_macos(self, _os_write): console.prepare() # needed to call restore() console.restore() # this should succeed + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_restore_in_thread(self, _os_write): + # gh-139391: ensure that console.restore() silently suppresses + # exceptions when calling signal.signal() from a non-main thread. + console = unix_console([]) + console.old_sigwinch = signal.SIG_DFL + thread = threading.Thread(target=console.restore) + thread.start() + thread.join() # this should not raise + @unittest.skipIf(sys.platform == "win32", "No Unix console on Windows") class TestUnixConsoleEIOHandling(TestCase): diff --git a/Misc/NEWS.d/next/Library/2025-09-28-16-34-11.gh-issue-139391.nRFnmx.rst b/Misc/NEWS.d/next/Library/2025-09-28-16-34-11.gh-issue-139391.nRFnmx.rst new file mode 100644 index 00000000000..93d1ce613bc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-28-16-34-11.gh-issue-139391.nRFnmx.rst @@ -0,0 +1,3 @@ +Fix an issue when, on non-Windows platforms, it was not possible to +gracefully exit a ``python -m asyncio`` process suspended by Ctrl+Z +and later resumed by :manpage:`fg` other than with :manpage:`kill`. From a310b3a99d05e107963023a5736b67afe4ae1968 Mon Sep 17 00:00:00 2001 From: yihong Date: Thu, 9 Oct 2025 23:24:52 +0800 Subject: [PATCH 081/112] gh-139845: do not print twice in default asyncio REPL (#139846) Co-authored-by: Kumar Aditya --- Lib/asyncio/__main__.py | 3 ++- .../Library/2025-10-09-21-37-20.gh-issue-139845.dzx5UP.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-09-21-37-20.gh-issue-139845.dzx5UP.rst diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 12d6f7714ee..10bfca3cf96 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -74,7 +74,8 @@ def callback(): return except BaseException: if keyboard_interrupted: - self.write("\nKeyboardInterrupt\n") + if not CAN_USE_PYREPL: + self.write("\nKeyboardInterrupt\n") else: self.showtraceback() return self.STATEMENT_FAILED diff --git a/Misc/NEWS.d/next/Library/2025-10-09-21-37-20.gh-issue-139845.dzx5UP.rst b/Misc/NEWS.d/next/Library/2025-10-09-21-37-20.gh-issue-139845.dzx5UP.rst new file mode 100644 index 00000000000..3cd294e49cd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-09-21-37-20.gh-issue-139845.dzx5UP.rst @@ -0,0 +1 @@ +Fix to not print KeyboardInterrupt twice in default asyncio REPL. From 9fc4366f09904fd9aac797d542e332e8a4c1a921 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Thu, 9 Oct 2025 09:53:14 -0700 Subject: [PATCH 082/112] GH-139809: Fix argparse subcommand prog not respecting color environment variables (#139818) --- Lib/argparse.py | 4 +++- Lib/test/test_argparse.py | 15 ++++++++++++++- ...2025-10-09-03-06-19.gh-issue-139809.lzHJNu.rst | 1 + 3 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-09-03-06-19.gh-issue-139809.lzHJNu.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index d71e551401c..1ddb7abbb35 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -1959,7 +1959,9 @@ def add_subparsers(self, **kwargs): # prog defaults to the usage message of this parser, skipping # optional arguments and with no "usage:" prefix if kwargs.get('prog') is None: - formatter = self._get_formatter() + # Create formatter without color to avoid storing ANSI codes in prog + formatter = self.formatter_class(prog=self.prog) + formatter._set_color(False) positionals = self._get_positional_actions() groups = self._mutually_exclusive_groups formatter.add_usage(None, positionals, groups, '') diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 27e38040a98..6a07f190285 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -22,6 +22,7 @@ captured_stderr, force_not_colorized, force_not_colorized_test_class, + swap_attr, ) from test.support import import_helper from test.support import os_helper @@ -7128,7 +7129,8 @@ class TestColorized(TestCase): def setUp(self): super().setUp() # Ensure color even if ran with NO_COLOR=1 - _colorize.can_colorize = lambda *args, **kwargs: True + self.enterContext(swap_attr(_colorize, 'can_colorize', + lambda *args, **kwargs: True)) self.theme = _colorize.get_theme(force_color=True).argparse def test_argparse_color(self): @@ -7355,6 +7357,17 @@ def __init__(self, prog): {short_b}+f{reset}, {long_b}++foo{reset} {label_b}FOO{reset} foo help ''')) + def test_subparser_prog_is_stored_without_color(self): + parser = argparse.ArgumentParser(prog='complex', color=True) + sub = parser.add_subparsers(dest='command') + demo_parser = sub.add_parser('demo') + + self.assertNotIn('\x1b[', demo_parser.prog) + + demo_parser.color = False + help_text = demo_parser.format_help() + self.assertNotIn('\x1b[', help_text) + class TestModule(unittest.TestCase): def test_deprecated__version__(self): diff --git a/Misc/NEWS.d/next/Library/2025-10-09-03-06-19.gh-issue-139809.lzHJNu.rst b/Misc/NEWS.d/next/Library/2025-10-09-03-06-19.gh-issue-139809.lzHJNu.rst new file mode 100644 index 00000000000..498b8f7fd27 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-09-03-06-19.gh-issue-139809.lzHJNu.rst @@ -0,0 +1 @@ +Prevent premature colorization of subparser ``prog`` in :meth:`argparse.ArgumentParser.add_subparsers` to respect color environment variable changes after parser creation. From 04461510fb8bedc054477c2634ffd0e575485b12 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Thu, 9 Oct 2025 20:13:38 +0300 Subject: [PATCH 083/112] gh-139672: Remove references to `passlib` (#139673) --- Doc/library/crypt.rst | 2 +- Doc/whatsnew/3.13.rst | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Doc/library/crypt.rst b/Doc/library/crypt.rst index 9ff37196ccf..647cb4925f3 100644 --- a/Doc/library/crypt.rst +++ b/Doc/library/crypt.rst @@ -13,7 +13,7 @@ being deprecated in Python 3.11. The removal was decided in :pep:`594`. Applications can use the :mod:`hashlib` module from the standard library. Other possible replacements are third-party libraries from PyPI: -:pypi:`legacycrypt`, :pypi:`bcrypt`, :pypi:`argon2-cffi`, or :pypi:`passlib`. +:pypi:`legacycrypt`, :pypi:`bcrypt`, or :pypi:`argon2-cffi`. These are not supported or maintained by the Python core team. The last version of Python that provided the :mod:`!crypt` module was diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 1548f128b5b..499f6f0ff4e 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1569,8 +1569,6 @@ and are now removed: * :pypi:`bcrypt`: Modern password hashing for your software and your servers. - * :pypi:`passlib`: - Comprehensive password hashing framework supporting over 30 schemes. * :pypi:`argon2-cffi`: The secure Argon2 password hashing algorithm. * :pypi:`legacycrypt`: From f575dd9ef815e79cb359f5466375363f0a5756ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Thu, 9 Oct 2025 19:53:42 +0200 Subject: [PATCH 084/112] gh-139842: Clarify `__module__` description in typing.rst (#139863) --- Doc/library/typing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index cfeeb19efbd..279ae3ef820 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2269,7 +2269,7 @@ without the dedicated syntax, as documented below. .. attribute:: __module__ - The module in which the type alias was defined:: + The name of the module in which the type alias was defined:: >>> type Alias = int >>> Alias.__module__ @@ -2449,7 +2449,7 @@ types. .. attribute:: __module__ - The module in which the new type is defined. + The name of the module in which the new type is defined. .. attribute:: __name__ From 34503111fe2724986701799dc994c6b0cb32f4f2 Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Thu, 9 Oct 2025 18:36:40 -0400 Subject: [PATCH 085/112] gh-137025: Update Emscripten Build Docs (#137312) Update Emscripten build docs to point at the devguide as the primary reference for managing an Emscripten build. --- ...-08-01-13-27-43.gh-issue-137025.ubuhQC.rst | 3 + Tools/wasm/README.md | 66 ++----------------- 2 files changed, 9 insertions(+), 60 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-08-01-13-27-43.gh-issue-137025.ubuhQC.rst diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-08-01-13-27-43.gh-issue-137025.ubuhQC.rst b/Misc/NEWS.d/next/Tools-Demos/2025-08-01-13-27-43.gh-issue-137025.ubuhQC.rst new file mode 100644 index 00000000000..73c79564006 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-08-01-13-27-43.gh-issue-137025.ubuhQC.rst @@ -0,0 +1,3 @@ +The ``wasm_build.py`` script has been removed. ``Tools/wasm/emscripten`` +and ``Tools/wasm/wasi`` should be used instead, as described in the `Dev +Guide `__. diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index efe9a3550c3..6615f1e2664 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -9,68 +9,14 @@ # Python WebAssembly (WASM) build run in modern browsers and JavaScript runtimes like *Node.js*. WASI builds use WASM runtimes such as *wasmtime*. -Users and developers are encouraged to use the script -`Tools/wasm/wasm_build.py`. The tool automates the build process and provides -assistance with installation of SDKs, running tests, etc. +**NOTE**: If you are looking for general information about WebAssembly that is +not directly related to CPython, please see https://github.com/psf/webassembly. -**NOTE**: If you are looking for information that is not directly related to -building CPython for WebAssembly (or the resulting build), please see -https://github.com/psf/webassembly for more information. - -## wasm32-emscripten +## Emscripten (wasm32-emscripten) ### Build -To cross compile to the ``wasm32-emscripten`` platform you need -[the Emscripten compiler toolchain](https://emscripten.org/), -a Python interpreter, and an installation of Node version 18 or newer. -Emscripten version 4.0.2 is recommended; newer versions may also work, but all -official testing is performed with that version. All commands below are relative -to a checkout of the Python repository. - -#### Install [the Emscripten compiler toolchain](https://emscripten.org/docs/getting_started/downloads.html) - -You can install the Emscripten toolchain as follows: -```shell -git clone https://github.com/emscripten-core/emsdk.git --depth 1 -./emsdk/emsdk install latest -./emsdk/emsdk activate latest -``` -To add the Emscripten compiler to your path: -```shell -source ./emsdk/emsdk_env.sh -``` -This adds `emcc` and `emconfigure` to your path. - -##### Optionally: enable ccache for EMSDK - -The ``EM_COMPILER_WRAPPER`` must be set after the EMSDK environment is -sourced. Otherwise the source script removes the environment variable. - -```shell -export EM_COMPILER_WRAPPER=ccache -``` - -#### Compile and build Python interpreter - -You can use `python Tools/wasm/emscripten` to compile and build targeting -Emscripten. You can do everything at once with: -```shell -python Tools/wasm/emscripten build -``` -or you can break it out into four separate steps: -```shell -python Tools/wasm/emscripten configure-build-python -python Tools/wasm/emscripten make-build-python -python Tools/wasm/emscripten make-libffi -python Tools/wasm/emscripten configure-host -python Tools/wasm/emscripten make-host -``` -Extra arguments to the configure steps are passed along to configure. For -instance, to do a debug build, you can use: -```shell -python Tools/wasm/emscripten build --with-py-debug -``` +See [the devguide instructions for building for Emscripten](https://devguide.python.org/getting-started/setup-building/#emscripten). ### Running from node @@ -97,8 +43,8 @@ ### Running tests ### The Web Example -When building for Emscripten, the web example will be built automatically. It -is in the ``web_example`` directory. To run the web example, ``cd`` into the +When building for Emscripten, a small web example will be built automatically +in the ``web_example`` directory. To run the web example, ``cd`` into the ``web_example`` directory, then run ``python server.py``. This will start a web server; you can then visit ``http://localhost:8000/`` in a browser to see a simple REPL example. From 744ec1d6c395f7a443ba4609d6023f86c245db58 Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Fri, 10 Oct 2025 08:08:51 +0300 Subject: [PATCH 086/112] gh-138614: `site._get_path` to respect non-default implementation name (#138610) * `site._get_path` to respect non-default implementation name * Add news entry * Remove NEWS entry --- Lib/site.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/site.py b/Lib/site.py index f9327197159..f0e74eeee2f 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -333,7 +333,7 @@ def _get_path(userbase): if sys.platform == 'darwin' and sys._framework: return f'{userbase}/lib/{implementation_lower}/site-packages' - return f'{userbase}/lib/python{version[0]}.{version[1]}{abi_thread}/site-packages' + return f'{userbase}/lib/{implementation_lower}{version[0]}.{version[1]}{abi_thread}/site-packages' def getuserbase(): From 33f32d6740810b7b97e311fe9790a4fa4b4920dc Mon Sep 17 00:00:00 2001 From: partev Date: Fri, 10 Oct 2025 01:38:13 -0400 Subject: [PATCH 087/112] Replace obsolete platforms with more recent examples (#132455) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Filipe Laíns 🇵🇸 --- Doc/library/sysconfig.rst | 3 +++ Lib/sysconfig/__init__.py | 3 +++ Tools/c-analyzer/distutils/util.py | 4 ++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 532facb45f8..3b0bfb85da7 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -382,6 +382,9 @@ Other functions Examples of returned values: + - linux-x86_64 + - linux-aarch64 + - solaris-2.6-sun4u Windows: diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 7374bde223e..8ff9c99435b 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -645,6 +645,9 @@ def get_platform(): isn't particularly important. Examples of returned values: + linux-x86_64 + linux-aarch64 + solaris-2.6-sun4u Windows: diff --git a/Tools/c-analyzer/distutils/util.py b/Tools/c-analyzer/distutils/util.py index 89ca094336f..f687a28ec2f 100644 --- a/Tools/c-analyzer/distutils/util.py +++ b/Tools/c-analyzer/distutils/util.py @@ -19,8 +19,8 @@ def get_host_platform(): particularly important. Examples of returned values: - linux-i586 - linux-alpha (?) + linux-x86_64 + linux-aarch64 solaris-2.6-sun4u Windows will return one of: From 9e15770d621780bf9f368f4a58379f9ff4a173fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Fri, 10 Oct 2025 06:52:13 +0100 Subject: [PATCH 088/112] GH-78870: copy test from GH-20439 (#139884) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/test/test_sysconfig.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 59ea623bb92..8101657b04a 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -770,6 +770,7 @@ def test_parse_makefile(self): print("var8=$$(var3)", file=makefile) print("var9=$(var10)(var3)", file=makefile) print("var10=$$", file=makefile) + print("var11=$${ORIGIN}${var5}", file=makefile) vars = _parse_makefile(TESTFN) self.assertEqual(vars, { 'var1': 'ab42', @@ -782,6 +783,7 @@ def test_parse_makefile(self): 'var8': '$(var3)', 'var9': '$(var3)', 'var10': '$', + 'var11': '${ORIGIN}dollar$5', }) def _test_parse_makefile_recursion(self): From 1f87d528a1fb999ff9cdb411a1ccc3f8ee674293 Mon Sep 17 00:00:00 2001 From: Weilin Du <108666168+LamentXU123@users.noreply.github.com> Date: Fri, 10 Oct 2025 14:24:41 +0800 Subject: [PATCH 089/112] gh-139843: Document `signal.SIGQUIT` to fix Sphinx references (#139844) --- Doc/library/signal.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 66f31b28da2..0c681c795c1 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -205,6 +205,12 @@ The variables defined in the :mod:`signal` module are: .. availability:: Unix. +.. data:: SIGQUIT + + Terminal quit signal. + + .. availability:: Unix. + .. data:: SIGSEGV Segmentation fault: invalid memory reference. From 8f14bddeae8935817166c4f594466f42c9f14139 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Fri, 10 Oct 2025 07:48:09 +0100 Subject: [PATCH 090/112] gh-139823: Extend list of optional dependencies in `configure.rst` (#139826) Co-authored-by: Emma Smith Co-authored-by: Victor Stinner --- Doc/using/configure.rst | 43 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index b05e0600114..01537951aeb 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -22,16 +22,51 @@ Features and minimum versions required to build CPython: * Support for threads. -* OpenSSL 1.1.1 is the minimum version and OpenSSL 3.0.18 is the recommended - minimum version for the :mod:`ssl` and :mod:`hashlib` extension modules. +To build optional modules: -* SQLite 3.15.2 for the :mod:`sqlite3` extension module. +* `libbz2 `_ for the :mod:`bz2` module. -* Tcl/Tk 8.5.12 for the :mod:`tkinter` module. +* `libb2 `_ (:ref:`BLAKE2 `), + used by :mod:`hashlib` module. + +* `libffi `_ 3.3.0 is the recommended + minimum version for the :mod:`ctypes` module. + +* ``liblzma``, for the :mod:`lzma` module. * `libmpdec `_ 2.5.0 for the :mod:`decimal` module. +* ``libncurses`` or ``libncursesw``, + for the :mod:`curses` module. + +* ``libpanel`` or ``libpanelw``, + for the :mod:`curses.panel` module. + +* `libreadline `_ or + `libedit `_ + for the :mod:`readline` module. + +* `libuuid `_, for the :mod:`uuid` module. + +* `OpenSSL `_ 1.1.1 is the minimum version and + OpenSSL 3.0.18 is the recommended minimum version for the + :mod:`ssl` and :mod:`hashlib` extension modules. + +* `SQLite `_ 3.15.2 for the :mod:`sqlite3` extension module. + +* `Tcl/Tk `_ 8.5.12 for the :mod:`tkinter` module. + +* `zlib `_ 1.1.4 is the reccomended minimum version for the + :mod:`zlib` module. + +* `zstd `_ 1.4.5 is the minimum version for + the :mod:`compression.zstd` module. + +For a full list of dependencies required to build all modules and how to install +them, see the +`devguide `_. + * Autoconf 2.72 and aclocal 1.16.5 are required to regenerate the :file:`configure` script. From e31c22dbf9b35bdc6b63871f2ca9a0a0d6786b28 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 10 Oct 2025 08:54:12 +0200 Subject: [PATCH 091/112] gh-111489: Add PyTuple_FromArray() function (#139691) --- Doc/c-api/tuple.rst | 13 +++++++++ Doc/whatsnew/3.15.rst | 3 ++ Include/cpython/tupleobject.h | 4 +++ Include/internal/pycore_tuple.h | 4 ++- Lib/test/test_capi/test_tuple.py | 22 +++++++++++++++ ...-10-07-12-51-32.gh-issue-111489.LCKKlg.rst | 2 ++ Modules/_testcapi/tuple.c | 28 +++++++++++++++++++ Objects/tupleobject.c | 2 +- 8 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-10-07-12-51-32.gh-issue-111489.LCKKlg.rst diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index 815afddad19..65f8334c437 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -37,6 +37,19 @@ Tuple Objects or ``NULL`` with an exception set on failure. +.. c:function:: PyObject* PyTuple_FromArray(PyObject *const *array, Py_ssize_t size) + + Create a tuple of *size* items and copy references from *array* to the new + tuple. + + *array* can be NULL if *size* is ``0``. + + On success, return a new reference. + On error, set an exception and return ``NULL``. + + .. versionadded:: next + + .. c:function:: PyObject* PyTuple_Pack(Py_ssize_t n, ...) Return a new tuple object of size *n*, diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 4b176d6c8e6..40286d4fe85 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -852,6 +852,9 @@ New features (Contributed by Victor Stinner in :gh:`129813`.) +* Add :c:func:`PyTuple_FromArray` to create a :class:`tuple` from an array. + (Contributed by Victor Stinner in :gh:`111489`.) + Porting to Python 3.15 ---------------------- diff --git a/Include/cpython/tupleobject.h b/Include/cpython/tupleobject.h index afb98ccbb81..888baaf3358 100644 --- a/Include/cpython/tupleobject.h +++ b/Include/cpython/tupleobject.h @@ -38,3 +38,7 @@ PyTuple_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) { } #define PyTuple_SET_ITEM(op, index, value) \ PyTuple_SET_ITEM(_PyObject_CAST(op), (index), _PyObject_CAST(value)) + +PyAPI_FUNC(PyObject*) PyTuple_FromArray( + PyObject *const *array, + Py_ssize_t size); diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index acf1bec4602..be1961cbf77 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -23,7 +23,9 @@ extern PyStatus _PyTuple_InitGlobalObjects(PyInterpreterState *); #define _PyTuple_ITEMS(op) _Py_RVALUE(_PyTuple_CAST(op)->ob_item) -PyAPI_FUNC(PyObject *)_PyTuple_FromArray(PyObject *const *, Py_ssize_t); +// Alias for backward compatibility +#define _PyTuple_FromArray PyTuple_FromArray + PyAPI_FUNC(PyObject *)_PyTuple_FromStackRefStealOnSuccess(const union _PyStackRef *, Py_ssize_t); PyAPI_FUNC(PyObject *)_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); diff --git a/Lib/test/test_capi/test_tuple.py b/Lib/test/test_capi/test_tuple.py index 7c07bc64e24..b6d6da008d0 100644 --- a/Lib/test/test_capi/test_tuple.py +++ b/Lib/test/test_capi/test_tuple.py @@ -62,6 +62,28 @@ def test_tuple_new(self): self.assertRaises(SystemError, tuple_new, PY_SSIZE_T_MIN) self.assertRaises(MemoryError, tuple_new, PY_SSIZE_T_MAX) + def test_tuple_fromarray(self): + # Test PyTuple_FromArray() + tuple_fromarray = _testcapi.tuple_fromarray + + tup = tuple([i] for i in range(5)) + copy = tuple_fromarray(tup) + self.assertEqual(copy, tup) + + tup = () + copy = tuple_fromarray(tup) + self.assertIs(copy, tup) + + copy = tuple_fromarray(NULL, 0) + self.assertIs(copy, ()) + + with self.assertRaises(SystemError): + tuple_fromarray(NULL, -1) + with self.assertRaises(SystemError): + tuple_fromarray(NULL, PY_SSIZE_T_MIN) + with self.assertRaises(MemoryError): + tuple_fromarray(NULL, PY_SSIZE_T_MAX) + def test_tuple_pack(self): # Test PyTuple_Pack() pack = _testlimitedcapi.tuple_pack diff --git a/Misc/NEWS.d/next/C_API/2025-10-07-12-51-32.gh-issue-111489.LCKKlg.rst b/Misc/NEWS.d/next/C_API/2025-10-07-12-51-32.gh-issue-111489.LCKKlg.rst new file mode 100644 index 00000000000..9c044f7796b --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-10-07-12-51-32.gh-issue-111489.LCKKlg.rst @@ -0,0 +1,2 @@ +Add :c:func:`PyTuple_FromArray` to create a :class:`tuple` from an array. +Patch by Victor Stinner. diff --git a/Modules/_testcapi/tuple.c b/Modules/_testcapi/tuple.c index d9c02ba0ff0..5de1c494c0a 100644 --- a/Modules/_testcapi/tuple.c +++ b/Modules/_testcapi/tuple.c @@ -104,12 +104,40 @@ _check_tuple_item_is_NULL(PyObject *Py_UNUSED(module), PyObject *args) } +static PyObject * +tuple_fromarray(PyObject* Py_UNUSED(module), PyObject *args) +{ + PyObject *src; + Py_ssize_t size = UNINITIALIZED_SIZE; + if (!PyArg_ParseTuple(args, "O|n", &src, &size)) { + return NULL; + } + if (src != Py_None && !PyTuple_Check(src)) { + PyErr_SetString(PyExc_TypeError, "expect a tuple"); + return NULL; + } + + PyObject **items; + if (src != Py_None) { + items = &PyTuple_GET_ITEM(src, 0); + if (size == UNINITIALIZED_SIZE) { + size = PyTuple_GET_SIZE(src); + } + } + else { + items = NULL; + } + return PyTuple_FromArray(items, size); +} + + static PyMethodDef test_methods[] = { {"tuple_get_size", tuple_get_size, METH_O}, {"tuple_get_item", tuple_get_item, METH_VARARGS}, {"tuple_set_item", tuple_set_item, METH_VARARGS}, {"_tuple_resize", _tuple_resize, METH_VARARGS}, {"_check_tuple_item_is_NULL", _check_tuple_item_is_NULL, METH_VARARGS}, + {"tuple_fromarray", tuple_fromarray, METH_VARARGS}, {NULL}, }; diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 9b31758485c..1fa4bae638a 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -366,7 +366,7 @@ tuple_item(PyObject *op, Py_ssize_t i) } PyObject * -_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) { if (n == 0) { return tuple_get_empty(); From f4104f5d74b99712253fceb39a4460ee3f7a281c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 10 Oct 2025 10:51:24 +0300 Subject: [PATCH 092/112] gh-139783: Fix inspect.getsourcelines() for the case when a decorator is followed by a comment or an empty line (GH-139836) --- Lib/inspect.py | 4 +++- Lib/test/test_inspect/inspect_fodder2.py | 12 ++++++++++++ Lib/test/test_inspect/test_inspect.py | 4 ++++ .../2025-10-09-13-48-28.gh-issue-139783.__NUgo.rst | 2 ++ 4 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-09-13-48-28.gh-issue-139783.__NUgo.rst diff --git a/Lib/inspect.py b/Lib/inspect.py index b345623b3fa..bb22bab3040 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1065,7 +1065,9 @@ def __init__(self): def tokeneater(self, type, token, srowcol, erowcol, line): if not self.started and not self.indecorator: - if type == tokenize.INDENT or token == "async": + if type in (tokenize.INDENT, tokenize.COMMENT, tokenize.NL): + pass + elif token == "async": pass # skip any decorators elif token == "@": diff --git a/Lib/test/test_inspect/inspect_fodder2.py b/Lib/test/test_inspect/inspect_fodder2.py index 1de283f672d..157e12167b5 100644 --- a/Lib/test/test_inspect/inspect_fodder2.py +++ b/Lib/test/test_inspect/inspect_fodder2.py @@ -388,4 +388,16 @@ def func383(): ) return ge385 +# line 391 +@decorator +# comment +def func394(): + return 395 + +# line 397 +@decorator + +def func400(): + return 401 + pass # end of file diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index e32e34c63b5..d42f2dbff99 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -1223,6 +1223,10 @@ def test_generator_expression(self): self.assertSourceEqual(next(mod2.ge377), 377, 380) self.assertSourceEqual(next(mod2.func383()), 385, 388) + def test_comment_or_empty_line_after_decorator(self): + self.assertSourceEqual(mod2.func394, 392, 395) + self.assertSourceEqual(mod2.func400, 398, 401) + class TestNoEOL(GetSourceBase): def setUp(self): diff --git a/Misc/NEWS.d/next/Library/2025-10-09-13-48-28.gh-issue-139783.__NUgo.rst b/Misc/NEWS.d/next/Library/2025-10-09-13-48-28.gh-issue-139783.__NUgo.rst new file mode 100644 index 00000000000..336653e73bf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-09-13-48-28.gh-issue-139783.__NUgo.rst @@ -0,0 +1,2 @@ +Fix :func:`inspect.getsourcelines` for the case when a decorator is followed +by a comment or an empty line. From 7cafd76a7fc6e52d93dd11e454a49c4dbebdf080 Mon Sep 17 00:00:00 2001 From: Nadeshiko Manju Date: Fri, 10 Oct 2025 16:56:10 +0800 Subject: [PATCH 093/112] gh-139184: Set O_CLOEXEC for master_fd when calling os.forkpty() (#139408) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Manjusaka Co-authored-by: Shamil Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/os.rst | 5 +++++ Doc/library/pty.rst | 5 +++++ Lib/test/test_pty.py | 1 + .../2025-09-29-14-15-20.gh-issue-139184.dNl9O4.rst | 1 + Modules/clinic/posixmodule.c.h | 5 +++-- Modules/posixmodule.c | 10 +++++++++- 6 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-09-29-14-15-20.gh-issue-139184.dNl9O4.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index ba3d1894549..540eaa09d0e 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -4592,6 +4592,8 @@ written in Python, such as a mail server's external command delivery program. master end of the pseudo-terminal. For a more portable approach, use the :mod:`pty` module. If an error occurs :exc:`OSError` is raised. + The returned file descriptor *fd* is :ref:`non-inheritable `. + .. audit-event:: os.forkpty "" os.forkpty .. warning:: @@ -4608,6 +4610,9 @@ written in Python, such as a mail server's external command delivery program. threads, this now raises a :exc:`DeprecationWarning`. See the longer explanation on :func:`os.fork`. + .. versionchanged:: next + The returned file descriptor is now made non-inheritable. + .. availability:: Unix, not WASI, not Android, not iOS. diff --git a/Doc/library/pty.rst b/Doc/library/pty.rst index 1a44bb13a84..9fef8760b62 100644 --- a/Doc/library/pty.rst +++ b/Doc/library/pty.rst @@ -33,9 +33,14 @@ The :mod:`pty` module defines the following functions: file descriptor connected to the child's controlling terminal (and also to the child's standard input and output). + The returned file descriptor *fd* is :ref:`non-inheritable `. + .. warning:: On macOS the use of this function is unsafe when mixed with using higher-level system APIs, and that includes using :mod:`urllib.request`. + .. versionchanged:: next + The returned file descriptor is now made non-inheritable. + .. function:: openpty() diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index fbba7025ac4..a2018e86444 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -230,6 +230,7 @@ def test_fork(self): os._exit(2) os._exit(4) else: + self.assertFalse(os.get_inheritable(master_fd)) debug("Waiting for child (%d) to finish." % pid) # In verbose mode, we have to consume the debug output from the # child or the child will block, causing this test to hang in the diff --git a/Misc/NEWS.d/next/Library/2025-09-29-14-15-20.gh-issue-139184.dNl9O4.rst b/Misc/NEWS.d/next/Library/2025-09-29-14-15-20.gh-issue-139184.dNl9O4.rst new file mode 100644 index 00000000000..d50cdeaadc4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-29-14-15-20.gh-issue-139184.dNl9O4.rst @@ -0,0 +1 @@ +:func:`os.forkpty` does now make the returned file descriptor non-inheritable. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index dddf98d127c..3d9863ad179 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -5035,7 +5035,8 @@ PyDoc_STRVAR(os_forkpty__doc__, "Returns a tuple of (pid, master_fd).\n" "Like fork(), return pid of 0 to the child process,\n" "and pid of child to the parent process.\n" -"To both, return fd of newly opened pseudo-terminal."); +"To both, return fd of newly opened pseudo-terminal.\n" +"The master_fd is non-inheritable."); #define OS_FORKPTY_METHODDEF \ {"forkpty", (PyCFunction)os_forkpty, METH_NOARGS, os_forkpty__doc__}, @@ -13446,4 +13447,4 @@ exit: #ifndef OS__EMSCRIPTEN_LOG_METHODDEF #define OS__EMSCRIPTEN_LOG_METHODDEF #endif /* !defined(OS__EMSCRIPTEN_LOG_METHODDEF) */ -/*[clinic end generated code: output=b5b370c499174f85 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=47ace1528820858b input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 4189d300856..7a2e36bf294 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -9018,11 +9018,12 @@ Returns a tuple of (pid, master_fd). Like fork(), return pid of 0 to the child process, and pid of child to the parent process. To both, return fd of newly opened pseudo-terminal. +The master_fd is non-inheritable. [clinic start generated code]*/ static PyObject * os_forkpty_impl(PyObject *module) -/*[clinic end generated code: output=60d0a5c7512e4087 input=f1f7f4bae3966010]*/ +/*[clinic end generated code: output=60d0a5c7512e4087 input=24765e0f33275b3b]*/ { int master_fd = -1; pid_t pid; @@ -9048,6 +9049,12 @@ os_forkpty_impl(PyObject *module) } else { /* parent: release the import lock. */ PyOS_AfterFork_Parent(); + /* set O_CLOEXEC on master_fd */ + if (_Py_set_inheritable(master_fd, 0, NULL) < 0) { + PyErr_FormatUnraisable("Exception ignored when setting master_fd " + "non-inheritable in forkpty()"); + } + // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. if (warn_about_fork_with_threads("forkpty") < 0) return NULL; @@ -9055,6 +9062,7 @@ os_forkpty_impl(PyObject *module) if (pid == -1) { return posix_error(); } + return Py_BuildValue("(Ni)", PyLong_FromPid(pid), master_fd); } #endif /* HAVE_FORKPTY */ From 4c119714d58912b10cb803a813da0364a0cc01e8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 10 Oct 2025 12:52:59 +0200 Subject: [PATCH 094/112] gh-139353: Add Objects/unicode_format.c file (#139491) * Move PyUnicode_Format() implementation from unicodeobject.c to unicode_format.c. * Replace unicode_modifiable() with _PyUnicode_IsModifiable() * Add empty lines to have two empty lines between functions. --- Include/internal/pycore_unicodeobject.h | 26 + Makefile.pre.in | 4 +- Objects/unicode_format.c | 1002 +++++++++++++++++++++++ Objects/unicodeobject.c | 980 +--------------------- PCbuild/_freeze_module.vcxproj | 1 + PCbuild/_freeze_module.vcxproj.filters | 3 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 + 8 files changed, 1047 insertions(+), 973 deletions(-) create mode 100644 Objects/unicode_format.c diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index f1c9bcd4788..b83039c1869 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -11,10 +11,14 @@ extern "C" { #include "pycore_fileutils.h" // _Py_error_handler #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI + // Maximum code point of Unicode 6.0: 0x10ffff (1,114,111). #define _Py_MAX_UNICODE 0x10ffff +extern int _PyUnicode_IsModifiable(PyObject *unicode); + + static inline void _PyUnicode_Fill(int kind, void *data, Py_UCS4 value, Py_ssize_t start, Py_ssize_t length) @@ -48,6 +52,28 @@ _PyUnicode_Fill(int kind, void *data, Py_UCS4 value, } } +static inline int +_PyUnicode_EnsureUnicode(PyObject *obj) +{ + if (!PyUnicode_Check(obj)) { + PyErr_Format(PyExc_TypeError, + "must be str, not %T", obj); + return -1; + } + return 0; +} + +static inline int +_PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch) +{ + assert(ch <= _Py_MAX_UNICODE); + if (_PyUnicodeWriter_Prepare(writer, 1, ch) < 0) + return -1; + PyUnicode_WRITE(writer->kind, writer->data, writer->pos, ch); + writer->pos++; + return 0; +} + /* --- Characters Type APIs ----------------------------------------------- */ diff --git a/Makefile.pre.in b/Makefile.pre.in index a5223246845..19423c11545 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -557,9 +557,10 @@ OBJECT_OBJS= \ Objects/tupleobject.o \ Objects/typeobject.o \ Objects/typevarobject.o \ + Objects/unicode_format.o \ Objects/unicode_formatter.o \ - Objects/unicodeobject.o \ Objects/unicodectype.o \ + Objects/unicodeobject.o \ Objects/unionobject.o \ Objects/weakrefobject.o \ @PERF_TRAMPOLINE_OBJ@ @@ -2105,6 +2106,7 @@ Objects/bytes_methods.o: $(srcdir)/Objects/bytes_methods.c $(BYTESTR_DEPS) Objects/bytesobject.o: $(srcdir)/Objects/bytesobject.c $(BYTESTR_DEPS) Objects/bytearrayobject.o: $(srcdir)/Objects/bytearrayobject.c $(BYTESTR_DEPS) +Objects/unicode_format.o: $(srcdir)/Objects/unicode_format.c $(UNICODE_DEPS) Objects/unicodeobject.o: $(srcdir)/Objects/unicodeobject.c $(UNICODE_DEPS) Objects/dictobject.o: $(srcdir)/Objects/stringlib/eq.h diff --git a/Objects/unicode_format.c b/Objects/unicode_format.c new file mode 100644 index 00000000000..26bdae55d8b --- /dev/null +++ b/Objects/unicode_format.c @@ -0,0 +1,1002 @@ +/* + +Unicode implementation based on original code by Fredrik Lundh, +modified by Marc-Andre Lemburg . + +Major speed upgrades to the method implementations at the Reykjavik +NeedForSpeed sprint, by Fredrik Lundh and Andrew Dalke. + +Copyright (c) Corporation for National Research Initiatives. + +-------------------------------------------------------------------- +The original string type implementation is: + + Copyright (c) 1999 by Secret Labs AB + Copyright (c) 1999 by Fredrik Lundh + +By obtaining, using, and/or copying this software and/or its +associated documentation, you agree that you have read, understood, +and will comply with the following terms and conditions: + +Permission to use, copy, modify, and distribute this software and its +associated documentation for any purpose and without fee is hereby +granted, provided that the above copyright notice appears in all +copies, and that both that copyright notice and this permission notice +appear in supporting documentation, and that the name of Secret Labs +AB or the author not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR +ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +-------------------------------------------------------------------- + +*/ + +// PyUnicode_Format() implementation + +#include "Python.h" +#include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_format.h" // F_ALT +#include "pycore_long.h" // _PyLong_FormatWriter() +#include "pycore_object.h" // _PyObject_IsUniquelyReferenced() +#include "pycore_unicodeobject.h" // _Py_MAX_UNICODE + + +#define MAX_UNICODE _Py_MAX_UNICODE +#define ensure_unicode _PyUnicode_EnsureUnicode + +struct unicode_formatter_t { + PyObject *args; + int args_owned; + Py_ssize_t arglen, argidx; + PyObject *dict; + + int fmtkind; + Py_ssize_t fmtcnt, fmtpos; + const void *fmtdata; + PyObject *fmtstr; + + _PyUnicodeWriter writer; +}; + + +struct unicode_format_arg_t { + Py_UCS4 ch; + int flags; + Py_ssize_t width; + int prec; + int sign; +}; + + +static PyObject * +unicode_format_getnextarg(struct unicode_formatter_t *ctx) +{ + Py_ssize_t argidx = ctx->argidx; + + if (argidx < ctx->arglen) { + ctx->argidx++; + if (ctx->arglen < 0) + return ctx->args; + else + return PyTuple_GetItem(ctx->args, argidx); + } + PyErr_SetString(PyExc_TypeError, + "not enough arguments for format string"); + return NULL; +} + + +/* Returns a new reference to a PyUnicode object, or NULL on failure. */ + +/* Format a float into the writer if the writer is not NULL, or into *p_output + otherwise. + + Return 0 on success, raise an exception and return -1 on error. */ +static int +formatfloat(PyObject *v, struct unicode_format_arg_t *arg, + PyObject **p_output, + _PyUnicodeWriter *writer) +{ + char *p; + double x; + Py_ssize_t len; + int prec; + int dtoa_flags = 0; + + x = PyFloat_AsDouble(v); + if (x == -1.0 && PyErr_Occurred()) + return -1; + + prec = arg->prec; + if (prec < 0) + prec = 6; + + if (arg->flags & F_ALT) + dtoa_flags |= Py_DTSF_ALT; + p = PyOS_double_to_string(x, arg->ch, prec, dtoa_flags, NULL); + if (p == NULL) + return -1; + len = strlen(p); + if (writer) { + if (_PyUnicodeWriter_WriteASCIIString(writer, p, len) < 0) { + PyMem_Free(p); + return -1; + } + } + else + *p_output = _PyUnicode_FromASCII(p, len); + PyMem_Free(p); + return 0; +} + + +/* formatlong() emulates the format codes d, u, o, x and X, and + * the F_ALT flag, for Python's long (unbounded) ints. It's not used for + * Python's regular ints. + * Return value: a new PyUnicodeObject*, or NULL if error. + * The output string is of the form + * "-"? ("0x" | "0X")? digit+ + * "0x"/"0X" are present only for x and X conversions, with F_ALT + * set in flags. The case of hex digits will be correct, + * There will be at least prec digits, zero-filled on the left if + * necessary to get that many. + * val object to be converted + * flags bitmask of format flags; only F_ALT is looked at + * prec minimum number of digits; 0-fill on left if needed + * type a character in [duoxX]; u acts the same as d + * + * CAUTION: o, x and X conversions on regular ints can never + * produce a '-' sign, but can for Python's unbounded ints. + */ +PyObject * +_PyUnicode_FormatLong(PyObject *val, int alt, int prec, int type) +{ + PyObject *result = NULL; + char *buf; + Py_ssize_t i; + int sign; /* 1 if '-', else 0 */ + int len; /* number of characters */ + Py_ssize_t llen; + int numdigits; /* len == numnondigits + numdigits */ + int numnondigits = 0; + + /* Avoid exceeding SSIZE_T_MAX */ + if (prec > INT_MAX-3) { + PyErr_SetString(PyExc_OverflowError, + "precision too large"); + return NULL; + } + + assert(PyLong_Check(val)); + + switch (type) { + default: + Py_UNREACHABLE(); + case 'd': + case 'i': + case 'u': + /* int and int subclasses should print numerically when a numeric */ + /* format code is used (see issue18780) */ + result = PyNumber_ToBase(val, 10); + break; + case 'o': + numnondigits = 2; + result = PyNumber_ToBase(val, 8); + break; + case 'x': + case 'X': + numnondigits = 2; + result = PyNumber_ToBase(val, 16); + break; + } + if (!result) + return NULL; + + assert(_PyUnicode_IsModifiable(result)); + assert(PyUnicode_IS_ASCII(result)); + + /* To modify the string in-place, there can only be one reference. */ + if (!_PyObject_IsUniquelyReferenced(result)) { + Py_DECREF(result); + PyErr_BadInternalCall(); + return NULL; + } + buf = PyUnicode_DATA(result); + llen = PyUnicode_GET_LENGTH(result); + if (llen > INT_MAX) { + Py_DECREF(result); + PyErr_SetString(PyExc_ValueError, + "string too large in _PyUnicode_FormatLong"); + return NULL; + } + len = (int)llen; + sign = buf[0] == '-'; + numnondigits += sign; + numdigits = len - numnondigits; + assert(numdigits > 0); + + /* Get rid of base marker unless F_ALT */ + if (((alt) == 0 && + (type == 'o' || type == 'x' || type == 'X'))) { + assert(buf[sign] == '0'); + assert(buf[sign+1] == 'x' || buf[sign+1] == 'X' || + buf[sign+1] == 'o'); + numnondigits -= 2; + buf += 2; + len -= 2; + if (sign) + buf[0] = '-'; + assert(len == numnondigits + numdigits); + assert(numdigits > 0); + } + + /* Fill with leading zeroes to meet minimum width. */ + if (prec > numdigits) { + PyObject *r1 = PyBytes_FromStringAndSize(NULL, + numnondigits + prec); + char *b1; + if (!r1) { + Py_DECREF(result); + return NULL; + } + b1 = PyBytes_AS_STRING(r1); + for (i = 0; i < numnondigits; ++i) + *b1++ = *buf++; + for (i = 0; i < prec - numdigits; i++) + *b1++ = '0'; + for (i = 0; i < numdigits; i++) + *b1++ = *buf++; + *b1 = '\0'; + Py_SETREF(result, r1); + buf = PyBytes_AS_STRING(result); + len = numnondigits + prec; + } + + /* Fix up case for hex conversions. */ + if (type == 'X') { + /* Need to convert all lower case letters to upper case. + and need to convert 0x to 0X (and -0x to -0X). */ + for (i = 0; i < len; i++) + if (buf[i] >= 'a' && buf[i] <= 'x') + buf[i] -= 'a'-'A'; + } + if (!PyUnicode_Check(result) + || buf != PyUnicode_DATA(result)) { + PyObject *unicode; + unicode = _PyUnicode_FromASCII(buf, len); + Py_SETREF(result, unicode); + } + else if (len != PyUnicode_GET_LENGTH(result)) { + if (PyUnicode_Resize(&result, len) < 0) + Py_CLEAR(result); + } + return result; +} + + +/* Format an integer or a float as an integer. + * Return 1 if the number has been formatted into the writer, + * 0 if the number has been formatted into *p_output + * -1 and raise an exception on error */ +static int +mainformatlong(PyObject *v, + struct unicode_format_arg_t *arg, + PyObject **p_output, + _PyUnicodeWriter *writer) +{ + PyObject *iobj, *res; + char type = (char)arg->ch; + + if (!PyNumber_Check(v)) + goto wrongtype; + + /* make sure number is a type of integer for o, x, and X */ + if (!PyLong_Check(v)) { + if (type == 'o' || type == 'x' || type == 'X') { + iobj = _PyNumber_Index(v); + } + else { + iobj = PyNumber_Long(v); + } + if (iobj == NULL ) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) + goto wrongtype; + return -1; + } + assert(PyLong_Check(iobj)); + } + else { + iobj = Py_NewRef(v); + } + + if (PyLong_CheckExact(v) + && arg->width == -1 && arg->prec == -1 + && !(arg->flags & (F_SIGN | F_BLANK)) + && type != 'X') + { + /* Fast path */ + int alternate = arg->flags & F_ALT; + int base; + + switch(type) + { + default: + Py_UNREACHABLE(); + case 'd': + case 'i': + case 'u': + base = 10; + break; + case 'o': + base = 8; + break; + case 'x': + case 'X': + base = 16; + break; + } + + if (_PyLong_FormatWriter(writer, v, base, alternate) == -1) { + Py_DECREF(iobj); + return -1; + } + Py_DECREF(iobj); + return 1; + } + + res = _PyUnicode_FormatLong(iobj, arg->flags & F_ALT, arg->prec, type); + Py_DECREF(iobj); + if (res == NULL) + return -1; + *p_output = res; + return 0; + +wrongtype: + switch(type) + { + case 'o': + case 'x': + case 'X': + PyErr_Format(PyExc_TypeError, + "%%%c format: an integer is required, " + "not %.200s", + type, Py_TYPE(v)->tp_name); + break; + default: + PyErr_Format(PyExc_TypeError, + "%%%c format: a real number is required, " + "not %.200s", + type, Py_TYPE(v)->tp_name); + break; + } + return -1; +} + + +static Py_UCS4 +formatchar(PyObject *v) +{ + /* presume that the buffer is at least 3 characters long */ + if (PyUnicode_Check(v)) { + if (PyUnicode_GET_LENGTH(v) == 1) { + return PyUnicode_READ_CHAR(v, 0); + } + PyErr_Format(PyExc_TypeError, + "%%c requires an int or a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(v)); + return (Py_UCS4) -1; + } + else { + int overflow; + long x = PyLong_AsLongAndOverflow(v, &overflow); + if (x == -1 && PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_Format(PyExc_TypeError, + "%%c requires an int or a unicode character, not %T", + v); + return (Py_UCS4) -1; + } + return (Py_UCS4) -1; + } + + if (x < 0 || x > MAX_UNICODE) { + /* this includes an overflow in converting to C long */ + PyErr_SetString(PyExc_OverflowError, + "%c arg not in range(0x110000)"); + return (Py_UCS4) -1; + } + + return (Py_UCS4) x; + } +} + + +/* Parse options of an argument: flags, width, precision. + Handle also "%(name)" syntax. + + Return 0 if the argument has been formatted into arg->str. + Return 1 if the argument has been written into ctx->writer, + Raise an exception and return -1 on error. */ +static int +unicode_format_arg_parse(struct unicode_formatter_t *ctx, + struct unicode_format_arg_t *arg) +{ +#define FORMAT_READ(ctx) \ + PyUnicode_READ((ctx)->fmtkind, (ctx)->fmtdata, (ctx)->fmtpos) + + PyObject *v; + + if (arg->ch == '(') { + /* Get argument value from a dictionary. Example: "%(name)s". */ + Py_ssize_t keystart; + Py_ssize_t keylen; + PyObject *key; + int pcount = 1; + + if (ctx->dict == NULL) { + PyErr_SetString(PyExc_TypeError, + "format requires a mapping"); + return -1; + } + ++ctx->fmtpos; + --ctx->fmtcnt; + keystart = ctx->fmtpos; + /* Skip over balanced parentheses */ + while (pcount > 0 && --ctx->fmtcnt >= 0) { + arg->ch = FORMAT_READ(ctx); + if (arg->ch == ')') + --pcount; + else if (arg->ch == '(') + ++pcount; + ctx->fmtpos++; + } + keylen = ctx->fmtpos - keystart - 1; + if (ctx->fmtcnt < 0 || pcount > 0) { + PyErr_SetString(PyExc_ValueError, + "incomplete format key"); + return -1; + } + key = PyUnicode_Substring(ctx->fmtstr, + keystart, keystart + keylen); + if (key == NULL) + return -1; + if (ctx->args_owned) { + ctx->args_owned = 0; + Py_DECREF(ctx->args); + } + ctx->args = PyObject_GetItem(ctx->dict, key); + Py_DECREF(key); + if (ctx->args == NULL) + return -1; + ctx->args_owned = 1; + ctx->arglen = -1; + ctx->argidx = -2; + } + + /* Parse flags. Example: "%+i" => flags=F_SIGN. */ + while (--ctx->fmtcnt >= 0) { + arg->ch = FORMAT_READ(ctx); + ctx->fmtpos++; + switch (arg->ch) { + case '-': arg->flags |= F_LJUST; continue; + case '+': arg->flags |= F_SIGN; continue; + case ' ': arg->flags |= F_BLANK; continue; + case '#': arg->flags |= F_ALT; continue; + case '0': arg->flags |= F_ZERO; continue; + } + break; + } + + /* Parse width. Example: "%10s" => width=10 */ + if (arg->ch == '*') { + v = unicode_format_getnextarg(ctx); + if (v == NULL) + return -1; + if (!PyLong_Check(v)) { + PyErr_SetString(PyExc_TypeError, + "* wants int"); + return -1; + } + arg->width = PyLong_AsSsize_t(v); + if (arg->width == -1 && PyErr_Occurred()) + return -1; + if (arg->width < 0) { + arg->flags |= F_LJUST; + arg->width = -arg->width; + } + if (--ctx->fmtcnt >= 0) { + arg->ch = FORMAT_READ(ctx); + ctx->fmtpos++; + } + } + else if (arg->ch >= '0' && arg->ch <= '9') { + arg->width = arg->ch - '0'; + while (--ctx->fmtcnt >= 0) { + arg->ch = FORMAT_READ(ctx); + ctx->fmtpos++; + if (arg->ch < '0' || arg->ch > '9') + break; + /* Since arg->ch is unsigned, the RHS would end up as unsigned, + mixing signed and unsigned comparison. Since arg->ch is between + '0' and '9', casting to int is safe. */ + if (arg->width > (PY_SSIZE_T_MAX - ((int)arg->ch - '0')) / 10) { + PyErr_SetString(PyExc_ValueError, + "width too big"); + return -1; + } + arg->width = arg->width*10 + (arg->ch - '0'); + } + } + + /* Parse precision. Example: "%.3f" => prec=3 */ + if (arg->ch == '.') { + arg->prec = 0; + if (--ctx->fmtcnt >= 0) { + arg->ch = FORMAT_READ(ctx); + ctx->fmtpos++; + } + if (arg->ch == '*') { + v = unicode_format_getnextarg(ctx); + if (v == NULL) + return -1; + if (!PyLong_Check(v)) { + PyErr_SetString(PyExc_TypeError, + "* wants int"); + return -1; + } + arg->prec = PyLong_AsInt(v); + if (arg->prec == -1 && PyErr_Occurred()) + return -1; + if (arg->prec < 0) + arg->prec = 0; + if (--ctx->fmtcnt >= 0) { + arg->ch = FORMAT_READ(ctx); + ctx->fmtpos++; + } + } + else if (arg->ch >= '0' && arg->ch <= '9') { + arg->prec = arg->ch - '0'; + while (--ctx->fmtcnt >= 0) { + arg->ch = FORMAT_READ(ctx); + ctx->fmtpos++; + if (arg->ch < '0' || arg->ch > '9') + break; + if (arg->prec > (INT_MAX - ((int)arg->ch - '0')) / 10) { + PyErr_SetString(PyExc_ValueError, + "precision too big"); + return -1; + } + arg->prec = arg->prec*10 + (arg->ch - '0'); + } + } + } + + /* Ignore "h", "l" and "L" format prefix (ex: "%hi" or "%ls") */ + if (ctx->fmtcnt >= 0) { + if (arg->ch == 'h' || arg->ch == 'l' || arg->ch == 'L') { + if (--ctx->fmtcnt >= 0) { + arg->ch = FORMAT_READ(ctx); + ctx->fmtpos++; + } + } + } + if (ctx->fmtcnt < 0) { + PyErr_SetString(PyExc_ValueError, + "incomplete format"); + return -1; + } + return 0; + +#undef FORMAT_READ +} + + +/* Format one argument. Supported conversion specifiers: + + - "s", "r", "a": any type + - "i", "d", "u": int or float + - "o", "x", "X": int + - "e", "E", "f", "F", "g", "G": float + - "c": int or str (1 character) + + When possible, the output is written directly into the Unicode writer + (ctx->writer). A string is created when padding is required. + + Return 0 if the argument has been formatted into *p_str, + 1 if the argument has been written into ctx->writer, + -1 on error. */ +static int +unicode_format_arg_format(struct unicode_formatter_t *ctx, + struct unicode_format_arg_t *arg, + PyObject **p_str) +{ + PyObject *v; + _PyUnicodeWriter *writer = &ctx->writer; + + if (ctx->fmtcnt == 0) + ctx->writer.overallocate = 0; + + v = unicode_format_getnextarg(ctx); + if (v == NULL) + return -1; + + + switch (arg->ch) { + case 's': + case 'r': + case 'a': + if (PyLong_CheckExact(v) && arg->width == -1 && arg->prec == -1) { + /* Fast path */ + if (_PyLong_FormatWriter(writer, v, 10, arg->flags & F_ALT) == -1) + return -1; + return 1; + } + + if (PyUnicode_CheckExact(v) && arg->ch == 's') { + *p_str = Py_NewRef(v); + } + else { + if (arg->ch == 's') + *p_str = PyObject_Str(v); + else if (arg->ch == 'r') + *p_str = PyObject_Repr(v); + else + *p_str = PyObject_ASCII(v); + } + break; + + case 'i': + case 'd': + case 'u': + case 'o': + case 'x': + case 'X': + { + int ret = mainformatlong(v, arg, p_str, writer); + if (ret != 0) + return ret; + arg->sign = 1; + break; + } + + case 'e': + case 'E': + case 'f': + case 'F': + case 'g': + case 'G': + if (arg->width == -1 && arg->prec == -1 + && !(arg->flags & (F_SIGN | F_BLANK))) + { + /* Fast path */ + if (formatfloat(v, arg, NULL, writer) == -1) + return -1; + return 1; + } + + arg->sign = 1; + if (formatfloat(v, arg, p_str, NULL) == -1) + return -1; + break; + + case 'c': + { + Py_UCS4 ch = formatchar(v); + if (ch == (Py_UCS4) -1) + return -1; + if (arg->width == -1 && arg->prec == -1) { + /* Fast path */ + if (_PyUnicodeWriter_WriteCharInline(writer, ch) < 0) + return -1; + return 1; + } + *p_str = PyUnicode_FromOrdinal(ch); + break; + } + + default: + PyErr_Format(PyExc_ValueError, + "unsupported format character '%c' (0x%x) " + "at index %zd", + (31<=arg->ch && arg->ch<=126) ? (char)arg->ch : '?', + (int)arg->ch, + ctx->fmtpos - 1); + return -1; + } + if (*p_str == NULL) + return -1; + assert (PyUnicode_Check(*p_str)); + return 0; +} + + +static int +unicode_format_arg_output(struct unicode_formatter_t *ctx, + struct unicode_format_arg_t *arg, + PyObject *str) +{ + Py_ssize_t len; + int kind; + const void *pbuf; + Py_ssize_t pindex; + Py_UCS4 signchar; + Py_ssize_t buflen; + Py_UCS4 maxchar; + Py_ssize_t sublen; + _PyUnicodeWriter *writer = &ctx->writer; + Py_UCS4 fill; + + fill = ' '; + if (arg->sign && arg->flags & F_ZERO) + fill = '0'; + + len = PyUnicode_GET_LENGTH(str); + if ((arg->width == -1 || arg->width <= len) + && (arg->prec == -1 || arg->prec >= len) + && !(arg->flags & (F_SIGN | F_BLANK))) + { + /* Fast path */ + if (_PyUnicodeWriter_WriteStr(writer, str) == -1) + return -1; + return 0; + } + + /* Truncate the string for "s", "r" and "a" formats + if the precision is set */ + if (arg->ch == 's' || arg->ch == 'r' || arg->ch == 'a') { + if (arg->prec >= 0 && len > arg->prec) + len = arg->prec; + } + + /* Adjust sign and width */ + kind = PyUnicode_KIND(str); + pbuf = PyUnicode_DATA(str); + pindex = 0; + signchar = '\0'; + if (arg->sign) { + Py_UCS4 ch = PyUnicode_READ(kind, pbuf, pindex); + if (ch == '-' || ch == '+') { + signchar = ch; + len--; + pindex++; + } + else if (arg->flags & F_SIGN) + signchar = '+'; + else if (arg->flags & F_BLANK) + signchar = ' '; + else + arg->sign = 0; + } + if (arg->width < len) + arg->width = len; + + /* Prepare the writer */ + maxchar = writer->maxchar; + if (!(arg->flags & F_LJUST)) { + if (arg->sign) { + if ((arg->width-1) > len) + maxchar = Py_MAX(maxchar, fill); + } + else { + if (arg->width > len) + maxchar = Py_MAX(maxchar, fill); + } + } + if (PyUnicode_MAX_CHAR_VALUE(str) > maxchar) { + Py_UCS4 strmaxchar = _PyUnicode_FindMaxChar(str, 0, pindex+len); + maxchar = Py_MAX(maxchar, strmaxchar); + } + + buflen = arg->width; + if (arg->sign && len == arg->width) + buflen++; + if (_PyUnicodeWriter_Prepare(writer, buflen, maxchar) == -1) + return -1; + + /* Write the sign if needed */ + if (arg->sign) { + if (fill != ' ') { + PyUnicode_WRITE(writer->kind, writer->data, writer->pos, signchar); + writer->pos += 1; + } + if (arg->width > len) + arg->width--; + } + + /* Write the numeric prefix for "x", "X" and "o" formats + if the alternate form is used. + For example, write "0x" for the "%#x" format. */ + if ((arg->flags & F_ALT) && (arg->ch == 'x' || arg->ch == 'X' || arg->ch == 'o')) { + assert(PyUnicode_READ(kind, pbuf, pindex) == '0'); + assert(PyUnicode_READ(kind, pbuf, pindex + 1) == arg->ch); + if (fill != ' ') { + PyUnicode_WRITE(writer->kind, writer->data, writer->pos, '0'); + PyUnicode_WRITE(writer->kind, writer->data, writer->pos+1, arg->ch); + writer->pos += 2; + pindex += 2; + } + arg->width -= 2; + if (arg->width < 0) + arg->width = 0; + len -= 2; + } + + /* Pad left with the fill character if needed */ + if (arg->width > len && !(arg->flags & F_LJUST)) { + sublen = arg->width - len; + _PyUnicode_Fill(writer->kind, writer->data, fill, writer->pos, sublen); + writer->pos += sublen; + arg->width = len; + } + + /* If padding with spaces: write sign if needed and/or numeric prefix if + the alternate form is used */ + if (fill == ' ') { + if (arg->sign) { + PyUnicode_WRITE(writer->kind, writer->data, writer->pos, signchar); + writer->pos += 1; + } + if ((arg->flags & F_ALT) && (arg->ch == 'x' || arg->ch == 'X' || arg->ch == 'o')) { + assert(PyUnicode_READ(kind, pbuf, pindex) == '0'); + assert(PyUnicode_READ(kind, pbuf, pindex+1) == arg->ch); + PyUnicode_WRITE(writer->kind, writer->data, writer->pos, '0'); + PyUnicode_WRITE(writer->kind, writer->data, writer->pos+1, arg->ch); + writer->pos += 2; + pindex += 2; + } + } + + /* Write characters */ + if (len) { + _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, + str, pindex, len); + writer->pos += len; + } + + /* Pad right with the fill character if needed */ + if (arg->width > len) { + sublen = arg->width - len; + _PyUnicode_Fill(writer->kind, writer->data, ' ', writer->pos, sublen); + writer->pos += sublen; + } + return 0; +} + + +/* Helper of PyUnicode_Format(): format one arg. + Return 0 on success, raise an exception and return -1 on error. */ +static int +unicode_format_arg(struct unicode_formatter_t *ctx) +{ + struct unicode_format_arg_t arg; + PyObject *str; + int ret; + + arg.ch = PyUnicode_READ(ctx->fmtkind, ctx->fmtdata, ctx->fmtpos); + if (arg.ch == '%') { + ctx->fmtpos++; + ctx->fmtcnt--; + if (_PyUnicodeWriter_WriteCharInline(&ctx->writer, '%') < 0) + return -1; + return 0; + } + arg.flags = 0; + arg.width = -1; + arg.prec = -1; + arg.sign = 0; + str = NULL; + + ret = unicode_format_arg_parse(ctx, &arg); + if (ret == -1) + return -1; + + ret = unicode_format_arg_format(ctx, &arg, &str); + if (ret == -1) + return -1; + + if (ret != 1) { + ret = unicode_format_arg_output(ctx, &arg, str); + Py_DECREF(str); + if (ret == -1) + return -1; + } + + if (ctx->dict && (ctx->argidx < ctx->arglen)) { + PyErr_SetString(PyExc_TypeError, + "not all arguments converted during string formatting"); + return -1; + } + return 0; +} + + +PyObject * +PyUnicode_Format(PyObject *format, PyObject *args) +{ + struct unicode_formatter_t ctx; + + if (format == NULL || args == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + + if (ensure_unicode(format) < 0) + return NULL; + + ctx.fmtstr = format; + ctx.fmtdata = PyUnicode_DATA(ctx.fmtstr); + ctx.fmtkind = PyUnicode_KIND(ctx.fmtstr); + ctx.fmtcnt = PyUnicode_GET_LENGTH(ctx.fmtstr); + ctx.fmtpos = 0; + + _PyUnicodeWriter_Init(&ctx.writer); + ctx.writer.min_length = ctx.fmtcnt + 100; + ctx.writer.overallocate = 1; + + if (PyTuple_Check(args)) { + ctx.arglen = PyTuple_Size(args); + ctx.argidx = 0; + } + else { + ctx.arglen = -1; + ctx.argidx = -2; + } + ctx.args_owned = 0; + if (PyMapping_Check(args) && !PyTuple_Check(args) && !PyUnicode_Check(args)) + ctx.dict = args; + else + ctx.dict = NULL; + ctx.args = args; + + while (--ctx.fmtcnt >= 0) { + if (PyUnicode_READ(ctx.fmtkind, ctx.fmtdata, ctx.fmtpos) != '%') { + Py_ssize_t nonfmtpos; + + nonfmtpos = ctx.fmtpos++; + while (ctx.fmtcnt >= 0 && + PyUnicode_READ(ctx.fmtkind, ctx.fmtdata, ctx.fmtpos) != '%') { + ctx.fmtpos++; + ctx.fmtcnt--; + } + if (ctx.fmtcnt < 0) { + ctx.fmtpos--; + ctx.writer.overallocate = 0; + } + + if (_PyUnicodeWriter_WriteSubstring(&ctx.writer, ctx.fmtstr, + nonfmtpos, ctx.fmtpos) < 0) + goto onError; + } + else { + ctx.fmtpos++; + if (unicode_format_arg(&ctx) == -1) + goto onError; + } + } + + if (ctx.argidx < ctx.arglen && !ctx.dict) { + PyErr_SetString(PyExc_TypeError, + "not all arguments converted during string formatting"); + goto onError; + } + + if (ctx.args_owned) { + Py_DECREF(ctx.args); + } + return _PyUnicodeWriter_Finish(&ctx.writer); + + onError: + _PyUnicodeWriter_Dealloc(&ctx.writer); + if (ctx.args_owned) { + Py_DECREF(ctx.args); + } + return NULL; +} diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index c71f9d3f71d..a67bf9b1c53 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -105,6 +105,7 @@ NOTE: In the interpreter's initialization phase, some globals are currently */ #define MAX_UNICODE _Py_MAX_UNICODE +#define ensure_unicode _PyUnicode_EnsureUnicode #ifdef Py_DEBUG # define _PyUnicode_CHECK(op) _PyUnicode_CheckConsistency(op, 0) @@ -452,7 +453,6 @@ const unsigned char _Py_ascii_whitespace[] = { /* forward */ static PyObject* get_latin1_char(unsigned char ch); -static int unicode_modifiable(PyObject *unicode); static PyObject * @@ -983,18 +983,6 @@ make_bloom_mask(int kind, const void* ptr, Py_ssize_t len) #undef BLOOM_UPDATE } -static int -ensure_unicode(PyObject *obj) -{ - if (!PyUnicode_Check(obj)) { - PyErr_Format(PyExc_TypeError, - "must be str, not %.100s", - Py_TYPE(obj)->tp_name); - return -1; - } - return 0; -} - /* Compilation of templated routines */ #define STRINGLIB_GET_EMPTY() unicode_get_empty() @@ -1120,7 +1108,7 @@ resize_compact(PyObject *unicode, Py_ssize_t length) Py_ssize_t old_length = _PyUnicode_LENGTH(unicode); #endif - if (!unicode_modifiable(unicode)) { + if (!_PyUnicode_IsModifiable(unicode)) { PyObject *copy = resize_copy(unicode, length); if (copy == NULL) { return NULL; @@ -1412,7 +1400,7 @@ PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar) static int unicode_check_modifiable(PyObject *unicode) { - if (!unicode_modifiable(unicode)) { + if (!_PyUnicode_IsModifiable(unicode)) { PyErr_SetString(PyExc_SystemError, "Cannot modify a string currently used"); return -1; @@ -1774,8 +1762,8 @@ unicode_is_singleton(PyObject *unicode) } #endif -static int -unicode_modifiable(PyObject *unicode) +int +_PyUnicode_IsModifiable(PyObject *unicode) { assert(_PyUnicode_CHECK(unicode)); if (!_PyObject_IsUniquelyReferenced(unicode)) @@ -1816,7 +1804,7 @@ unicode_resize(PyObject **p_unicode, Py_ssize_t length) return 0; } - if (!unicode_modifiable(unicode)) { + if (!_PyUnicode_IsModifiable(unicode)) { PyObject *copy = resize_copy(unicode, length); if (copy == NULL) return -1; @@ -10252,7 +10240,7 @@ _PyUnicode_FastFill(PyObject *unicode, Py_ssize_t start, Py_ssize_t length, { const int kind = PyUnicode_KIND(unicode); void *data = PyUnicode_DATA(unicode); - assert(unicode_modifiable(unicode)); + assert(_PyUnicode_IsModifiable(unicode)); assert(fill_char <= PyUnicode_MAX_CHAR_VALUE(unicode)); assert(start >= 0); assert(start + length <= PyUnicode_GET_LENGTH(unicode)); @@ -11524,7 +11512,7 @@ PyUnicode_Append(PyObject **p_left, PyObject *right) } new_len = left_len + right_len; - if (unicode_modifiable(left) + if (_PyUnicode_IsModifiable(left) && PyUnicode_CheckExact(right) && PyUnicode_KIND(right) <= PyUnicode_KIND(left) /* Don't resize for ascii += latin1. Convert ascii to latin1 requires @@ -13722,17 +13710,6 @@ _PyUnicodeWriter_PrepareKindInternal(_PyUnicodeWriter *writer, return _PyUnicodeWriter_PrepareInternal(writer, 0, maxchar); } -static inline int -_PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch) -{ - assert(ch <= MAX_UNICODE); - if (_PyUnicodeWriter_Prepare(writer, 1, ch) < 0) - return -1; - PyUnicode_WRITE(writer->kind, writer->data, writer->pos, ch); - writer->pos++; - return 0; -} - int _PyUnicodeWriter_WriteChar(_PyUnicodeWriter *writer, Py_UCS4 ch) { @@ -14461,947 +14438,6 @@ static PyMappingMethods unicode_as_mapping = { }; -/* Helpers for PyUnicode_Format() */ - -struct unicode_formatter_t { - PyObject *args; - int args_owned; - Py_ssize_t arglen, argidx; - PyObject *dict; - - int fmtkind; - Py_ssize_t fmtcnt, fmtpos; - const void *fmtdata; - PyObject *fmtstr; - - _PyUnicodeWriter writer; -}; - -struct unicode_format_arg_t { - Py_UCS4 ch; - int flags; - Py_ssize_t width; - int prec; - int sign; -}; - -static PyObject * -unicode_format_getnextarg(struct unicode_formatter_t *ctx) -{ - Py_ssize_t argidx = ctx->argidx; - - if (argidx < ctx->arglen) { - ctx->argidx++; - if (ctx->arglen < 0) - return ctx->args; - else - return PyTuple_GetItem(ctx->args, argidx); - } - PyErr_SetString(PyExc_TypeError, - "not enough arguments for format string"); - return NULL; -} - -/* Returns a new reference to a PyUnicode object, or NULL on failure. */ - -/* Format a float into the writer if the writer is not NULL, or into *p_output - otherwise. - - Return 0 on success, raise an exception and return -1 on error. */ -static int -formatfloat(PyObject *v, struct unicode_format_arg_t *arg, - PyObject **p_output, - _PyUnicodeWriter *writer) -{ - char *p; - double x; - Py_ssize_t len; - int prec; - int dtoa_flags = 0; - - x = PyFloat_AsDouble(v); - if (x == -1.0 && PyErr_Occurred()) - return -1; - - prec = arg->prec; - if (prec < 0) - prec = 6; - - if (arg->flags & F_ALT) - dtoa_flags |= Py_DTSF_ALT; - p = PyOS_double_to_string(x, arg->ch, prec, dtoa_flags, NULL); - if (p == NULL) - return -1; - len = strlen(p); - if (writer) { - if (_PyUnicodeWriter_WriteASCIIString(writer, p, len) < 0) { - PyMem_Free(p); - return -1; - } - } - else - *p_output = _PyUnicode_FromASCII(p, len); - PyMem_Free(p); - return 0; -} - -/* formatlong() emulates the format codes d, u, o, x and X, and - * the F_ALT flag, for Python's long (unbounded) ints. It's not used for - * Python's regular ints. - * Return value: a new PyUnicodeObject*, or NULL if error. - * The output string is of the form - * "-"? ("0x" | "0X")? digit+ - * "0x"/"0X" are present only for x and X conversions, with F_ALT - * set in flags. The case of hex digits will be correct, - * There will be at least prec digits, zero-filled on the left if - * necessary to get that many. - * val object to be converted - * flags bitmask of format flags; only F_ALT is looked at - * prec minimum number of digits; 0-fill on left if needed - * type a character in [duoxX]; u acts the same as d - * - * CAUTION: o, x and X conversions on regular ints can never - * produce a '-' sign, but can for Python's unbounded ints. - */ -PyObject * -_PyUnicode_FormatLong(PyObject *val, int alt, int prec, int type) -{ - PyObject *result = NULL; - char *buf; - Py_ssize_t i; - int sign; /* 1 if '-', else 0 */ - int len; /* number of characters */ - Py_ssize_t llen; - int numdigits; /* len == numnondigits + numdigits */ - int numnondigits = 0; - - /* Avoid exceeding SSIZE_T_MAX */ - if (prec > INT_MAX-3) { - PyErr_SetString(PyExc_OverflowError, - "precision too large"); - return NULL; - } - - assert(PyLong_Check(val)); - - switch (type) { - default: - Py_UNREACHABLE(); - case 'd': - case 'i': - case 'u': - /* int and int subclasses should print numerically when a numeric */ - /* format code is used (see issue18780) */ - result = PyNumber_ToBase(val, 10); - break; - case 'o': - numnondigits = 2; - result = PyNumber_ToBase(val, 8); - break; - case 'x': - case 'X': - numnondigits = 2; - result = PyNumber_ToBase(val, 16); - break; - } - if (!result) - return NULL; - - assert(unicode_modifiable(result)); - assert(PyUnicode_IS_ASCII(result)); - - /* To modify the string in-place, there can only be one reference. */ - if (!_PyObject_IsUniquelyReferenced(result)) { - Py_DECREF(result); - PyErr_BadInternalCall(); - return NULL; - } - buf = PyUnicode_DATA(result); - llen = PyUnicode_GET_LENGTH(result); - if (llen > INT_MAX) { - Py_DECREF(result); - PyErr_SetString(PyExc_ValueError, - "string too large in _PyUnicode_FormatLong"); - return NULL; - } - len = (int)llen; - sign = buf[0] == '-'; - numnondigits += sign; - numdigits = len - numnondigits; - assert(numdigits > 0); - - /* Get rid of base marker unless F_ALT */ - if (((alt) == 0 && - (type == 'o' || type == 'x' || type == 'X'))) { - assert(buf[sign] == '0'); - assert(buf[sign+1] == 'x' || buf[sign+1] == 'X' || - buf[sign+1] == 'o'); - numnondigits -= 2; - buf += 2; - len -= 2; - if (sign) - buf[0] = '-'; - assert(len == numnondigits + numdigits); - assert(numdigits > 0); - } - - /* Fill with leading zeroes to meet minimum width. */ - if (prec > numdigits) { - PyObject *r1 = PyBytes_FromStringAndSize(NULL, - numnondigits + prec); - char *b1; - if (!r1) { - Py_DECREF(result); - return NULL; - } - b1 = PyBytes_AS_STRING(r1); - for (i = 0; i < numnondigits; ++i) - *b1++ = *buf++; - for (i = 0; i < prec - numdigits; i++) - *b1++ = '0'; - for (i = 0; i < numdigits; i++) - *b1++ = *buf++; - *b1 = '\0'; - Py_SETREF(result, r1); - buf = PyBytes_AS_STRING(result); - len = numnondigits + prec; - } - - /* Fix up case for hex conversions. */ - if (type == 'X') { - /* Need to convert all lower case letters to upper case. - and need to convert 0x to 0X (and -0x to -0X). */ - for (i = 0; i < len; i++) - if (buf[i] >= 'a' && buf[i] <= 'x') - buf[i] -= 'a'-'A'; - } - if (!PyUnicode_Check(result) - || buf != PyUnicode_DATA(result)) { - PyObject *unicode; - unicode = _PyUnicode_FromASCII(buf, len); - Py_SETREF(result, unicode); - } - else if (len != PyUnicode_GET_LENGTH(result)) { - if (PyUnicode_Resize(&result, len) < 0) - Py_CLEAR(result); - } - return result; -} - -/* Format an integer or a float as an integer. - * Return 1 if the number has been formatted into the writer, - * 0 if the number has been formatted into *p_output - * -1 and raise an exception on error */ -static int -mainformatlong(PyObject *v, - struct unicode_format_arg_t *arg, - PyObject **p_output, - _PyUnicodeWriter *writer) -{ - PyObject *iobj, *res; - char type = (char)arg->ch; - - if (!PyNumber_Check(v)) - goto wrongtype; - - /* make sure number is a type of integer for o, x, and X */ - if (!PyLong_Check(v)) { - if (type == 'o' || type == 'x' || type == 'X') { - iobj = _PyNumber_Index(v); - } - else { - iobj = PyNumber_Long(v); - } - if (iobj == NULL ) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) - goto wrongtype; - return -1; - } - assert(PyLong_Check(iobj)); - } - else { - iobj = Py_NewRef(v); - } - - if (PyLong_CheckExact(v) - && arg->width == -1 && arg->prec == -1 - && !(arg->flags & (F_SIGN | F_BLANK)) - && type != 'X') - { - /* Fast path */ - int alternate = arg->flags & F_ALT; - int base; - - switch(type) - { - default: - Py_UNREACHABLE(); - case 'd': - case 'i': - case 'u': - base = 10; - break; - case 'o': - base = 8; - break; - case 'x': - case 'X': - base = 16; - break; - } - - if (_PyLong_FormatWriter(writer, v, base, alternate) == -1) { - Py_DECREF(iobj); - return -1; - } - Py_DECREF(iobj); - return 1; - } - - res = _PyUnicode_FormatLong(iobj, arg->flags & F_ALT, arg->prec, type); - Py_DECREF(iobj); - if (res == NULL) - return -1; - *p_output = res; - return 0; - -wrongtype: - switch(type) - { - case 'o': - case 'x': - case 'X': - PyErr_Format(PyExc_TypeError, - "%%%c format: an integer is required, " - "not %.200s", - type, Py_TYPE(v)->tp_name); - break; - default: - PyErr_Format(PyExc_TypeError, - "%%%c format: a real number is required, " - "not %.200s", - type, Py_TYPE(v)->tp_name); - break; - } - return -1; -} - -static Py_UCS4 -formatchar(PyObject *v) -{ - /* presume that the buffer is at least 3 characters long */ - if (PyUnicode_Check(v)) { - if (PyUnicode_GET_LENGTH(v) == 1) { - return PyUnicode_READ_CHAR(v, 0); - } - PyErr_Format(PyExc_TypeError, - "%%c requires an int or a unicode character, " - "not a string of length %zd", - PyUnicode_GET_LENGTH(v)); - return (Py_UCS4) -1; - } - else { - int overflow; - long x = PyLong_AsLongAndOverflow(v, &overflow); - if (x == -1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) { - PyErr_Format(PyExc_TypeError, - "%%c requires an int or a unicode character, not %T", - v); - return (Py_UCS4) -1; - } - return (Py_UCS4) -1; - } - - if (x < 0 || x > MAX_UNICODE) { - /* this includes an overflow in converting to C long */ - PyErr_SetString(PyExc_OverflowError, - "%c arg not in range(0x110000)"); - return (Py_UCS4) -1; - } - - return (Py_UCS4) x; - } -} - -/* Parse options of an argument: flags, width, precision. - Handle also "%(name)" syntax. - - Return 0 if the argument has been formatted into arg->str. - Return 1 if the argument has been written into ctx->writer, - Raise an exception and return -1 on error. */ -static int -unicode_format_arg_parse(struct unicode_formatter_t *ctx, - struct unicode_format_arg_t *arg) -{ -#define FORMAT_READ(ctx) \ - PyUnicode_READ((ctx)->fmtkind, (ctx)->fmtdata, (ctx)->fmtpos) - - PyObject *v; - - if (arg->ch == '(') { - /* Get argument value from a dictionary. Example: "%(name)s". */ - Py_ssize_t keystart; - Py_ssize_t keylen; - PyObject *key; - int pcount = 1; - - if (ctx->dict == NULL) { - PyErr_SetString(PyExc_TypeError, - "format requires a mapping"); - return -1; - } - ++ctx->fmtpos; - --ctx->fmtcnt; - keystart = ctx->fmtpos; - /* Skip over balanced parentheses */ - while (pcount > 0 && --ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - if (arg->ch == ')') - --pcount; - else if (arg->ch == '(') - ++pcount; - ctx->fmtpos++; - } - keylen = ctx->fmtpos - keystart - 1; - if (ctx->fmtcnt < 0 || pcount > 0) { - PyErr_SetString(PyExc_ValueError, - "incomplete format key"); - return -1; - } - key = PyUnicode_Substring(ctx->fmtstr, - keystart, keystart + keylen); - if (key == NULL) - return -1; - if (ctx->args_owned) { - ctx->args_owned = 0; - Py_DECREF(ctx->args); - } - ctx->args = PyObject_GetItem(ctx->dict, key); - Py_DECREF(key); - if (ctx->args == NULL) - return -1; - ctx->args_owned = 1; - ctx->arglen = -1; - ctx->argidx = -2; - } - - /* Parse flags. Example: "%+i" => flags=F_SIGN. */ - while (--ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - ctx->fmtpos++; - switch (arg->ch) { - case '-': arg->flags |= F_LJUST; continue; - case '+': arg->flags |= F_SIGN; continue; - case ' ': arg->flags |= F_BLANK; continue; - case '#': arg->flags |= F_ALT; continue; - case '0': arg->flags |= F_ZERO; continue; - } - break; - } - - /* Parse width. Example: "%10s" => width=10 */ - if (arg->ch == '*') { - v = unicode_format_getnextarg(ctx); - if (v == NULL) - return -1; - if (!PyLong_Check(v)) { - PyErr_SetString(PyExc_TypeError, - "* wants int"); - return -1; - } - arg->width = PyLong_AsSsize_t(v); - if (arg->width == -1 && PyErr_Occurred()) - return -1; - if (arg->width < 0) { - arg->flags |= F_LJUST; - arg->width = -arg->width; - } - if (--ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - ctx->fmtpos++; - } - } - else if (arg->ch >= '0' && arg->ch <= '9') { - arg->width = arg->ch - '0'; - while (--ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - ctx->fmtpos++; - if (arg->ch < '0' || arg->ch > '9') - break; - /* Since arg->ch is unsigned, the RHS would end up as unsigned, - mixing signed and unsigned comparison. Since arg->ch is between - '0' and '9', casting to int is safe. */ - if (arg->width > (PY_SSIZE_T_MAX - ((int)arg->ch - '0')) / 10) { - PyErr_SetString(PyExc_ValueError, - "width too big"); - return -1; - } - arg->width = arg->width*10 + (arg->ch - '0'); - } - } - - /* Parse precision. Example: "%.3f" => prec=3 */ - if (arg->ch == '.') { - arg->prec = 0; - if (--ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - ctx->fmtpos++; - } - if (arg->ch == '*') { - v = unicode_format_getnextarg(ctx); - if (v == NULL) - return -1; - if (!PyLong_Check(v)) { - PyErr_SetString(PyExc_TypeError, - "* wants int"); - return -1; - } - arg->prec = PyLong_AsInt(v); - if (arg->prec == -1 && PyErr_Occurred()) - return -1; - if (arg->prec < 0) - arg->prec = 0; - if (--ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - ctx->fmtpos++; - } - } - else if (arg->ch >= '0' && arg->ch <= '9') { - arg->prec = arg->ch - '0'; - while (--ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - ctx->fmtpos++; - if (arg->ch < '0' || arg->ch > '9') - break; - if (arg->prec > (INT_MAX - ((int)arg->ch - '0')) / 10) { - PyErr_SetString(PyExc_ValueError, - "precision too big"); - return -1; - } - arg->prec = arg->prec*10 + (arg->ch - '0'); - } - } - } - - /* Ignore "h", "l" and "L" format prefix (ex: "%hi" or "%ls") */ - if (ctx->fmtcnt >= 0) { - if (arg->ch == 'h' || arg->ch == 'l' || arg->ch == 'L') { - if (--ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - ctx->fmtpos++; - } - } - } - if (ctx->fmtcnt < 0) { - PyErr_SetString(PyExc_ValueError, - "incomplete format"); - return -1; - } - return 0; - -#undef FORMAT_READ -} - -/* Format one argument. Supported conversion specifiers: - - - "s", "r", "a": any type - - "i", "d", "u": int or float - - "o", "x", "X": int - - "e", "E", "f", "F", "g", "G": float - - "c": int or str (1 character) - - When possible, the output is written directly into the Unicode writer - (ctx->writer). A string is created when padding is required. - - Return 0 if the argument has been formatted into *p_str, - 1 if the argument has been written into ctx->writer, - -1 on error. */ -static int -unicode_format_arg_format(struct unicode_formatter_t *ctx, - struct unicode_format_arg_t *arg, - PyObject **p_str) -{ - PyObject *v; - _PyUnicodeWriter *writer = &ctx->writer; - - if (ctx->fmtcnt == 0) - ctx->writer.overallocate = 0; - - v = unicode_format_getnextarg(ctx); - if (v == NULL) - return -1; - - - switch (arg->ch) { - case 's': - case 'r': - case 'a': - if (PyLong_CheckExact(v) && arg->width == -1 && arg->prec == -1) { - /* Fast path */ - if (_PyLong_FormatWriter(writer, v, 10, arg->flags & F_ALT) == -1) - return -1; - return 1; - } - - if (PyUnicode_CheckExact(v) && arg->ch == 's') { - *p_str = Py_NewRef(v); - } - else { - if (arg->ch == 's') - *p_str = PyObject_Str(v); - else if (arg->ch == 'r') - *p_str = PyObject_Repr(v); - else - *p_str = PyObject_ASCII(v); - } - break; - - case 'i': - case 'd': - case 'u': - case 'o': - case 'x': - case 'X': - { - int ret = mainformatlong(v, arg, p_str, writer); - if (ret != 0) - return ret; - arg->sign = 1; - break; - } - - case 'e': - case 'E': - case 'f': - case 'F': - case 'g': - case 'G': - if (arg->width == -1 && arg->prec == -1 - && !(arg->flags & (F_SIGN | F_BLANK))) - { - /* Fast path */ - if (formatfloat(v, arg, NULL, writer) == -1) - return -1; - return 1; - } - - arg->sign = 1; - if (formatfloat(v, arg, p_str, NULL) == -1) - return -1; - break; - - case 'c': - { - Py_UCS4 ch = formatchar(v); - if (ch == (Py_UCS4) -1) - return -1; - if (arg->width == -1 && arg->prec == -1) { - /* Fast path */ - if (_PyUnicodeWriter_WriteCharInline(writer, ch) < 0) - return -1; - return 1; - } - *p_str = PyUnicode_FromOrdinal(ch); - break; - } - - default: - PyErr_Format(PyExc_ValueError, - "unsupported format character '%c' (0x%x) " - "at index %zd", - (31<=arg->ch && arg->ch<=126) ? (char)arg->ch : '?', - (int)arg->ch, - ctx->fmtpos - 1); - return -1; - } - if (*p_str == NULL) - return -1; - assert (PyUnicode_Check(*p_str)); - return 0; -} - -static int -unicode_format_arg_output(struct unicode_formatter_t *ctx, - struct unicode_format_arg_t *arg, - PyObject *str) -{ - Py_ssize_t len; - int kind; - const void *pbuf; - Py_ssize_t pindex; - Py_UCS4 signchar; - Py_ssize_t buflen; - Py_UCS4 maxchar; - Py_ssize_t sublen; - _PyUnicodeWriter *writer = &ctx->writer; - Py_UCS4 fill; - - fill = ' '; - if (arg->sign && arg->flags & F_ZERO) - fill = '0'; - - len = PyUnicode_GET_LENGTH(str); - if ((arg->width == -1 || arg->width <= len) - && (arg->prec == -1 || arg->prec >= len) - && !(arg->flags & (F_SIGN | F_BLANK))) - { - /* Fast path */ - if (_PyUnicodeWriter_WriteStr(writer, str) == -1) - return -1; - return 0; - } - - /* Truncate the string for "s", "r" and "a" formats - if the precision is set */ - if (arg->ch == 's' || arg->ch == 'r' || arg->ch == 'a') { - if (arg->prec >= 0 && len > arg->prec) - len = arg->prec; - } - - /* Adjust sign and width */ - kind = PyUnicode_KIND(str); - pbuf = PyUnicode_DATA(str); - pindex = 0; - signchar = '\0'; - if (arg->sign) { - Py_UCS4 ch = PyUnicode_READ(kind, pbuf, pindex); - if (ch == '-' || ch == '+') { - signchar = ch; - len--; - pindex++; - } - else if (arg->flags & F_SIGN) - signchar = '+'; - else if (arg->flags & F_BLANK) - signchar = ' '; - else - arg->sign = 0; - } - if (arg->width < len) - arg->width = len; - - /* Prepare the writer */ - maxchar = writer->maxchar; - if (!(arg->flags & F_LJUST)) { - if (arg->sign) { - if ((arg->width-1) > len) - maxchar = Py_MAX(maxchar, fill); - } - else { - if (arg->width > len) - maxchar = Py_MAX(maxchar, fill); - } - } - if (PyUnicode_MAX_CHAR_VALUE(str) > maxchar) { - Py_UCS4 strmaxchar = _PyUnicode_FindMaxChar(str, 0, pindex+len); - maxchar = Py_MAX(maxchar, strmaxchar); - } - - buflen = arg->width; - if (arg->sign && len == arg->width) - buflen++; - if (_PyUnicodeWriter_Prepare(writer, buflen, maxchar) == -1) - return -1; - - /* Write the sign if needed */ - if (arg->sign) { - if (fill != ' ') { - PyUnicode_WRITE(writer->kind, writer->data, writer->pos, signchar); - writer->pos += 1; - } - if (arg->width > len) - arg->width--; - } - - /* Write the numeric prefix for "x", "X" and "o" formats - if the alternate form is used. - For example, write "0x" for the "%#x" format. */ - if ((arg->flags & F_ALT) && (arg->ch == 'x' || arg->ch == 'X' || arg->ch == 'o')) { - assert(PyUnicode_READ(kind, pbuf, pindex) == '0'); - assert(PyUnicode_READ(kind, pbuf, pindex + 1) == arg->ch); - if (fill != ' ') { - PyUnicode_WRITE(writer->kind, writer->data, writer->pos, '0'); - PyUnicode_WRITE(writer->kind, writer->data, writer->pos+1, arg->ch); - writer->pos += 2; - pindex += 2; - } - arg->width -= 2; - if (arg->width < 0) - arg->width = 0; - len -= 2; - } - - /* Pad left with the fill character if needed */ - if (arg->width > len && !(arg->flags & F_LJUST)) { - sublen = arg->width - len; - _PyUnicode_Fill(writer->kind, writer->data, fill, writer->pos, sublen); - writer->pos += sublen; - arg->width = len; - } - - /* If padding with spaces: write sign if needed and/or numeric prefix if - the alternate form is used */ - if (fill == ' ') { - if (arg->sign) { - PyUnicode_WRITE(writer->kind, writer->data, writer->pos, signchar); - writer->pos += 1; - } - if ((arg->flags & F_ALT) && (arg->ch == 'x' || arg->ch == 'X' || arg->ch == 'o')) { - assert(PyUnicode_READ(kind, pbuf, pindex) == '0'); - assert(PyUnicode_READ(kind, pbuf, pindex+1) == arg->ch); - PyUnicode_WRITE(writer->kind, writer->data, writer->pos, '0'); - PyUnicode_WRITE(writer->kind, writer->data, writer->pos+1, arg->ch); - writer->pos += 2; - pindex += 2; - } - } - - /* Write characters */ - if (len) { - _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, - str, pindex, len); - writer->pos += len; - } - - /* Pad right with the fill character if needed */ - if (arg->width > len) { - sublen = arg->width - len; - _PyUnicode_Fill(writer->kind, writer->data, ' ', writer->pos, sublen); - writer->pos += sublen; - } - return 0; -} - -/* Helper of PyUnicode_Format(): format one arg. - Return 0 on success, raise an exception and return -1 on error. */ -static int -unicode_format_arg(struct unicode_formatter_t *ctx) -{ - struct unicode_format_arg_t arg; - PyObject *str; - int ret; - - arg.ch = PyUnicode_READ(ctx->fmtkind, ctx->fmtdata, ctx->fmtpos); - if (arg.ch == '%') { - ctx->fmtpos++; - ctx->fmtcnt--; - if (_PyUnicodeWriter_WriteCharInline(&ctx->writer, '%') < 0) - return -1; - return 0; - } - arg.flags = 0; - arg.width = -1; - arg.prec = -1; - arg.sign = 0; - str = NULL; - - ret = unicode_format_arg_parse(ctx, &arg); - if (ret == -1) - return -1; - - ret = unicode_format_arg_format(ctx, &arg, &str); - if (ret == -1) - return -1; - - if (ret != 1) { - ret = unicode_format_arg_output(ctx, &arg, str); - Py_DECREF(str); - if (ret == -1) - return -1; - } - - if (ctx->dict && (ctx->argidx < ctx->arglen)) { - PyErr_SetString(PyExc_TypeError, - "not all arguments converted during string formatting"); - return -1; - } - return 0; -} - -PyObject * -PyUnicode_Format(PyObject *format, PyObject *args) -{ - struct unicode_formatter_t ctx; - - if (format == NULL || args == NULL) { - PyErr_BadInternalCall(); - return NULL; - } - - if (ensure_unicode(format) < 0) - return NULL; - - ctx.fmtstr = format; - ctx.fmtdata = PyUnicode_DATA(ctx.fmtstr); - ctx.fmtkind = PyUnicode_KIND(ctx.fmtstr); - ctx.fmtcnt = PyUnicode_GET_LENGTH(ctx.fmtstr); - ctx.fmtpos = 0; - - _PyUnicodeWriter_Init(&ctx.writer); - ctx.writer.min_length = ctx.fmtcnt + 100; - ctx.writer.overallocate = 1; - - if (PyTuple_Check(args)) { - ctx.arglen = PyTuple_Size(args); - ctx.argidx = 0; - } - else { - ctx.arglen = -1; - ctx.argidx = -2; - } - ctx.args_owned = 0; - if (PyMapping_Check(args) && !PyTuple_Check(args) && !PyUnicode_Check(args)) - ctx.dict = args; - else - ctx.dict = NULL; - ctx.args = args; - - while (--ctx.fmtcnt >= 0) { - if (PyUnicode_READ(ctx.fmtkind, ctx.fmtdata, ctx.fmtpos) != '%') { - Py_ssize_t nonfmtpos; - - nonfmtpos = ctx.fmtpos++; - while (ctx.fmtcnt >= 0 && - PyUnicode_READ(ctx.fmtkind, ctx.fmtdata, ctx.fmtpos) != '%') { - ctx.fmtpos++; - ctx.fmtcnt--; - } - if (ctx.fmtcnt < 0) { - ctx.fmtpos--; - ctx.writer.overallocate = 0; - } - - if (_PyUnicodeWriter_WriteSubstring(&ctx.writer, ctx.fmtstr, - nonfmtpos, ctx.fmtpos) < 0) - goto onError; - } - else { - ctx.fmtpos++; - if (unicode_format_arg(&ctx) == -1) - goto onError; - } - } - - if (ctx.argidx < ctx.arglen && !ctx.dict) { - PyErr_SetString(PyExc_TypeError, - "not all arguments converted during string formatting"); - goto onError; - } - - if (ctx.args_owned) { - Py_DECREF(ctx.args); - } - return _PyUnicodeWriter_Finish(&ctx.writer); - - onError: - _PyUnicodeWriter_Dealloc(&ctx.writer); - if (ctx.args_owned) { - Py_DECREF(ctx.args); - } - return NULL; -} - static PyObject * unicode_subtype_new(PyTypeObject *type, PyObject *unicode); diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index c4a11fa9b24..02b6f35798f 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -164,6 +164,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 7bbbec2c988..39462a6380c 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -481,6 +481,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index e2e1e415827..2657ee5c444 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -558,6 +558,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 7e7ed9c2ae6..9c12be6e935 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1271,6 +1271,9 @@ Objects + + Objects + Objects From d0b18b19fa0ed73ce99e49751604f3e7d409b8ba Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 10 Oct 2025 13:01:06 +0200 Subject: [PATCH 095/112] gh-129813: Fix PyBytesWriter tests (#139892) --- Doc/c-api/bytes.rst | 2 +- Modules/_testcapi/bytes.c | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index 177ebf6c6b3..0a73c4748f4 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -260,7 +260,7 @@ Create, Finish, Discard writer size to *size*. The caller is responsible to write *size* bytes using :c:func:`PyBytesWriter_GetData`. - On error, set an exception and return NULL. + On error, set an exception and return ``NULL``. *size* must be positive or zero. diff --git a/Modules/_testcapi/bytes.c b/Modules/_testcapi/bytes.c index 388e65456c3..f12fc7f5f3a 100644 --- a/Modules/_testcapi/bytes.c +++ b/Modules/_testcapi/bytes.c @@ -79,11 +79,6 @@ writer_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) static int writer_init(PyObject *self_raw, PyObject *args, PyObject *kwargs) { - WriterObject *self = (WriterObject *)self_raw; - if (self->writer) { - PyBytesWriter_Discard(self->writer); - } - if (kwargs && PyDict_GET_SIZE(kwargs)) { PyErr_Format(PyExc_TypeError, "PyBytesWriter() takes exactly no keyword arguments"); @@ -99,6 +94,10 @@ writer_init(PyObject *self_raw, PyObject *args, PyObject *kwargs) return -1; } + WriterObject *self = (WriterObject *)self_raw; + if (self->writer) { + PyBytesWriter_Discard(self->writer); + } if (use_bytearray) { self->writer = _PyBytesWriter_CreateByteArray(alloc); } From 5c942f11cdf5f9d7313200983fa0c58b3bc670a2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 10 Oct 2025 15:51:19 +0300 Subject: [PATCH 096/112] gh-63161: Fix PEP 263 support (GH-139481) * Support non-UTF-8 shebang and comments if non-UTF-8 encoding is specified. * Detect decoding error in comments for UTF-8 encoding. * Include the decoding error position for default encoding in SyntaxError. --- Lib/test/test_exceptions.py | 8 +- Lib/test/test_source_encoding.py | 123 ++++++++++++++++-- ...5-10-01-18-21-19.gh-issue-63161.ef1S6N.rst | 5 + Parser/pegen_errors.c | 8 ++ Parser/tokenizer/file_tokenizer.c | 49 +++++-- Parser/tokenizer/helpers.c | 53 +++++--- Parser/tokenizer/helpers.h | 2 +- Parser/tokenizer/readline_tokenizer.c | 2 +- Parser/tokenizer/string_tokenizer.c | 6 + 9 files changed, 210 insertions(+), 46 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-01-18-21-19.gh-issue-63161.ef1S6N.rst diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 59f77f91d85..323a8c401bd 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -224,6 +224,8 @@ def check(self, src, lineno, offset, end_lineno=None, end_offset=None, encoding= if not isinstance(src, str): src = src.decode(encoding, 'replace') line = src.split('\n')[lineno-1] + if lineno == 1: + line = line.removeprefix('\ufeff') self.assertIn(line, cm.exception.text) def test_error_offset_continuation_characters(self): @@ -239,7 +241,9 @@ def testSyntaxErrorOffset(self): check('Python = "\u1e54\xfd\u0163\u0125\xf2\xf1" +', 1, 20) check(b'# -*- coding: cp1251 -*-\nPython = "\xcf\xb3\xf2\xee\xed" +', 2, 19, encoding='cp1251') - check(b'Python = "\xcf\xb3\xf2\xee\xed" +', 1, 10) + check(b'Python = "\xcf\xb3\xf2\xee\xed" +', 1, 12) + check(b'\n\n\nPython = "\xcf\xb3\xf2\xee\xed" +', 4, 12) + check(b'\xef\xbb\xbfPython = "\xcf\xb3\xf2\xee\xed" +', 1, 12) check('x = "a', 1, 5) check('lambda x: x = 2', 1, 1) check('f{a + b + c}', 1, 2) @@ -287,7 +291,7 @@ def baz(): check("pass\npass\npass\n(1+)\npass\npass\npass", 4, 4) check("(1+)", 1, 4) check("[interesting\nfoo()\n", 1, 1) - check(b"\xef\xbb\xbf#coding: utf8\nprint('\xe6\x88\x91')\n", 0, -1) + check(b"\xef\xbb\xbf#coding: utf8\nprint('\xe6\x88\x91')\n", 1, 0) check("""f''' { (123_a) diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index 5df40782382..46b291192df 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- import unittest -from test.support import script_helper, captured_stdout, requires_subprocess, requires_resource +from test import support +from test.support import script_helper from test.support.os_helper import TESTFN, unlink, rmtree from test.support.import_helper import unload import importlib @@ -64,7 +65,7 @@ def test_issue7820(self): # two bytes in common with the UTF-8 BOM self.assertRaises(SyntaxError, eval, b'\xef\xbb\x20') - @requires_subprocess() + @support.requires_subprocess() def test_20731(self): sub = subprocess.Popen([sys.executable, os.path.join(os.path.dirname(__file__), @@ -267,6 +268,17 @@ def test_second_non_utf8_coding_line(self): b'print(ascii("\xc3\xa4"))\n') self.check_script_output(src, br"'\xc3\u20ac'") + def test_first_utf8_coding_line_error(self): + src = (b'#coding:ascii \xc3\xa4\n' + b'raise RuntimeError\n') + self.check_script_error(src, br"(\(unicode error\) )?'ascii' codec can't decode byte") + + def test_second_utf8_coding_line_error(self): + src = (b'#!/usr/bin/python\n' + b'#coding:ascii \xc3\xa4\n' + b'raise RuntimeError\n') + self.check_script_error(src, br"(\(unicode error\) )?'ascii' codec can't decode byte") + def test_utf8_bom(self): src = (b'\xef\xbb\xbfprint(ascii("\xc3\xa4"))\n') self.check_script_output(src, br"'\xe4'") @@ -282,10 +294,80 @@ def test_utf8_bom_and_utf8_coding_line(self): b'print(ascii("\xc3\xa4"))\n') self.check_script_output(src, br"'\xe4'") - def test_utf8_non_utf8_comment_line_error(self): + def test_utf8_bom_and_non_utf8_first_coding_line(self): + src = (b'\xef\xbb\xbf#coding:iso-8859-15\n' + b'raise RuntimeError\n') + self.check_script_error(src, + br"encoding problem: iso-8859-15 with BOM", + lineno=1) + + def test_utf8_bom_and_non_utf8_second_coding_line(self): + src = (b'\xef\xbb\xbf#first\n' + b'#coding:iso-8859-15\n' + b'raise RuntimeError\n') + self.check_script_error(src, + br"encoding problem: iso-8859-15 with BOM", + lineno=2) + + def test_non_utf8_shebang(self): + src = (b'#!/home/\xa4/bin/python\n' + b'#coding:iso-8859-15\n' + b'print(ascii("\xc3\xa4"))\n') + self.check_script_output(src, br"'\xc3\u20ac'") + + def test_utf8_shebang_error(self): + src = (b'#!/home/\xc3\xa4/bin/python\n' + b'#coding:ascii\n' + b'raise RuntimeError\n') + self.check_script_error(src, br"(\(unicode error\) )?'ascii' codec can't decode byte") + + def test_non_utf8_shebang_error(self): + src = (b'#!/home/\xa4/bin/python\n' + b'raise RuntimeError\n') + self.check_script_error(src, br"Non-UTF-8 code starting with .* on line 1", + lineno=1) + + def test_non_utf8_second_line_error(self): + src = (b'#first\n' + b'#second\xa4\n' + b'raise RuntimeError\n') + self.check_script_error(src, + br"Non-UTF-8 code starting with .* on line 2", + lineno=2) + + def test_non_utf8_third_line_error(self): + src = (b'#first\n' + b'#second\n' + b'#third\xa4\n' + b'raise RuntimeError\n') + self.check_script_error(src, + br"Non-UTF-8 code starting with .* on line 3", + lineno=3) + + def test_utf8_bom_non_utf8_third_line_error(self): + src = (b'\xef\xbb\xbf#first\n' + b'#second\n' + b'#third\xa4\n' + b'raise RuntimeError\n') + self.check_script_error(src, + br"Non-UTF-8 code starting with .* on line 3|" + br"'utf-8' codec can't decode byte", + lineno=3) + + def test_utf_8_non_utf8_third_line_error(self): + src = (b'#coding: utf-8\n' + b'#second\n' + b'#third\xa4\n' + b'raise RuntimeError\n') + self.check_script_error(src, + br"Non-UTF-8 code starting with .* on line 3|" + br"'utf-8' codec can't decode byte", + lineno=3) + + def test_utf8_non_utf8_third_line_error(self): src = (b'#coding: utf8\n' - b'#\n' - b'#\xa4\n' + b'#second\n' + b'#third\xa4\n' b'raise RuntimeError\n') self.check_script_error(src, br"'utf-8' codec can't decode byte|" @@ -326,7 +408,7 @@ def test_nul_in_second_coding_line(self): class UTF8ValidatorTest(unittest.TestCase): @unittest.skipIf(not sys.platform.startswith("linux"), "Too slow to run on non-Linux platforms") - @requires_resource('cpu') + @support.requires_resource('cpu') def test_invalid_utf8(self): # This is a port of test_utf8_decode_invalid_sequences in # test_unicode.py to exercise the separate utf8 validator in @@ -392,19 +474,29 @@ def check(content): check(b'\xF4'+cb+b'\xBF\xBF') +@support.force_not_colorized_test_class class BytesSourceEncodingTest(AbstractSourceEncodingTest, unittest.TestCase): def check_script_output(self, src, expected): - with captured_stdout() as stdout: + with support.captured_stdout() as stdout: exec(src) out = stdout.getvalue().encode('latin1') self.assertEqual(out.rstrip(), expected) - def check_script_error(self, src, expected): - with self.assertRaisesRegex(SyntaxError, expected.decode()) as cm: + def check_script_error(self, src, expected, lineno=...): + with self.assertRaises(SyntaxError) as cm: exec(src) + exc = cm.exception + self.assertRegex(str(exc), expected.decode()) + if lineno is not ...: + self.assertEqual(exc.lineno, lineno) + line = src.splitlines()[lineno-1].decode(errors='replace') + if lineno == 1: + line = line.removeprefix('\ufeff') + self.assertEqual(line, exc.text) +@support.force_not_colorized_test_class class FileSourceEncodingTest(AbstractSourceEncodingTest, unittest.TestCase): def check_script_output(self, src, expected): @@ -415,13 +507,22 @@ def check_script_output(self, src, expected): res = script_helper.assert_python_ok(fn) self.assertEqual(res.out.rstrip(), expected) - def check_script_error(self, src, expected): + def check_script_error(self, src, expected, lineno=...): with tempfile.TemporaryDirectory() as tmpd: fn = os.path.join(tmpd, 'test.py') with open(fn, 'wb') as fp: fp.write(src) res = script_helper.assert_python_failure(fn) - self.assertRegex(res.err.rstrip().splitlines()[-1], b'SyntaxError.*?' + expected) + err = res.err.rstrip() + self.assertRegex(err.splitlines()[-1], b'SyntaxError: ' + expected) + if lineno is not ...: + self.assertIn(f', line {lineno}\n'.encode(), + err.replace(os.linesep.encode(), b'\n')) + line = src.splitlines()[lineno-1].decode(errors='replace') + if lineno == 1: + line = line.removeprefix('\ufeff') + self.assertIn(line.encode(), err) + if __name__ == "__main__": diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-01-18-21-19.gh-issue-63161.ef1S6N.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-01-18-21-19.gh-issue-63161.ef1S6N.rst new file mode 100644 index 00000000000..5eafe0813dc --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-01-18-21-19.gh-issue-63161.ef1S6N.rst @@ -0,0 +1,5 @@ +Support non-UTF-8 shebang and comments in Python source files if non-UTF-8 +encoding is specified. Detect decoding error in comments for default (UTF-8) +encoding. Show the line and position of decoding error for default encoding +in a traceback. Show the line containing the coding cookie when it conflicts +with the BOM in a traceback. diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index f62b8695995..0639a4e4243 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -2,6 +2,7 @@ #include #include "pycore_pyerrors.h" // _PyErr_ProgramDecodedTextObject() +#include "pycore_runtime.h" // _Py_ID() #include "lexer/state.h" #include "lexer/lexer.h" #include "pegen.h" @@ -23,6 +24,13 @@ _PyPegen_raise_tokenizer_init_error(PyObject *filename) PyObject *value; PyObject *tback; PyErr_Fetch(&type, &value, &tback); + if (PyErr_GivenExceptionMatches(value, PyExc_SyntaxError)) { + if (PyObject_SetAttr(value, &_Py_ID(filename), filename)) { + goto error; + } + PyErr_Restore(type, value, tback); + return; + } errstr = PyObject_Str(value); if (!errstr) { goto error; diff --git a/Parser/tokenizer/file_tokenizer.c b/Parser/tokenizer/file_tokenizer.c index 01e473f58a0..8c836a3f725 100644 --- a/Parser/tokenizer/file_tokenizer.c +++ b/Parser/tokenizer/file_tokenizer.c @@ -282,10 +282,8 @@ tok_underflow_interactive(struct tok_state *tok) { } static int -tok_underflow_file(struct tok_state *tok) { - if (tok->start == NULL && !INSIDE_FSTRING(tok)) { - tok->cur = tok->inp = tok->buf; - } +tok_underflow_file(struct tok_state *tok) +{ if (tok->decoding_state == STATE_INIT) { /* We have not yet determined the encoding. If an encoding is found, use the file-pointer @@ -296,8 +294,16 @@ tok_underflow_file(struct tok_state *tok) { } assert(tok->decoding_state != STATE_INIT); } + int raw = tok->decoding_readline == NULL; + if (raw && tok->decoding_state != STATE_NORMAL) { + /* Keep the first line in the buffer to validate it later if + * the encoding has not yet been determined. */ + } + else if (tok->start == NULL && !INSIDE_FSTRING(tok)) { + tok->cur = tok->inp = tok->buf; + } /* Read until '\n' or EOF */ - if (tok->decoding_readline != NULL) { + if (!raw) { /* We already have a codec associated with this input. */ if (!tok_readline_recode(tok)) { return 0; @@ -328,20 +334,35 @@ tok_underflow_file(struct tok_state *tok) { ADVANCE_LINENO(); if (tok->decoding_state != STATE_NORMAL) { - if (tok->lineno > 2) { - tok->decoding_state = STATE_NORMAL; - } - else if (!_PyTokenizer_check_coding_spec(tok->cur, strlen(tok->cur), + if (!_PyTokenizer_check_coding_spec(tok->cur, strlen(tok->cur), tok, fp_setreadl)) { return 0; } + if (tok->lineno >= 2) { + tok->decoding_state = STATE_NORMAL; + } } - /* The default encoding is UTF-8, so make sure we don't have any - non-UTF-8 sequences in it. */ - if (!tok->encoding && !_PyTokenizer_ensure_utf8(tok->cur, tok)) { - _PyTokenizer_error_ret(tok); - return 0; + if (raw && tok->decoding_state == STATE_NORMAL) { + const char *line = tok->lineno <= 2 ? tok->buf : tok->cur; + int lineno = tok->lineno <= 2 ? 1 : tok->lineno; + if (!tok->encoding) { + /* The default encoding is UTF-8, so make sure we don't have any + non-UTF-8 sequences in it. */ + if (!_PyTokenizer_ensure_utf8(line, tok, lineno)) { + _PyTokenizer_error_ret(tok); + return 0; + } + } + else { + PyObject *tmp = PyUnicode_Decode(line, strlen(line), + tok->encoding, NULL); + if (tmp == NULL) { + _PyTokenizer_error_ret(tok); + return 0; + } + Py_DECREF(tmp); + } } assert(tok->done == E_OK); return tok->done == E_OK; diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index 5a416adb875..e5e2eed2d34 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -47,8 +47,10 @@ _syntaxerror_range(struct tok_state *tok, const char *format, goto error; } - args = Py_BuildValue("(O(OiiNii))", errmsg, tok->filename, tok->lineno, - col_offset, errtext, tok->lineno, end_col_offset); + args = Py_BuildValue("(O(OiiNii))", errmsg, + tok->filename ? tok->filename : Py_None, + tok->lineno, col_offset, errtext, + tok->lineno, end_col_offset); if (args) { PyErr_SetObject(PyExc_SyntaxError, args); Py_DECREF(args); @@ -422,10 +424,13 @@ _PyTokenizer_check_coding_spec(const char* line, Py_ssize_t size, struct tok_sta tok->encoding = cs; } else { /* then, compare cs with BOM */ if (strcmp(tok->encoding, cs) != 0) { - _PyTokenizer_error_ret(tok); - PyErr_Format(PyExc_SyntaxError, - "encoding problem: %s with BOM", cs); + tok->line_start = line; + tok->cur = (char *)line; + assert(size <= INT_MAX); + _PyTokenizer_syntaxerror_known_range(tok, 0, (int)size, + "encoding problem: %s with BOM", cs); PyMem_Free(cs); + _PyTokenizer_error_ret(tok); return 0; } PyMem_Free(cs); @@ -496,24 +501,38 @@ valid_utf8(const unsigned char* s) } int -_PyTokenizer_ensure_utf8(char *line, struct tok_state *tok) +_PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno) { - int badchar = 0; - unsigned char *c; + const char *badchar = NULL; + const char *c; int length; - for (c = (unsigned char *)line; *c; c += length) { - if (!(length = valid_utf8(c))) { - badchar = *c; + int col_offset = 0; + const char *line_start = line; + for (c = line; *c; c += length) { + if (!(length = valid_utf8((const unsigned char *)c))) { + badchar = c; break; } + col_offset++; + if (*c == '\n') { + lineno++; + col_offset = 0; + line_start = c + 1; + } } if (badchar) { - PyErr_Format(PyExc_SyntaxError, - "Non-UTF-8 code starting with '\\x%.2x' " - "in file %U on line %i, " - "but no encoding declared; " - "see https://peps.python.org/pep-0263/ for details", - badchar, tok->filename, tok->lineno); + tok->lineno = lineno; + tok->line_start = line_start; + tok->cur = (char *)badchar; + _PyTokenizer_syntaxerror_known_range(tok, + col_offset + 1, col_offset + 1, + "Non-UTF-8 code starting with '\\x%.2x'" + "%s%V on line %i, " + "but no encoding declared; " + "see https://peps.python.org/pep-0263/ for details", + (unsigned char)*badchar, + tok->filename ? " in file " : "", tok->filename, "", + lineno); return 0; } return 1; diff --git a/Parser/tokenizer/helpers.h b/Parser/tokenizer/helpers.h index 42ea13cd1f8..98f6445d5a3 100644 --- a/Parser/tokenizer/helpers.h +++ b/Parser/tokenizer/helpers.h @@ -26,7 +26,7 @@ int _PyTokenizer_check_bom(int get_char(struct tok_state *), struct tok_state *tok); int _PyTokenizer_check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok, int set_readline(struct tok_state *, const char *)); -int _PyTokenizer_ensure_utf8(char *line, struct tok_state *tok); +int _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno); #ifdef Py_DEBUG void _PyTokenizer_print_escape(FILE *f, const char *s, Py_ssize_t size); diff --git a/Parser/tokenizer/readline_tokenizer.c b/Parser/tokenizer/readline_tokenizer.c index 22f84c77a12..0f7769aeb8f 100644 --- a/Parser/tokenizer/readline_tokenizer.c +++ b/Parser/tokenizer/readline_tokenizer.c @@ -97,7 +97,7 @@ tok_underflow_readline(struct tok_state* tok) { ADVANCE_LINENO(); /* The default encoding is UTF-8, so make sure we don't have any non-UTF-8 sequences in it. */ - if (!tok->encoding && !_PyTokenizer_ensure_utf8(tok->cur, tok)) { + if (!tok->encoding && !_PyTokenizer_ensure_utf8(tok->cur, tok, tok->lineno)) { _PyTokenizer_error_ret(tok); return 0; } diff --git a/Parser/tokenizer/string_tokenizer.c b/Parser/tokenizer/string_tokenizer.c index 0c26d5df8d4..7299ecf483c 100644 --- a/Parser/tokenizer/string_tokenizer.c +++ b/Parser/tokenizer/string_tokenizer.c @@ -86,15 +86,18 @@ decode_str(const char *input, int single, struct tok_state *tok, int preserve_cr /* need to check line 1 and 2 separately since check_coding_spec assumes a single line as input */ if (newl[0]) { + tok->lineno = 1; if (!_PyTokenizer_check_coding_spec(str, newl[0] - str, tok, buf_setreadl)) { return NULL; } if (tok->enc == NULL && tok->decoding_state != STATE_NORMAL && newl[1]) { + tok->lineno = 2; if (!_PyTokenizer_check_coding_spec(newl[0]+1, newl[1] - newl[0], tok, buf_setreadl)) return NULL; } } + tok->lineno = 0; if (tok->enc != NULL) { assert(utf8 == NULL); utf8 = _PyTokenizer_translate_into_utf8(str, tok->enc); @@ -102,6 +105,9 @@ decode_str(const char *input, int single, struct tok_state *tok, int preserve_cr return _PyTokenizer_error_ret(tok); str = PyBytes_AS_STRING(utf8); } + else if (!_PyTokenizer_ensure_utf8(str, tok, 1)) { + return _PyTokenizer_error_ret(tok); + } assert(tok->decoding_buffer == NULL); tok->decoding_buffer = utf8; /* CAUTION */ return str; From 1c598e04361dbfc9cf465f3a02f83715c11b028c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 10 Oct 2025 16:29:18 +0300 Subject: [PATCH 097/112] gh-139065: Fix trailing space before long word in textwrap (GH-139070) Fix trailing space before a wrapped long word if the line length with a space is exactly "width". --- Lib/test/test_textwrap.py | 4 ++-- Lib/textwrap.py | 2 +- .../Library/2025-09-17-19-08-34.gh-issue-139065.Hu8fM5.rst | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-09-17-19-08-34.gh-issue-139065.Hu8fM5.rst diff --git a/Lib/test/test_textwrap.py b/Lib/test/test_textwrap.py index cbd383ea4e2..aca1f427656 100644 --- a/Lib/test/test_textwrap.py +++ b/Lib/test/test_textwrap.py @@ -605,7 +605,7 @@ def test_break_long(self): # bug 1146. Prevent a long word to be wrongly wrapped when the # preceding word is exactly one character shorter than the width self.check_wrap(self.text, 12, - ['Did you say ', + ['Did you say', '"supercalifr', 'agilisticexp', 'ialidocious?', @@ -633,7 +633,7 @@ def test_nobreak_long(self): def test_max_lines_long(self): self.check_wrap(self.text, 12, - ['Did you say ', + ['Did you say', '"supercalifr', 'agilisticexp', '[...]'], diff --git a/Lib/textwrap.py b/Lib/textwrap.py index 5ae439f5cd3..41366fbf443 100644 --- a/Lib/textwrap.py +++ b/Lib/textwrap.py @@ -211,7 +211,7 @@ def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width): # If we're allowed to break long words, then do so: put as much # of the next chunk onto the current line as will fit. - if self.break_long_words: + if self.break_long_words and space_left > 0: end = space_left chunk = reversed_chunks[-1] if self.break_on_hyphens and len(chunk) > space_left: diff --git a/Misc/NEWS.d/next/Library/2025-09-17-19-08-34.gh-issue-139065.Hu8fM5.rst b/Misc/NEWS.d/next/Library/2025-09-17-19-08-34.gh-issue-139065.Hu8fM5.rst new file mode 100644 index 00000000000..20c00603959 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-17-19-08-34.gh-issue-139065.Hu8fM5.rst @@ -0,0 +1,2 @@ +Fix trailing space before a wrapped long word if the line length is exactly +*width* in :mod:`textwrap`. From 302f19d1f1880dd29deb5ad02fa2c21639319559 Mon Sep 17 00:00:00 2001 From: Weilin Du <108666168+LamentXU123@users.noreply.github.com> Date: Fri, 10 Oct 2025 23:28:38 +0800 Subject: [PATCH 098/112] gh-139843: Document signals (SIGSTOP, SIGVTALRM, SIGPROF) to fix sphinx references (GH-139896) --- Doc/library/signal.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 0c681c795c1..ccdf5027ff9 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -205,6 +205,12 @@ The variables defined in the :mod:`signal` module are: .. availability:: Unix. +.. data:: SIGPROF + + Profiling timer expired. + + .. availability:: Unix. + .. data:: SIGQUIT Terminal quit signal. @@ -215,6 +221,10 @@ The variables defined in the :mod:`signal` module are: Segmentation fault: invalid memory reference. +.. data:: SIGSTOP + + Stop executing (cannot be caught or ignored). + .. data:: SIGSTKFLT Stack fault on coprocessor. The Linux kernel does not raise this signal: it @@ -243,6 +253,12 @@ The variables defined in the :mod:`signal` module are: .. availability:: Unix. +.. data:: SIGVTALRM + + Virtual timer expired. + + .. availability:: Unix. + .. data:: SIGWINCH Window resize signal. From c7f1da97eb4639a17fb01ef122155bab2f262a34 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Fri, 10 Oct 2025 16:32:44 +0100 Subject: [PATCH 099/112] gh-101100: Document `zlib` public constants to fix reference warnings (#139835) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: Victor Stinner --- Doc/library/zlib.rst | 136 ++++++++++++++++++++++++++++++++++++++++++- Doc/tools/.nitignore | 1 - 2 files changed, 133 insertions(+), 4 deletions(-) diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index 7c5e9b086e1..d4727f366d0 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -16,7 +16,7 @@ earlier than 1.1.3; 1.1.3 has a `security vulnerability `_ for authoritative information. For reading and writing ``.gz`` files see the :mod:`gzip` module. @@ -340,6 +340,136 @@ Decompression objects support the following methods and attributes: objects. +The following constants are available to configure compression and decompression +behavior: + +.. data:: DEFLATED + + The deflate compression method. + + +.. data:: MAX_WBITS + + The maximum window size, expressed as a power of 2. + For example, if :const:`!MAX_WBITS` is ``15`` it results in a window size + of ``32 KiB``. + + +.. data:: DEF_MEM_LEVEL + + The default memory level for compression objects. + + +.. data:: DEF_BUF_SIZE + + The default buffer size for decompression operations. + + +.. data:: Z_NO_COMPRESSION + + Compression level ``0``. + + .. versionadded:: 3.6 + + +.. data:: Z_BEST_SPEED + + Compression level ``1``. + + +.. data:: Z_BEST_COMPRESSION + + Compression level ``9``. + + +.. data:: Z_DEFAULT_COMPRESSION + + Default compression level (``-1``). + + +.. data:: Z_DEFAULT_STRATEGY + + Default compression strategy, for normal data. + + +.. data:: Z_FILTERED + + Compression strategy for data produced by a filter (or predictor). + + +.. data:: Z_HUFFMAN_ONLY + + Compression strategy that forces Huffman coding only. + + +.. data:: Z_RLE + + Compression strategy that limits match distances to one (run-length encoding). + + This constant is only available if Python was compiled with zlib + 1.2.0.1 or greater. + + .. versionadded:: 3.6 + + +.. data:: Z_FIXED + + Compression strategy that prevents the use of dynamic Huffman codes. + + This constant is only available if Python was compiled with zlib + 1.2.2.2 or greater. + + .. versionadded:: 3.6 + + +.. data:: Z_NO_FLUSH + + Flush mode ``0``. No special flushing behavior. + + .. versionadded:: 3.6 + + +.. data:: Z_PARTIAL_FLUSH + + Flush mode ``1``. Flush as much output as possible. + + +.. data:: Z_SYNC_FLUSH + + Flush mode ``2``. All output is flushed and the output is aligned to a byte boundary. + + +.. data:: Z_FULL_FLUSH + + Flush mode ``3``. All output is flushed and the compression state is reset. + + +.. data:: Z_FINISH + + Flush mode ``4``. All pending input is processed, no more input is expected. + + +.. data:: Z_BLOCK + + Flush mode ``5``. A deflate block is completed and emitted. + + This constant is only available if Python was compiled with zlib + 1.2.2.2 or greater. + + .. versionadded:: 3.6 + + +.. data:: Z_TREES + + Flush mode ``6``, for inflate operations. Instructs inflate to return when + it gets to the next deflate block boundary. + + This constant is only available if Python was compiled with zlib + 1.2.3.4 or greater. + + .. versionadded:: 3.6 + + Information about the version of the zlib library in use is available through the following constants: @@ -375,10 +505,10 @@ the following constants: Module :mod:`gzip` Reading and writing :program:`gzip`\ -format files. - http://www.zlib.net + https://www.zlib.net The zlib library home page. - http://www.zlib.net/manual.html + https://www.zlib.net/manual.html The zlib manual explains the semantics and usage of the library's many functions. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 827c5808fa5..657d720e766 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -51,7 +51,6 @@ Doc/library/xml.sax.reader.rst Doc/library/xml.sax.rst Doc/library/xmlrpc.client.rst Doc/library/xmlrpc.server.rst -Doc/library/zlib.rst Doc/whatsnew/2.4.rst Doc/whatsnew/2.5.rst Doc/whatsnew/2.6.rst From b881df47ff1adca515d1de04f689160ddae72142 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Fri, 10 Oct 2025 21:58:23 +0530 Subject: [PATCH 100/112] gh-139894: fix incorrect sharing of current task while forking in `asyncio` (#139897) Fix incorrect sharing of current task with the forked child process by clearing thread state's current task and current loop in `PyOS_AfterFork_Child`. --- Lib/test/test_asyncio/test_unix_events.py | 86 +++++++++++++------ ...-10-10-11-22-50.gh-issue-139894.ECAXqj.rst | 1 + Modules/posixmodule.c | 10 +++ 3 files changed, 72 insertions(+), 25 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-10-11-22-50.gh-issue-139894.ECAXqj.rst diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index a69a5e32b1b..d2b3de3b9a4 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1180,32 +1180,68 @@ async def runner(): @support.requires_fork() -class TestFork(unittest.IsolatedAsyncioTestCase): +class TestFork(unittest.TestCase): - async def test_fork_not_share_event_loop(self): - with warnings_helper.ignore_fork_in_thread_deprecation_warnings(): - # The forked process should not share the event loop with the parent - loop = asyncio.get_running_loop() - r, w = os.pipe() - self.addCleanup(os.close, r) - self.addCleanup(os.close, w) - pid = os.fork() - if pid == 0: - # child - try: - loop = asyncio.get_event_loop() - os.write(w, b'LOOP:' + str(id(loop)).encode()) - except RuntimeError: - os.write(w, b'NO LOOP') - except BaseException as e: - os.write(w, b'ERROR:' + ascii(e).encode()) - finally: - os._exit(0) - else: - # parent - result = os.read(r, 100) - self.assertEqual(result, b'NO LOOP') - wait_process(pid, exitcode=0) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + def test_fork_not_share_current_task(self): + loop = object() + task = object() + asyncio._set_running_loop(loop) + self.addCleanup(asyncio._set_running_loop, None) + asyncio.tasks._enter_task(loop, task) + self.addCleanup(asyncio.tasks._leave_task, loop, task) + self.assertIs(asyncio.current_task(), task) + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + pid = os.fork() + if pid == 0: + # child + try: + asyncio._set_running_loop(loop) + current_task = asyncio.current_task() + if current_task is None: + os.write(w, b'NO TASK') + else: + os.write(w, b'TASK:' + str(id(current_task)).encode()) + except BaseException as e: + os.write(w, b'ERROR:' + ascii(e).encode()) + finally: + asyncio._set_running_loop(None) + os._exit(0) + else: + # parent + result = os.read(r, 100) + self.assertEqual(result, b'NO TASK') + wait_process(pid, exitcode=0) + + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + def test_fork_not_share_event_loop(self): + # The forked process should not share the event loop with the parent + loop = object() + asyncio._set_running_loop(loop) + self.assertIs(asyncio.get_running_loop(), loop) + self.addCleanup(asyncio._set_running_loop, None) + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + pid = os.fork() + if pid == 0: + # child + try: + loop = asyncio.get_event_loop() + os.write(w, b'LOOP:' + str(id(loop)).encode()) + except RuntimeError: + os.write(w, b'NO LOOP') + except BaseException as e: + os.write(w, b'ERROR:' + ascii(e).encode()) + finally: + os._exit(0) + else: + # parent + result = os.read(r, 100) + self.assertEqual(result, b'NO LOOP') + wait_process(pid, exitcode=0) @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @hashlib_helper.requires_hashdigest('md5') diff --git a/Misc/NEWS.d/next/Library/2025-10-10-11-22-50.gh-issue-139894.ECAXqj.rst b/Misc/NEWS.d/next/Library/2025-10-10-11-22-50.gh-issue-139894.ECAXqj.rst new file mode 100644 index 00000000000..05a977ad119 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-10-11-22-50.gh-issue-139894.ECAXqj.rst @@ -0,0 +1 @@ +Fix incorrect sharing of current task with the child process while forking in :mod:`asyncio`. Patch by Kumar Aditya. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 7a2e36bf294..8278902cbeb 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -689,6 +689,14 @@ reset_remotedebug_data(PyThreadState *tstate) _Py_MAX_SCRIPT_PATH_SIZE); } +static void +reset_asyncio_state(_PyThreadStateImpl *tstate) +{ + llist_init(&tstate->asyncio_tasks_head); + tstate->asyncio_running_loop = NULL; + tstate->asyncio_running_task = NULL; +} + void PyOS_AfterFork_Child(void) @@ -725,6 +733,8 @@ PyOS_AfterFork_Child(void) reset_remotedebug_data(tstate); + reset_asyncio_state((_PyThreadStateImpl *)tstate); + // Remove the dead thread states. We "start the world" once we are the only // thread state left to undo the stop the world call in `PyOS_BeforeFork`. // That needs to happen before `_PyThreadState_DeleteList`, because that From 49aaee7978c54211967392678072accc403d15f2 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 10 Oct 2025 19:08:55 +0100 Subject: [PATCH 101/112] pathlib ABCs: restore `relative_to()` and `is_relative_to()` (#138853) Restore `JoinablePath.[is_]relative_to()`, which were deleted in ef63cca494571f50906baae1d176469a3dcf8838. These methods are too useful to forgo. Restore old tests, and add new tests covering path classes with non-overridden `__eq__()` and `__hash__()`. Slightly simplify `PurePath.relative_to()` while we're in the area. No change to public APIs, because the pathlib ABCs are still private. --- Lib/pathlib/__init__.py | 7 ++-- Lib/pathlib/types.py | 27 +++++++++++++++ Lib/test/test_pathlib/test_join.py | 55 ++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 2 deletions(-) diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index 8a892102cc0..6c07cd9ab01 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -490,16 +490,19 @@ def relative_to(self, other, *, walk_up=False): """ if not hasattr(other, 'with_segments'): other = self.with_segments(other) - for step, path in enumerate(chain([other], other.parents)): + parts = [] + for path in chain([other], other.parents): if path == self or path in self.parents: break elif not walk_up: raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") elif path.name == '..': raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") + else: + parts.append('..') else: raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") - parts = ['..'] * step + self._tail[len(path._tail):] + parts.extend(self._tail[len(path._tail):]) return self._from_parsed_parts('', '', parts) def is_relative_to(self, other): diff --git a/Lib/pathlib/types.py b/Lib/pathlib/types.py index fea0dd305fe..f21ce077454 100644 --- a/Lib/pathlib/types.py +++ b/Lib/pathlib/types.py @@ -234,6 +234,33 @@ def parents(self): parent = split(path)[0] return tuple(parents) + def relative_to(self, other, *, walk_up=False): + """Return the relative path to another path identified by the passed + arguments. If the operation is not possible (because this is not + related to the other path), raise ValueError. + + The *walk_up* parameter controls whether `..` may be used to resolve + the path. + """ + parts = [] + for path in (other,) + other.parents: + if self.is_relative_to(path): + break + elif not walk_up: + raise ValueError(f"{self!r} is not in the subpath of {other!r}") + elif path.name == '..': + raise ValueError(f"'..' segment in {other!r} cannot be walked") + else: + parts.append('..') + else: + raise ValueError(f"{self!r} and {other!r} have different anchors") + return self.with_segments(*parts, *self.parts[len(path.parts):]) + + def is_relative_to(self, other): + """Return True if the path is relative to another path or False. + """ + return other == self or other in self.parents + def full_match(self, pattern): """ Return True if this path matches the given glob-style pattern. The diff --git a/Lib/test/test_pathlib/test_join.py b/Lib/test/test_pathlib/test_join.py index f1a24204b4c..2f4e79345f3 100644 --- a/Lib/test/test_pathlib/test_join.py +++ b/Lib/test/test_pathlib/test_join.py @@ -354,6 +354,61 @@ def test_with_suffix(self): self.assertRaises(ValueError, P('a/b').with_suffix, '.d/.') self.assertRaises(TypeError, P('a/b').with_suffix, None) + def test_relative_to(self): + P = self.cls + p = P('a/b') + self.assertEqual(p.relative_to(P('')), P('a', 'b')) + self.assertEqual(p.relative_to(P('a')), P('b')) + self.assertEqual(p.relative_to(P('a/b')), P('')) + self.assertEqual(p.relative_to(P(''), walk_up=True), P('a', 'b')) + self.assertEqual(p.relative_to(P('a'), walk_up=True), P('b')) + self.assertEqual(p.relative_to(P('a/b'), walk_up=True), P('')) + self.assertEqual(p.relative_to(P('a/c'), walk_up=True), P('..', 'b')) + self.assertEqual(p.relative_to(P('a/b/c'), walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('c'), walk_up=True), P('..', 'a', 'b')) + self.assertRaises(ValueError, p.relative_to, P('c')) + self.assertRaises(ValueError, p.relative_to, P('a/b/c')) + self.assertRaises(ValueError, p.relative_to, P('a/c')) + self.assertRaises(ValueError, p.relative_to, P('/a')) + self.assertRaises(ValueError, p.relative_to, P('../a')) + self.assertRaises(ValueError, p.relative_to, P('a/..')) + self.assertRaises(ValueError, p.relative_to, P('/a/..')) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/a'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('../a'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('a/..'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/a/..'), walk_up=True) + class Q(self.cls): + __eq__ = object.__eq__ + __hash__ = object.__hash__ + q = Q('a/b') + self.assertTrue(q.relative_to(q)) + self.assertRaises(ValueError, q.relative_to, Q('')) + self.assertRaises(ValueError, q.relative_to, Q('a')) + self.assertRaises(ValueError, q.relative_to, Q('a'), walk_up=True) + self.assertRaises(ValueError, q.relative_to, Q('a/b')) + self.assertRaises(ValueError, q.relative_to, Q('c')) + + def test_is_relative_to(self): + P = self.cls + p = P('a/b') + self.assertTrue(p.is_relative_to(P(''))) + self.assertTrue(p.is_relative_to(P('a'))) + self.assertTrue(p.is_relative_to(P('a/b'))) + self.assertFalse(p.is_relative_to(P('c'))) + self.assertFalse(p.is_relative_to(P('a/b/c'))) + self.assertFalse(p.is_relative_to(P('a/c'))) + self.assertFalse(p.is_relative_to(P('/a'))) + class Q(self.cls): + __eq__ = object.__eq__ + __hash__ = object.__hash__ + q = Q('a/b') + self.assertTrue(q.is_relative_to(q)) + self.assertFalse(q.is_relative_to(Q(''))) + self.assertFalse(q.is_relative_to(Q('a'))) + self.assertFalse(q.is_relative_to(Q('a/b'))) + self.assertFalse(q.is_relative_to(Q('c'))) + class LexicalPathJoinTest(JoinTestBase, unittest.TestCase): cls = LexicalPath From aa840f500ca901deea29669df68992193a273a62 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Fri, 10 Oct 2025 20:22:34 +0100 Subject: [PATCH 102/112] gh-138843: Removing "Unpacking" section from Download page (GH-139918) --- Doc/tools/templates/download.html | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index 523b505f596..f914ad86211 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -75,16 +75,6 @@

{% trans %}Download Python {{ dl_version }} documentation{% endtrans %}

See the directory listing for file sizes.{% endtrans %}

-

{% trans %}Unpacking{% endtrans %}

- -

{% trans %}Unix users should download the .tar.bz2 archives; these are bzipped tar -archives and can be handled in the usual way using tar and the bzip2 -program. The Info-ZIP unzip program can be -used to handle the ZIP archives if desired. The .tar.bz2 archives provide the -best compression and fastest download times.{% endtrans %}

- -

{% trans %}Windows users can use the ZIP archives since those are customary on that -platform. These are created on Unix using the Info-ZIP zip program.{% endtrans %}

{% trans %}Problems{% endtrans %}

{% set bugs = pathto('bugs') %} From d9b4eef71e7904fbe3a3786a908e493be7debbff Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 10 Oct 2025 17:20:18 -0400 Subject: [PATCH 103/112] gh-139001: Fix thread-safety issue in `pathlib.Path` (gh-139066) Don't cache the joined path in `_raw_path` because the caching isn't thread safe. --- Lib/pathlib/__init__.py | 7 +----- Lib/test/test_pathlib/test_join.py | 22 +++++++++++++++++++ ...-09-17-12-07-21.gh-issue-139001.O6tseN.rst | 2 ++ 3 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-09-17-12-07-21.gh-issue-139001.O6tseN.rst diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index 6c07cd9ab01..51359cec8b0 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -336,13 +336,8 @@ def _raw_path(self): return paths[0] elif paths: # Join path segments from the initializer. - path = self.parser.join(*paths) - # Cache the joined path. - paths.clear() - paths.append(path) - return path + return self.parser.join(*paths) else: - paths.append('') return '' @property diff --git a/Lib/test/test_pathlib/test_join.py b/Lib/test/test_pathlib/test_join.py index 2f4e79345f3..4630210e492 100644 --- a/Lib/test/test_pathlib/test_join.py +++ b/Lib/test/test_pathlib/test_join.py @@ -3,6 +3,8 @@ """ import unittest +import threading +from test.support import threading_helper from .support import is_pypi from .support.lexical_path import LexicalPath @@ -158,6 +160,26 @@ def test_parts(self): parts = p.parts self.assertEqual(parts, (sep, 'a', 'b')) + @threading_helper.requires_working_threading() + def test_parts_multithreaded(self): + P = self.cls + + NUM_THREADS = 10 + NUM_ITERS = 10 + + for _ in range(NUM_ITERS): + b = threading.Barrier(NUM_THREADS) + path = P('a') / 'b' / 'c' / 'd' / 'e' + expected = ('a', 'b', 'c', 'd', 'e') + + def check_parts(): + b.wait() + self.assertEqual(path.parts, expected) + + threads = [threading.Thread(target=check_parts) for _ in range(NUM_THREADS)] + with threading_helper.start_threads(threads): + pass + def test_parent(self): # Relative P = self.cls diff --git a/Misc/NEWS.d/next/Library/2025-09-17-12-07-21.gh-issue-139001.O6tseN.rst b/Misc/NEWS.d/next/Library/2025-09-17-12-07-21.gh-issue-139001.O6tseN.rst new file mode 100644 index 00000000000..3ad5a1272df --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-17-12-07-21.gh-issue-139001.O6tseN.rst @@ -0,0 +1,2 @@ +Fix race condition in :class:`pathlib.Path` on the internal ``_raw_paths`` +field. From ff7bb565d836162eed0851c36afa325a107a5a56 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Fri, 10 Oct 2025 15:25:38 -0700 Subject: [PATCH 104/112] gh-139924: Add PyFunction_PYFUNC_EVENT_MODIFY_QUALNAME event for function watchers (#139925) Add PyFunction_PYFUNC_EVENT_MODIFY_QUALNAME event for function watchers --- Doc/c-api/function.rst | 3 +++ Include/cpython/funcobject.h | 3 ++- Lib/test/test_capi/test_watchers.py | 4 ++++ .../next/C_API/2025-10-10-20-59-07.gh-issue-139924.ALByCb.rst | 1 + Objects/funcobject.c | 2 ++ 5 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-10-10-20-59-07.gh-issue-139924.ALByCb.rst diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 5fb8567ef8c..764b2ac610b 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -175,6 +175,9 @@ There are a few functions specific to Python functions. .. versionadded:: 3.12 + - ``PyFunction_PYFUNC_EVENT_MODIFY_QUALNAME`` + + .. versionadded:: 3.15 .. c:type:: int (*PyFunction_WatchCallback)(PyFunction_WatchEvent event, PyFunctionObject *func, PyObject *new_value) diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h index 598cd330bc9..9e1599a7648 100644 --- a/Include/cpython/funcobject.h +++ b/Include/cpython/funcobject.h @@ -134,7 +134,8 @@ PyAPI_FUNC(PyObject *) PyStaticMethod_New(PyObject *); V(DESTROY) \ V(MODIFY_CODE) \ V(MODIFY_DEFAULTS) \ - V(MODIFY_KWDEFAULTS) + V(MODIFY_KWDEFAULTS) \ + V(MODIFY_QUALNAME) typedef enum { #define PY_DEF_EVENT(EVENT) PyFunction_EVENT_##EVENT, diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 8644479d83d..bef72032513 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -514,6 +514,10 @@ def myfunc(): _testcapi.set_func_kwdefaults_via_capi(myfunc, new_kwdefaults) self.assertIn((_testcapi.PYFUNC_EVENT_MODIFY_KWDEFAULTS, myfunc, new_kwdefaults), events) + new_qualname = "foo.bar" + myfunc.__qualname__ = new_qualname + self.assertIn((_testcapi.PYFUNC_EVENT_MODIFY_QUALNAME, myfunc, new_qualname), events) + # Clear events reference to func events = [] del myfunc diff --git a/Misc/NEWS.d/next/C_API/2025-10-10-20-59-07.gh-issue-139924.ALByCb.rst b/Misc/NEWS.d/next/C_API/2025-10-10-20-59-07.gh-issue-139924.ALByCb.rst new file mode 100644 index 00000000000..a53d5d068d7 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-10-10-20-59-07.gh-issue-139924.ALByCb.rst @@ -0,0 +1 @@ +Function watchers can now receive a PyFunction_PYFUNC_EVENT_MODIFY_QUALNAME event when a watched functions qualname is changed. diff --git a/Objects/funcobject.c b/Objects/funcobject.c index d8a10075578..43198aaf8a7 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -62,6 +62,7 @@ handle_func_event(PyFunction_WatchEvent event, PyFunctionObject *func, case PyFunction_EVENT_MODIFY_CODE: case PyFunction_EVENT_MODIFY_DEFAULTS: case PyFunction_EVENT_MODIFY_KWDEFAULTS: + case PyFunction_EVENT_MODIFY_QUALNAME: RARE_EVENT_INTERP_INC(interp, func_modification); break; default: @@ -747,6 +748,7 @@ func_set_qualname(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) "__qualname__ must be set to a string object"); return -1; } + handle_func_event(PyFunction_EVENT_MODIFY_QUALNAME, (PyFunctionObject *) op, value); Py_XSETREF(op->func_qualname, Py_NewRef(value)); return 0; } From d4e5802588db3459f04d4b8013cc571a8988e203 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sat, 11 Oct 2025 02:37:48 +0100 Subject: [PATCH 105/112] gh-96491: Deduplicate version in IDLE shell title (#139841) Saving to a file added both the filename and repeated the version. --------- Co-authored-by: Terry Jan Reedy --- Lib/idlelib/CREDITS.txt | 1 + Lib/idlelib/editor.py | 7 +++++-- Lib/idlelib/idle_test/test_outwin.py | 3 ++- Lib/idlelib/pyshell.py | 3 +-- .../IDLE/2025-10-09-12-53-47.gh-issue-96491.4YKxvy.rst | 1 + 5 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2025-10-09-12-53-47.gh-issue-96491.4YKxvy.rst diff --git a/Lib/idlelib/CREDITS.txt b/Lib/idlelib/CREDITS.txt index bea3ba7c20d..1b853e8cc1c 100644 --- a/Lib/idlelib/CREDITS.txt +++ b/Lib/idlelib/CREDITS.txt @@ -37,6 +37,7 @@ Major contributors since 2005: - 2014: Saimadhav Heblikar - 2015: Mark Roseman - 2017: Louie Lu, Cheryl Sabella, and Serhiy Storchaka +- 2025: Stan Ulbrych For additional details refer to NEWS.txt and Changelog. diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index b4d6d25871b..83112d85575 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -33,7 +33,6 @@ # The default tab setting for a Text widget, in average-width characters. TK_TABWIDTH_DEFAULT = 8 -_py_version = ' (%s)' % platform.python_version() darwin = sys.platform == 'darwin' def _sphinx_version(): @@ -1008,12 +1007,16 @@ def open_recent_file(fn_closure=file_name): def saved_change_hook(self): short = self.short_title() long = self.long_title() + _py_version = ' (%s)' % platform.python_version() if short and long and not macosx.isCocoaTk(): # Don't use both values on macOS because # that doesn't match platform conventions. title = short + " - " + long + _py_version elif short: - title = short + if short == "IDLE Shell": + title = short + " " + platform.python_version() + else: + title = short + _py_version elif long: title = long else: diff --git a/Lib/idlelib/idle_test/test_outwin.py b/Lib/idlelib/idle_test/test_outwin.py index 81f4aad7e95..0f13363f84f 100644 --- a/Lib/idlelib/idle_test/test_outwin.py +++ b/Lib/idlelib/idle_test/test_outwin.py @@ -1,6 +1,7 @@ "Test outwin, coverage 76%." from idlelib import outwin +import platform import sys import unittest from test.support import requires @@ -41,7 +42,7 @@ def test_ispythonsource(self): self.assertFalse(w.ispythonsource(__file__)) def test_window_title(self): - self.assertEqual(self.window.top.title(), 'Output') + self.assertEqual(self.window.top.title(), 'Output' + ' (%s)' % platform.python_version()) def test_maybesave(self): w = self.window diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 74a0e03994f..1b7c2af1a92 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -22,7 +22,6 @@ import linecache import os import os.path -from platform import python_version import re import socket import subprocess @@ -841,7 +840,7 @@ def display_executing_dialog(self): class PyShell(OutputWindow): from idlelib.squeezer import Squeezer - shell_title = "IDLE Shell " + python_version() + shell_title = "IDLE Shell" # Override classes ColorDelegator = ModifiedColorDelegator diff --git a/Misc/NEWS.d/next/IDLE/2025-10-09-12-53-47.gh-issue-96491.4YKxvy.rst b/Misc/NEWS.d/next/IDLE/2025-10-09-12-53-47.gh-issue-96491.4YKxvy.rst new file mode 100644 index 00000000000..beb6ef5ade5 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2025-10-09-12-53-47.gh-issue-96491.4YKxvy.rst @@ -0,0 +1 @@ +Deduplicate version number in IDLE shell title bar after saving to a file. From 897a36baddb3611865a839c6345315da689c52a1 Mon Sep 17 00:00:00 2001 From: yihong Date: Sat, 11 Oct 2025 20:32:57 +0800 Subject: [PATCH 106/112] gh-139935: fix `test_os.test_getlogin` on some platforms (#139936) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This amends 4e7e2dd043c1da85b0c157d3ed24866b77e83a4f to catch errors that `os.getlogin` can raise as specified by POSIX and Linux/glibc [1]. [1]: https://man7.org/linux/man-pages/man3/getlogin.3.html#ERRORS --------- Signed-off-by: yihong0618 Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_os/test_os.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_os/test_os.py b/Lib/test/test_os/test_os.py index 371771087aa..95b175db6fc 100644 --- a/Lib/test/test_os/test_os.py +++ b/Lib/test/test_os/test_os.py @@ -3203,7 +3203,14 @@ def test_getlogin(self): try: user_name = os.getlogin() except OSError as exc: - if exc.errno in (errno.ENOTTY, errno.ENXIO): + # See https://man7.org/linux/man-pages/man3/getlogin.3.html#ERRORS. + allowed_errors = ( + # defined by POSIX + errno.EMFILE, errno.ENFILE, errno.ENXIO, errno.ERANGE, + # defined by Linux/glibc + errno.ENOENT, errno.ENOMEM, errno.ENOTTY, + ) + if exc.errno in allowed_errors: self.skipTest(str(exc)) else: raise From 2eb32add92d553b320850975442fa2e55addc377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 11 Oct 2025 16:31:34 +0200 Subject: [PATCH 107/112] gh-139935: do not skip test on real errors in `os.getlogin` (#139953) --- Lib/test/test_os/test_os.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Lib/test/test_os/test_os.py b/Lib/test/test_os/test_os.py index 95b175db6fc..e074858fe2a 100644 --- a/Lib/test/test_os/test_os.py +++ b/Lib/test/test_os/test_os.py @@ -3204,13 +3204,7 @@ def test_getlogin(self): user_name = os.getlogin() except OSError as exc: # See https://man7.org/linux/man-pages/man3/getlogin.3.html#ERRORS. - allowed_errors = ( - # defined by POSIX - errno.EMFILE, errno.ENFILE, errno.ENXIO, errno.ERANGE, - # defined by Linux/glibc - errno.ENOENT, errno.ENOMEM, errno.ENOTTY, - ) - if exc.errno in allowed_errors: + if exc.errno in (errno.ENXIO, errno.ENOENT, errno.ENOTTY): self.skipTest(str(exc)) else: raise From 5776d0d2e08f4d93dcd62d875bae8c1396a04ba4 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sat, 11 Oct 2025 16:14:29 +0100 Subject: [PATCH 108/112] gh-139905: Provide suggestion in error message if `Generic.__init_subclass__` was not called (#139943) --- Lib/test/test_typing.py | 28 +++++++++++++++++++ Lib/typing.py | 16 +++++++++-- ...-10-11-10-02-56.gh-issue-139905.UyJIR_.rst | 3 ++ 3 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-11-10-02-56.gh-issue-139905.UyJIR_.rst diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 428089d88cc..bc7f14f90a7 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -4722,6 +4722,34 @@ class D(Generic[T]): pass with self.assertRaises(TypeError): D[()] + def test_generic_init_subclass_not_called_error(self): + notes = ["Note: this exception may have been caused by " + r"'GenericTests.test_generic_init_subclass_not_called_error..Base.__init_subclass__' " + "(or the '__init_subclass__' method on a superclass) not calling 'super().__init_subclass__()'"] + + class Base: + def __init_subclass__(cls) -> None: + # Oops, I forgot super().__init_subclass__()! + pass + + with self.subTest(): + class Sub(Base, Generic[T]): + pass + + with self.assertRaises(AttributeError) as cm: + Sub[int] + + self.assertEqual(cm.exception.__notes__, notes) + + with self.subTest(): + class Sub[U](Base): + pass + + with self.assertRaises(AttributeError) as cm: + Sub[int] + + self.assertEqual(cm.exception.__notes__, notes) + def test_generic_subclass_checks(self): for typ in [list[int], List[int], tuple[int, str], Tuple[int, str], diff --git a/Lib/typing.py b/Lib/typing.py index 4311a77b8db..03d5357d4cf 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1158,14 +1158,26 @@ def _generic_class_getitem(cls, args): f"Parameters to {cls.__name__}[...] must all be unique") else: # Subscripting a regular Generic subclass. - for param in cls.__parameters__: + try: + parameters = cls.__parameters__ + except AttributeError as e: + init_subclass = getattr(cls, '__init_subclass__', None) + if init_subclass not in {None, Generic.__init_subclass__}: + e.add_note( + f"Note: this exception may have been caused by " + f"{init_subclass.__qualname__!r} (or the " + f"'__init_subclass__' method on a superclass) not " + f"calling 'super().__init_subclass__()'" + ) + raise + for param in parameters: prepare = getattr(param, '__typing_prepare_subst__', None) if prepare is not None: args = prepare(cls, args) _check_generic_specialization(cls, args) new_args = [] - for param, new_arg in zip(cls.__parameters__, args): + for param, new_arg in zip(parameters, args): if isinstance(param, TypeVarTuple): new_args.extend(new_arg) else: diff --git a/Misc/NEWS.d/next/Library/2025-10-11-10-02-56.gh-issue-139905.UyJIR_.rst b/Misc/NEWS.d/next/Library/2025-10-11-10-02-56.gh-issue-139905.UyJIR_.rst new file mode 100644 index 00000000000..a6876ca2df8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-11-10-02-56.gh-issue-139905.UyJIR_.rst @@ -0,0 +1,3 @@ +Add suggestion to error message for :class:`typing.Generic` subclasses when +``cls.__parameters__`` is missing due to a parent class failing to call +:meth:`super().__init_subclass__() ` in its ``__init_subclass__``. From cdd3eee7fc26538c8365dcbf2dd844ec7cdf7fb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 11 Oct 2025 19:34:08 +0200 Subject: [PATCH 109/112] gh-139929: fix incorrect OpenSSL version-based guard in `_ssl.c` (GH-139945) fix OpenSSL version-based guards --- Modules/_ssl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 1fa44ef1de4..4b75e455f40 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -937,7 +937,7 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, } /* bpo43522 and OpenSSL < 1.1.1l: copy hostflags manually */ -#if OPENSSL_VERSION < 0x101010cf +#if OPENSSL_VERSION_NUMBER < 0x101010cf X509_VERIFY_PARAM *ssl_verification_params = SSL_get0_param(self->ssl); X509_VERIFY_PARAM *ssl_ctx_verification_params = SSL_CTX_get0_param(ctx); From 447c7a89fb41b7fa84b9b26f111aedd649bc5400 Mon Sep 17 00:00:00 2001 From: Weilin Du <108666168+LamentXU123@users.noreply.github.com> Date: Sun, 12 Oct 2025 02:26:48 +0800 Subject: [PATCH 110/112] gh-101100: Fix Sphinx warnings in `Doc/library/signal.rst` (GH-139930) --- Doc/library/signal.rst | 7 +++---- Doc/tools/.nitignore | 1 - 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index ccdf5027ff9..47f824488f5 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -270,7 +270,7 @@ The variables defined in the :mod:`signal` module are: All the signal numbers are defined symbolically. For example, the hangup signal is defined as :const:`signal.SIGHUP`; the variable names are identical to the names used in C programs, as found in ````. The Unix man page for - ':c:func:`signal`' lists the existing signals (on some systems this is + '``signal``' lists the existing signals (on some systems this is :manpage:`signal(2)`, on others the list is in :manpage:`signal(7)`). Note that not all systems define the same set of signal names; only those names defined by the system are defined by this module. @@ -666,9 +666,8 @@ The :mod:`signal` module defines the following functions: *sigset*. The return value is an object representing the data contained in the - :c:type:`siginfo_t` structure, namely: :attr:`si_signo`, :attr:`si_code`, - :attr:`si_errno`, :attr:`si_pid`, :attr:`si_uid`, :attr:`si_status`, - :attr:`si_band`. + ``siginfo_t`` structure, namely: ``si_signo``, ``si_code``, + ``si_errno``, ``si_pid``, ``si_uid``, ``si_status``, ``si_band``. .. availability:: Unix. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 657d720e766..60a80fe2c72 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -29,7 +29,6 @@ Doc/library/profile.rst Doc/library/pyexpat.rst Doc/library/resource.rst Doc/library/select.rst -Doc/library/signal.rst Doc/library/smtplib.rst Doc/library/socket.rst Doc/library/ssl.rst From 166cdaa6fb06c0ec802fd6910c117d809f818ede Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 11 Oct 2025 22:58:14 +0200 Subject: [PATCH 111/112] gh-111489: Remove _PyTuple_FromArray() alias (#139973) Replace _PyTuple_FromArray() with PyTuple_FromArray(). Remove pycore_tuple.h includes. --- Include/internal/pycore_tuple.h | 3 --- Lib/test/clinic.test.c | 16 ++++++------- Modules/_testclinic.c | 6 ++--- Modules/clinic/_testclinic.c.h | 34 ++++++++++++++-------------- Modules/clinic/_testclinic_depr.c.h | 4 ++-- Modules/clinic/_testclinic_kwds.c.h | 4 ++-- Modules/clinic/gcmodule.c.h | 7 +++--- Modules/gcmodule.c | 1 - Modules/itertoolsmodule.c | 8 +++---- Objects/call.c | 2 +- Objects/descrobject.c | 4 ++-- Objects/exceptions.c | 3 +-- Objects/listobject.c | 10 ++++---- Objects/structseq.c | 4 ++-- Objects/tupleobject.c | 2 +- Objects/unicodeobject.c | 3 +-- Python/bltinmodule.c | 6 ++--- Python/ceval.c | 2 +- Python/clinic/sysmodule.c.h | 5 ++-- Python/intrinsics.c | 3 +-- Python/optimizer.c | 2 +- Python/optimizer_symbols.c | 3 +-- Tools/clinic/libclinic/converters.py | 5 ++-- 23 files changed, 63 insertions(+), 74 deletions(-) diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index be1961cbf77..46db02593ad 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -23,9 +23,6 @@ extern PyStatus _PyTuple_InitGlobalObjects(PyInterpreterState *); #define _PyTuple_ITEMS(op) _Py_RVALUE(_PyTuple_CAST(op)->ob_item) -// Alias for backward compatibility -#define _PyTuple_FromArray PyTuple_FromArray - PyAPI_FUNC(PyObject *)_PyTuple_FromStackRefStealOnSuccess(const union _PyStackRef *, Py_ssize_t); PyAPI_FUNC(PyObject *)_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index ed9058899c0..4729708efd3 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -4341,7 +4341,7 @@ test_vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t narg goto exit; } a = args[0]; - __clinic_args = _PyTuple_FromArray(args + 1, nargs - 1); + __clinic_args = PyTuple_FromArray(args + 1, nargs - 1); if (__clinic_args == NULL) { goto exit; } @@ -4356,7 +4356,7 @@ test_vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t narg static PyObject * test_vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=0c11c475e240869e input=2c49a482f68545c0]*/ +/*[clinic end generated code: output=83cbe9554d04add2 input=2c49a482f68545c0]*/ /*[clinic input] test_vararg @@ -4421,7 +4421,7 @@ test_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject } a = fastargs[0]; __clinic_args = nargs > 1 - ? _PyTuple_FromArray(args + 1, nargs - 1) + ? PyTuple_FromArray(args + 1, nargs - 1) : PyTuple_New(0); if (__clinic_args == NULL) { goto exit; @@ -4437,7 +4437,7 @@ test_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject static PyObject * test_vararg_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=17ba625cdd0369c1 input=7448995636d9186a]*/ +/*[clinic end generated code: output=d773f7b54e61f73a input=7448995636d9186a]*/ /*[clinic input] test_vararg_with_default @@ -4514,7 +4514,7 @@ test_vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nar } skip_optional_kwonly: __clinic_args = nargs > 1 - ? _PyTuple_FromArray(args + 1, nargs - 1) + ? PyTuple_FromArray(args + 1, nargs - 1) : PyTuple_New(0); if (__clinic_args == NULL) { goto exit; @@ -4531,7 +4531,7 @@ test_vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nar static PyObject * test_vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, int b) -/*[clinic end generated code: output=3f2b06ab08d5d0be input=3a0f9f557ce1f712]*/ +/*[clinic end generated code: output=d25e56802c197344 input=3a0f9f557ce1f712]*/ /*[clinic input] test_vararg_with_only_defaults @@ -4612,7 +4612,7 @@ test_vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize } c = fastargs[1]; skip_optional_kwonly: - __clinic_args = _PyTuple_FromArray(args, nargs); + __clinic_args = PyTuple_FromArray(args, nargs); if (__clinic_args == NULL) { goto exit; } @@ -4628,7 +4628,7 @@ test_vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize static PyObject * test_vararg_with_only_defaults_impl(PyObject *module, PyObject *args, int b, PyObject *c) -/*[clinic end generated code: output=f46666f0b1bf86b9 input=6983e66817f82924]*/ +/*[clinic end generated code: output=7366943a7df42e05 input=6983e66817f82924]*/ /*[clinic input] test_paramname_module diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 5c196c0dd0f..890f2201b46 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -63,7 +63,7 @@ pack_arguments_2pos_varpos(PyObject *a, PyObject *b, PyObject * const *args, Py_ssize_t args_length) /*[clinic end generated code: output=267032f41bd039cc input=86ee3064b7853e86]*/ { - PyObject *tuple = _PyTuple_FromArray(args, args_length); + PyObject *tuple = PyTuple_FromArray(args, args_length); if (tuple == NULL) { return NULL; } @@ -1174,7 +1174,7 @@ varpos_array_impl(PyObject *module, PyObject * const *args, Py_ssize_t args_length) /*[clinic end generated code: output=a25f42f39c9b13ad input=97b8bdcf87e019c7]*/ { - return _PyTuple_FromArray(args, args_length); + return PyTuple_FromArray(args, args_length); } @@ -1610,7 +1610,7 @@ _testclinic_TestClass_varpos_array_no_fastcall_impl(PyTypeObject *type, Py_ssize_t args_length) /*[clinic end generated code: output=27c9da663e942617 input=9ba5ae1f1eb58777]*/ { - return _PyTuple_FromArray(args, args_length); + return PyTuple_FromArray(args, args_length); } diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 7e971f7ad73..9bcd0eeb008 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -9,7 +9,7 @@ preserve #include "pycore_long.h" // _PyLong_UnsignedShort_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_runtime.h" // _Py_ID() -#include "pycore_tuple.h" // _PyTuple_FromArray() +#include "pycore_tuple.h" // _PyTuple_ITEMS() PyDoc_STRVAR(test_empty_function__doc__, "test_empty_function($module, /)\n" @@ -2764,7 +2764,7 @@ varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *return_value = NULL; PyObject *__clinic_args = NULL; - __clinic_args = _PyTuple_FromArray(args, nargs); + __clinic_args = PyTuple_FromArray(args, nargs); if (__clinic_args == NULL) { goto exit; } @@ -2802,7 +2802,7 @@ posonly_varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } a = args[0]; b = args[1]; - __clinic_args = _PyTuple_FromArray(args + 2, nargs - 2); + __clinic_args = PyTuple_FromArray(args + 2, nargs - 2); if (__clinic_args == NULL) { goto exit; } @@ -2845,7 +2845,7 @@ posonly_req_opt_varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs b = args[1]; skip_optional: __clinic_args = nargs > 2 - ? _PyTuple_FromArray(args + 2, nargs - 2) + ? PyTuple_FromArray(args + 2, nargs - 2) : PyTuple_New(0); if (__clinic_args == NULL) { goto exit; @@ -2916,7 +2916,7 @@ posonly_poskw_varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs, a = fastargs[0]; b = fastargs[1]; __clinic_args = nargs > 2 - ? _PyTuple_FromArray(args + 2, nargs - 2) + ? PyTuple_FromArray(args + 2, nargs - 2) : PyTuple_New(0); if (__clinic_args == NULL) { goto exit; @@ -2984,7 +2984,7 @@ poskw_varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject } a = fastargs[0]; __clinic_args = nargs > 1 - ? _PyTuple_FromArray(args + 1, nargs - 1) + ? PyTuple_FromArray(args + 1, nargs - 1) : PyTuple_New(0); if (__clinic_args == NULL) { goto exit; @@ -3063,7 +3063,7 @@ poskw_varpos_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t narg } skip_optional_kwonly: __clinic_args = nargs > 1 - ? _PyTuple_FromArray(args + 1, nargs - 1) + ? PyTuple_FromArray(args + 1, nargs - 1) : PyTuple_New(0); if (__clinic_args == NULL) { goto exit; @@ -3146,7 +3146,7 @@ poskw_varpos_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ssize_t nar c = fastargs[2]; skip_optional_kwonly: __clinic_args = nargs > 1 - ? _PyTuple_FromArray(args + 1, nargs - 1) + ? PyTuple_FromArray(args + 1, nargs - 1) : PyTuple_New(0); if (__clinic_args == NULL) { goto exit; @@ -3218,7 +3218,7 @@ varpos_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO } b = fastargs[0]; skip_optional_kwonly: - __clinic_args = _PyTuple_FromArray(args, nargs); + __clinic_args = PyTuple_FromArray(args, nargs); if (__clinic_args == NULL) { goto exit; } @@ -3299,7 +3299,7 @@ varpos_kwonly_req_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, } c = fastargs[2]; skip_optional_kwonly: - __clinic_args = _PyTuple_FromArray(args, nargs); + __clinic_args = PyTuple_FromArray(args, nargs); if (__clinic_args == NULL) { goto exit; } @@ -3549,7 +3549,7 @@ gh_32092_oob(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject kw2 = fastargs[3]; skip_optional_kwonly: varargs = nargs > 2 - ? _PyTuple_FromArray(args + 2, nargs - 2) + ? PyTuple_FromArray(args + 2, nargs - 2) : PyTuple_New(0); if (varargs == NULL) { goto exit; @@ -3626,7 +3626,7 @@ gh_32092_kw_pass(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb kw = fastargs[1]; skip_optional_kwonly: __clinic_args = nargs > 1 - ? _PyTuple_FromArray(args + 1, nargs - 1) + ? PyTuple_FromArray(args + 1, nargs - 1) : PyTuple_New(0); if (__clinic_args == NULL) { goto exit; @@ -3658,7 +3658,7 @@ gh_99233_refcount(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *return_value = NULL; PyObject *__clinic_args = NULL; - __clinic_args = _PyTuple_FromArray(args, nargs); + __clinic_args = PyTuple_FromArray(args, nargs); if (__clinic_args == NULL) { goto exit; } @@ -3771,7 +3771,7 @@ null_or_tuple_for_varargs(PyObject *module, PyObject *const *args, Py_ssize_t na } skip_optional_kwonly: constraints = nargs > 1 - ? _PyTuple_FromArray(args + 1, nargs - 1) + ? PyTuple_FromArray(args + 1, nargs - 1) : PyTuple_New(0); if (constraints == NULL) { goto exit; @@ -4174,7 +4174,7 @@ _testclinic_TestClass_defclass_varpos(PyObject *self, PyTypeObject *cls, PyObjec if (!fastargs) { goto exit; } - __clinic_args = _PyTuple_FromArray(args, nargs); + __clinic_args = PyTuple_FromArray(args, nargs); if (__clinic_args == NULL) { goto exit; } @@ -4231,7 +4231,7 @@ _testclinic_TestClass_defclass_posonly_varpos(PyObject *self, PyTypeObject *cls, } a = fastargs[0]; b = fastargs[1]; - __clinic_args = _PyTuple_FromArray(args + 2, nargs - 2); + __clinic_args = PyTuple_FromArray(args + 2, nargs - 2); if (__clinic_args == NULL) { goto exit; } @@ -4600,4 +4600,4 @@ _testclinic_TestClass_posonly_poskw_varpos_array_no_fastcall(PyObject *type, PyO exit: return return_value; } -/*[clinic end generated code: output=0764e6f8c9d94057 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=290d2e346ea7bfa1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic_depr.c.h b/Modules/clinic/_testclinic_depr.c.h index a46d238801b..e2db4fd87ed 100644 --- a/Modules/clinic/_testclinic_depr.c.h +++ b/Modules/clinic/_testclinic_depr.c.h @@ -9,7 +9,7 @@ preserve #include "pycore_long.h" // _PyLong_UnsignedShort_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_runtime.h" // _Py_ID() -#include "pycore_tuple.h" // _PyTuple_FromArray() +#include "pycore_tuple.h" // _PyTuple_ITEMS() PyDoc_STRVAR(depr_star_new__doc__, "DeprStarNew(a=None)\n" @@ -2474,4 +2474,4 @@ depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * exit: return return_value; } -/*[clinic end generated code: output=4e60af44fd6b7b94 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2231bec0ed196830 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic_kwds.c.h b/Modules/clinic/_testclinic_kwds.c.h index e2fd4d9f3b4..86cad50c56c 100644 --- a/Modules/clinic/_testclinic_kwds.c.h +++ b/Modules/clinic/_testclinic_kwds.c.h @@ -9,7 +9,7 @@ preserve #include "pycore_long.h" // _PyLong_UnsignedShort_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_runtime.h" // _Py_ID() -#include "pycore_tuple.h" // _PyTuple_FromArray() +#include "pycore_tuple.h" // _PyTuple_ITEMS() PyDoc_STRVAR(lone_kwds__doc__, "lone_kwds($module, /, **kwds)\n" @@ -181,4 +181,4 @@ exit: return return_value; } -/*[clinic end generated code: output=e4dea1070e003f5d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3e5251b10aa44382 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index 53ff9e4faf8..08275e35413 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -8,7 +8,6 @@ preserve #endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() -#include "pycore_tuple.h" // _PyTuple_FromArray() PyDoc_STRVAR(gc_enable__doc__, "enable($module, /)\n" @@ -324,7 +323,7 @@ gc_get_referrers(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *return_value = NULL; PyObject *objs = NULL; - objs = _PyTuple_FromArray(args, nargs); + objs = PyTuple_FromArray(args, nargs); if (objs == NULL) { goto exit; } @@ -355,7 +354,7 @@ gc_get_referents(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *return_value = NULL; PyObject *objs = NULL; - objs = _PyTuple_FromArray(args, nargs); + objs = PyTuple_FromArray(args, nargs); if (objs == NULL) { goto exit; } @@ -584,4 +583,4 @@ gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=96d057eac558e6ca input=a9049054013a1b77]*/ +/*[clinic end generated code: output=19738854607938db input=a9049054013a1b77]*/ diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 19b3f2a2eb6..8a8c7284283 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -8,7 +8,6 @@ #include "pycore_gc.h" #include "pycore_object.h" // _PyObject_IS_GC() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_tuple.h" // _PyTuple_FromArray() typedef struct _gc_runtime_state GCState; diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 5d2506f48e3..60ef6f9ff4c 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -2131,7 +2131,7 @@ product_next_lock_held(PyObject *op) /* Copy the previous result tuple or re-use it if available */ if (Py_REFCNT(result) > 1) { PyObject *old_result = result; - result = _PyTuple_FromArray(_PyTuple_ITEMS(old_result), npools); + result = PyTuple_FromArray(_PyTuple_ITEMS(old_result), npools); if (result == NULL) goto empty; lz->result = result; @@ -2366,7 +2366,7 @@ combinations_next_lock_held(PyObject *op) /* Copy the previous result tuple or re-use it if available */ if (Py_REFCNT(result) > 1) { PyObject *old_result = result; - result = _PyTuple_FromArray(_PyTuple_ITEMS(old_result), r); + result = PyTuple_FromArray(_PyTuple_ITEMS(old_result), r); if (result == NULL) goto empty; co->result = result; @@ -2620,7 +2620,7 @@ cwr_next(PyObject *op) /* Copy the previous result tuple or re-use it if available */ if (Py_REFCNT(result) > 1) { PyObject *old_result = result; - result = _PyTuple_FromArray(_PyTuple_ITEMS(old_result), r); + result = PyTuple_FromArray(_PyTuple_ITEMS(old_result), r); if (result == NULL) goto empty; co->result = result; @@ -2881,7 +2881,7 @@ permutations_next(PyObject *op) /* Copy the previous result tuple or re-use it if available */ if (Py_REFCNT(result) > 1) { PyObject *old_result = result; - result = _PyTuple_FromArray(_PyTuple_ITEMS(old_result), r); + result = PyTuple_FromArray(_PyTuple_ITEMS(old_result), r); if (result == NULL) goto empty; po->result = result; diff --git a/Objects/call.c b/Objects/call.c index c9a18bcc3da..bd8617825b5 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -213,7 +213,7 @@ _PyObject_MakeTpCall(PyThreadState *tstate, PyObject *callable, return NULL; } - PyObject *argstuple = _PyTuple_FromArray(args, nargs); + PyObject *argstuple = PyTuple_FromArray(args, nargs); if (argstuple == NULL) { return NULL; } diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 06a81a4fdbd..5ac4fbd8129 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -313,7 +313,7 @@ method_vectorcall_VARARGS( if (method_check_args(func, args, nargs, kwnames)) { return NULL; } - PyObject *argstuple = _PyTuple_FromArray(args+1, nargs-1); + PyObject *argstuple = PyTuple_FromArray(args+1, nargs-1); if (argstuple == NULL) { return NULL; } @@ -338,7 +338,7 @@ method_vectorcall_VARARGS_KEYWORDS( if (method_check_args(func, args, nargs, NULL)) { return NULL; } - PyObject *argstuple = _PyTuple_FromArray(args+1, nargs-1); + PyObject *argstuple = PyTuple_FromArray(args+1, nargs-1); if (argstuple == NULL) { return NULL; } diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 531ee48eaf8..244d8f39e2b 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -13,7 +13,6 @@ #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" #include "pycore_pyerrors.h" // struct _PyErr_SetRaisedException -#include "pycore_tuple.h" // _PyTuple_FromArray() #include "osdefs.h" // SEP #include "clinic/exceptions.c.h" @@ -119,7 +118,7 @@ BaseException_vectorcall(PyObject *type_obj, PyObject * const*args, self->context = NULL; self->suppress_context = 0; - self->args = _PyTuple_FromArray(args, PyVectorcall_NARGS(nargsf)); + self->args = PyTuple_FromArray(args, PyVectorcall_NARGS(nargsf)); if (!self->args) { Py_DECREF(self); return NULL; diff --git a/Objects/listobject.c b/Objects/listobject.c index 5905a6d335b..b2903e5c93e 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -6,16 +6,16 @@ #include "pycore_critical_section.h" // _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED() #include "pycore_dict.h" // _PyDictViewObject #include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() -#include "pycore_pyatomic_ft_wrappers.h" #include "pycore_interp.h" // PyInterpreterState.list #include "pycore_list.h" // struct _Py_list_freelist, _PyListIterObject #include "pycore_long.h" // _PyLong_DigitCount #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() -#include "pycore_stackref.h" // _Py_TryIncrefCompareStackRef() -#include "pycore_tuple.h" // _PyTuple_FromArray() -#include "pycore_typeobject.h" // _Py_TYPE_VERSION_LIST +#include "pycore_pyatomic_ft_wrappers.h" #include "pycore_setobject.h" // _PySet_NextEntry() +#include "pycore_stackref.h" // _Py_TryIncrefCompareStackRef() +#include "pycore_tuple.h" // _PyTuple_FromArraySteal() +#include "pycore_typeobject.h" // _Py_TYPE_VERSION_LIST #include /*[clinic input] @@ -3221,7 +3221,7 @@ PyList_AsTuple(PyObject *v) PyObject *ret; PyListObject *self = (PyListObject *)v; Py_BEGIN_CRITICAL_SECTION(self); - ret = _PyTuple_FromArray(self->ob_item, Py_SIZE(v)); + ret = PyTuple_FromArray(self->ob_item, Py_SIZE(v)); Py_END_CRITICAL_SECTION(); return ret; } diff --git a/Objects/structseq.c b/Objects/structseq.c index c05bcde24c4..7a159338b9b 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -12,7 +12,7 @@ #include "pycore_modsupport.h" // _PyArg_NoPositional() #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_structseq.h" // PyStructSequence_InitType() -#include "pycore_tuple.h" // _PyTuple_FromArray() +#include "pycore_tuple.h" // _PyTuple_RESET_HASH_CACHE() #include "pycore_typeobject.h" // _PyStaticType_FiniBuiltin() static const char visible_length_key[] = "n_sequence_fields"; @@ -353,7 +353,7 @@ structseq_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) if (n_unnamed_fields < 0) { return NULL; } - tup = _PyTuple_FromArray(self->ob_item, n_visible_fields); + tup = PyTuple_FromArray(self->ob_item, n_visible_fields); if (!tup) goto error; diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 1fa4bae638a..94b7ae7e642 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -438,7 +438,7 @@ tuple_slice(PyTupleObject *a, Py_ssize_t ilow, if (ilow == 0 && ihigh == Py_SIZE(a) && PyTuple_CheckExact(a)) { return Py_NewRef(a); } - return _PyTuple_FromArray(a->ob_item + ilow, ihigh - ilow); + return PyTuple_FromArray(a->ob_item + ilow, ihigh - ilow); } PyObject * diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index a67bf9b1c53..d4549b70d4d 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -56,7 +56,6 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "pycore_pyhash.h" // _Py_HashSecret_t #include "pycore_pylifecycle.h" // _Py_SetFileSystemEncoding() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_tuple.h" // _PyTuple_FromArray() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI #include "pycore_unicodeobject.h" // struct _Py_unicode_state #include "pycore_unicodeobject_generated.h" // _PyUnicode_InitStaticStrings() @@ -14494,7 +14493,7 @@ unicode_vectorcall(PyObject *type, PyObject *const *args, Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); if (kwnames != NULL && PyTuple_GET_SIZE(kwnames) != 0) { // Fallback to unicode_new() - PyObject *tuple = _PyTuple_FromArray(args, nargs); + PyObject *tuple = PyTuple_FromArray(args, nargs); if (tuple == NULL) { return NULL; } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 2551c2c961b..64249177eec 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_ast.h" // _PyAST_Validate() #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_cell.h" // PyCell_GetRef() #include "pycore_ceval.h" // _PyEval_Vector() #include "pycore_compile.h" // _PyAST_Compile() #include "pycore_fileutils.h" // _PyFile_Flush @@ -14,8 +15,7 @@ #include "pycore_pyerrors.h" // _PyErr_NoMemory() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_pythonrun.h" // _Py_SourceAsString() -#include "pycore_tuple.h" // _PyTuple_FromArray() -#include "pycore_cell.h" // PyCell_GetRef() +#include "pycore_tuple.h" // _PyTuple_Recycle() #include "clinic/bltinmodule.c.h" @@ -123,7 +123,7 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, "__build_class__: name is not a string"); return NULL; } - orig_bases = _PyTuple_FromArray(args + 2, nargs - 2); + orig_bases = PyTuple_FromArray(args + 2, nargs - 2); if (orig_bases == NULL) return NULL; diff --git a/Python/ceval.c b/Python/ceval.c index 1b52128c858..f48f412fab8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2011,7 +2011,7 @@ PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals, { PyThreadState *tstate = _PyThreadState_GET(); PyObject *res = NULL; - PyObject *defaults = _PyTuple_FromArray(defs, defcount); + PyObject *defaults = PyTuple_FromArray(defs, defcount); if (defaults == NULL) { return NULL; } diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index a47e4d11b54..4c4a86de2f9 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -7,7 +7,6 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() -#include "pycore_tuple.h" // _PyTuple_FromArray() PyDoc_STRVAR(sys_addaudithook__doc__, "addaudithook($module, /, hook)\n" @@ -102,7 +101,7 @@ sys_audit(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - __clinic_args = _PyTuple_FromArray(args + 1, nargs - 1); + __clinic_args = PyTuple_FromArray(args + 1, nargs - 1); if (__clinic_args == NULL) { goto exit; } @@ -1948,4 +1947,4 @@ exit: #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=449d16326e69dcf6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5f7d84c5bf00d557 input=a9049054013a1b77]*/ diff --git a/Python/intrinsics.c b/Python/intrinsics.c index 8ea920e690c..9cfc285c6a5 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -9,7 +9,6 @@ #include "pycore_intrinsics.h" // INTRINSIC_PRINT #include "pycore_pyerrors.h" // _PyErr_SetString() #include "pycore_runtime.h" // _Py_ID() -#include "pycore_tuple.h" // _PyTuple_FromArray() #include "pycore_typevarobject.h" // _Py_make_typevar() #include "pycore_unicodeobject.h" // _PyUnicode_FromASCII() @@ -192,7 +191,7 @@ static PyObject * list_to_tuple(PyThreadState* unused, PyObject *v) { assert(PyList_Check(v)); - return _PyTuple_FromArray(((PyListObject *)v)->ob_item, Py_SIZE(v)); + return PyTuple_FromArray(((PyListObject *)v)->ob_item, Py_SIZE(v)); } static PyObject * diff --git a/Python/optimizer.c b/Python/optimizer.c index 7b76cddeabf..83b0b1a5deb 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -14,7 +14,7 @@ #include "pycore_opcode_utils.h" // MAX_REAL_OPCODE #include "pycore_optimizer.h" // _Py_uop_analyze_and_optimize() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_tuple.h" // _PyTuple_FromArraySteal +#include "pycore_tuple.h" // _PyTuple_FromArraySteal #include "pycore_unicodeobject.h" // _PyUnicode_FromASCII #include "pycore_uop_ids.h" #include "pycore_jit.h" diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 0e6884b9923..01cff0b014c 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -7,7 +7,6 @@ #include "pycore_long.h" #include "pycore_optimizer.h" #include "pycore_stats.h" -#include "pycore_tuple.h" // _PyTuple_FromArray() #include #include @@ -1044,7 +1043,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) "tuple item does not match value used to create tuple" ); PyObject *pair[2] = { val_42, val_43 }; - tuple = _PyTuple_FromArray(pair, 2); + tuple = PyTuple_FromArray(pair, 2); ref = _Py_uop_sym_new_const(ctx, tuple); TEST_PREDICATE( _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, ref, 1)) == val_43, diff --git a/Tools/clinic/libclinic/converters.py b/Tools/clinic/libclinic/converters.py index 201125b9165..bc21ae84e1c 100644 --- a/Tools/clinic/libclinic/converters.py +++ b/Tools/clinic/libclinic/converters.py @@ -1266,13 +1266,12 @@ def parse_vararg(self, *, pos_only: int, min_pos: int, max_pos: int, }}}} """ else: - self.add_include('pycore_tuple.h', '_PyTuple_FromArray()') start = f'args + {max_pos}' if max_pos else 'args' size = f'nargs - {max_pos}' if max_pos else 'nargs' if min(pos_only, min_pos) < max_pos: return f""" {paramname} = nargs > {max_pos} - ? _PyTuple_FromArray({start}, {size}) + ? PyTuple_FromArray({start}, {size}) : PyTuple_New(0); if ({paramname} == NULL) {{{{ goto exit; @@ -1280,7 +1279,7 @@ def parse_vararg(self, *, pos_only: int, min_pos: int, max_pos: int, """ else: return f""" - {paramname} = _PyTuple_FromArray({start}, {size}); + {paramname} = PyTuple_FromArray({start}, {size}); if ({paramname} == NULL) {{{{ goto exit; }}}} From 35e9d41a9cc3999672ba7440847b16ec71bd9ddd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 11 Oct 2025 22:58:43 +0200 Subject: [PATCH 112/112] gh-139482: Add `posix._clearenv()` function (#139965) --- Lib/os.py | 11 ++++++++ Lib/test/test_os/test_os.py | 8 ++++++ ...-10-11-20-03-13.gh-issue-139482.du2Stg.rst | 3 +++ Modules/clinic/posixmodule.c.h | 27 ++++++++++++++++++- Modules/posixmodule.c | 20 ++++++++++++++ configure | 6 +++++ configure.ac | 5 ++-- pyconfig.h.in | 3 +++ 8 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-11-20-03-13.gh-issue-139482.du2Stg.rst diff --git a/Lib/os.py b/Lib/os.py index 710d6f8cfcd..328d13c303b 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -58,6 +58,11 @@ def _get_exports_list(module): __all__.append('_exit') except ImportError: pass + try: + from posix import _clearenv + __all__.append('_clearenv') + except ImportError: + pass import posixpath as path try: @@ -768,6 +773,12 @@ def __ror__(self, other): new.update(self) return new + if _exists("_clearenv"): + def clear(self): + _clearenv() + self._data.clear() + + def _create_environ_mapping(): if name == 'nt': # Where Env Var Names Must Be UPPERCASE diff --git a/Lib/test/test_os/test_os.py b/Lib/test/test_os/test_os.py index e074858fe2a..86880a6d281 100644 --- a/Lib/test/test_os/test_os.py +++ b/Lib/test/test_os/test_os.py @@ -1494,6 +1494,14 @@ def test_reload_environ(self): self.assertNotIn(b'test_env', os.environb) self.assertNotIn('test_env', os.environ) + def test_clearenv(self): + os.environ['REMOVEME'] = '1' + os.environ.clear() + self.assertEqual(os.environ, {}) + + self.assertRaises(TypeError, os.environ.clear, None) + + class WalkTests(unittest.TestCase): """Tests for os.walk().""" is_fwalk = False diff --git a/Misc/NEWS.d/next/Library/2025-10-11-20-03-13.gh-issue-139482.du2Stg.rst b/Misc/NEWS.d/next/Library/2025-10-11-20-03-13.gh-issue-139482.du2Stg.rst new file mode 100644 index 00000000000..4edd3d238bf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-11-20-03-13.gh-issue-139482.du2Stg.rst @@ -0,0 +1,3 @@ +Optimize :data:`os.environ.clear() ` by calling +:manpage:`clearenv(3)` when this function is available. +Patch by Victor Stinner. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 3d9863ad179..71f87ac8ec7 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -9539,6 +9539,27 @@ exit: #endif /* !defined(MS_WINDOWS) */ +#if defined(HAVE_CLEARENV) + +PyDoc_STRVAR(os__clearenv__doc__, +"_clearenv($module, /)\n" +"--\n" +"\n"); + +#define OS__CLEARENV_METHODDEF \ + {"_clearenv", (PyCFunction)os__clearenv, METH_NOARGS, os__clearenv__doc__}, + +static PyObject * +os__clearenv_impl(PyObject *module); + +static PyObject * +os__clearenv(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os__clearenv_impl(module); +} + +#endif /* defined(HAVE_CLEARENV) */ + PyDoc_STRVAR(os_strerror__doc__, "strerror($module, code, /)\n" "--\n" @@ -13292,6 +13313,10 @@ exit: #define OS_UNSETENV_METHODDEF #endif /* !defined(OS_UNSETENV_METHODDEF) */ +#ifndef OS__CLEARENV_METHODDEF + #define OS__CLEARENV_METHODDEF +#endif /* !defined(OS__CLEARENV_METHODDEF) */ + #ifndef OS_WCOREDUMP_METHODDEF #define OS_WCOREDUMP_METHODDEF #endif /* !defined(OS_WCOREDUMP_METHODDEF) */ @@ -13447,4 +13472,4 @@ exit: #ifndef OS__EMSCRIPTEN_LOG_METHODDEF #define OS__EMSCRIPTEN_LOG_METHODDEF #endif /* !defined(OS__EMSCRIPTEN_LOG_METHODDEF) */ -/*[clinic end generated code: output=47ace1528820858b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=67f0df7cd5a7de20 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 8278902cbeb..38ddc3ec4ff 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -13201,6 +13201,25 @@ os_unsetenv_impl(PyObject *module, PyObject *name) #endif /* !MS_WINDOWS */ +#ifdef HAVE_CLEARENV +/*[clinic input] +os._clearenv +[clinic start generated code]*/ + +static PyObject * +os__clearenv_impl(PyObject *module) +/*[clinic end generated code: output=2d6705d62c014b51 input=47d2fa7f323c43ca]*/ +{ + errno = 0; + int err = clearenv(); + if (err) { + return posix_error(); + } + Py_RETURN_NONE; +} +#endif + + /*[clinic input] os.strerror @@ -17167,6 +17186,7 @@ static PyMethodDef posix_methods[] = { OS_POSIX_FADVISE_METHODDEF OS_PUTENV_METHODDEF OS_UNSETENV_METHODDEF + OS__CLEARENV_METHODDEF OS_STRERROR_METHODDEF OS_FCHDIR_METHODDEF OS_FSYNC_METHODDEF diff --git a/configure b/configure index d80340e3015..211f8439906 100755 --- a/configure +++ b/configure @@ -19225,6 +19225,12 @@ if test "x$ac_cv_func_chown" = xyes then : printf "%s\n" "#define HAVE_CHOWN 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "clearenv" "ac_cv_func_clearenv" +if test "x$ac_cv_func_clearenv" = xyes +then : + printf "%s\n" "#define HAVE_CLEARENV 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "clock" "ac_cv_func_clock" if test "x$ac_cv_func_clock" = xyes diff --git a/configure.ac b/configure.ac index 1e0c0f71b7c..35bf153a898 100644 --- a/configure.ac +++ b/configure.ac @@ -5226,7 +5226,8 @@ fi # checks for library functions AC_CHECK_FUNCS([ \ - accept4 alarm bind_textdomain_codeset chmod chown clock closefrom close_range confstr \ + accept4 alarm bind_textdomain_codeset chmod chown clearenv \ + clock closefrom close_range confstr \ copy_file_range ctermid dladdr dup dup3 execv explicit_bzero explicit_memset \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fork fork1 fpathconf fstatat ftime ftruncate futimens futimes futimesat \ @@ -8173,7 +8174,7 @@ PY_STDLIB_MOD([xxlimited_35], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_d # Determine JIT stencils header files based on target platform JIT_STENCILS_H="" -AS_VAR_IF([enable_experimental_jit], [no], +AS_VAR_IF([enable_experimental_jit], [no], [], [case "$host" in aarch64-apple-darwin*) diff --git a/pyconfig.h.in b/pyconfig.h.in index 60bff4a9f26..72870411bc0 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -141,6 +141,9 @@ /* Define if you have the 'chroot' function. */ #undef HAVE_CHROOT +/* Define to 1 if you have the 'clearenv' function. */ +#undef HAVE_CLEARENV + /* Define to 1 if you have the 'clock' function. */ #undef HAVE_CLOCK