From 769cc8338f35eb134508aca701a59342bcb6a84b Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Fri, 17 Apr 2026 15:09:09 +0300 Subject: [PATCH 001/152] gh-148464: Add missing ``__ctype_le/be__`` attributes for complex types in the ctype module (GH-148485) Co-authored-by: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> --- Lib/test/test_ctypes/test_byteswap.py | 43 ++++++++++ ...-04-13-06-22-27.gh-issue-148464.Bj_NZy.rst | 3 + Modules/_ctypes/_ctypes.c | 45 +++++++---- Modules/_ctypes/cfield.c | 78 +++++++++++++++++++ 4 files changed, 155 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-04-13-06-22-27.gh-issue-148464.Bj_NZy.rst diff --git a/Lib/test/test_ctypes/test_byteswap.py b/Lib/test/test_ctypes/test_byteswap.py index f14e1aa32e1..6a1bae14773 100644 --- a/Lib/test/test_ctypes/test_byteswap.py +++ b/Lib/test/test_ctypes/test_byteswap.py @@ -1,4 +1,5 @@ import binascii +import ctypes import math import struct import sys @@ -165,6 +166,48 @@ def test_endian_double(self): self.assertEqual(s.value, math.pi) self.assertEqual(bin(struct.pack(">d", math.pi)), bin(s)) + @unittest.skipUnless(hasattr(ctypes, 'c_float_complex'), "No complex types") + def test_endian_float_complex(self): + c_float_complex = ctypes.c_float_complex + if sys.byteorder == "little": + self.assertIs(c_float_complex.__ctype_le__, c_float_complex) + self.assertIs(c_float_complex.__ctype_be__.__ctype_le__, + c_float_complex) + else: + self.assertIs(c_float_complex.__ctype_be__, c_float_complex) + self.assertIs(c_float_complex.__ctype_le__.__ctype_be__, + c_float_complex) + s = c_float_complex(math.pi+1j) + self.assertEqual(bin(struct.pack("F", math.pi+1j)), bin(s)) + self.assertAlmostEqual(s.value, math.pi+1j, places=6) + s = c_float_complex.__ctype_le__(math.pi+1j) + self.assertAlmostEqual(s.value, math.pi+1j, places=6) + self.assertEqual(bin(struct.pack("F", math.pi+1j)), bin(s)) + + @unittest.skipUnless(hasattr(ctypes, 'c_double_complex'), "No complex types") + def test_endian_double_complex(self): + c_double_complex = ctypes.c_double_complex + if sys.byteorder == "little": + self.assertIs(c_double_complex.__ctype_le__, c_double_complex) + self.assertIs(c_double_complex.__ctype_be__.__ctype_le__, + c_double_complex) + else: + self.assertIs(c_double_complex.__ctype_be__, c_double_complex) + self.assertIs(c_double_complex.__ctype_le__.__ctype_be__, + c_double_complex) + s = c_double_complex(math.pi+1j) + self.assertEqual(bin(struct.pack("D", math.pi+1j)), bin(s)) + self.assertAlmostEqual(s.value, math.pi+1j, places=6) + s = c_double_complex.__ctype_le__(math.pi+1j) + self.assertAlmostEqual(s.value, math.pi+1j, places=6) + self.assertEqual(bin(struct.pack("D", math.pi+1j)), bin(s)) + def test_endian_other(self): self.assertIs(c_byte.__ctype_le__, c_byte) self.assertIs(c_byte.__ctype_be__, c_byte) diff --git a/Misc/NEWS.d/next/Library/2026-04-13-06-22-27.gh-issue-148464.Bj_NZy.rst b/Misc/NEWS.d/next/Library/2026-04-13-06-22-27.gh-issue-148464.Bj_NZy.rst new file mode 100644 index 00000000000..85b99531d03 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-13-06-22-27.gh-issue-148464.Bj_NZy.rst @@ -0,0 +1,3 @@ +Add missing ``__ctype_le/be__`` attributes for +:class:`~ctypes.c_float_complex` and :class:`~ctypes.c_double_complex`. Patch +by Sergey B Kirpichev. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 55eade1c830..0bdc30a0cb3 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2222,6 +2222,31 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) return NULL; } +static int +set_stginfo_ffi_type_pointer(StgInfo *stginfo, struct fielddesc *fmt) +{ + if (!fmt->pffi_type->elements) { + stginfo->ffi_type_pointer = *fmt->pffi_type; + } + else { + /* From primitive types - only complex types have the elements + struct field as non-NULL (two element array). */ + assert(fmt->pffi_type->type == FFI_TYPE_COMPLEX); + const size_t els_size = 2 * sizeof(ffi_type *); + stginfo->ffi_type_pointer.size = fmt->pffi_type->size; + stginfo->ffi_type_pointer.alignment = fmt->pffi_type->alignment; + stginfo->ffi_type_pointer.type = fmt->pffi_type->type; + stginfo->ffi_type_pointer.elements = PyMem_Malloc(els_size); + if (!stginfo->ffi_type_pointer.elements) { + PyErr_NoMemory(); + return -1; + } + memcpy(stginfo->ffi_type_pointer.elements, + fmt->pffi_type->elements, els_size); + } + return 0; +} + static PyMethodDef c_void_p_methods[] = {C_VOID_P_FROM_PARAM_METHODDEF {0}}; static PyMethodDef c_char_p_methods[] = {C_CHAR_P_FROM_PARAM_METHODDEF {0}}; static PyMethodDef c_wchar_p_methods[] = {C_WCHAR_P_FROM_PARAM_METHODDEF {0}}; @@ -2266,8 +2291,10 @@ static PyObject *CreateSwappedType(ctypes_state *st, PyTypeObject *type, Py_DECREF(result); return NULL; } - - stginfo->ffi_type_pointer = *fmt->pffi_type; + if (set_stginfo_ffi_type_pointer(stginfo, fmt)) { + Py_DECREF(result); + return NULL; + } stginfo->align = fmt->pffi_type->alignment; stginfo->length = 0; stginfo->size = fmt->pffi_type->size; @@ -2362,18 +2389,8 @@ PyCSimpleType_init(PyObject *self, PyObject *args, PyObject *kwds) if (!stginfo) { goto error; } - - if (!fmt->pffi_type->elements) { - stginfo->ffi_type_pointer = *fmt->pffi_type; - } - else { - const size_t els_size = sizeof(fmt->pffi_type->elements); - stginfo->ffi_type_pointer.size = fmt->pffi_type->size; - stginfo->ffi_type_pointer.alignment = fmt->pffi_type->alignment; - stginfo->ffi_type_pointer.type = fmt->pffi_type->type; - stginfo->ffi_type_pointer.elements = PyMem_Malloc(els_size); - memcpy(stginfo->ffi_type_pointer.elements, - fmt->pffi_type->elements, els_size); + if (set_stginfo_ffi_type_pointer(stginfo, fmt)) { + goto error; } stginfo->align = fmt->pffi_type->alignment; stginfo->length = 0; diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 4ebca0e0b3d..b0dc11fdddc 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -792,6 +792,44 @@ D_get(void *ptr, Py_ssize_t size) return PyComplex_FromDoubles(x[0], x[1]); } +static PyObject * +D_set_sw(void *ptr, PyObject *value, Py_ssize_t size) +{ + assert(NUM_BITS(size) || (size == 2*sizeof(double))); + Py_complex c = PyComplex_AsCComplex(value); + + if (c.real == -1 && PyErr_Occurred()) { + return NULL; + } +#ifdef WORDS_BIGENDIAN + if (PyFloat_Pack8(c.real, ptr, 1) + || PyFloat_Pack8(c.imag, ptr + sizeof(double), 1)) + { + return NULL; + } +#else + if (PyFloat_Pack8(c.real, ptr, 0) + || PyFloat_Pack8(c.imag, ptr + sizeof(double), 0)) + { + return NULL; + } +#endif + _RET(value); +} + +static PyObject * +D_get_sw(void *ptr, Py_ssize_t size) +{ + assert(NUM_BITS(size) || (size == 2*sizeof(double))); +#ifdef WORDS_BIGENDIAN + return PyComplex_FromDoubles(PyFloat_Unpack8(ptr, 1), + PyFloat_Unpack8(ptr + sizeof(double), 1)); +#else + return PyComplex_FromDoubles(PyFloat_Unpack8(ptr, 0), + PyFloat_Unpack8(ptr + sizeof(double), 0)); +#endif +} + /* F: float complex */ static PyObject * F_set(void *ptr, PyObject *value, Py_ssize_t size) @@ -817,6 +855,44 @@ F_get(void *ptr, Py_ssize_t size) return PyComplex_FromDoubles(x[0], x[1]); } +static PyObject * +F_set_sw(void *ptr, PyObject *value, Py_ssize_t size) +{ + assert(NUM_BITS(size) || (size == 2*sizeof(float))); + Py_complex c = PyComplex_AsCComplex(value); + + if (c.real == -1 && PyErr_Occurred()) { + return NULL; + } +#ifdef WORDS_BIGENDIAN + if (PyFloat_Pack4(c.real, ptr, 1) + || PyFloat_Pack4(c.imag, ptr + sizeof(float), 1)) + { + return NULL; + } +#else + if (PyFloat_Pack4(c.real, ptr, 0) + || PyFloat_Pack4(c.imag, ptr + sizeof(float), 0)) + { + return NULL; + } +#endif + _RET(value); +} + +static PyObject * +F_get_sw(void *ptr, Py_ssize_t size) +{ + assert(NUM_BITS(size) || (size == 2*sizeof(float))); +#ifdef WORDS_BIGENDIAN + return PyComplex_FromDoubles(PyFloat_Unpack4(ptr, 1), + PyFloat_Unpack4(ptr + sizeof(float), 1)); +#else + return PyComplex_FromDoubles(PyFloat_Unpack4(ptr, 0), + PyFloat_Unpack4(ptr + sizeof(float), 0)); +#endif +} + /* G: long double complex */ static PyObject * G_set(void *ptr, PyObject *value, Py_ssize_t size) @@ -1602,7 +1678,9 @@ for base_code, base_c_type in [ #if defined(_Py_FFI_SUPPORT_C_COMPLEX) if (Py_FFI_COMPLEX_AVAILABLE) { TABLE_ENTRY(D, &ffi_type_complex_double); + TABLE_ENTRY_SW(D, &ffi_type_complex_double); TABLE_ENTRY(F, &ffi_type_complex_float); + TABLE_ENTRY_SW(F, &ffi_type_complex_float); TABLE_ENTRY(G, &ffi_type_complex_longdouble); } #endif From afde75664eb3ff3e147806f027c9da54c7eb77d4 Mon Sep 17 00:00:00 2001 From: Gleb Popov Date: Fri, 17 Apr 2026 15:13:44 +0300 Subject: [PATCH 002/152] gh-148484: Fix memory leak of iterator in array.array constructor (GH-148523) --- Modules/arraymodule.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index a86a7561271..b01e92eb887 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -3053,8 +3053,10 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) len = 0; a = newarrayobject(type, len, descr); - if (a == NULL) + if (a == NULL) { + Py_XDECREF(it); return NULL; + } if (len > 0 && !array_Check(initial, state)) { Py_ssize_t i; @@ -3063,11 +3065,13 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PySequence_GetItem(initial, i); if (v == NULL) { Py_DECREF(a); + Py_XDECREF(it); return NULL; } if (setarrayitem(a, i, v) != 0) { Py_DECREF(v); Py_DECREF(a); + Py_XDECREF(it); return NULL; } Py_DECREF(v); @@ -3079,6 +3083,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) v = array_array_frombytes((PyObject *)a, initial); if (v == NULL) { Py_DECREF(a); + Py_XDECREF(it); return NULL; } Py_DECREF(v); @@ -3089,6 +3094,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) wchar_t *ustr = PyUnicode_AsWideCharString(initial, &n); if (ustr == NULL) { Py_DECREF(a); + Py_XDECREF(it); return NULL; } @@ -3109,6 +3115,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_UCS4 *ustr = PyUnicode_AsUCS4Copy(initial); if (ustr == NULL) { Py_DECREF(a); + Py_XDECREF(it); return NULL; } @@ -3136,6 +3143,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return a; } } + Py_XDECREF(it); PyErr_SetString(PyExc_ValueError, "bad typecode (must be b, B, u, w, h, H, i, I, l, L, q, Q, f or d)"); return NULL; From a86234ea853431bd3eae014bb3eef71227aeab27 Mon Sep 17 00:00:00 2001 From: Xuwz Date: Fri, 17 Apr 2026 22:13:41 +0800 Subject: [PATCH 003/152] gh-148683: Doc: fix misplaced pprint entries in What's New 3.15 (#148685) Doc: fix misplaced pprint entries in What's New 3.15 --- Doc/whatsnew/3.15.rst | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 7ea7c901ece..56cc71b40fc 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -978,6 +978,20 @@ pickle (Contributed by Zackery Spytz and Serhiy Storchaka in :gh:`77188`.) +pprint +------ + +* Add an *expand* keyword argument for :func:`pprint.pprint`, + :func:`pprint.pformat`, :func:`pprint.pp`. If true, the output will be + formatted similar to pretty-printed :func:`json.dumps` when + *indent* is supplied. + (Contributed by Stefan Todoran, Semyon Moroz and Hugo van Kemenade in + :gh:`112632`.) + +* Add t-string support to :mod:`pprint`. + (Contributed by Loïc Simon and Hugo van Kemenade in :gh:`134551`.) + + re -- @@ -1594,20 +1608,6 @@ platform (Contributed by Alexey Makridenko in :gh:`133604`.) -pprint ------- - -* Add an *expand* keyword argument for :func:`pprint.pprint`, - :func:`pprint.pformat`, :func:`pprint.pp`. If true, the output will be - formatted similar to pretty-printed :func:`json.dumps` when - *indent* is supplied. - (Contributed by Stefan Todoran, Semyon Moroz and Hugo van Kemenade in - :gh:`112632`.) - -* Add t-string support to :mod:`pprint`. - (Contributed by Loïc Simon and Hugo van Kemenade in :gh:`134551`.) - - sre_* ----- From 446edda20919447fdc8b5a43f2f2ae686df82e6a Mon Sep 17 00:00:00 2001 From: Michael Bommarito Date: Fri, 17 Apr 2026 11:42:41 -0400 Subject: [PATCH 004/152] gh-148651: Fix refcount leak in _zstd decompressor options (#148657) The option parsing in Modules/_zstd/decompressor.c had a missing Py_DECREF(value) before the early return -1 when PyLong_AsInt(key) fails. The identical code in Modules/_zstd/compressor.c line 158 has the fix. --- .../next/Library/2026-04-16-13-30-00.gh-issue-148651.ZsTdLk.rst | 2 ++ Modules/_zstd/decompressor.c | 1 + 2 files changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-04-16-13-30-00.gh-issue-148651.ZsTdLk.rst diff --git a/Misc/NEWS.d/next/Library/2026-04-16-13-30-00.gh-issue-148651.ZsTdLk.rst b/Misc/NEWS.d/next/Library/2026-04-16-13-30-00.gh-issue-148651.ZsTdLk.rst new file mode 100644 index 00000000000..b69f94a1766 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-16-13-30-00.gh-issue-148651.ZsTdLk.rst @@ -0,0 +1,2 @@ +Fix reference leak in :class:`compression.zstd.ZstdDecompressor` when an +invalid option key is passed. diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index 0186ee92f5b..46682b483ad 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -111,6 +111,7 @@ _zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options) int key_v = PyLong_AsInt(key); Py_DECREF(key); if (key_v == -1 && PyErr_Occurred()) { + Py_DECREF(value); return -1; } From db3e990b98fd12fee1bf851f4185d4e857318d76 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Fri, 17 Apr 2026 18:52:16 +0200 Subject: [PATCH 005/152] gh-146393: Remove special character in optimizer_bytecodes.c (#148693) --- Python/optimizer_bytecodes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 6e4882143fb..daebef4a043 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -300,7 +300,7 @@ dummy_func(void) { // narrowing unlocks a meaningful downstream win: // - NB_TRUE_DIVIDE: enables the specialized float path below. // - NB_REMAINDER: lets the float result type propagate. - // NB_POWER is excluded — speculative guards there regressed + // NB_POWER is excluded: speculative guards there regressed // test_power_type_depends_on_input_values (GH-127844). if (is_truediv || is_remainder) { if (!sym_has_type(rhs) From 634568d030f18183212c01bd4544aa7f97e05442 Mon Sep 17 00:00:00 2001 From: Prakash Sellathurai Date: Sat, 18 Apr 2026 05:51:13 +0530 Subject: [PATCH 006/152] gh-148222: Fix NULL dereference bugs in genericaliasobject.c (#148226) --- .../2026-04-07-20-37-23.gh-issue-148222.uF4D4E.rst | 1 + Objects/genericaliasobject.c | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-04-07-20-37-23.gh-issue-148222.uF4D4E.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-07-20-37-23.gh-issue-148222.uF4D4E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-07-20-37-23.gh-issue-148222.uF4D4E.rst new file mode 100644 index 00000000000..2c273fc4dab --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-07-20-37-23.gh-issue-148222.uF4D4E.rst @@ -0,0 +1 @@ +Fix vectorcall support in :class:`types.GenericAlias` when the underlying type does not support the vectorcall protocol. Fix possible leaks in :class:`types.GenericAlias` and :class:`types.UnionType` in case of memory error. diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 7aef56cf4e9..e3bc8eb2739 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -242,7 +242,6 @@ _Py_make_parameters(PyObject *args) len += needed; if (_PyTuple_Resize(¶meters, len) < 0) { Py_DECREF(subparams); - Py_DECREF(parameters); Py_XDECREF(tuple_args); return NULL; } @@ -650,7 +649,7 @@ ga_vectorcall(PyObject *self, PyObject *const *args, size_t nargsf, PyObject *kwnames) { gaobject *alias = (gaobject *) self; - PyObject *obj = PyVectorcall_Function(alias->origin)(alias->origin, args, nargsf, kwnames); + PyObject *obj = PyObject_Vectorcall(alias->origin, args, nargsf, kwnames); return set_orig_class(obj, self); } From 92164dc91712dcf27e2d526fa6ef8735a8356a36 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 17 Apr 2026 19:20:41 -0700 Subject: [PATCH 007/152] gh-148639: Implement PEP 800 (typing.disjoint_base) (#148640) Co-authored-by: Victorien <65306057+Viicos@users.noreply.github.com> Co-authored-by: Petr Viktorin --- Doc/library/typing.rst | 30 +++++++++++++++++++ Doc/whatsnew/3.15.rst | 8 +++++ Lib/test/test_typing.py | 14 ++++++++- Lib/typing.py | 24 +++++++++++++++ ...-04-15-20-32-55.gh-issue-148639.-dwsjB.rst | 2 ++ 5 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-04-15-20-32-55.gh-issue-148639.-dwsjB.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 2ce868cf84d..9150385bd58 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -3358,6 +3358,36 @@ Functions and decorators .. versionadded:: 3.12 +.. decorator:: disjoint_base + + Decorator to mark a class as a disjoint base. + + Type checkers do not allow child classes of a disjoint base ``C`` to + inherit from other disjoint bases that are not parent or child classes of ``C``. + + For example:: + + @disjoint_base + class Disjoint1: pass + + @disjoint_base + class Disjoint2: pass + + class Disjoint3(Disjoint1, Disjoint2): pass # Type checker error + + Type checkers can use knowledge of disjoint bases to detect unreachable code + and determine when two types can overlap. + + The corresponding runtime concept is a solid base (see :ref:`multiple-inheritance`). + Classes that are solid bases at runtime can be marked with ``@disjoint_base`` in stub files. + Users may also mark other classes as disjoint bases to indicate to type checkers that + multiple inheritance with other disjoint bases should not be allowed. + + Note that the concept of a solid base is a CPython implementation + detail, and the exact set of standard library classes that are + disjoint bases at runtime may change in future versions of Python. + + .. versionadded:: next .. decorator:: type_check_only diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 56cc71b40fc..7cf4dc3701f 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -80,6 +80,7 @@ Summary -- Release highlights * :pep:`728`: ``TypedDict`` with typed extra items * :pep:`747`: :ref:`Annotating type forms with TypeForm ` +* :pep:`800`: Disjoint bases in the type system * :pep:`782`: :ref:`A new PyBytesWriter C API to create a Python bytes object ` * :pep:`803`: :ref:`Stable ABI for Free-Threaded Builds ` @@ -1290,6 +1291,13 @@ typing as it was incorrectly inferred in runtime before. (Contributed by Nikita Sobolev in :gh:`137191`.) +* :pep:`800`: Add :deco:`typing.disjoint_base`, a new decorator marking a class + as a disjoint base. This is an advanced feature primarily intended to allow + type checkers to faithfully reflect the runtime semantics of types defined + as builtins or in compiled extensions. If a class ``C`` is a disjoint base, then + child classes of that class cannot inherit from other disjoint bases that are + not parent or child classes of ``C``. (Contributed by Jelle Zijlstra in :gh:`148639`.) + unicodedata ----------- diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 9c0172f6ba7..3fb974c517d 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -29,7 +29,7 @@ from typing import assert_type, cast, runtime_checkable from typing import get_type_hints from typing import get_origin, get_args, get_protocol_members -from typing import override +from typing import override, disjoint_base from typing import is_typeddict, is_protocol from typing import reveal_type from typing import dataclass_transform @@ -10920,6 +10920,18 @@ def bar(self): self.assertNotIn('__magic__', dir_items) +class DisjointBaseTests(BaseTestCase): + def test_disjoint_base_unmodified(self): + class C: ... + self.assertIs(C, disjoint_base(C)) + + def test_dunder_disjoint_base(self): + @disjoint_base + class C: ... + + self.assertIs(C.__disjoint_base__, True) + + class RevealTypeTests(BaseTestCase): def test_reveal_type(self): obj = object() diff --git a/Lib/typing.py b/Lib/typing.py index e78fb8b71a9..868fec9e088 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -126,6 +126,7 @@ 'cast', 'clear_overloads', 'dataclass_transform', + 'disjoint_base', 'evaluate_forward_ref', 'final', 'get_args', @@ -2794,6 +2795,29 @@ class Other(Leaf): # Error reported by type checker return f +def disjoint_base(cls): + """This decorator marks a class as a disjoint base. + + Child classes of a disjoint base cannot inherit from other disjoint bases that are + not parent or child classes of the disjoint base. + + For example: + + @disjoint_base + class Disjoint1: pass + + @disjoint_base + class Disjoint2: pass + + class Disjoint3(Disjoint1, Disjoint2): pass # Type checker error + + Type checkers can use knowledge of disjoint bases to detect unreachable code + and determine when two types can overlap. + """ + cls.__disjoint_base__ = True + return cls + + # Some unconstrained type variables. These were initially used by the container types. # They were never meant for export and are now unused, but we keep them around to # avoid breaking compatibility with users who import them. diff --git a/Misc/NEWS.d/next/Library/2026-04-15-20-32-55.gh-issue-148639.-dwsjB.rst b/Misc/NEWS.d/next/Library/2026-04-15-20-32-55.gh-issue-148639.-dwsjB.rst new file mode 100644 index 00000000000..d7acdb09838 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-15-20-32-55.gh-issue-148639.-dwsjB.rst @@ -0,0 +1,2 @@ +Implement :pep:`800`, adding the :deco:`typing.disjoint_base` decorator. +Patch by Jelle Zijlstra. From 2e37d836411e99cff7bb341ba14be5ea95fac08c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 18 Apr 2026 11:24:33 +0300 Subject: [PATCH 008/152] gh-148653: Fix some marshal errors related to recursive immutable objects (GH-148698) Forbid marshalling recursive code, slice and frozendict objects which cannot be correctly unmarshalled. Reject invalid marshal data produced by marshalling recursive frozendict objects which was previously incorrectly unmarshalled. Add multiple tests for recursive data structures. --- Lib/test/test_marshal.py | 127 ++++++++++++++++++ ...-04-17-20-37-02.gh-issue-148653.nbbHMh.rst | 2 + Python/marshal.c | 69 +++++++--- 3 files changed, 182 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-04-17-20-37-02.gh-issue-148653.nbbHMh.rst diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 78db4219e29..9ec37d27dfa 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -317,6 +317,133 @@ def test_recursion_limit(self): last.append([0]) self.assertRaises(ValueError, marshal.dumps, head) + def test_reference_loop_list(self): + a = [] + a.append(a) + for v in range(3): + self.assertRaises(ValueError, marshal.dumps, a, v) + for v in range(3, marshal.version + 1): + d = marshal.dumps(a, v) + b = marshal.loads(d) + self.assertIsInstance(b, list) + self.assertIs(b[0], b) + + def test_reference_loop_dict(self): + a = {} + a[None] = a + for v in range(3): + self.assertRaises(ValueError, marshal.dumps, a, v) + for v in range(3, marshal.version + 1): + d = marshal.dumps(a, v) + b = marshal.loads(d) + self.assertIsInstance(b, dict) + self.assertIs(b[None], b) + + def test_reference_loop_tuple(self): + a = ([],) + a[0].append(a) + for v in range(3): + self.assertRaises(ValueError, marshal.dumps, a, v) + for v in range(3, marshal.version + 1): + d = marshal.dumps(a, v) + b = marshal.loads(d) + self.assertIsInstance(b, tuple) + self.assertIsInstance(b[0], list) + self.assertIs(b[0][0], b) + + def test_reference_loop_code(self): + def f(): + return 1234.5 + code = f.__code__ + a = [] + code = code.replace(co_consts=code.co_consts + (a,)) + a.append(code) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, code, v) + + def test_reference_loop_slice(self): + a = slice([], None) + a.start.append(a) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, a, v) + + a = slice(None, []) + a.stop.append(a) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, a, v) + + a = slice(None, None, []) + a.step.append(a) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, a, v) + + def test_reference_loop_frozendict(self): + a = frozendict({None: []}) + a[None].append(a) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, a, v) + + def test_loads_reference_loop_list(self): + data = b'\xdb\x01\x00\x00\x00r\x00\x00\x00\x00' # [] + a = marshal.loads(data) + self.assertIsInstance(a, list) + self.assertIs(a[0], a) + + def test_loads_reference_loop_dict(self): + data = b'\xfbNr\x00\x00\x00\x000' # {None: } + a = marshal.loads(data) + self.assertIsInstance(a, dict) + self.assertIs(a[None], a) + + def test_loads_abnormal_reference_loops(self): + # Indirect self-references of tuples. + data = b'\xa8\x01\x00\x00\x00[\x01\x00\x00\x00r\x00\x00\x00\x00' # ([],) + a = marshal.loads(data) + self.assertIsInstance(a, tuple) + self.assertIsInstance(a[0], list) + self.assertIs(a[0][0], a) + + data = b'\xa8\x01\x00\x00\x00{Nr\x00\x00\x00\x000' # ({None: },) + a = marshal.loads(data) + self.assertIsInstance(a, tuple) + self.assertIsInstance(a[0], dict) + self.assertIs(a[0][None], a) + + # Direct self-reference which cannot be created in Python. + data = b'\xa8\x01\x00\x00\x00r\x00\x00\x00\x00' # (,) + a = marshal.loads(data) + self.assertIsInstance(a, tuple) + self.assertIs(a[0], a) + + # Direct self-references which cannot be created in Python + # because of unhashability. + data = b'\xfbr\x00\x00\x00\x00N0' # {: None} + self.assertRaises(TypeError, marshal.loads, data) + data = b'\xbc\x01\x00\x00\x00r\x00\x00\x00\x00' # {} + self.assertRaises(TypeError, marshal.loads, data) + + for data in [ + # Indirect self-references of immutable objects. + b'\xba[\x01\x00\x00\x00r\x00\x00\x00\x00NN', # slice([], None) + b'\xbaN[\x01\x00\x00\x00r\x00\x00\x00\x00N', # slice(None, []) + b'\xbaNN[\x01\x00\x00\x00r\x00\x00\x00\x00', # slice(None, None, []) + b'\xba{Nr\x00\x00\x00\x000NN', # slice({None: }, None) + b'\xbaN{Nr\x00\x00\x00\x000N', # slice(None, {None: }) + b'\xbaNN{Nr\x00\x00\x00\x000', # slice(None, None, {None: }) + b'\xfdN[\x01\x00\x00\x00r\x00\x00\x00\x000', # frozendict({None: []}) + b'\xfdN{Nr\x00\x00\x00\x0000', # frozendict({None: {None: }) + + # Direct self-references which cannot be created in Python. + b'\xbe\x01\x00\x00\x00r\x00\x00\x00\x00', # frozenset({}) + b'\xfdNr\x00\x00\x00\x000', # frozendict({None: }) + b'\xfdr\x00\x00\x00\x00N0', # frozendict({: None}) + b'\xbar\x00\x00\x00\x00NN', # slice(, None) + b'\xbaNr\x00\x00\x00\x00N', # slice(None, ) + b'\xbaNNr\x00\x00\x00\x00', # slice(None, None, ) + ]: + with self.subTest(data=data): + self.assertRaises(ValueError, marshal.loads, data) + def test_exact_type_match(self): # Former bug: # >>> class Int(int): pass diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-17-20-37-02.gh-issue-148653.nbbHMh.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-17-20-37-02.gh-issue-148653.nbbHMh.rst new file mode 100644 index 00000000000..d3242235c60 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-17-20-37-02.gh-issue-148653.nbbHMh.rst @@ -0,0 +1,2 @@ +Forbid :mod:`marshalling ` recursive code objects, :class:`slice` +and :class:`frozendict` objects which cannot be correctly unmarshalled. diff --git a/Python/marshal.c b/Python/marshal.c index b60a36e128c..dace22da0d4 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -382,7 +382,6 @@ static int w_ref(PyObject *v, char *flag, WFILE *p) { _Py_hashtable_entry_t *entry; - int w; if (p->version < 3 || p->hashtable == NULL) return 0; /* not writing object references */ @@ -399,20 +398,28 @@ w_ref(PyObject *v, char *flag, WFILE *p) entry = _Py_hashtable_get_entry(p->hashtable, v); if (entry != NULL) { /* write the reference index to the stream */ - w = (int)(uintptr_t)entry->value; + uintptr_t w = (uintptr_t)entry->value; + if (w & 0x80000000LU) { + PyErr_Format(PyExc_ValueError, "cannot marshal recursion %T objects", v); + goto err; + } /* we don't store "long" indices in the dict */ - assert(0 <= w && w <= 0x7fffffff); + assert(w <= 0x7fffffff); w_byte(TYPE_REF, p); - w_long(w, p); + w_long((int)w, p); return 1; } else { - size_t s = p->hashtable->nentries; + size_t w = p->hashtable->nentries; /* we don't support long indices */ - if (s >= 0x7fffffff) { + if (w >= 0x7fffffff) { PyErr_SetString(PyExc_ValueError, "too many objects"); goto err; } - w = (int)s; + // Corresponding code should call w_complete() after + // writing the object. + if (PyCode_Check(v) || PySlice_Check(v) || PyFrozenDict_CheckExact(v)) { + w |= 0x80000000LU; + } if (_Py_hashtable_set(p->hashtable, Py_NewRef(v), (void *)(uintptr_t)w) < 0) { Py_DECREF(v); @@ -426,6 +433,27 @@ w_ref(PyObject *v, char *flag, WFILE *p) return 1; } +static void +w_complete(PyObject *v, WFILE *p) +{ + if (p->version < 3 || p->hashtable == NULL) { + return; + } + if (_PyObject_IsUniquelyReferenced(v)) { + return; + } + + _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(p->hashtable, v); + if (entry == NULL) { + return; + } + assert(entry != NULL); + uintptr_t w = (uintptr_t)entry->value; + assert(w & 0x80000000LU); + w &= ~0x80000000LU; + entry->value = (void *)(uintptr_t)w; +} + static void w_complex_object(PyObject *v, char flag, WFILE *p); @@ -599,6 +627,9 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(value, p); } w_object((PyObject *)NULL, p); + if (PyFrozenDict_CheckExact(v)) { + w_complete(v, p); + } } else if (PyAnySet_CheckExact(v)) { PyObject *value; @@ -684,6 +715,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(co->co_linetable, p); w_object(co->co_exceptiontable, p); Py_DECREF(co_code); + w_complete(v, p); } else if (PyObject_CheckBuffer(v)) { /* Write unknown bytes-like objects as a bytes object */ @@ -709,6 +741,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(slice->start, p); w_object(slice->stop, p); w_object(slice->step, p); + w_complete(v, p); } else { W_TYPE(TYPE_UNKNOWN, p); @@ -1433,9 +1466,19 @@ r_object(RFILE *p) case TYPE_DICT: case TYPE_FROZENDICT: v = PyDict_New(); - R_REF(v); - if (v == NULL) + if (v == NULL) { break; + } + if (type == TYPE_DICT) { + R_REF(v); + } + else { + idx = r_ref_reserve(flag, p); + if (idx < 0) { + Py_CLEAR(v); + break; + } + } for (;;) { PyObject *key, *val; key = r_object(p); @@ -1458,13 +1501,7 @@ r_object(RFILE *p) Py_CLEAR(v); } if (type == TYPE_FROZENDICT && v != NULL) { - PyObject *frozendict = PyFrozenDict_New(v); - if (frozendict != NULL) { - Py_SETREF(v, frozendict); - } - else { - Py_CLEAR(v); - } + Py_SETREF(v, PyFrozenDict_New(v)); } retval = v; break; From e9bbf8617dff942360b5d800769c00440dc93bac Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sat, 18 Apr 2026 11:37:54 +0300 Subject: [PATCH 009/152] Add a new Sphinx `soft-deprecated` directive (#148630) Co-authored-by: Stan Ulbrych --- Doc/c-api/allocation.rst | 12 ++--- Doc/c-api/bytes.rst | 11 +++-- Doc/c-api/extension-modules.rst | 11 +++-- Doc/c-api/file.rst | 9 ++-- Doc/c-api/float.rst | 18 ++++---- Doc/c-api/frame.rst | 22 +++------- Doc/c-api/intro.rst | 27 ++++-------- Doc/c-api/long.rst | 4 +- Doc/c-api/module.rst | 4 +- Doc/c-api/monitoring.rst | 4 +- Doc/c-api/sequence.rst | 5 +-- Doc/library/ctypes.rst | 7 ++- Doc/library/math.rst | 5 +-- Doc/library/mimetypes.rst | 2 +- Doc/library/os.rst | 10 ++--- Doc/library/re.rst | 18 +++++--- Doc/tools/extensions/changes.py | 78 ++++++++++++++++++++++++++++++++- Doc/tools/removed-ids.txt | 4 ++ Doc/tools/templates/dummy.html | 1 + 19 files changed, 155 insertions(+), 97 deletions(-) diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 59044d2d88c..09c9ed3ca54 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -2,7 +2,7 @@ .. _allocating-objects: -Allocating Objects on the Heap +Allocating objects on the heap ============================== @@ -153,10 +153,12 @@ Allocating Objects on the Heap To allocate and create extension modules. -Deprecated aliases -^^^^^^^^^^^^^^^^^^ +Soft-deprecated aliases +^^^^^^^^^^^^^^^^^^^^^^^ -These are :term:`soft deprecated` aliases to existing functions and macros. +.. soft-deprecated:: 3.15 + +These are aliases to existing functions and macros. They exist solely for backwards compatibility. @@ -164,7 +166,7 @@ They exist solely for backwards compatibility. :widths: auto :header-rows: 1 - * * Deprecated alias + * * Soft-deprecated alias * Function * * .. c:macro:: PyObject_NEW(type, typeobj) * :c:macro:`PyObject_New` diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index d1fde1baf71..f56bcd6333a 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -47,9 +47,9 @@ called with a non-bytes parameter. *len* on success, and ``NULL`` on failure. If *v* is ``NULL``, the contents of the bytes object are uninitialized. - .. deprecated:: 3.15 - ``PyBytes_FromStringAndSize(NULL, len)`` is :term:`soft deprecated`, - use the :c:type:`PyBytesWriter` API instead. + .. soft-deprecated:: 3.15 + Use the :c:type:`PyBytesWriter` API instead of + ``PyBytes_FromStringAndSize(NULL, len)``. .. c:function:: PyObject* PyBytes_FromFormat(const char *format, ...) @@ -238,9 +238,8 @@ called with a non-bytes parameter. *\*bytes* is set to ``NULL``, :exc:`MemoryError` is set, and ``-1`` is returned. - .. deprecated:: 3.15 - The function is :term:`soft deprecated`, - use the :c:type:`PyBytesWriter` API instead. + .. soft-deprecated:: 3.15 + Use the :c:type:`PyBytesWriter` API instead. .. c:function:: PyObject *PyBytes_Repr(PyObject *bytes, int smartquotes) diff --git a/Doc/c-api/extension-modules.rst b/Doc/c-api/extension-modules.rst index 92b531665e1..7bc04970b19 100644 --- a/Doc/c-api/extension-modules.rst +++ b/Doc/c-api/extension-modules.rst @@ -191,10 +191,10 @@ the :c:data:`Py_mod_multiple_interpreters` slot. ``PyInit`` function ................... -.. deprecated:: 3.15 +.. soft-deprecated:: 3.15 - This functionality is :term:`soft deprecated`. - It will not get new features, but there are no plans to remove it. + This functionality will not get new features, + but there are no plans to remove it. Instead of :c:func:`PyModExport_modulename`, an extension module can define an older-style :dfn:`initialization function` with the signature: @@ -272,10 +272,9 @@ For example, a module called ``spam`` would be defined like this:: Legacy single-phase initialization ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. deprecated:: 3.15 +.. soft-deprecated:: 3.15 - Single-phase initialization is :term:`soft deprecated`. - It is a legacy mechanism to initialize extension + Single-phase initialization is a legacy mechanism to initialize extension modules, with known drawbacks and design flaws. Extension module authors are encouraged to use multi-phase initialization instead. diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index d89072ab24e..dcafefdc045 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -2,7 +2,7 @@ .. _fileobjects: -File Objects +File objects ------------ .. index:: pair: object; file @@ -136,11 +136,12 @@ the :mod:`io` APIs instead. failure; the appropriate exception will be set. -Deprecated API -^^^^^^^^^^^^^^ +Soft-deprecated API +^^^^^^^^^^^^^^^^^^^ +.. soft-deprecated:: 3.15 -These are :term:`soft deprecated` APIs that were included in Python's C API +These are APIs that were included in Python's C API by mistake. They are documented solely for completeness; use other ``PyFile*`` APIs instead. diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 929b56bd8e8..a12ad11abb1 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -86,8 +86,7 @@ Floating-Point Objects It is equivalent to the :c:macro:`!INFINITY` macro from the C11 standard ```` header. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_NAN @@ -103,8 +102,7 @@ Floating-Point Objects Equivalent to :c:macro:`!INFINITY`. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.14 .. c:macro:: Py_MATH_E @@ -161,8 +159,8 @@ Floating-Point Objects that is, it is normal, subnormal or zero, but not infinite or NaN. Return ``0`` otherwise. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. Use :c:macro:`!isfinite` instead. + .. soft-deprecated:: 3.14 + Use :c:macro:`!isfinite` instead. .. c:macro:: Py_IS_INFINITY(X) @@ -170,8 +168,8 @@ Floating-Point Objects Return ``1`` if the given floating-point number *X* is positive or negative infinity. Return ``0`` otherwise. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. Use :c:macro:`!isinf` instead. + .. soft-deprecated:: 3.14 + Use :c:macro:`!isinf` instead. .. c:macro:: Py_IS_NAN(X) @@ -179,8 +177,8 @@ Floating-Point Objects Return ``1`` if the given floating-point number *X* is a not-a-number (NaN) value. Return ``0`` otherwise. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. Use :c:macro:`!isnan` instead. + .. soft-deprecated:: 3.14 + Use :c:macro:`!isnan` instead. Pack and Unpack functions diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 967cfc72765..4159ff6e596 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -1,6 +1,6 @@ .. highlight:: c -Frame Objects +Frame objects ------------- .. c:type:: PyFrameObject @@ -147,7 +147,7 @@ See also :ref:`Reflection `. Return the line number that *frame* is currently executing. -Frame Locals Proxies +Frame locals proxies ^^^^^^^^^^^^^^^^^^^^ .. versionadded:: 3.13 @@ -169,7 +169,7 @@ See :pep:`667` for more information. Return non-zero if *obj* is a frame :func:`locals` proxy. -Legacy Local Variable APIs +Legacy local variable APIs ^^^^^^^^^^^^^^^^^^^^^^^^^^ These APIs are :term:`soft deprecated`. As of Python 3.13, they do nothing. @@ -178,40 +178,34 @@ They exist solely for backwards compatibility. .. c:function:: void PyFrame_LocalsToFast(PyFrameObject *f, int clear) - This function is :term:`soft deprecated` and does nothing. - Prior to Python 3.13, this function would copy the :attr:`~frame.f_locals` attribute of *f* to the internal "fast" array of local variables, allowing changes in frame objects to be visible to the interpreter. If *clear* was true, this function would process variables that were unset in the locals dictionary. - .. versionchanged:: 3.13 + .. soft-deprecated:: 3.13 This function now does nothing. .. c:function:: void PyFrame_FastToLocals(PyFrameObject *f) - This function is :term:`soft deprecated` and does nothing. - Prior to Python 3.13, this function would copy the internal "fast" array of local variables (which is used by the interpreter) to the :attr:`~frame.f_locals` attribute of *f*, allowing changes in local variables to be visible to frame objects. - .. versionchanged:: 3.13 + .. soft-deprecated:: 3.13 This function now does nothing. .. c:function:: int PyFrame_FastToLocalsWithError(PyFrameObject *f) - This function is :term:`soft deprecated` and does nothing. - Prior to Python 3.13, this function was similar to :c:func:`PyFrame_FastToLocals`, but would return ``0`` on success, and ``-1`` with an exception set on failure. - .. versionchanged:: 3.13 + .. soft-deprecated:: 3.13 This function now does nothing. @@ -219,7 +213,7 @@ They exist solely for backwards compatibility. :pep:`667` -Internal Frames +Internal frames ^^^^^^^^^^^^^^^ Unless using :pep:`523`, you will not need this. @@ -249,5 +243,3 @@ Unless using :pep:`523`, you will not need this. Return the currently executing line number, or -1 if there is no line number. .. versionadded:: 3.12 - - diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 2a22a023bda..0e6fd3421f2 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -536,16 +536,14 @@ have been standardized in C11 (or previous standards). Use the standard ``alignas`` specifier rather than this macro. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: PY_FORMAT_SIZE_T The :c:func:`printf` formatting modifier for :c:type:`size_t`. Use ``"z"`` directly instead. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_LL(number) Py_ULL(number) @@ -558,8 +556,7 @@ have been standardized in C11 (or previous standards). Consider using the C99 standard suffixes ``LL`` and ``LLU`` directly. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: PY_LONG_LONG PY_INT32_T @@ -572,8 +569,7 @@ have been standardized in C11 (or previous standards). respectively. Historically, these types needed compiler-specific extensions. - .. deprecated:: 3.15 - These macros are :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: PY_LLONG_MIN PY_LLONG_MAX @@ -587,16 +583,14 @@ have been standardized in C11 (or previous standards). The required header, ````, :ref:`is included ` in ``Python.h``. - .. deprecated:: 3.15 - These macros are :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_MEMCPY(dest, src, n) This is a :term:`soft deprecated` alias to :c:func:`!memcpy`. Use :c:func:`!memcpy` directly instead. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.14 .. c:macro:: Py_UNICODE_SIZE @@ -606,16 +600,14 @@ have been standardized in C11 (or previous standards). The required header for the latter, ````, :ref:`is included ` in ``Python.h``. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_UNICODE_WIDE Defined if ``wchar_t`` can hold a Unicode character (UCS-4). Use ``sizeof(wchar_t) >= 4`` instead - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_VA_COPY @@ -627,8 +619,7 @@ have been standardized in C11 (or previous standards). .. versionchanged:: 3.6 This is now an alias to ``va_copy``. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. _api-objects: diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 790ec8da109..60e3ae4a064 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -197,12 +197,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: long PyLong_AS_LONG(PyObject *obj) - A :term:`soft deprecated` alias. Exactly equivalent to the preferred ``PyLong_AsLong``. In particular, it can fail with :exc:`OverflowError` or another exception. - .. deprecated:: 3.14 - The function is soft deprecated. + .. soft-deprecated:: 3.14 .. c:function:: int PyLong_AsInt(PyObject *obj) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index a66a1bfd7f8..b67ca671a2a 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -965,9 +965,7 @@ or code that creates modules dynamically. // PyModule_AddObject() stole a reference to obj: // Py_XDECREF(obj) is not needed here. - .. deprecated:: 3.13 - - :c:func:`PyModule_AddObject` is :term:`soft deprecated`. + .. soft-deprecated:: 3.13 .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst index b0227c2f4fa..4bfcb86abf5 100644 --- a/Doc/c-api/monitoring.rst +++ b/Doc/c-api/monitoring.rst @@ -205,6 +205,4 @@ would typically correspond to a Python function. .. versionadded:: 3.13 - .. deprecated:: 3.14 - - This function is :term:`soft deprecated`. + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index df5bf6b64a9..6bae8f25ad7 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -109,9 +109,8 @@ Sequence Protocol Alias for :c:func:`PySequence_Contains`. - .. deprecated:: 3.14 - The function is :term:`soft deprecated` and should no longer be used to - write new code. + .. soft-deprecated:: 3.14 + The function should no longer be used to write new code. .. c:function:: Py_ssize_t PySequence_Index(PyObject *o, PyObject *value) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 571975d4674..ff09bb8d884 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1756,11 +1756,10 @@ as a default or fallback. (or by) Python. It is recommended to only use this function as a default or fallback, - .. deprecated:: 3.15 + .. soft-deprecated:: 3.15 - This function is :term:`soft deprecated`. - It is kept for use in cases where it works, but not expected to be - updated for additional platforms and configurations. + This function is kept for use in cases where it works, but not expected to + be updated for additional platforms and configurations. On Linux, :func:`!find_library` tries to run external programs (``/sbin/ldconfig``, ``gcc``, ``objdump`` and ``ld``) to find the diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 4a11aec15df..9cc8c5d6886 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -781,9 +781,8 @@ the following functions from the :mod:`math.integer` module: Floats with integral values (like ``5.0``) are no longer accepted in the :func:`factorial` function. -.. deprecated:: 3.15 - These aliases are :term:`soft deprecated` in favor of the - :mod:`math.integer` functions. +.. soft-deprecated:: 3.15 + Use the :mod:`math.integer` functions instead of these aliases. Constants diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 1e599bde8bc..af9098c4970 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -54,7 +54,7 @@ the information :func:`init` sets up. .. versionchanged:: 3.8 Added support for *url* being a :term:`path-like object`. - .. deprecated:: 3.13 + .. soft-deprecated:: 3.13 Passing a file path instead of URL is :term:`soft deprecated`. Use :func:`guess_file_type` for this. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 7547967c6b3..d2534b3e974 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -5110,9 +5110,8 @@ written in Python, such as a mail server's external command delivery program. Use :class:`subprocess.Popen` or :func:`subprocess.run` to control options like encodings. - .. deprecated:: 3.14 - The function is :term:`soft deprecated` and should no longer be used to - write new code. The :mod:`subprocess` module is recommended instead. + .. soft-deprecated:: 3.14 + The :mod:`subprocess` module is recommended instead. .. function:: posix_spawn(path, argv, env, *, file_actions=None, \ @@ -5340,9 +5339,8 @@ written in Python, such as a mail server's external command delivery program. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. - .. deprecated:: 3.14 - These functions are :term:`soft deprecated` and should no longer be used - to write new code. The :mod:`subprocess` module is recommended instead. + .. soft-deprecated:: 3.14 + The :mod:`subprocess` module is recommended instead. .. data:: P_NOWAIT diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 7e0a00cba2f..6ed285c4b11 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -931,7 +931,6 @@ Functions .. function:: prefixmatch(pattern, string, flags=0) -.. function:: match(pattern, string, flags=0) If zero or more characters at the beginning of *string* match the regular expression *pattern*, return a corresponding :class:`~re.Match`. Return @@ -954,7 +953,11 @@ Functions :func:`~re.match`. Use that name when you need to retain compatibility with older Python versions. - .. deprecated:: 3.15 + .. versionadded:: 3.15 + +.. function:: match(pattern, string, flags=0) + + .. soft-deprecated:: 3.15 :func:`~re.match` has been :term:`soft deprecated` in favor of the alternate :func:`~re.prefixmatch` name of this API which is more explicitly descriptive. Use it to better @@ -1285,7 +1288,6 @@ Regular expression objects .. method:: Pattern.prefixmatch(string[, pos[, endpos]]) -.. method:: Pattern.match(string[, pos[, endpos]]) If zero or more characters at the *beginning* of *string* match this regular expression, return a corresponding :class:`~re.Match`. Return ``None`` if the @@ -1310,7 +1312,11 @@ Regular expression objects :meth:`~Pattern.match`. Use that name when you need to retain compatibility with older Python versions. - .. deprecated:: 3.15 + .. versionadded:: 3.15 + +.. method:: Pattern.match(string[, pos[, endpos]]) + + .. soft-deprecated:: 3.15 :meth:`~Pattern.match` has been :term:`soft deprecated` in favor of the alternate :meth:`~Pattern.prefixmatch` name of this API which is more explicitly descriptive. Use it to @@ -1794,8 +1800,8 @@ while new code should prefer :func:`!prefixmatch`. .. versionadded:: 3.15 :func:`!prefixmatch` -.. deprecated:: 3.15 - :func:`!match` is :term:`soft deprecated` +.. soft-deprecated:: 3.15 + :func:`!match` Making a phonebook ^^^^^^^^^^^^^^^^^^ diff --git a/Doc/tools/extensions/changes.py b/Doc/tools/extensions/changes.py index 8de5e7f78c6..02dc51b3a76 100644 --- a/Doc/tools/extensions/changes.py +++ b/Doc/tools/extensions/changes.py @@ -2,8 +2,10 @@ from __future__ import annotations -from typing import TYPE_CHECKING +import re +from docutils import nodes +from sphinx import addnodes from sphinx.domains.changeset import ( VersionChange, versionlabel_classes, @@ -11,6 +13,7 @@ ) from sphinx.locale import _ as sphinx_gettext +TYPE_CHECKING = False if TYPE_CHECKING: from docutils.nodes import Node from sphinx.application import Sphinx @@ -73,6 +76,76 @@ def run(self) -> list[Node]: versionlabel_classes[self.name] = "" +class SoftDeprecated(PyVersionChange): + """Directive for soft deprecations that auto-links to the glossary term. + + Usage:: + + .. soft-deprecated:: 3.15 + + Use :func:`new_thing` instead. + + Renders as: "Soft deprecated since version 3.15: Use new_thing() instead." + with "Soft deprecated" linking to the glossary definition. + """ + + _TERM_RE = re.compile(r":term:`([^`]+)`") + + def run(self) -> list[Node]: + versionlabels[self.name] = sphinx_gettext( + ":term:`Soft deprecated` since version %s" + ) + versionlabel_classes[self.name] = "soft-deprecated" + try: + result = super().run() + finally: + versionlabels[self.name] = "" + versionlabel_classes[self.name] = "" + + for node in result: + # Add "versionchanged" class so existing theme CSS applies + node["classes"] = node.get("classes", []) + ["versionchanged"] + # Replace the plain-text "Soft deprecated" with a glossary reference + for inline in node.findall(nodes.inline): + if "versionmodified" in inline.get("classes", []): + self._add_glossary_link(inline) + + return result + + @classmethod + def _add_glossary_link(cls, inline: nodes.inline) -> None: + """Replace :term:`soft deprecated` text with a cross-reference to the + 'Soft deprecated' glossary entry.""" + for child in inline.children: + if not isinstance(child, nodes.Text): + continue + + text = str(child) + match = cls._TERM_RE.search(text) + if match is None: + continue + + ref = addnodes.pending_xref( + "", + nodes.Text(match.group(1)), + refdomain="std", + reftype="term", + reftarget="soft deprecated", + refwarn=True, + ) + + start, end = match.span() + new_nodes: list[nodes.Node] = [] + if start > 0: + new_nodes.append(nodes.Text(text[:start])) + new_nodes.append(ref) + if end < len(text): + new_nodes.append(nodes.Text(text[end:])) + + child.parent.replace(child, new_nodes) + break + + def setup(app: Sphinx) -> ExtensionMetadata: # Override Sphinx's directives with support for 'next' app.add_directive("versionadded", PyVersionChange, override=True) @@ -83,6 +156,9 @@ def setup(app: Sphinx) -> ExtensionMetadata: # Register the ``.. deprecated-removed::`` directive app.add_directive("deprecated-removed", DeprecatedRemoved) + # Register the ``.. soft-deprecated::`` directive + app.add_directive("soft-deprecated", SoftDeprecated) + return { "version": "1.0", "parallel_read_safe": True, diff --git a/Doc/tools/removed-ids.txt b/Doc/tools/removed-ids.txt index f3cd8bf0ef5..7bffbb8d861 100644 --- a/Doc/tools/removed-ids.txt +++ b/Doc/tools/removed-ids.txt @@ -1 +1,5 @@ # HTML IDs excluded from the check-html-ids.py check. + +# Remove from here in 3.16 +c-api/allocation.html: deprecated-aliases +c-api/file.html: deprecated-api diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html index 75f6607d8f3..699e518801c 100644 --- a/Doc/tools/templates/dummy.html +++ b/Doc/tools/templates/dummy.html @@ -29,6 +29,7 @@ {% trans %}Deprecated since version %s, will be removed in version %s{% endtrans %} {% trans %}Deprecated since version %s, removed in version %s{% endtrans %} +{% trans %}:term:`Soft deprecated` since version %s{% endtrans %} In docsbuild-scripts, when rewriting indexsidebar.html with actual versions: From d61fcf834d197f0113a6a507fdbecc1545d9d483 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 18 Apr 2026 11:56:56 +0200 Subject: [PATCH 010/152] gh-148688: Fix _BlocksOutputBuffer_Finish() double free (#148689) If _BlocksOutputBuffer_Finish() fails (memory allocation failure), PyBytesWriter_Discard() is called on the writer. Then if _BlocksOutputBuffer_OnError() is called, it calls again PyBytesWriter_Discard() causing a double free. Fix _BlocksOutputBuffer_Finish() by setting buffer->writer to NULL, so _BlocksOutputBuffer_OnError() does nothing instead of calling PyBytesWriter_Discard() again. --- Include/internal/pycore_blocks_output_buffer.h | 7 +++++-- .../Library/2026-04-17-16-31-58.gh-issue-148688.vVugFn.rst | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-04-17-16-31-58.gh-issue-148688.vVugFn.rst diff --git a/Include/internal/pycore_blocks_output_buffer.h b/Include/internal/pycore_blocks_output_buffer.h index 016e7a18665..322c1e93344 100644 --- a/Include/internal/pycore_blocks_output_buffer.h +++ b/Include/internal/pycore_blocks_output_buffer.h @@ -242,9 +242,12 @@ static inline PyObject * _BlocksOutputBuffer_Finish(_BlocksOutputBuffer *buffer, const Py_ssize_t avail_out) { + PyObject *obj; assert(buffer->writer != NULL); - return PyBytesWriter_FinishWithSize(buffer->writer, - buffer->allocated - avail_out); + obj = PyBytesWriter_FinishWithSize(buffer->writer, + buffer->allocated - avail_out); + buffer->writer = NULL; + return obj; } /* Clean up the buffer when an error occurred. */ diff --git a/Misc/NEWS.d/next/Library/2026-04-17-16-31-58.gh-issue-148688.vVugFn.rst b/Misc/NEWS.d/next/Library/2026-04-17-16-31-58.gh-issue-148688.vVugFn.rst new file mode 100644 index 00000000000..1e367716e5a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-17-16-31-58.gh-issue-148688.vVugFn.rst @@ -0,0 +1,2 @@ +:mod:`bz2`, :mod:`compression.zstd`, :mod:`lzma`, :mod:`zlib`: Fix a double +free on memory allocation failure. Patch by Victor Stinner. From 7ce737ea11919aebf7eef174f910759e74d0ea50 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 18 Apr 2026 15:11:14 +0300 Subject: [PATCH 011/152] gh-148653: Fix reference leaks in test_marshal introduced in gh-148698 (GH-148725) --- Lib/test/test_marshal.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 9ec37d27dfa..9c4d91c456d 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -357,6 +357,9 @@ def f(): code = f.__code__ a = [] code = code.replace(co_consts=code.co_consts + (a,)) + # This test creates a reference loop which leads to reference leaks, + # so we need to break the loop manually. See gh-148722. + self.addCleanup(a.clear) a.append(code) for v in range(marshal.version + 1): self.assertRaises(ValueError, marshal.dumps, code, v) @@ -410,10 +413,12 @@ def test_loads_abnormal_reference_loops(self): self.assertIs(a[0][None], a) # Direct self-reference which cannot be created in Python. - data = b'\xa8\x01\x00\x00\x00r\x00\x00\x00\x00' # (,) - a = marshal.loads(data) - self.assertIsInstance(a, tuple) - self.assertIs(a[0], a) + # This creates a reference loop which cannot be collected. + if False: + data = b'\xa8\x01\x00\x00\x00r\x00\x00\x00\x00' # (,) + a = marshal.loads(data) + self.assertIsInstance(a, tuple) + self.assertIs(a[0], a) # Direct self-references which cannot be created in Python # because of unhashability. From d81599eeb73b4b8adcbcd5a1532c175d92fbf526 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Sat, 18 Apr 2026 11:32:22 -0700 Subject: [PATCH 012/152] gh-148659: Export a few more functions required for external JITs (#148704) Export a few more functions required for external JITs --- Include/internal/pycore_genobject.h | 3 +++ Include/internal/pycore_instruments.h | 9 ++++++++ Objects/genobject.c | 11 ++++----- Python/instrumentation.c | 32 +++++++++++++-------------- 4 files changed, 32 insertions(+), 23 deletions(-) diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index a3badb59cb7..2c264c39ae9 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -33,6 +33,9 @@ PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); PyAPI_FUNC(PyObject *)_PyCoro_GetAwaitableIter(PyObject *o); PyAPI_FUNC(PyObject *)_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *); +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyCoro_ComputeOrigin(int origin_depth, _PyInterpreterFrame *current_frame); + extern PyTypeObject _PyCoroWrapper_Type; extern PyTypeObject _PyAsyncGenWrappedValue_Type; extern PyTypeObject _PyAsyncGenAThrow_Type; diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 1da8237e93f..cb1f50e441c 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -122,6 +122,15 @@ typedef struct _PyCoMonitoringData { extern int _Py_Instrumentation_GetLine(PyCodeObject *code, _PyCoLineInstrumentationData *line_data, int index); +static inline uint8_t +_PyCode_GetOriginalOpcode(_PyCoLineInstrumentationData *line_data, int index) +{ + return line_data->data[index*line_data->bytes_per_entry]; +} + +// Exported for external JIT support +PyAPI_FUNC(uint8_t) _PyCode_Deinstrument(uint8_t opcode); + #ifdef __cplusplus } #endif diff --git a/Objects/genobject.c b/Objects/genobject.c index 2895833b4ff..2bbe79c253d 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1110,9 +1110,6 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) return (PyObject *)gen; } -static PyObject * -compute_cr_origin(int origin_depth, _PyInterpreterFrame *current_frame); - PyObject * _Py_MakeCoro(PyFunctionObject *func) { @@ -1150,7 +1147,7 @@ _Py_MakeCoro(PyFunctionObject *func) assert(frame); assert(_PyFrame_IsIncomplete(frame)); frame = _PyFrame_GetFirstComplete(frame->previous); - PyObject *cr_origin = compute_cr_origin(origin_depth, frame); + PyObject *cr_origin = _PyCoro_ComputeOrigin(origin_depth, frame); ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin; if (!cr_origin) { Py_DECREF(coro); @@ -1535,8 +1532,8 @@ PyTypeObject _PyCoroWrapper_Type = { 0, /* tp_free */ }; -static PyObject * -compute_cr_origin(int origin_depth, _PyInterpreterFrame *current_frame) +PyObject * +_PyCoro_ComputeOrigin(int origin_depth, _PyInterpreterFrame *current_frame) { _PyInterpreterFrame *frame = current_frame; /* First count how many frames we have */ @@ -1581,7 +1578,7 @@ PyCoro_New(PyFrameObject *f, PyObject *name, PyObject *qualname) if (origin_depth == 0) { ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL; } else { - PyObject *cr_origin = compute_cr_origin(origin_depth, _PyEval_GetFrame()); + PyObject *cr_origin = _PyCoro_ComputeOrigin(origin_depth, _PyEval_GetFrame()); ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin; if (!cr_origin) { Py_DECREF(coro); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 256e2a3d3a2..4041aa0d8ae 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -185,6 +185,12 @@ opcode_has_event(int opcode) ); } +uint8_t +_PyCode_Deinstrument(uint8_t opcode) +{ + return DE_INSTRUMENT[opcode]; +} + static inline bool is_instrumented(int opcode) { @@ -330,12 +336,6 @@ _PyInstruction_GetLength(PyCodeObject *code, int offset) return 1 + _PyOpcode_Caches[inst.op.code]; } -static inline uint8_t -get_original_opcode(_PyCoLineInstrumentationData *line_data, int index) -{ - return line_data->data[index*line_data->bytes_per_entry]; -} - static inline uint8_t * get_original_opcode_ptr(_PyCoLineInstrumentationData *line_data, int index) { @@ -401,7 +401,7 @@ dump_instrumentation_data_lines(PyCodeObject *code, _PyCoLineInstrumentationData fprintf(out, ", lines = NULL"); } else { - int opcode = get_original_opcode(lines, i); + int opcode = _PyCode_GetOriginalOpcode(lines, i); int line_delta = get_line_delta(lines, i); if (opcode == 0) { fprintf(out, ", lines = {original_opcode = No LINE (0), line_delta = %d)", line_delta); @@ -571,7 +571,7 @@ sanity_check_instrumentation(PyCodeObject *code) } if (opcode == INSTRUMENTED_LINE) { CHECK(data->lines); - opcode = get_original_opcode(data->lines, i); + opcode = _PyCode_GetOriginalOpcode(data->lines, i); CHECK(valid_opcode(opcode)); CHECK(opcode != END_FOR); CHECK(opcode != RESUME); @@ -588,7 +588,7 @@ sanity_check_instrumentation(PyCodeObject *code) * *and* we are executing a INSTRUMENTED_LINE instruction * that has de-instrumented itself, then we will execute * an invalid INSTRUMENTED_INSTRUCTION */ - CHECK(get_original_opcode(data->lines, i) != INSTRUMENTED_INSTRUCTION); + CHECK(_PyCode_GetOriginalOpcode(data->lines, i) != INSTRUMENTED_INSTRUCTION); } if (opcode == INSTRUMENTED_INSTRUCTION) { CHECK(data->per_instruction_opcodes[i] != 0); @@ -603,7 +603,7 @@ sanity_check_instrumentation(PyCodeObject *code) } CHECK(active_monitors.tools[event] != 0); } - if (data->lines && get_original_opcode(data->lines, i)) { + if (data->lines && _PyCode_GetOriginalOpcode(data->lines, i)) { int line1 = compute_line(code, get_line_delta(data->lines, i)); int line2 = _PyCode_CheckLineNumber(i*sizeof(_Py_CODEUNIT), &range); CHECK(line1 == line2); @@ -655,7 +655,7 @@ _Py_GetBaseCodeUnit(PyCodeObject *code, int i) return inst; } if (opcode == INSTRUMENTED_LINE) { - opcode = get_original_opcode(code->_co_monitoring->lines, i); + opcode = _PyCode_GetOriginalOpcode(code->_co_monitoring->lines, i); } if (opcode == INSTRUMENTED_INSTRUCTION) { opcode = code->_co_monitoring->per_instruction_opcodes[i]; @@ -714,7 +714,7 @@ de_instrument_line(PyCodeObject *code, _Py_CODEUNIT *bytecode, _PyCoMonitoringDa return; } _PyCoLineInstrumentationData *lines = monitoring->lines; - int original_opcode = get_original_opcode(lines, i); + int original_opcode = _PyCode_GetOriginalOpcode(lines, i); if (original_opcode == INSTRUMENTED_INSTRUCTION) { set_original_opcode(lines, i, monitoring->per_instruction_opcodes[i]); } @@ -1391,7 +1391,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, Py_DECREF(line_obj); uint8_t original_opcode; done: - original_opcode = get_original_opcode(line_data, i); + original_opcode = _PyCode_GetOriginalOpcode(line_data, i); assert(original_opcode != 0); assert(original_opcode != INSTRUMENTED_LINE); assert(_PyOpcode_Deopt[original_opcode] == original_opcode); @@ -1464,7 +1464,7 @@ initialize_tools(PyCodeObject *code) int opcode = instr->op.code; assert(opcode != ENTER_EXECUTOR); if (opcode == INSTRUMENTED_LINE) { - opcode = get_original_opcode(code->_co_monitoring->lines, i); + opcode = _PyCode_GetOriginalOpcode(code->_co_monitoring->lines, i); } if (opcode == INSTRUMENTED_INSTRUCTION) { opcode = code->_co_monitoring->per_instruction_opcodes[i]; @@ -1849,7 +1849,7 @@ force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) if (removed_line_tools) { _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; for (int i = code->_co_firsttraceable; i < code_len;) { - if (get_original_opcode(line_data, i)) { + if (_PyCode_GetOriginalOpcode(line_data, i)) { remove_line_tools(code, i, removed_line_tools); } i += _PyInstruction_GetLength(code, i); @@ -1876,7 +1876,7 @@ force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) if (new_line_tools) { _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; for (int i = code->_co_firsttraceable; i < code_len;) { - if (get_original_opcode(line_data, i)) { + if (_PyCode_GetOriginalOpcode(line_data, i)) { add_line_tools(code, i, new_line_tools); } i += _PyInstruction_GetLength(code, i); From 28b8d5ffccd355dad7c8fd2fbf7b7552083c7e14 Mon Sep 17 00:00:00 2001 From: John Seong <39040639+sandole@users.noreply.github.com> Date: Sun, 19 Apr 2026 02:50:17 +0800 Subject: [PATCH 013/152] gh-133403: Add type annotations to generate_levenshtein_examples.py (#143317) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/mypy.yml | 1 + Tools/build/generate_levenshtein_examples.py | 8 ++++---- Tools/build/mypy.ini | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index e5a5b3939e5..7f6571ef954 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -19,6 +19,7 @@ on: - "Tools/build/consts_getter.py" - "Tools/build/deepfreeze.py" - "Tools/build/generate-build-details.py" + - "Tools/build/generate_levenshtein_examples.py" - "Tools/build/generate_sbom.py" - "Tools/build/generate_stdlib_module_names.py" - "Tools/build/mypy.ini" diff --git a/Tools/build/generate_levenshtein_examples.py b/Tools/build/generate_levenshtein_examples.py index 30dcc7cf1a1..2396c8040ca 100644 --- a/Tools/build/generate_levenshtein_examples.py +++ b/Tools/build/generate_levenshtein_examples.py @@ -13,7 +13,7 @@ _CASE_COST = 1 -def _substitution_cost(ch_a, ch_b): +def _substitution_cost(ch_a: str, ch_b: str) -> int: if ch_a == ch_b: return 0 if ch_a.lower() == ch_b.lower(): @@ -22,7 +22,7 @@ def _substitution_cost(ch_a, ch_b): @lru_cache(None) -def levenshtein(a, b): +def levenshtein(a: str, b: str) -> int: if not a or not b: return (len(a) + len(b)) * _MOVE_COST option1 = levenshtein(a[:-1], b[:-1]) + _substitution_cost(a[-1], b[-1]) @@ -31,7 +31,7 @@ def levenshtein(a, b): return min(option1, option2, option3) -def main(): +def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('output_path', metavar='FILE', type=str) parser.add_argument('--overwrite', dest='overwrite', action='store_const', @@ -48,7 +48,7 @@ def main(): ) return - examples = set() + examples: set[tuple[str, str, int]] = set() # Create a lot of non-empty examples, which should end up with a Gauss-like # distribution for even costs (moves) and odd costs (case substitutions). while len(examples) < 9990: diff --git a/Tools/build/mypy.ini b/Tools/build/mypy.ini index 7d341afd1cd..5465e2d4b61 100644 --- a/Tools/build/mypy.ini +++ b/Tools/build/mypy.ini @@ -9,6 +9,7 @@ files = Tools/build/consts_getter.py, Tools/build/deepfreeze.py, Tools/build/generate-build-details.py, + Tools/build/generate_levenshtein_examples.py, Tools/build/generate_sbom.py, Tools/build/generate_stdlib_module_names.py, Tools/build/verify_ensurepip_wheels.py, From 4b3330813760a3e3c75cd03023d252742168683b Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Sat, 18 Apr 2026 19:51:58 +0100 Subject: [PATCH 014/152] gh-148406: Fix annotations of _colorize.FancyCompleter (#148408) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Lib/_colorize.py | 31 ++++++++++++++++--------------- Lib/test/test__colorize.py | 10 ++++++++++ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 478f8189491..852ad38f086 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -1,3 +1,4 @@ +import builtins import os import sys @@ -202,25 +203,25 @@ class Difflib(ThemeSection): @dataclass(frozen=True, kw_only=True) class FancyCompleter(ThemeSection): # functions and methods - function: str = ANSIColors.BOLD_BLUE - builtin_function_or_method: str = ANSIColors.BOLD_BLUE - method: str = ANSIColors.BOLD_CYAN - method_wrapper: str = ANSIColors.BOLD_CYAN - wrapper_descriptor: str = ANSIColors.BOLD_CYAN - method_descriptor: str = ANSIColors.BOLD_CYAN + function: builtins.str = ANSIColors.BOLD_BLUE + builtin_function_or_method: builtins.str = ANSIColors.BOLD_BLUE + method: builtins.str = ANSIColors.BOLD_CYAN + method_wrapper: builtins.str = ANSIColors.BOLD_CYAN + wrapper_descriptor: builtins.str = ANSIColors.BOLD_CYAN + method_descriptor: builtins.str = ANSIColors.BOLD_CYAN # numbers - int: str = ANSIColors.BOLD_YELLOW - float: str = ANSIColors.BOLD_YELLOW - complex: str = ANSIColors.BOLD_YELLOW - bool: str = ANSIColors.BOLD_YELLOW + int: builtins.str = ANSIColors.BOLD_YELLOW + float: builtins.str = ANSIColors.BOLD_YELLOW + complex: builtins.str = ANSIColors.BOLD_YELLOW + bool: builtins.str = ANSIColors.BOLD_YELLOW # others - type: str = ANSIColors.BOLD_MAGENTA - module: str = ANSIColors.CYAN - NoneType: str = ANSIColors.GREY - bytes: str = ANSIColors.BOLD_GREEN - str: str = ANSIColors.BOLD_GREEN + type: builtins.str = ANSIColors.BOLD_MAGENTA + module: builtins.str = ANSIColors.CYAN + NoneType: builtins.str = ANSIColors.GREY + bytes: builtins.str = ANSIColors.BOLD_GREEN + str: builtins.str = ANSIColors.BOLD_GREEN @dataclass(frozen=True, kw_only=True) diff --git a/Lib/test/test__colorize.py b/Lib/test/test__colorize.py index 67e0595943d..0353ff7530b 100644 --- a/Lib/test/test__colorize.py +++ b/Lib/test/test__colorize.py @@ -5,6 +5,7 @@ import unittest import unittest.mock import _colorize +from test.support import cpython_only, import_helper from test.support.os_helper import EnvironmentVarGuard @@ -22,6 +23,15 @@ def supports_virtual_terminal(): return contextlib.nullcontext() +class TestImportTime(unittest.TestCase): + + @cpython_only + def test_lazy_import(self): + import_helper.ensure_lazy_imports( + "_colorize", {"copy", "re"} + ) + + class TestTheme(unittest.TestCase): def test_attributes(self): From 9e236522302a003ae659a825da74501f3aa1c4c1 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sun, 19 Apr 2026 12:21:17 +0300 Subject: [PATCH 015/152] Prevent GitHub's web conflict editor from converting LF to CRLF (#148739) --- .gitattributes | 3 +++ ...26-02-22-00-00-00.gh-issue-145105.csv-reader-reentrant.rst | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitattributes b/.gitattributes index b8189f12ded..f4d65dfd1df 100644 --- a/.gitattributes +++ b/.gitattributes @@ -34,6 +34,9 @@ Lib/test/xmltestdata/* noeol Lib/venv/scripts/common/activate text eol=lf Lib/venv/scripts/posix/* text eol=lf +# Prevent GitHub's web conflict editor from converting LF to CRLF +*.rst text eol=lf + # CRLF files [attr]dos text eol=crlf diff --git a/Misc/NEWS.d/next/Library/2026-02-22-00-00-00.gh-issue-145105.csv-reader-reentrant.rst b/Misc/NEWS.d/next/Library/2026-02-22-00-00-00.gh-issue-145105.csv-reader-reentrant.rst index bc61cc43a5a..1c2e06c86f6 100644 --- a/Misc/NEWS.d/next/Library/2026-02-22-00-00-00.gh-issue-145105.csv-reader-reentrant.rst +++ b/Misc/NEWS.d/next/Library/2026-02-22-00-00-00.gh-issue-145105.csv-reader-reentrant.rst @@ -1,2 +1,2 @@ -Fix crash in :mod:`csv` reader when iterating with a re-entrant iterator -that calls :func:`next` on the same reader from within ``__next__``. +Fix crash in :mod:`csv` reader when iterating with a re-entrant iterator +that calls :func:`next` on the same reader from within ``__next__``. From ad7d3616c6cc21c5ec032a726e4c5e819628aa6e Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Sun, 19 Apr 2026 08:13:47 -0400 Subject: [PATCH 016/152] gh-121946: Use clang-20 for TSan build (#148570) --- .github/workflows/reusable-san.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index 9d4f412cfcf..33f6f0ef455 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -40,17 +40,15 @@ jobs: # Install clang wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh + sudo ./llvm.sh 20 + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-20 100 + sudo update-alternatives --set clang /usr/bin/clang-20 + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-20 100 + sudo update-alternatives --set clang++ /usr/bin/clang++-20 if [ "${SANITIZER}" = "TSan" ]; then - sudo ./llvm.sh 17 # gh-121946: llvm-18 package is temporarily broken - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 - sudo update-alternatives --set clang /usr/bin/clang-17 - sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 - sudo update-alternatives --set clang++ /usr/bin/clang++-17 # Reduce ASLR to avoid TSan crashing sudo sysctl -w vm.mmap_rnd_bits=28 - else - sudo ./llvm.sh 20 fi - name: Sanitizer option setup From a8c9aa924b9795facc6bf1cafb37d2832289c9e6 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sun, 19 Apr 2026 21:42:23 +0300 Subject: [PATCH 017/152] gh-133879: Copyedit "What's new in Python 3.15" (#148686) --- Doc/deprecations/pending-removal-in-3.17.rst | 2 +- Doc/whatsnew/3.15.rst | 126 +++++++++---------- 2 files changed, 57 insertions(+), 71 deletions(-) diff --git a/Doc/deprecations/pending-removal-in-3.17.rst b/Doc/deprecations/pending-removal-in-3.17.rst index ea9fb93ddd8..952ffad6435 100644 --- a/Doc/deprecations/pending-removal-in-3.17.rst +++ b/Doc/deprecations/pending-removal-in-3.17.rst @@ -35,7 +35,7 @@ Pending removal in Python 3.17 - Passing non-ascii *encoding* names to :func:`encodings.normalize_encoding` is deprecated and scheduled for removal in Python 3.17. - (Contributed by Stan Ulbrych in :gh:`136702`) + (Contributed by Stan Ulbrych in :gh:`136702`.) * :mod:`typing`: diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 7cf4dc3701f..c4dac339be6 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -142,7 +142,7 @@ In the case where loading a lazily imported module fails (for example, if the module does not exist), Python raises the exception at the point of first use rather than at import time. The associated traceback includes both the location where the name was accessed and the original import statement, -making it straightforward to diagnose & debug the failure. +making it straightforward to diagnose and debug the failure. For cases where you want to enable lazy loading globally without modifying source code, Python provides the :option:`-X lazy_imports <-X>` command-line @@ -451,14 +451,36 @@ Improved error messages Running this code now produces a clearer suggestion: - .. code-block:: pycon + .. code-block:: pytb Traceback (most recent call last): - File "/home/pablogsal/github/python/main/lel.py", line 42, in - print(container.area) - ^^^^^^^^^^^^^^ + File "/home/pablogsal/github/python/main/lel.py", line 42, in + print(container.area) + ^^^^^^^^^^^^^^ AttributeError: 'Container' object has no attribute 'area'. Did you mean '.inner.area' instead of '.area'? +* The interpreter now tries to provide a suggestion when + :func:`delattr` fails due to a missing attribute. + When an attribute name that closely resembles an existing attribute is used, + the interpreter will suggest the correct attribute name in the error message. + For example: + + .. doctest:: + + >>> class A: + ... pass + >>> a = A() + >>> a.abcde = 1 + >>> del a.abcdf # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'A' object has no attribute 'abcdf'. Did you mean: 'abcde'? + + (Contributed by Nikita Sobolev and Pranjal Prajapati in :gh:`136588`.) + +* Several error messages incorrectly using the term "argument" have been corrected. + (Contributed by Stan Ulbrych in :gh:`133382`.) + Other language changes ====================== @@ -490,28 +512,6 @@ Other language changes (Contributed by Adam Turner in :gh:`133711`; PEP 686 written by Inada Naoki.) -* Several error messages incorrectly using the term "argument" have been corrected. - (Contributed by Stan Ulbrych in :gh:`133382`.) - -* The interpreter now tries to provide a suggestion when - :func:`delattr` fails due to a missing attribute. - When an attribute name that closely resembles an existing attribute is used, - the interpreter will suggest the correct attribute name in the error message. - For example: - - .. doctest:: - - >>> class A: - ... pass - >>> a = A() - >>> a.abcde = 1 - >>> del a.abcdf # doctest: +ELLIPSIS - Traceback (most recent call last): - ... - AttributeError: 'A' object has no attribute 'abcdf'. Did you mean: 'abcde'? - - (Contributed by Nikita Sobolev and Pranjal Prajapati in :gh:`136588`.) - * Unraisable exceptions are now highlighted with color by default. This can be controlled by :ref:`environment variables `. (Contributed by Peter Bierma in :gh:`134170`.) @@ -708,7 +708,7 @@ base64 (Contributed by Serhiy Storchaka in :gh:`143214` and :gh:`146431`.) * Added the *ignorechars* parameter in :func:`~base64.b16decode`, - :func:`~base64.b32decode`, :func:`~base64.b32hexdecode`, + :func:`~base64.b32decode`, :func:`~base64.b32hexdecode`, :func:`~base64.b64decode`, :func:`~base64.b85decode`, and :func:`~base64.z85decode`. (Contributed by Serhiy Storchaka in :gh:`144001` and :gh:`146431`.) @@ -880,13 +880,13 @@ inspect json ---- -* Add the *array_hook* parameter to :func:`~json.load` and +* Add the *array_hook* parameter to :func:`~json.load` and :func:`~json.loads` functions: allow a callback for JSON literal array types to customize Python lists in the resulting decoded object. Passing combined :class:`frozendict` to *object_pairs_hook* param and :class:`tuple` to ``array_hook`` will yield a deeply nested immutable Python structure representing the JSON data. - (Contributed by Joao S. O. Bueno in :gh:`146440`) + (Contributed by Joao S. O. Bueno in :gh:`146440`.) locale @@ -914,23 +914,11 @@ math mimetypes --------- -* Add ``application/dicom`` MIME type for ``.dcm`` extension. - (Contributed by Benedikt Johannes in :gh:`144217`.) -* Add ``application/efi``. (Contributed by Charlie Lin in :gh:`145720`.) -* Add ``application/node`` MIME type for ``.cjs`` extension. - (Contributed by John Franey in :gh:`140937`.) -* Add ``application/toml``. (Contributed by Gil Forcada in :gh:`139959`.) -* Add ``application/sql`` and ``application/vnd.sqlite3``. - (Contributed by Charlie Lin in :gh:`145698`.) -* Add the following MIME types: - - - ``application/vnd.ms-cab-compressed`` for ``.cab`` extension - - ``application/vnd.ms-htmlhelp`` for ``.chm`` extension - - ``application/vnd.ms-officetheme`` for ``.thmx`` extension - - (Contributed by Charlie Lin in :gh:`145718`.) - -* Add ``image/jxl``. (Contributed by Foolbar in :gh:`144213`.) +* Add more MIME types. + (Contributed by Benedikt Johannes, Charlie Lin, Foolbar, Gil Forcada and + John Franey + in :gh:`144217`, :gh:`145720`, :gh:`140937`, :gh:`139959`, :gh:`145698`, + :gh:`145718` and :gh:`144213`.) * Rename ``application/x-texinfo`` to ``application/texinfo``. (Contributed by Charlie Lin in :gh:`140165`.) * Changed the MIME type for ``.ai`` files to ``application/pdf``. @@ -1114,7 +1102,7 @@ subprocess If none of these mechanisms are available, the function falls back to the traditional busy loop (non-blocking call and short sleeps). - (Contributed by Giampaolo Rodola in :gh:`83069`). + (Contributed by Giampaolo Rodola in :gh:`83069`.) symtable @@ -1171,7 +1159,7 @@ timeit * Make the target time of :meth:`timeit.Timer.autorange` configurable and add ``--target-time`` option to the command-line interface. - (Contributed by Alessandro Cucci and Miikka Koskinen in :gh:`140283`.) + (Contributed by Alessandro Cucci and Miikka Koskinen in :gh:`80642`.) tkinter @@ -1216,10 +1204,7 @@ tomllib Previously an inline table had to be on a single line and couldn't end with a trailing comma. This is now relaxed so that the following is valid: - .. syntax highlighting needs TOML 1.1.0 support in Pygments, - see https://github.com/pygments/pygments/issues/3026 - - .. code-block:: text + .. code-block:: toml tbl = { key = "a string", @@ -1231,7 +1216,7 @@ tomllib - Add ``\xHH`` notation to basic strings for codepoints under 255, and the ``\e`` escape for the escape character: - .. code-block:: text + .. code-block:: toml null = "null byte: \x00; letter a: \x61" csi = "\e[" @@ -1239,7 +1224,7 @@ tomllib - Seconds in datetime and time values are now optional. The following are now valid: - .. code-block:: text + .. code-block:: toml dt = 2010-02-03 14:15 t = 14:15 @@ -1421,7 +1406,7 @@ Optimizations ============= * ``mimalloc`` is now used as the default allocator for - for raw memory allocations such as via :c:func:`PyMem_RawMalloc` + raw memory allocations such as via :c:func:`PyMem_RawMalloc` for better performance on :term:`free-threaded builds `. (Contributed by Kumar Aditya in :gh:`144914`.) @@ -1440,7 +1425,7 @@ base64 & binascii * Implementation for Base32 has been rewritten in C. Encoding and decoding is now two orders of magnitude faster. - (Contributed by James Seo in :gh:`146192`) + (Contributed by James Seo in :gh:`146192`.) csv @@ -1534,7 +1519,7 @@ The JIT compiler's machine code generator now produces better machine code for x86-64 and AArch64 macOS and Linux targets. In general, users should experience lower memory usage for generated machine code and more efficient machine code versus 3.14. -(Contributed by Brandt Bucher in :gh:`136528` and :gh:`136528`. +(Contributed by Brandt Bucher in :gh:`136528` and :gh:`135905`. Implementation for AArch64 contributed by Mark Shannon in :gh:`139855`. Additional optimizations for AArch64 contributed by Mark Shannon and Diego Russo in :gh:`140683` and :gh:`142305`.) @@ -1557,6 +1542,14 @@ collections.abc deprecated since Python 3.12, and is scheduled for removal in Python 3.17. +ctypes +------ + +* Removed the undocumented function :func:`!ctypes.SetPointerType`, + which has been deprecated since Python 3.13. + (Contributed by Bénédikt Tran in :gh:`133866`.) + + datetime -------- @@ -1566,14 +1559,6 @@ datetime (Contributed by Stan Ulbrych and Gregory P. Smith in :gh:`70647`.) -ctypes ------- - -* Removed the undocumented function :func:`!ctypes.SetPointerType`, - which has been deprecated since Python 3.13. - (Contributed by Bénédikt Tran in :gh:`133866`.) - - glob ---- @@ -1658,6 +1643,9 @@ typing or ``TD = TypedDict("TD", {})`` instead. (Contributed by Bénédikt Tran in :gh:`133823`.) +* Deprecated :func:`!typing.no_type_check_decorator` has been removed. + (Contributed by Nikita Sobolev in :gh:`133601`.) + wave ---- @@ -1773,8 +1761,6 @@ New deprecations :func:`issubclass`, but warnings were not previously emitted if it was merely imported or accessed from the :mod:`!typing` module. - * Deprecated :func:`!typing.no_type_check_decorator` has been removed. - (Contributed by Nikita Sobolev in :gh:`133601`.) * ``__version__`` @@ -2061,8 +2047,8 @@ Deprecated C APIs - :c:macro:`Py_ALIGNED`: Prefer ``alignas`` instead. - :c:macro:`PY_FORMAT_SIZE_T`: Use ``"z"`` directly. - - :c:macro:`Py_LL` & :c:macro:`Py_ULL`: - Use standard suffixes, ``LL`` & ``ULL``. + - :c:macro:`Py_LL` and :c:macro:`Py_ULL`: + Use standard suffixes, ``LL`` and ``ULL``. - :c:macro:`PY_LONG_LONG`, :c:macro:`PY_LLONG_MIN`, :c:macro:`PY_LLONG_MAX`, :c:macro:`PY_ULLONG_MAX`, :c:macro:`PY_INT32_T`, :c:macro:`PY_UINT32_T`, :c:macro:`PY_INT64_T`, :c:macro:`PY_UINT64_T`, :c:macro:`PY_SIZE_MAX`: From 82767780f8de2fc492567ceb6a590101ae3b19ad Mon Sep 17 00:00:00 2001 From: partev Date: Sun, 19 Apr 2026 17:44:08 -0400 Subject: [PATCH 018/152] gh-148779: Update Briefcase link in android.rst documentation (#148777) Use canonical beeware.org URL for link to Briefcase. --- Doc/using/android.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/using/android.rst b/Doc/using/android.rst index 45345d045dd..60a13569318 100644 --- a/Doc/using/android.rst +++ b/Doc/using/android.rst @@ -30,7 +30,7 @@ Adding Python to an Android app Most app developers should use one of the following tools, which will provide a much easier experience: -* `Briefcase `__, from the BeeWare project +* `Briefcase `__, from the BeeWare project * `Buildozer `__, from the Kivy project * `Chaquopy `__ * `pyqtdeploy `__ From e50acef0b2c2057874a9eec98c37ca6cf8ee98e1 Mon Sep 17 00:00:00 2001 From: Matthew Davis <7035647+mdavis-xyz@users.noreply.github.com> Date: Mon, 20 Apr 2026 02:05:50 +0200 Subject: [PATCH 019/152] gh-148763: Fix paramter name in `multiprocessing.connection.send_bytes/recv_bytes_into` docs (GH-126603) Doc: Fix buf argument name in multiprocessing connection send_bytes --- Doc/library/multiprocessing.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 63bc252e129..3ceb5e717c4 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1336,12 +1336,12 @@ Connection objects are usually created using Note that multiple connection objects may be polled at once by using :func:`multiprocessing.connection.wait`. - .. method:: send_bytes(buffer[, offset[, size]]) + .. method:: send_bytes(buf[, offset[, size]]) Send byte data from a :term:`bytes-like object` as a complete message. - If *offset* is given then data is read from that position in *buffer*. If - *size* is given then that many bytes will be read from buffer. Very large + If *offset* is given then data is read from that position in *buf*. If + *size* is given then that many bytes will be read from *buf*. Very large buffers (approximately 32 MiB+, though it depends on the OS) may raise a :exc:`ValueError` exception @@ -1361,18 +1361,18 @@ Connection objects are usually created using alias of :exc:`OSError`. - .. method:: recv_bytes_into(buffer[, offset]) + .. method:: recv_bytes_into(buf[, offset]) - Read into *buffer* a complete message of byte data sent from the other end + Read into *buf* a complete message of byte data sent from the other end of the connection and return the number of bytes in the message. Blocks until there is something to receive. Raises :exc:`EOFError` if there is nothing left to receive and the other end was closed. - *buffer* must be a writable :term:`bytes-like object`. If + *buf* must be a writable :term:`bytes-like object`. If *offset* is given then the message will be written into the buffer from that position. Offset must be a non-negative integer less than the - length of *buffer* (in bytes). + length of *buf* (in bytes). If the buffer is too short then a :exc:`BufferTooShort` exception is raised and the complete message is available as ``e.args[0]`` where ``e`` From a00b24ec6832f0972823fb0a453a547113fbd55f Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Mon, 20 Apr 2026 03:17:50 +0100 Subject: [PATCH 020/152] gh-148788: Update Emscripten example post move to Platforms dir (#148761) Update Emscripten example post move to Platforms dir. --- Platforms/emscripten/web_example/index.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Platforms/emscripten/web_example/index.html b/Platforms/emscripten/web_example/index.html index 9c89c9c0ed3..3a207b92015 100644 --- a/Platforms/emscripten/web_example/index.html +++ b/Platforms/emscripten/web_example/index.html @@ -663,9 +663,9 @@

Simple REPL for Python WASM

The simple REPL provides a limited Python experience in the browser. - Tools/wasm/README.md + Platforms/emscripten/README.md contains a list of known limitations and issues. Networking, subprocesses, and threading are not available. @@ -679,9 +679,9 @@

Simple REPL for Python WASM

your browser instead of using server.py as described in - Tools/wasm/README.md + Platforms/emscripten/README.md .

From bfe6f9f590849f0d9f08a6fe94a5b4e76d8ed29f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 20 Apr 2026 11:47:37 +0300 Subject: [PATCH 021/152] gh-123853: Update locale.windows_locale (GH-123901) Update the table of Windows language code identifiers (LCIDs) to protocol version 16.0 (2024-04-23). --- Lib/locale.py | 477 +++++++++++++----- ...-09-09-12-48-37.gh-issue-123853.e-zFxb.rst | 3 + 2 files changed, 349 insertions(+), 131 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-09-12-48-37.gh-issue-123853.e-zFxb.rst diff --git a/Lib/locale.py b/Lib/locale.py index e7382796905..4ff6f8c0f0a 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -1505,8 +1505,8 @@ def getpreferredencoding(do_setlocale=True): # This maps Windows language identifiers to locale strings. # # This list has been updated from -# http://msdn.microsoft.com/library/default.asp?url=/library/en-us/intl/nls_238z.asp -# to include every locale up to Windows Vista. +# https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-lcid/70feba9f-294e-491e-b6eb-56532684c37f +# to include every locale up to protocol revision 16.0 (2024-04-23). # # NOTE: this mapping is incomplete. If your language is missing, please # submit a bug report as detailed in the Python devguide at: @@ -1516,10 +1516,15 @@ def getpreferredencoding(do_setlocale=True): # windows_locale = { - 0x0436: "af_ZA", # Afrikaans - 0x041c: "sq_AL", # Albanian - 0x0484: "gsw_FR",# Alsatian - France + 0x0036: "af", # Afrikaans + 0x0436: "af_ZA", # Afrikaans - South Africa + 0x001c: "sq", # Albanian + 0x041c: "sq_AL", # Albanian - Albania + 0x0084: "gsw", # Alsatian + 0x0484: "gsw_FR", # Alsatian - France + 0x005e: "am", # Amharic 0x045e: "am_ET", # Amharic - Ethiopia + 0x0001: "ar", # Arabic 0x0401: "ar_SA", # Arabic - Saudi Arabia 0x0801: "ar_IQ", # Arabic - Iraq 0x0c01: "ar_EG", # Arabic - Egypt @@ -1533,39 +1538,72 @@ def getpreferredencoding(do_setlocale=True): 0x2c01: "ar_JO", # Arabic - Jordan 0x3001: "ar_LB", # Arabic - Lebanon 0x3401: "ar_KW", # Arabic - Kuwait - 0x3801: "ar_AE", # Arabic - United Arab Emirates + 0x3801: "ar_AE", # Arabic - U.A.E. 0x3c01: "ar_BH", # Arabic - Bahrain 0x4001: "ar_QA", # Arabic - Qatar - 0x042b: "hy_AM", # Armenian + 0x002b: "hy", # Armenian + 0x042b: "hy_AM", # Armenian - Armenia + 0x004d: "as", # Assamese 0x044d: "as_IN", # Assamese - India - 0x042c: "az_AZ", # Azeri - Latin - 0x082c: "az_AZ", # Azeri - Cyrillic - 0x046d: "ba_RU", # Bashkir - 0x042d: "eu_ES", # Basque - Russia - 0x0423: "be_BY", # Belarusian - 0x0445: "bn_IN", # Begali - 0x201a: "bs_BA", # Bosnian - Cyrillic - 0x141a: "bs_BA", # Bosnian - Latin + 0x002c: "az", # Azerbaijani (Latin) + 0x742c: "az", # Azerbaijani (Cyrillic) + 0x782c: "az", # Azerbaijani (Latin) + 0x042c: "az_AZ", # Azerbaijani (Latin) - Azerbaijan + 0x0045: "bn", # Bangla + 0x0445: "bn_IN", # Bangla - India + 0x0845: "bn_BD", # Bangla - Bangladesh + 0x006d: "ba", # Bashkir + 0x046d: "ba_RU", # Bashkir - Russia + 0x002d: "eu", # Basque + 0x042d: "eu_ES", # Basque - Spain + 0x0023: "be", # Belarusian + 0x0423: "be_BY", # Belarusian - Belarus + 0x641a: "bs", # Bosnian (Cyrillic) + 0x681a: "bs", # Bosnian (Latin) + 0x141a: "bs_BA", # Bosnian (Latin) - Bosnia and Herzegovina + 0x201a: "bs_BA", # Bosnian (Cyrillic) - Bosnia and Herzegovina + 0x781a: "bs", # Bosnian (Latin) + 0x007e: "br", # Breton 0x047e: "br_FR", # Breton - France - 0x0402: "bg_BG", # Bulgarian -# 0x0455: "my_MM", # Burmese - Not supported - 0x0403: "ca_ES", # Catalan - 0x0004: "zh_CHS",# Chinese - Simplified - 0x0404: "zh_TW", # Chinese - Taiwan - 0x0804: "zh_CN", # Chinese - PRC - 0x0c04: "zh_HK", # Chinese - Hong Kong S.A.R. - 0x1004: "zh_SG", # Chinese - Singapore - 0x1404: "zh_MO", # Chinese - Macao S.A.R. - 0x7c04: "zh_CHT",# Chinese - Traditional + 0x0002: "bg", # Bulgarian + 0x0402: "bg_BG", # Bulgarian - Bulgaria + 0x0055: "my", # Burmese + 0x0455: "my_MM", # Burmese - Myanmar + 0x0003: "ca", # Catalan + 0x0403: "ca_ES", # Catalan - Spain + 0x0803: "ca_ES", # Valencian - Spain + 0x0092: "ku", # Central Kurdish + 0x7c92: "ku", # Central Kurdish + 0x0492: "ku_IQ", # Central Kurdish - Iraq + 0x005c: "chr", # Cherokee + 0x7c5c: "chr", # Cherokee + 0x045c: "chr_US", # Cherokee - United States + 0x0004: "zh", # Chinese (Simplified) + 0x7804: "zh", # Chinese (Simplified) + 0x7c04: "zh", # Chinese (Traditional) + 0x0404: "zh_TW", # Chinese (Traditional) - Taiwan + 0x0804: "zh_CN", # Chinese (Simplified) - People's Republic of China + 0x0c04: "zh_HK", # Chinese (Traditional) - Hong Kong S.A.R. + 0x1004: "zh_SG", # Chinese (Simplified) - Singapore + 0x1404: "zh_MO", # Chinese (Traditional) - Macao S.A.R. + 0x0083: "co", # Corsican 0x0483: "co_FR", # Corsican - France - 0x041a: "hr_HR", # Croatian - 0x101a: "hr_BA", # Croatian - Bosnia - 0x0405: "cs_CZ", # Czech - 0x0406: "da_DK", # Danish - 0x048c: "gbz_AF",# Dari - Afghanistan - 0x0465: "div_MV",# Divehi - Maldives - 0x0413: "nl_NL", # Dutch - The Netherlands + 0x001a: "hr", # Croatian + 0x041a: "hr_HR", # Croatian - Croatia + 0x101a: "hr_BA", # Croatian (Latin) - Bosnia and Herzegovina + 0x0005: "cs", # Czech + 0x0405: "cs_CZ", # Czech - Czech Republic + 0x0006: "da", # Danish + 0x0406: "da_DK", # Danish - Denmark + 0x008c: "prs", # Dari + 0x048c: "prs_AF", # Dari - Afghanistan + 0x0065: "dv", # Divehi + 0x0465: "dv_MV", # Divehi - Maldives + 0x0013: "nl", # Dutch + 0x0413: "nl_NL", # Dutch - Netherlands 0x0813: "nl_BE", # Dutch - Belgium + 0x0c51: "dz_BT", # Dzongkha - Bhutan + 0x0009: "en", # English 0x0409: "en_US", # English - United States 0x0809: "en_GB", # English - United Kingdom 0x0c09: "en_AU", # English - Australia @@ -1573,122 +1611,248 @@ def getpreferredencoding(do_setlocale=True): 0x1409: "en_NZ", # English - New Zealand 0x1809: "en_IE", # English - Ireland 0x1c09: "en_ZA", # English - South Africa - 0x2009: "en_JA", # English - Jamaica - 0x2409: "en_CB", # English - Caribbean + 0x2009: "en_JM", # English - Jamaica 0x2809: "en_BZ", # English - Belize - 0x2c09: "en_TT", # English - Trinidad + 0x2c09: "en_TT", # English - Trinidad and Tobago 0x3009: "en_ZW", # English - Zimbabwe - 0x3409: "en_PH", # English - Philippines + 0x3409: "en_PH", # English - Republic of the Philippines + 0x3c09: "en_HK", # English - Hong Kong 0x4009: "en_IN", # English - India 0x4409: "en_MY", # English - Malaysia - 0x4809: "en_IN", # English - Singapore - 0x0425: "et_EE", # Estonian - 0x0438: "fo_FO", # Faroese - 0x0464: "fil_PH",# Filipino - 0x040b: "fi_FI", # Finnish + 0x4809: "en_SG", # English - Singapore + 0x4c09: "en_AE", # English - United Arab Emirates + 0x0025: "et", # Estonian + 0x0425: "et_EE", # Estonian - Estonia + 0x0038: "fo", # Faroese + 0x0438: "fo_FO", # Faroese - Faroe Islands + 0x0064: "fil", # Filipino + 0x0464: "fil_PH", # Filipino - Philippines + 0x000b: "fi", # Finnish + 0x040b: "fi_FI", # Finnish - Finland + 0x000c: "fr", # French 0x040c: "fr_FR", # French - France 0x080c: "fr_BE", # French - Belgium 0x0c0c: "fr_CA", # French - Canada 0x100c: "fr_CH", # French - Switzerland 0x140c: "fr_LU", # French - Luxembourg - 0x180c: "fr_MC", # French - Monaco + 0x180c: "fr_MC", # French - Principality of Monaco + 0x1c0c: "fr_029", # French - Caribbean + 0x200c: "fr_RE", # French - Reunion + 0x240c: "fr_CD", # French - Congo, DRC + 0x280c: "fr_SN", # French - Senegal + 0x2c0c: "fr_CM", # French - Cameroon + 0x300c: "fr_CI", # French - Côte d'Ivoire + 0x340c: "fr_ML", # French - Mali + 0x380c: "fr_MA", # French - Morocco + 0x3c0c: "fr_HT", # French - Haiti + 0x0062: "fy", # Frisian 0x0462: "fy_NL", # Frisian - Netherlands - 0x0456: "gl_ES", # Galician - 0x0437: "ka_GE", # Georgian + 0x0067: "ff", # Fulah + 0x7c67: "ff", # Fulah (Latin) + 0x0467: "ff_NG", + 0x0867: "ff_SN", # Fulah - Senegal + 0x0056: "gl", # Galician + 0x0456: "gl_ES", # Galician - Spain + 0x0037: "ka", # Georgian + 0x0437: "ka_GE", # Georgian - Georgia + 0x0007: "de", # German 0x0407: "de_DE", # German - Germany 0x0807: "de_CH", # German - Switzerland 0x0c07: "de_AT", # German - Austria 0x1007: "de_LU", # German - Luxembourg 0x1407: "de_LI", # German - Liechtenstein - 0x0408: "el_GR", # Greek + 0x0008: "el", # Greek + 0x0408: "el_GR", # Greek - Greece + 0x006f: "kl", # Greenlandic 0x046f: "kl_GL", # Greenlandic - Greenland - 0x0447: "gu_IN", # Gujarati - 0x0468: "ha_NG", # Hausa - Latin - 0x040d: "he_IL", # Hebrew - 0x0439: "hi_IN", # Hindi - 0x040e: "hu_HU", # Hungarian - 0x040f: "is_IS", # Icelandic - 0x0421: "id_ID", # Indonesian - 0x045d: "iu_CA", # Inuktitut - Syllabics - 0x085d: "iu_CA", # Inuktitut - Latin + 0x0074: "gn", # Guarani + 0x0474: "gn_PY", # Guarani - Paraguay + 0x0047: "gu", # Gujarati + 0x0447: "gu_IN", # Gujarati - India + 0x0068: "ha", # Hausa (Latin) + 0x7c68: "ha", # Hausa (Latin) + 0x0468: "ha_NG", # Hausa (Latin) - Nigeria + 0x0075: "haw", # Hawaiian + 0x0475: "haw_US", # Hawaiian - United States + 0x000d: "he", # Hebrew + 0x040d: "he_IL", # Hebrew - Israel + 0x0039: "hi", # Hindi + 0x0439: "hi_IN", # Hindi - India + 0x000e: "hu", # Hungarian + 0x040e: "hu_HU", # Hungarian - Hungary + 0x000f: "is", # Icelandic + 0x040f: "is_IS", # Icelandic - Iceland + 0x0070: "ig", # Igbo + 0x0470: "ig_NG", # Igbo - Nigeria + 0x0021: "id", # Indonesian + 0x0421: "id_ID", # Indonesian - Indonesia + 0x005d: "iu", # Inuktitut (Latin) + 0x785d: "iu", # Inuktitut (Syllabics) + 0x7c5d: "iu", # Inuktitut (Latin) + 0x045d: "iu_CA", # Inuktitut (Syllabics) - Canada + 0x085d: "iu_CA", # Inuktitut (Latin) - Canada + 0x003c: "ga", # Irish 0x083c: "ga_IE", # Irish - Ireland + 0x0010: "it", # Italian 0x0410: "it_IT", # Italian - Italy 0x0810: "it_CH", # Italian - Switzerland - 0x0411: "ja_JP", # Japanese + 0x0011: "ja", # Japanese + 0x0411: "ja_JP", # Japanese - Japan + 0x004b: "kn", # Kannada 0x044b: "kn_IN", # Kannada - India - 0x043f: "kk_KZ", # Kazakh - 0x0453: "kh_KH", # Khmer - Cambodia - 0x0486: "qut_GT",# K'iche - Guatemala + 0x0471: "kr_NG", # Kanuri (Latin) - Nigeria + 0x0060: "ks", # Kashmiri + 0x0460: "ks", # Kashmiri - Perso_Arabic + 0x0860: "ks_IN", # Kashmiri (Devanagari) - India + 0x003f: "kk", # Kazakh + 0x043f: "kk_KZ", # Kazakh - Kazakhstan + 0x0053: "km", # Khmer + 0x0453: "km_KH", # Khmer - Cambodia + 0x0087: "rw", # Kinyarwanda 0x0487: "rw_RW", # Kinyarwanda - Rwanda - 0x0457: "kok_IN",# Konkani - 0x0412: "ko_KR", # Korean - 0x0440: "ky_KG", # Kyrgyz - 0x0454: "lo_LA", # Lao - Lao PDR - 0x0426: "lv_LV", # Latvian - 0x0427: "lt_LT", # Lithuanian - 0x082e: "dsb_DE",# Lower Sorbian - Germany - 0x046e: "lb_LU", # Luxembourgish - 0x042f: "mk_MK", # FYROM Macedonian + 0x0041: "sw", # Kiswahili + 0x0441: "sw_KE", # Kiswahili - Kenya + 0x0057: "kok", # Konkani + 0x0457: "kok_IN", # Konkani - India + 0x0012: "ko", # Korean + 0x0412: "ko_KR", # Korean - Korea + 0x0040: "ky", # Kyrgyz + 0x0440: "ky_KG", # Kyrgyz - Kyrgyzstan + 0x0054: "lo", # Lao + 0x0454: "lo_LA", # Lao - Lao P.D.R. + 0x0476: "la_VA", # Latin - Vatican City + 0x0026: "lv", # Latvian + 0x0426: "lv_LV", # Latvian - Latvia + 0x0027: "lt", # Lithuanian + 0x0427: "lt_LT", # Lithuanian - Lithuania + 0x7c2e: "dsb", # Lower Sorbian + 0x082e: "dsb_DE", # Lower Sorbian - Germany + 0x006e: "lb", # Luxembourgish + 0x046e: "lb_LU", # Luxembourgish - Luxembourg + 0x002f: "mk", # Macedonian + 0x042f: "mk_MK", # Macedonian - North Macedonia + 0x003e: "ms", # Malay 0x043e: "ms_MY", # Malay - Malaysia 0x083e: "ms_BN", # Malay - Brunei Darussalam + 0x004c: "ml", # Malayalam 0x044c: "ml_IN", # Malayalam - India - 0x043a: "mt_MT", # Maltese - 0x0481: "mi_NZ", # Maori - 0x047a: "arn_CL",# Mapudungun - 0x044e: "mr_IN", # Marathi - 0x047c: "moh_CA",# Mohawk - Canada - 0x0450: "mn_MN", # Mongolian - Cyrillic - 0x0850: "mn_CN", # Mongolian - PRC - 0x0461: "ne_NP", # Nepali - 0x0414: "nb_NO", # Norwegian - Bokmal - 0x0814: "nn_NO", # Norwegian - Nynorsk + 0x003a: "mt", # Maltese + 0x043a: "mt_MT", # Maltese - Malta + 0x0081: "mi", # Maori + 0x0481: "mi_NZ", # Maori - New Zealand + 0x007a: "arn", # Mapudungun + 0x047a: "arn_CL", # Mapudungun - Chile + 0x004e: "mr", # Marathi + 0x044e: "mr_IN", # Marathi - India + 0x007c: "moh", # Mohawk + 0x047c: "moh_CA", # Mohawk - Canada + 0x0050: "mn", # Mongolian (Cyrillic) + 0x7850: "mn", # Mongolian (Cyrillic) + 0x7c50: "mn", # Mongolian (Traditional Mongolian) + 0x0450: "mn_MN", # Mongolian (Cyrillic) - Mongolia + 0x0c50: "mn_MN", # Mongolian (Traditional Mongolian) - Mongolia + 0x0061: "ne", # Nepali + 0x0461: "ne_NP", # Nepali - Nepal + 0x0861: "ne_IN", # Nepali - India + 0x0014: "no", # Norwegian (Bokmal) + 0x0414: "nb_NO", # Norwegian (Bokmal) - Norway + 0x0814: "nn_NO", # Norwegian (Nynorsk) - Norway + 0x7814: "nn", # Norwegian (Nynorsk) + 0x7c14: "nb", # Norwegian (Bokmal) + 0x0082: "oc", # Occitan 0x0482: "oc_FR", # Occitan - France - 0x0448: "or_IN", # Oriya - India + 0x0048: "or", # Odia + 0x0448: "or_IN", # Odia - India + 0x0072: "om", # Oromo + 0x0472: "om_ET", # Oromo - Ethiopia + 0x0063: "ps", # Pashto 0x0463: "ps_AF", # Pashto - Afghanistan - 0x0429: "fa_IR", # Persian - 0x0415: "pl_PL", # Polish + 0x0029: "fa", # Persian + 0x0429: "fa_IR", # Persian - Iran + 0x0015: "pl", # Polish + 0x0415: "pl_PL", # Polish - Poland + 0x0016: "pt", # Portuguese 0x0416: "pt_BR", # Portuguese - Brazil 0x0816: "pt_PT", # Portuguese - Portugal - 0x0446: "pa_IN", # Punjabi - 0x046b: "quz_BO",# Quechua (Bolivia) - 0x086b: "quz_EC",# Quechua (Ecuador) - 0x0c6b: "quz_PE",# Quechua (Peru) + 0x0046: "pa", # Punjabi + 0x7c46: "pa", # Punjabi + 0x0446: "pa_IN", # Punjabi - India + 0x0846: "pa_PK", # Punjabi - Islamic Republic of Pakistan + 0x006b: "quz", # Quechua + 0x046b: "quz_BO", # Quechua - Bolivia + 0x086b: "quz_EC", # Quechua - Ecuador + 0x0c6b: "quz_PE", # Quechua - Peru + 0x0018: "ro", # Romanian 0x0418: "ro_RO", # Romanian - Romania - 0x0417: "rm_CH", # Romansh - 0x0419: "ru_RU", # Russian - 0x243b: "smn_FI",# Sami Finland - 0x103b: "smj_NO",# Sami Norway - 0x143b: "smj_SE",# Sami Sweden - 0x043b: "se_NO", # Sami Northern Norway - 0x083b: "se_SE", # Sami Northern Sweden - 0x0c3b: "se_FI", # Sami Northern Finland - 0x203b: "sms_FI",# Sami Skolt - 0x183b: "sma_NO",# Sami Southern Norway - 0x1c3b: "sma_SE",# Sami Southern Sweden - 0x044f: "sa_IN", # Sanskrit - 0x0c1a: "sr_SP", # Serbian - Cyrillic - 0x1c1a: "sr_BA", # Serbian - Bosnia Cyrillic - 0x081a: "sr_SP", # Serbian - Latin - 0x181a: "sr_BA", # Serbian - Bosnia Latin + 0x0818: "ro_MD", # Romanian - Moldova + 0x0017: "rm", # Romansh + 0x0417: "rm_CH", # Romansh - Switzerland + 0x0019: "ru", # Russian + 0x0419: "ru_RU", # Russian - Russia + 0x0819: "ru_MD", # Russian - Moldova + 0x0085: "sah", # Sakha + 0x0485: "sah_RU", # Sakha - Russia + 0x003b: "se", # Sami (Northern) + 0x043b: "se_NO", # Sami (Northern) - Norway + 0x083b: "se_SE", # Sami (Northern) - Sweden + 0x0c3b: "se_FI", # Sami (Northern) - Finland + 0x7c3b: "smj", # Sami (Lule) + 0x103b: "smj_NO", # Sami (Lule) - Norway + 0x143b: "smj_SE", # Sami (Lule) - Sweden + 0x783b: "sma", # Sami (Southern) + 0x183b: "sma_NO", # Sami (Southern) - Norway + 0x1c3b: "sma_SE", # Sami (Southern) - Sweden + 0x743b: "sms", # Sami (Skolt) + 0x203b: "sms_FI", # Sami (Skolt) - Finland + 0x703b: "smn", # Sami (Inari) + 0x243b: "smn_FI", # Sami (Inari) - Finland + 0x004f: "sa", # Sanskrit + 0x044f: "sa_IN", # Sanskrit - India + 0x0091: "gd", # Scottish Gaelic + 0x0491: "gd_GB", # Scottish Gaelic - United Kingdom + 0x6c1a: "sr", # Serbian (Cyrillic) + 0x701a: "sr", # Serbian (Latin) + 0x7c1a: "sr", # Serbian (Latin) + 0x081a: "sr_CS", # Serbian (Latin) - Serbia and Montenegro (Former) + 0x0c1a: "sr_CS", # Serbian (Cyrillic) - Serbia and Montenegro (Former) + 0x181a: "sr_BA", # Serbian (Latin) - Bosnia and Herzegovina + 0x1c1a: "sr_BA", # Serbian (Cyrillic) - Bosnia and Herzegovina + 0x241a: "sr_RS", # Serbian (Latin) - Serbia + 0x281a: "sr_RS", # Serbian (Cyrillic) - Serbia + 0x2c1a: "sr_ME", # Serbian (Latin) - Montenegro + 0x301a: "sr_ME", # Serbian (Cyrillic) - Montenegro + 0x006c: "nso", # Sesotho sa Leboa + 0x046c: "nso_ZA", # Sesotho sa Leboa - South Africa + 0x0032: "tn", # Setswana + 0x0432: "tn_ZA", # Setswana - South Africa + 0x0832: "tn_BW", # Setswana - Botswana + 0x0059: "sd", # Sindhi + 0x7c59: "sd", # Sindhi + 0x0859: "sd_PK", # Sindhi - Islamic Republic of Pakistan + 0x005b: "si", # Sinhala 0x045b: "si_LK", # Sinhala - Sri Lanka - 0x046c: "ns_ZA", # Northern Sotho - 0x0432: "tn_ZA", # Setswana - Southern Africa - 0x041b: "sk_SK", # Slovak - 0x0424: "sl_SI", # Slovenian + 0x001b: "sk", # Slovak + 0x041b: "sk_SK", # Slovak - Slovakia + 0x0024: "sl", # Slovenian + 0x0424: "sl_SI", # Slovenian - Slovenia + 0x0477: "so_SO", # Somali - Somalia + 0x0030: "st", # Sotho + 0x0430: "st_ZA", # Sotho - South Africa + 0x000a: "es", # Spanish 0x040a: "es_ES", # Spanish - Spain 0x080a: "es_MX", # Spanish - Mexico - 0x0c0a: "es_ES", # Spanish - Spain (Modern) + 0x0c0a: "es_ES", # Spanish - Spain 0x100a: "es_GT", # Spanish - Guatemala 0x140a: "es_CR", # Spanish - Costa Rica 0x180a: "es_PA", # Spanish - Panama 0x1c0a: "es_DO", # Spanish - Dominican Republic - 0x200a: "es_VE", # Spanish - Venezuela + 0x200a: "es_VE", # Spanish - Bolivarian Republic of Venezuela 0x240a: "es_CO", # Spanish - Colombia 0x280a: "es_PE", # Spanish - Peru 0x2c0a: "es_AR", # Spanish - Argentina 0x300a: "es_EC", # Spanish - Ecuador 0x340a: "es_CL", # Spanish - Chile - 0x380a: "es_UR", # Spanish - Uruguay + 0x380a: "es_UY", # Spanish - Uruguay 0x3c0a: "es_PY", # Spanish - Paraguay 0x400a: "es_BO", # Spanish - Bolivia 0x440a: "es_SV", # Spanish - El Salvador @@ -1696,36 +1860,87 @@ def getpreferredencoding(do_setlocale=True): 0x4c0a: "es_NI", # Spanish - Nicaragua 0x500a: "es_PR", # Spanish - Puerto Rico 0x540a: "es_US", # Spanish - United States -# 0x0430: "", # Sutu - Not supported - 0x0441: "sw_KE", # Swahili + 0x5c0a: "es_CU", # Spanish - Cuba + 0x001d: "sv", # Swedish 0x041d: "sv_SE", # Swedish - Sweden 0x081d: "sv_FI", # Swedish - Finland - 0x045a: "syr_SY",# Syriac - 0x0428: "tg_TJ", # Tajik - Cyrillic - 0x085f: "tmz_DZ",# Tamazight - Latin - 0x0449: "ta_IN", # Tamil - 0x0444: "tt_RU", # Tatar - 0x044a: "te_IN", # Telugu - 0x041e: "th_TH", # Thai - 0x0851: "bo_BT", # Tibetan - Bhutan - 0x0451: "bo_CN", # Tibetan - PRC - 0x041f: "tr_TR", # Turkish - 0x0442: "tk_TM", # Turkmen - Cyrillic - 0x0480: "ug_CN", # Uighur - Arabic - 0x0422: "uk_UA", # Ukrainian - 0x042e: "wen_DE",# Upper Sorbian - Germany - 0x0420: "ur_PK", # Urdu + 0x005a: "syr", # Syriac + 0x045a: "syr_SY", # Syriac - Syria + 0x0028: "tg", # Tajik (Cyrillic) + 0x7c28: "tg", # Tajik (Cyrillic) + 0x0428: "tg_TJ", # Tajik (Cyrillic) - Tajikistan + 0x005f: "tzm", # Tamazight (Latin) + 0x785f: "tzm", + 0x7c5f: "tzm", # Tamazight (Latin) + 0x085f: "tzm_DZ", # Tamazight (Latin) - Algeria + 0x045f: "tzm_MA", # Central Atlas Tamazight (Arabic) - Morocco + 0x105f: "tzm_MA", + 0x0049: "ta", # Tamil + 0x0449: "ta_IN", # Tamil - India + 0x0849: "ta_LK", # Tamil - Sri Lanka + 0x0044: "tt", # Tatar + 0x0444: "tt_RU", # Tatar - Russia + 0x004a: "te", # Telugu + 0x044a: "te_IN", # Telugu - India + 0x001e: "th", # Thai + 0x041e: "th_TH", # Thai - Thailand + 0x0051: "bo", # Tibetan + 0x0451: "bo_CN", # Tibetan - People's Republic of China + 0x0073: "ti", # Tigrinya + 0x0473: "ti_ET", # Tigrinya - Ethiopia + 0x0873: "ti_ER", # Tigrinya - Eritrea + 0x0031: "ts", # Tsonga + 0x0431: "ts_ZA", # Tsonga - South Africa + 0x001f: "tr", # Turkish + 0x041f: "tr_TR", # Turkish - Turkey + 0x0042: "tk", # Turkmen + 0x0442: "tk_TM", # Turkmen - Turkmenistan + 0x0022: "uk", # Ukrainian + 0x0422: "uk_UA", # Ukrainian - Ukraine + 0x002e: "hsb", # Upper Sorbian + 0x042e: "hsb_DE", # Upper Sorbian - Germany + 0x0020: "ur", # Urdu + 0x0420: "ur_PK", # Urdu - Islamic Republic of Pakistan 0x0820: "ur_IN", # Urdu - India - 0x0443: "uz_UZ", # Uzbek - Latin - 0x0843: "uz_UZ", # Uzbek - Cyrillic - 0x042a: "vi_VN", # Vietnamese - 0x0452: "cy_GB", # Welsh + 0x0080: "ug", # Uyghur + 0x0480: "ug_CN", # Uyghur - People's Republic of China + 0x0043: "uz", # Uzbek (Latin) + 0x7843: "uz", # Uzbek (Cyrillic) + 0x7c43: "uz", # Uzbek (Latin) + 0x0443: "uz_UZ", # Uzbek (Latin) - Uzbekistan + 0x0033: "ve", # Venda + 0x0433: "ve_ZA", # Venda - South Africa + 0x002a: "vi", # Vietnamese + 0x042a: "vi_VN", # Vietnamese - Vietnam + 0x0052: "cy", # Welsh + 0x0452: "cy_GB", # Welsh - United Kingdom + 0x0088: "wo", # Wolof 0x0488: "wo_SN", # Wolof - Senegal + 0x0034: "xh", # Xhosa 0x0434: "xh_ZA", # Xhosa - South Africa - 0x0485: "sah_RU",# Yakut - Cyrillic - 0x0478: "ii_CN", # Yi - PRC + 0x0078: "ii", # Yi + 0x0478: "ii_CN", # Yi - People's Republic of China + 0x043d: "yi_001", # Yiddish - World + 0x006a: "yo", # Yoruba 0x046a: "yo_NG", # Yoruba - Nigeria - 0x0435: "zu_ZA", # Zulu + 0x0035: "zu", # Zulu + 0x0435: "zu_ZA", # Zulu - South Africa + 0x0086: "qut", + +# 0x0001007f: "x-IV-mathan", # math alphanumeric sorting + 0x00010407: "de_DE", + 0x0001040e: "hu_HU", + 0x00010437: "ka_GE", + 0x00020804: "zh_CN", + 0x00021004: "zh_SG", + 0x00021404: "zh_MO", + 0x00030404: "zh_TW", + 0x00040404: "zh_TW", + 0x00040411: "ja_JP", + 0x00040c04: "zh_HK", + 0x00041404: "zh_MO", + 0x00050804: "zh_CN", + 0x00051004: "zh_SG", } def _print_locale(): diff --git a/Misc/NEWS.d/next/Library/2024-09-09-12-48-37.gh-issue-123853.e-zFxb.rst b/Misc/NEWS.d/next/Library/2024-09-09-12-48-37.gh-issue-123853.e-zFxb.rst new file mode 100644 index 00000000000..d7204c28936 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-09-12-48-37.gh-issue-123853.e-zFxb.rst @@ -0,0 +1,3 @@ +Update the table of Windows language code identifiers (LCIDs) used by +:func:`locale.getdefaultlocale` on Windows to protocol version 16.0 +(2024-04-23). From 22c8590e40a13070d75b1e7f9af01252b1b2e9ce Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Mon, 20 Apr 2026 21:55:03 +0900 Subject: [PATCH 022/152] gh-148718: Fix Py_STACKREF_DEBUG build by defining macros (#148719) --- Include/internal/pycore_stackref.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 329045b5faa..ca4a7c216ed 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -71,8 +71,10 @@ static const _PyStackRef PyStackRef_NULL = { .index = 0 }; static const _PyStackRef PyStackRef_ERROR = { .index = (1 << Py_TAGGED_SHIFT) }; #define PyStackRef_None ((_PyStackRef){ .index = (2 << Py_TAGGED_SHIFT) } ) -#define PyStackRef_False ((_PyStackRef){ .index = (3 << Py_TAGGED_SHIFT) }) -#define PyStackRef_True ((_PyStackRef){ .index = (4 << Py_TAGGED_SHIFT) }) +#define _Py_STACKREF_FALSE_INDEX (3 << Py_TAGGED_SHIFT) +#define _Py_STACKREF_TRUE_INDEX (4 << Py_TAGGED_SHIFT) +#define PyStackRef_False ((_PyStackRef){ .index = _Py_STACKREF_FALSE_INDEX }) +#define PyStackRef_True ((_PyStackRef){ .index = _Py_STACKREF_TRUE_INDEX }) #define INITIAL_STACKREF_INDEX (5 << Py_TAGGED_SHIFT) From 513db7211056f6ba5453eb32f12c88887425f2e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Mon, 20 Apr 2026 14:41:10 +0100 Subject: [PATCH 023/152] GH-145278: also filter mmap2 in strace_helper.filter_memory (GH-148648) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/test/support/strace_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/support/strace_helper.py b/Lib/test/support/strace_helper.py index cf95f7bdc7d..bf15283d302 100644 --- a/Lib/test/support/strace_helper.py +++ b/Lib/test/support/strace_helper.py @@ -74,7 +74,7 @@ def sections(self): def _filter_memory_call(call): # mmap can operate on a fd or "MAP_ANONYMOUS" which gives a block of memory. # Ignore "MAP_ANONYMOUS + the "MAP_ANON" alias. - if call.syscall == "mmap" and "MAP_ANON" in call.args[3]: + if call.syscall in ("mmap", "mmap2") and "MAP_ANON" in call.args[3]: return True if call.syscall in ("munmap", "mprotect"): From 789120e82609a50a94c683a62d043f89e383d23f Mon Sep 17 00:00:00 2001 From: AraHaan Date: Mon, 20 Apr 2026 10:01:06 -0400 Subject: [PATCH 024/152] gh-148790: Eliminate redundant call to `_PyRuntime_Initialize` in `Py_InitializeEx` (GH-121628) --- Python/pylifecycle.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index d9fc28475a4..0232ed6c382 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1641,18 +1641,12 @@ Py_InitializeFromConfig(const PyConfig *config) void Py_InitializeEx(int install_sigs) { - PyStatus status; - - status = _PyRuntime_Initialize(); - if (_PyStatus_EXCEPTION(status)) { - Py_ExitStatusException(status); - } - if (Py_IsInitialized()) { /* bpo-33932: Calling Py_Initialize() twice does nothing. */ return; } + PyStatus status; PyConfig config; _PyConfig_InitCompatConfig(&config); From 5c5dae0282c5649886b6e37dd6d487a779fa70c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Desgroppes?= Date: Mon, 20 Apr 2026 16:18:10 +0200 Subject: [PATCH 025/152] gh-148644: Propagate the PGO job exit code in PCbuild/build.bat (GH-148645) --- ...6-04-17-21-45-32.gh-issue-148644.vwkknh.rst | 1 + PCbuild/build.bat | 18 +++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2026-04-17-21-45-32.gh-issue-148644.vwkknh.rst diff --git a/Misc/NEWS.d/next/Build/2026-04-17-21-45-32.gh-issue-148644.vwkknh.rst b/Misc/NEWS.d/next/Build/2026-04-17-21-45-32.gh-issue-148644.vwkknh.rst new file mode 100644 index 00000000000..a0cc9c9358c --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-04-17-21-45-32.gh-issue-148644.vwkknh.rst @@ -0,0 +1 @@ +Errors during the PGO training job on Windows are no longer ignored, and a non-zero return code will cause the build to fail. diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 8fb2f096c93..9d2f032f5a9 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -170,16 +170,20 @@ if "%do_pgo%"=="true" ( del /s "%dir%\*.pgc" del /s "%dir%\..\Lib\*.pyc" set conf=PGUpdate - if "%clean%"=="false" ( - echo on - call "%dir%\..\python.bat" %pgo_job% - @echo off - call :Kill - set target=Build - ) + if "%clean%"=="false" goto :RunPgoJob ) goto :Build +:RunPgoJob +echo on +call "%dir%\..\python.bat" %pgo_job% +@echo off +set pgo_errorlevel=%ERRORLEVEL% +call :Kill +if %pgo_errorlevel% NEQ 0 exit /B %pgo_errorlevel% +set target=Build +goto :Build + :Kill echo on %MSBUILD% "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^ From 983c7462d65abc82d80345aa4769c1907522f310 Mon Sep 17 00:00:00 2001 From: Manoj K M Date: Mon, 20 Apr 2026 20:07:12 +0530 Subject: [PATCH 026/152] Docs: Fix some typos in `calendar.rst` (GH-148756) --- Doc/library/calendar.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 54cafaf4fe4..2ddef79eab0 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -54,13 +54,13 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is .. method:: setfirstweekday(firstweekday) - Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6) + Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6). Identical to setting the :attr:`~Calendar.firstweekday` property. .. method:: iterweekdays() - Return an iterator for the week day numbers that will be used for one + Return an iterator for the weekday numbers that will be used for one week. The first value from the iterator will be the same as the value of the :attr:`~Calendar.firstweekday` property. @@ -86,7 +86,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is Return an iterator for the month *month* in the year *year* similar to :meth:`itermonthdates`, but not restricted by the :class:`datetime.date` range. Days returned will be tuples consisting of a day of the month - number and a week day number. + number and a weekday number. .. method:: itermonthdays3(year, month) @@ -408,7 +408,7 @@ For simple text calendars this module provides the following functions. .. function:: monthrange(year, month) - Returns weekday of first day of the month and number of days in month, for the + Returns weekday of first day of the month and number of days in month, for the specified *year* and *month*. @@ -446,7 +446,7 @@ For simple text calendars this module provides the following functions. An unrelated but handy function that takes a time tuple such as returned by the :func:`~time.gmtime` function in the :mod:`time` module, and returns the corresponding Unix timestamp value, assuming an epoch of 1970, and the POSIX - encoding. In fact, :func:`time.gmtime` and :func:`timegm` are each others' + encoding. In fact, :func:`time.gmtime` and :func:`timegm` are each other's inverse. From 3fd61b74de97b152ed424aaea1295292dcb181fd Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Mon, 20 Apr 2026 17:00:35 +0100 Subject: [PATCH 027/152] Use `soft-deprecated` in more places (#148769) --- Doc/c-api/buffer.rst | 4 +++- Doc/c-api/code.rst | 4 +++- Doc/c-api/descriptor.rst | 4 +++- Doc/c-api/exceptions.rst | 4 +++- Doc/c-api/gen.rst | 4 +++- Doc/c-api/intro.rst | 7 +++---- Doc/c-api/set.rst | 4 +++- Doc/c-api/typeobj.rst | 8 +++++--- Doc/library/ctypes.rst | 4 ++-- Doc/library/mimetypes.rst | 2 +- 10 files changed, 29 insertions(+), 16 deletions(-) diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index fe950196297..dc3e0f37c36 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -258,7 +258,9 @@ readonly, format .. c:macro:: PyBUF_WRITEABLE - This is a :term:`soft deprecated` alias to :c:macro:`PyBUF_WRITABLE`. + This is an alias to :c:macro:`PyBUF_WRITABLE`. + + .. soft-deprecated:: 3.13 .. c:macro:: PyBUF_FORMAT diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index be2c85ec974..57b77f92a7d 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -212,7 +212,7 @@ bound into a function. .. c:function:: PyObject *PyCode_Optimize(PyObject *code, PyObject *consts, PyObject *names, PyObject *lnotab_obj) - This is a :term:`soft deprecated` function that does nothing. + This is a function that does nothing. Prior to Python 3.10, this function would perform basic optimizations to a code object. @@ -220,6 +220,8 @@ bound into a function. .. versionchanged:: 3.10 This function now does nothing. + .. soft-deprecated:: 3.13 + .. _c_codeobject_flags: diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index b913e24b3c7..539c4610ce4 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -140,7 +140,7 @@ found in the dictionary of type objects. .. c:macro:: PyDescr_COMMON - This is a :term:`soft deprecated` macro including the common fields for a + This is a macro including the common fields for a descriptor object. This was included in Python's C API by mistake; do not use it in extensions. @@ -148,6 +148,8 @@ found in the dictionary of type objects. descriptor protocol (:c:member:`~PyTypeObject.tp_descr_get` and :c:member:`~PyTypeObject.tp_descr_set`). + .. soft-deprecated:: 3.15 + Built-in descriptors ^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 8ecd7c62517..7a07818b7b4 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -818,7 +818,7 @@ Exception Classes .. c:macro:: PyException_HEAD - This is a :term:`soft deprecated` macro including the base fields for an + This is a macro including the base fields for an exception object. This was included in Python's C API by mistake and is not designed for use @@ -826,6 +826,8 @@ Exception Classes :c:func:`PyErr_NewException` or otherwise create a class inheriting from :c:data:`PyExc_BaseException`. + .. soft-deprecated:: 3.15 + Exception Objects ================= diff --git a/Doc/c-api/gen.rst b/Doc/c-api/gen.rst index 74db49a6814..ed121726b89 100644 --- a/Doc/c-api/gen.rst +++ b/Doc/c-api/gen.rst @@ -90,7 +90,9 @@ Deprecated API .. c:macro:: PyAsyncGenASend_CheckExact(op) - This is a :term:`soft deprecated` API that was included in Python's C API + This is an API that was included in Python's C API by mistake. It is solely here for completeness; do not use this API. + + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 0e6fd3421f2..500f2818e2e 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -587,10 +587,10 @@ have been standardized in C11 (or previous standards). .. c:macro:: Py_MEMCPY(dest, src, n) - This is a :term:`soft deprecated` alias to :c:func:`!memcpy`. - Use :c:func:`!memcpy` directly instead. + This is an alias to :c:func:`!memcpy`. .. soft-deprecated:: 3.14 + Use :c:func:`!memcpy` directly instead. .. c:macro:: Py_UNICODE_SIZE @@ -611,8 +611,7 @@ have been standardized in C11 (or previous standards). .. c:macro:: Py_VA_COPY - This is a :term:`soft deprecated` alias to the C99-standard ``va_copy`` - function. + This is an alias to the C99-standard ``va_copy`` function. Historically, this would use a compiler-specific method to copy a ``va_list``. diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index 53febd0c4c1..db537aff2e6 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -201,7 +201,7 @@ Deprecated API .. c:macro:: PySet_MINSIZE - A :term:`soft deprecated` constant representing the size of an internal + A constant representing the size of an internal preallocated table inside :c:type:`PySetObject` instances. This is documented solely for completeness, as there are no guarantees @@ -211,3 +211,5 @@ Deprecated API :c:macro:`!PySet_MINSIZE` can be replaced with a small constant like ``8``. If looking for the size of a set, use :c:func:`PySet_Size` instead. + + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index c3960d6ff87..d3d8239365f 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1391,8 +1391,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. versionchanged:: 3.9 - Renamed to the current name, without the leading underscore. - The old provisional name is :term:`soft deprecated`. + Renamed to the current name, without the leading underscore. + The old provisional name is :term:`soft deprecated`. .. versionchanged:: 3.12 @@ -1501,11 +1501,13 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_HAVE_VERSION_TAG - This is a :term:`soft deprecated` macro that does nothing. + This macro does nothing. Historically, this would indicate that the :c:member:`~PyTypeObject.tp_version_tag` field was available and initialized. + .. soft-deprecated:: 3.13 + .. c:macro:: Py_TPFLAGS_INLINE_VALUES diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index ff09bb8d884..e71d1d06d49 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -3190,8 +3190,8 @@ Arrays and pointers Equivalent to ``type * length``, where *type* is a :mod:`!ctypes` data type and *length* an integer. - This function is :term:`soft deprecated` in favor of multiplication. - There are no plans to remove it. + .. soft-deprecated:: 3.14 + In favor of multiplication. .. class:: _Pointer diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index af9098c4970..0facacd50fd 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -55,7 +55,7 @@ the information :func:`init` sets up. Added support for *url* being a :term:`path-like object`. .. soft-deprecated:: 3.13 - Passing a file path instead of URL is :term:`soft deprecated`. + Passing a file path instead of URL. Use :func:`guess_file_type` for this. From 5a3f479601e52d694cc21415cb925037dffc1138 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Mon, 20 Apr 2026 18:45:53 +0200 Subject: [PATCH 028/152] gh-138451: Support custom LLVM installation path (#138452) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Steve Dower Co-authored-by: Savannah Ostrowski --- ...-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst | 1 + PCbuild/regen.targets | 2 +- Tools/jit/README.md | 9 +++++-- Tools/jit/_llvm.py | 25 ++++++++++++++++--- Tools/jit/_targets.py | 25 ++++++++++++++++--- Tools/jit/build.py | 5 ++++ configure | 2 +- configure.ac | 2 +- 8 files changed, 59 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst diff --git a/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst b/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst new file mode 100644 index 00000000000..d83aee08025 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst @@ -0,0 +1 @@ +Allow for custom LLVM path using ``LLVM_TOOLS_INSTALL_DIR`` during JIT build. diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 41af9cacfb9..bb059f382eb 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -129,7 +129,7 @@ x86_64-pc-windows-msvc $(JITArgs) --debug - + diff --git a/Tools/jit/README.md b/Tools/jit/README.md index 8eadb3349ba..fd7154d0e76 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -9,7 +9,12 @@ ## Installing LLVM The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). -LLVM version 21 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. +LLVM version 21 is the officially supported version. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-21`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. + +You can customize the LLVM configuration using environment variables before running configure: + +- LLVM_VERSION: Specify a different LLVM version (default: 21) +- LLVM_TOOLS_INSTALL_DIR: Point to a specific LLVM installation prefix when multiple installations exist (the tools are expected in `

/bin`) It's easy to install all of the required tools: @@ -62,7 +67,7 @@ ### Windows ### Dev Containers -If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no +If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no need to install LLVM as the Fedora 43 base image includes LLVM 21 out of the box. ## Building diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index a4aaacdf412..601752bf1f6 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -80,7 +80,18 @@ async def _get_brew_llvm_prefix(llvm_version: str, *, echo: bool = False) -> str @_async_cache -async def _find_tool(tool: str, llvm_version: str, *, echo: bool = False) -> str | None: +async def _find_tool( + tool: str, + llvm_version: str, + llvm_tools_install_dir: str | None, + *, + echo: bool = False, +) -> str | None: + # Explicitly defined LLVM installation location + if llvm_tools_install_dir: + path = os.path.join(llvm_tools_install_dir, "bin", tool) + if await _check_tool_version(path, llvm_version, echo=echo): + return path # Unversioned executables: path = tool if await _check_tool_version(path, llvm_version, echo=echo): @@ -114,10 +125,11 @@ async def maybe_run( args: typing.Iterable[str], echo: bool = False, llvm_version: str = _LLVM_VERSION, + llvm_tools_install_dir: str | None = None, ) -> str | None: """Run an LLVM tool if it can be found. Otherwise, return None.""" - path = await _find_tool(tool, llvm_version, echo=echo) + path = await _find_tool(tool, llvm_version, llvm_tools_install_dir, echo=echo) return path and await _run(path, args, echo=echo) @@ -126,10 +138,17 @@ async def run( args: typing.Iterable[str], echo: bool = False, llvm_version: str = _LLVM_VERSION, + llvm_tools_install_dir: str | None = None, ) -> str: """Run an LLVM tool if it can be found. Otherwise, raise RuntimeError.""" - output = await maybe_run(tool, args, echo=echo, llvm_version=llvm_version) + output = await maybe_run( + tool, + args, + echo=echo, + llvm_version=llvm_version, + llvm_tools_install_dir=llvm_tools_install_dir, + ) if output is None: raise RuntimeError(f"Can't find {tool}-{llvm_version}!") return output diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index fd5c143b8a8..f78e80db165 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -53,6 +53,7 @@ class _Target(typing.Generic[_S, _R]): cflags: str = "" frame_pointers: bool = False llvm_version: str = _llvm._LLVM_VERSION + llvm_tools_install_dir: str | None = None known_symbols: dict[str, int] = dataclasses.field(default_factory=dict) pyconfig_dir: pathlib.Path = pathlib.Path.cwd().resolve() @@ -85,7 +86,11 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: group = _stencils.StencilGroup() args = ["--disassemble", "--reloc", f"{path}"] output = await _llvm.maybe_run( - "llvm-objdump", args, echo=self.verbose, llvm_version=self.llvm_version + "llvm-objdump", + args, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) if output is not None: # Make sure that full paths don't leak out (for reproducibility): @@ -105,7 +110,11 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: f"{path}", ] output = await _llvm.run( - "llvm-readobj", args, echo=self.verbose, llvm_version=self.llvm_version + "llvm-readobj", + args, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) # --elf-output-style=JSON is only *slightly* broken on Mach-O... output = output.replace("PrivateExtern\n", "\n") @@ -184,7 +193,11 @@ async def _compile( # Allow user-provided CFLAGS to override any defaults args_s += shlex.split(self.cflags) await _llvm.run( - "clang", args_s, echo=self.verbose, llvm_version=self.llvm_version + "clang", + args_s, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) if not is_shim: self.optimizer( @@ -196,7 +209,11 @@ async def _compile( ).run() args_o = [f"--target={self.triple}", "-c", "-o", f"{o}", f"{s}"] await _llvm.run( - "clang", args_o, echo=self.verbose, llvm_version=self.llvm_version + "clang", + args_o, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) return await self._parse(o) diff --git a/Tools/jit/build.py b/Tools/jit/build.py index 127d93b317f..5e1b05a3d86 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -43,6 +43,9 @@ "--cflags", help="additional flags to pass to the compiler", default="" ) parser.add_argument("--llvm-version", help="LLVM version to use") + parser.add_argument( + "--llvm-tools-install-dir", help="Installation location of LLVM tools" + ) args = parser.parse_args() for target in args.target: target.debug = args.debug @@ -52,6 +55,8 @@ target.pyconfig_dir = args.pyconfig_dir if args.llvm_version: target.llvm_version = args.llvm_version + if args.llvm_tools_install_dir: + target.llvm_tools_install_dir = args.llvm_tools_install_dir target.build( comment=comment, force=args.force, diff --git a/configure b/configure index 562bb6860c7..49319bc2aa4 100755 --- a/configure +++ b/configure @@ -11046,7 +11046,7 @@ then : else case e in #( e) as_fn_append CFLAGS_NODIST " $jit_flags" - REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\"" + REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\" --llvm-tools-install-dir=\"$LLVM_TOOLS_INSTALL_DIR\"" if test "x$Py_DEBUG" = xtrue then : as_fn_append REGEN_JIT_COMMAND " --debug" diff --git a/configure.ac b/configure.ac index 20e1afc2e9e..7b6f3c5e0ed 100644 --- a/configure.ac +++ b/configure.ac @@ -2850,7 +2850,7 @@ AS_VAR_IF([jit_flags], [], [AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"]) AS_VAR_SET([REGEN_JIT_COMMAND], - ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\""]) + ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\" --llvm-tools-install-dir=\"$LLVM_TOOLS_INSTALL_DIR\""]) AS_VAR_IF([Py_DEBUG], [true], [AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])], From 9a1c70c639f2ac60ae3756bec20c2584303f748e Mon Sep 17 00:00:00 2001 From: ByteFlow Date: Tue, 21 Apr 2026 03:22:37 +0800 Subject: [PATCH 029/152] Fix typos in asyncio, ctypes, and importlib documentation (#148747) --- Doc/library/asyncio-dev.rst | 2 +- Doc/library/ctypes.rst | 2 +- Doc/library/importlib.rst | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index f3409bcd2df..713b40d7466 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -304,7 +304,7 @@ generator can occur in an unexpected order:: try: yield 2 finally: - await asyncio.sleep(0.1) # immitate some async work + await asyncio.sleep(0.1) # imitate some async work work_done = True diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index e71d1d06d49..b8d615565a5 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1735,7 +1735,7 @@ If wrapping a shared library with :mod:`!ctypes`, consider determining the shared library name at development time, and hardcoding it into the wrapper module instead of using :func:`!find_library` to locate the library at runtime. -Also consider addding a configuration option or environment variable to let +Also consider adding a configuration option or environment variable to let users select a library to use, and then perhaps use :func:`!find_library` as a default or fallback. diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 785f6c614b4..0b76020eacc 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -286,7 +286,7 @@ ABC hierarchy:: This method can potentially yield a very large number of objects, and it may carry out IO operations when computing these values. - Because of this, it will generaly be desirable to compute the result + Because of this, it will generally be desirable to compute the result values on-the-fly, as they are needed. As such, the returned object is only guaranteed to be an :class:`iterable `, instead of a :class:`list` or other @@ -340,7 +340,7 @@ ABC hierarchy:: This method can potentially yield a very large number of objects, and it may carry out IO operations when computing these values. - Because of this, it will generaly be desirable to compute the result + Because of this, it will generally be desirable to compute the result values on-the-fly, as they are needed. As such, the returned object is only guaranteed to be an :class:`iterable `, instead of a :class:`list` or other From f6ed7c0acbd9234226cab5cca12f9d312809103e Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Mon, 20 Apr 2026 23:19:43 +0300 Subject: [PATCH 030/152] gh-108411: Make typing.IO/BinaryIO arguments positional-only (#142906) `IO` is purported to be the type of the file objects returned by `open`. However, all methods on those objects take positional-only arguments, while `IO`'s methods are declared with regular arguments. As such, the file objects cannot actually be considered to implement `IO`. The same thing applies to `BinaryIO`. Fix this by adjusting the definition of these ABCs to match the file objects. This is technically a breaking change, but it is unlikely to actually break anything: * These methods should never be called at runtime, since they are abstract. Therefore, this should not cause any runtime errors. * In typeshed these arguments are already positional-only, so this should not cause any errors during typechecking either. --- Lib/typing.py | 18 +++++++++--------- ...5-12-17-02-55-03.gh-issue-108411.up7MAc.rst | 2 ++ 2 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-17-02-55-03.gh-issue-108411.up7MAc.rst diff --git a/Lib/typing.py b/Lib/typing.py index 868fec9e088..3e7661dd2f8 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -3612,7 +3612,7 @@ def isatty(self) -> bool: pass @abstractmethod - def read(self, n: int = -1) -> AnyStr: + def read(self, n: int = -1, /) -> AnyStr: pass @abstractmethod @@ -3620,15 +3620,15 @@ def readable(self) -> bool: pass @abstractmethod - def readline(self, limit: int = -1) -> AnyStr: + def readline(self, limit: int = -1, /) -> AnyStr: pass @abstractmethod - def readlines(self, hint: int = -1) -> list[AnyStr]: + def readlines(self, hint: int = -1, /) -> list[AnyStr]: pass @abstractmethod - def seek(self, offset: int, whence: int = 0) -> int: + def seek(self, offset: int, whence: int = 0, /) -> int: pass @abstractmethod @@ -3640,7 +3640,7 @@ def tell(self) -> int: pass @abstractmethod - def truncate(self, size: int | None = None) -> int: + def truncate(self, size: int | None = None, /) -> int: pass @abstractmethod @@ -3648,11 +3648,11 @@ def writable(self) -> bool: pass @abstractmethod - def write(self, s: AnyStr) -> int: + def write(self, s: AnyStr, /) -> int: pass @abstractmethod - def writelines(self, lines: list[AnyStr]) -> None: + def writelines(self, lines: list[AnyStr], /) -> None: pass @abstractmethod @@ -3660,7 +3660,7 @@ def __enter__(self) -> IO[AnyStr]: pass @abstractmethod - def __exit__(self, type, value, traceback) -> None: + def __exit__(self, type, value, traceback, /) -> None: pass @@ -3670,7 +3670,7 @@ class BinaryIO(IO[bytes]): __slots__ = () @abstractmethod - def write(self, s: bytes | bytearray) -> int: + def write(self, s: bytes | bytearray, /) -> int: pass @abstractmethod diff --git a/Misc/NEWS.d/next/Library/2025-12-17-02-55-03.gh-issue-108411.up7MAc.rst b/Misc/NEWS.d/next/Library/2025-12-17-02-55-03.gh-issue-108411.up7MAc.rst new file mode 100644 index 00000000000..95aa41e9226 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-17-02-55-03.gh-issue-108411.up7MAc.rst @@ -0,0 +1,2 @@ +``typing.IO`` and ``typing.BinaryIO`` method arguments are now +positional-only. From d206d42834b2a34aee11b048357131371cf6947d Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Tue, 21 Apr 2026 00:04:50 +0100 Subject: [PATCH 031/152] gh-148814: Fix an issue in Emscripten README (#148752) Correct the description of the default state of test module compilation. --- Platforms/emscripten/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Platforms/emscripten/README.md b/Platforms/emscripten/README.md index 017bb3c8977..ce230c4b74a 100644 --- a/Platforms/emscripten/README.md +++ b/Platforms/emscripten/README.md @@ -186,8 +186,8 @@ #### In the browser are not shipped. All other modules are bundled as pre-compiled ``pyc`` files. - In-memory file system (MEMFS) is not persistent and limited. -- Test modules are disabled by default. Use ``--enable-test-modules`` build - test modules like ``_testcapi``. +- Test modules are built by default. Use ``--disable-test-modules`` to disable + building test modules like ``_testcapi``. ## Detecting Emscripten builds From 1274766d3c29007ab77245a72abbf8dce2a9db4d Mon Sep 17 00:00:00 2001 From: Seth Larson Date: Tue, 21 Apr 2026 09:29:07 -0500 Subject: [PATCH 032/152] =?UTF-8?q?gh-148808:=20Add=20boundary=20check=20t?= =?UTF-8?q?o=20asyncio.AbstractEventLoop.sock=5Frecvf=E2=80=A6=20(#148809)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Lib/test/test_asyncio/test_sock_lowlevel.py | 21 +++++++++++++++++++ ...-04-20-15-31-37.gh-issue-148808._Z8JL0.rst | 3 +++ Modules/overlapped.c | 5 +++++ 3 files changed, 29 insertions(+) create mode 100644 Misc/NEWS.d/next/Security/2026-04-20-15-31-37.gh-issue-148808._Z8JL0.rst diff --git a/Lib/test/test_asyncio/test_sock_lowlevel.py b/Lib/test/test_asyncio/test_sock_lowlevel.py index df4ec794897..f32dcd589e2 100644 --- a/Lib/test/test_asyncio/test_sock_lowlevel.py +++ b/Lib/test/test_asyncio/test_sock_lowlevel.py @@ -427,6 +427,27 @@ def test_recvfrom_into(self): self.loop.run_until_complete( self._basetest_datagram_recvfrom_into(server_address)) + async def _basetest_datagram_recvfrom_into_wrong_size(self, server_address): + # Call sock_sendto() with a size larger than the buffer + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + sock.setblocking(False) + + buf = bytearray(5000) + data = b'\x01' * 4096 + wrong_size = len(buf) + 1 + await self.loop.sock_sendto(sock, data, server_address) + with self.assertRaises(ValueError): + await self.loop.sock_recvfrom_into( + sock, buf, wrong_size) + + size, addr = await self.loop.sock_recvfrom_into(sock, buf) + self.assertEqual(buf[:size], data) + + def test_recvfrom_into_wrong_size(self): + with test_utils.run_udp_echo_server() as server_address: + self.loop.run_until_complete( + self._basetest_datagram_recvfrom_into_wrong_size(server_address)) + async def _basetest_datagram_sendto_blocking(self, server_address): # Sad path, sock.sendto() raises BlockingIOError # This involves patching sock.sendto() to raise BlockingIOError but diff --git a/Misc/NEWS.d/next/Security/2026-04-20-15-31-37.gh-issue-148808._Z8JL0.rst b/Misc/NEWS.d/next/Security/2026-04-20-15-31-37.gh-issue-148808._Z8JL0.rst new file mode 100644 index 00000000000..0b5cf85fedf --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-04-20-15-31-37.gh-issue-148808._Z8JL0.rst @@ -0,0 +1,3 @@ +Added buffer boundary check when using ``nbytes`` parameter with +:meth:`!asyncio.AbstractEventLoop.sock_recvfrom_into`. Only +relevant for Windows and the :class:`asyncio.ProactorEventLoop`. diff --git a/Modules/overlapped.c b/Modules/overlapped.c index 822e1ce4bdc..51aee5afd35 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -1910,6 +1910,11 @@ _overlapped_Overlapped_WSARecvFromInto_impl(OverlappedObject *self, } #endif + if (bufobj->len < (Py_ssize_t)size) { + PyErr_SetString(PyExc_ValueError, "nbytes is greater than the length of the buffer"); + return NULL; + } + wsabuf.buf = bufobj->buf; wsabuf.len = size; From 33e82be1746a964b595b2bba64f38a5787681eb3 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Tue, 21 Apr 2026 17:20:18 +0100 Subject: [PATCH 033/152] gh-148801: Fix unbound C recursion in `Element.__deepcopy__()` (#148802) --- Lib/test/test_xml_etree.py | 13 +++++++++++ ...-04-20-18-29-21.gh-issue-148801.ROeNqs.rst | 2 ++ Modules/_elementtree.c | 23 +++++++++++++------ 3 files changed, 31 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-04-20-18-29-21.gh-issue-148801.ROeNqs.rst diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index b380d0276b0..51af46f124c 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -3190,6 +3190,19 @@ def __deepcopy__(self, memo): self.assertEqual([c.tag for c in children[3:]], [a.tag, b.tag, a.tag, b.tag]) + @support.skip_if_unlimited_stack_size + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() + def test_deeply_nested_deepcopy(self): + # This should raise a RecursionError and not crash. + # See https://github.com/python/cpython/issues/148801. + root = cur = ET.Element('s') + for _ in range(150_000): + cur = ET.SubElement(cur, 'u') + with support.infinite_recursion(): + with self.assertRaises(RecursionError): + copy.deepcopy(root) + class MutationDeleteElementPath(str): def __new__(cls, elem, *args): diff --git a/Misc/NEWS.d/next/Library/2026-04-20-18-29-21.gh-issue-148801.ROeNqs.rst b/Misc/NEWS.d/next/Library/2026-04-20-18-29-21.gh-issue-148801.ROeNqs.rst new file mode 100644 index 00000000000..6fcd30e8f05 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-20-18-29-21.gh-issue-148801.ROeNqs.rst @@ -0,0 +1,2 @@ +:mod:`xml.etree.ElementTree`: Fix a crash in :meth:`Element.__deepcopy__ +` on deeply nested trees. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index e2185c4bd03..32150924292 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -16,6 +16,7 @@ #endif #include "Python.h" +#include "pycore_ceval.h" // _Py_EnterRecursiveCall() #include "pycore_dict.h" // _PyDict_CopyAsDict() #include "pycore_pyhash.h" // _Py_HashSecret #include "pycore_tuple.h" // _PyTuple_FromPair @@ -811,26 +812,31 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) /*[clinic end generated code: output=eefc3df50465b642 input=a2d40348c0aade10]*/ { Py_ssize_t i; - ElementObject* element; + ElementObject* element = NULL; PyObject* tag; PyObject* attrib; PyObject* text; PyObject* tail; PyObject* id; + if (_Py_EnterRecursiveCall(" in Element.__deepcopy__")) { + return NULL; + } + PyTypeObject *tp = Py_TYPE(self); elementtreestate *st = get_elementtree_state_by_type(tp); // The deepcopy() helper takes care of incrementing the refcount // of the object to copy so to avoid use-after-frees. tag = deepcopy(st, self->tag, memo); - if (!tag) - return NULL; + if (!tag) { + goto error; + } if (self->extra && self->extra->attrib) { attrib = deepcopy(st, self->extra->attrib, memo); if (!attrib) { Py_DECREF(tag); - return NULL; + goto error; } } else { attrib = NULL; @@ -841,8 +847,9 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) Py_DECREF(tag); Py_XDECREF(attrib); - if (!element) - return NULL; + if (!element) { + goto error; + } text = deepcopy(st, JOIN_OBJ(self->text), memo); if (!text) @@ -904,10 +911,12 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) if (i < 0) goto error; + _Py_LeaveRecursiveCall(); return (PyObject*) element; error: - Py_DECREF(element); + _Py_LeaveRecursiveCall(); + Py_XDECREF(element); return NULL; } From 0b9146e90b4969af8cd6ed39c3d97bb71bfc6a7a Mon Sep 17 00:00:00 2001 From: Rida Zouga <96395950+ZougaRida@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:17:02 +0100 Subject: [PATCH 034/152] [Enum] Improve clarity of comparison sentence (GH-148753) Co-authored-by: Ethan Furman --- Doc/howto/enum.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 5260c2ca4ad..2fe5814bb04 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -371,7 +371,7 @@ Equality comparisons are defined though:: >>> Color.BLUE == Color.BLUE True -Comparisons against non-enumeration values will always compare not equal +Equality comparisons against non-enumeration values will always return ``False`` (again, :class:`IntEnum` was explicitly designed to behave differently, see below):: From 09233bd19879284395aff97d7357b693893e6dd7 Mon Sep 17 00:00:00 2001 From: cui Date: Wed, 22 Apr 2026 03:49:44 +0800 Subject: [PATCH 035/152] gh-146578: _zstd: Fix printf format for pledged size errors (#146576) Use %llu instead of %ull for unsigned long long in zstd_contentsize_converter ValueError messages. --- Modules/_zstd/compressor.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index f90bc9c5ab5..8a3cd182ab1 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -74,7 +74,7 @@ zstd_contentsize_converter(PyObject *size, unsigned long long *p) if (PyErr_ExceptionMatches(PyExc_OverflowError)) { PyErr_Format(PyExc_ValueError, "size argument should be a positive int less " - "than %ull", ZSTD_CONTENTSIZE_ERROR); + "than %llu", ZSTD_CONTENTSIZE_ERROR); return 0; } return 0; @@ -83,7 +83,7 @@ zstd_contentsize_converter(PyObject *size, unsigned long long *p) *p = ZSTD_CONTENTSIZE_ERROR; PyErr_Format(PyExc_ValueError, "size argument should be a positive int less " - "than %ull", ZSTD_CONTENTSIZE_ERROR); + "than %llu", ZSTD_CONTENTSIZE_ERROR); return 0; } *p = pledged_size; From 858e69eab0949852cc41733e8465250fc80d0b66 Mon Sep 17 00:00:00 2001 From: "Gabriele N. Tornetta" Date: Wed, 22 Apr 2026 09:08:23 +0100 Subject: [PATCH 036/152] gh-142186: Allow all PEP-669 events to be per-code object and disableable (GH-146182) * Make the `PY_UNWIND` monitoring event available as a code-local event to allow trapping on function exit events when an exception bubbles up. This complements the PY_RETURN event by allowing to catch any function exit event. * Allow `PY_UNWIND` to be `DISABLE`d; disabling it disables the event for the whole code object. * Do the above for `PY_THROW`, `RAISE`, `EXCEPTION_HANDLED`, and `RERAISE` events. --- Doc/library/sys.monitoring.rst | 32 +- Doc/whatsnew/3.15.rst | 13 + Include/cpython/monitoring.h | 12 +- Include/internal/pycore_ceval.h | 2 +- Include/internal/pycore_instruments.h | 7 +- Lib/test/test_monitoring.py | 347 +++++++++++++++++- ...-03-23-11-34-37.gh-issue-142186.v8Yp3W.rst | 3 + Objects/genobject.c | 2 +- Python/ceval.c | 7 +- Python/ceval.h | 10 +- Python/instrumentation.c | 72 ++-- 11 files changed, 436 insertions(+), 71 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-03-23-11-34-37.gh-issue-142186.v8Yp3W.rst diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 16e6b1d6dc7..7cca6f2bcda 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -180,8 +180,8 @@ Local events '''''''''''' Local events are associated with normal execution of the program and happen -at clearly defined locations. All local events can be disabled. -The local events are: +at clearly defined locations. All local events can be disabled +per location. The local events are: * :monitoring-event:`PY_START` * :monitoring-event:`PY_RESUME` @@ -205,6 +205,8 @@ Using :monitoring-event:`BRANCH_LEFT` and :monitoring-event:`BRANCH_RIGHT` events will give much better performance as they can be disabled independently. +.. _monitoring-ancillary-events: + Ancillary events '''''''''''''''' @@ -226,7 +228,7 @@ Other events '''''''''''' Other events are not necessarily tied to a specific location in the -program and cannot be individually disabled via :data:`DISABLE`. +program and cannot be individually disabled per location. The other events that can be monitored are: @@ -234,6 +236,12 @@ The other events that can be monitored are: * :monitoring-event:`PY_UNWIND` * :monitoring-event:`RAISE` * :monitoring-event:`EXCEPTION_HANDLED` +* :monitoring-event:`RERAISE` + +.. versionchanged:: 3.15 + Other events can now be turned on and disabled on a per code object + basis. Returning :data:`DISABLE` from a callback disables the event + for the entire code object (for the current tool). The STOP_ITERATION event @@ -247,8 +255,7 @@ raise an exception unless it would be visible to other code. To allow tools to monitor for real exceptions without slowing down generators and coroutines, the :monitoring-event:`STOP_ITERATION` event is provided. -:monitoring-event:`STOP_ITERATION` can be locally disabled, unlike -:monitoring-event:`RAISE`. +:monitoring-event:`STOP_ITERATION` can be locally disabled. Note that the :monitoring-event:`STOP_ITERATION` event and the :monitoring-event:`RAISE` event for a :exc:`StopIteration` exception are @@ -314,15 +321,14 @@ location by returning :data:`sys.monitoring.DISABLE` from a callback function. This does not change which events are set, or any other code locations for the same event. -Disabling events for specific locations is very important for high -performance monitoring. For example, a program can be run under a -debugger with no overhead if the debugger disables all monitoring -except for a few breakpoints. +:ref:`Other events ` can be disabled on a per code +object basis by returning :data:`sys.monitoring.DISABLE` from a callback +function. This disables the event for the entire code object (for the current +tool). -If :data:`DISABLE` is returned by a callback for a -:ref:`global event `, :exc:`ValueError` will be raised -by the interpreter in a non-specific location (that is, no traceback will be -provided). +Disabling events for specific locations is very important for high performance +monitoring. For example, a program can be run under a debugger with no overhead +if the debugger disables all monitoring except for a few breakpoints. .. function:: restart_events() -> None diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index c4dac339be6..9630df9aad3 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1119,6 +1119,19 @@ sys (Contributed by Klaus Zimmermann in :gh:`137476`.) +sys.monitoring +-------------- + +* The :ref:`other events ` + (:monitoring-event:`PY_THROW`, :monitoring-event:`PY_UNWIND`, + :monitoring-event:`RAISE`, :monitoring-event:`EXCEPTION_HANDLED`, and + :monitoring-event:`RERAISE`) can now be turned on and disabled on a per code + object basis. Returning :data:`~sys.monitoring.DISABLE` from a callback for + one of these events disables the event for the entire code object (for the + current tool), rather than raising :exc:`ValueError` as in prior versions. + (Contributed by Gabriele N. Tornetta in :gh:`146182`.) + + tarfile ------- diff --git a/Include/cpython/monitoring.h b/Include/cpython/monitoring.h index 5094c8c23ae..fa6168d95cd 100644 --- a/Include/cpython/monitoring.h +++ b/Include/cpython/monitoring.h @@ -24,16 +24,20 @@ extern "C" { #define PY_MONITORING_EVENT_STOP_ITERATION 10 #define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \ - ((ev) < _PY_MONITORING_LOCAL_EVENTS) +((ev) <= PY_MONITORING_EVENT_STOP_ITERATION) -/* Other events, mainly exceptions */ +/* Other events, mainly exceptions. + * These can now be turned on and disabled on a per code object basis. */ -#define PY_MONITORING_EVENT_RAISE 11 +#define PY_MONITORING_EVENT_PY_UNWIND 11 #define PY_MONITORING_EVENT_EXCEPTION_HANDLED 12 -#define PY_MONITORING_EVENT_PY_UNWIND 13 +#define PY_MONITORING_EVENT_RAISE 13 #define PY_MONITORING_EVENT_PY_THROW 14 #define PY_MONITORING_EVENT_RERAISE 15 +#define _PY_MONITORING_IS_UNGROUPED_EVENT(ev) \ +((ev) < _PY_MONITORING_UNGROUPED_EVENTS) + /* Ancillary events */ diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 94a1f687b7b..ee8eb1095fe 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -320,7 +320,7 @@ PyObject * _PyEval_ImportNameWithImport( PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); PyAPI_FUNC(void) _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); -PyAPI_FUNC(bool) _PyEval_NoToolsForUnwind(PyThreadState *tstate); +PyAPI_FUNC(bool) _PyEval_NoToolsForUnwind(PyThreadState *tstate, _PyInterpreterFrame *frame); PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, _PyStackRef *sp); PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch); diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index cb1f50e441c..56b55e93a01 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -70,16 +70,15 @@ PyAPI_DATA(PyObject) _PyInstrumentation_DISABLE; /* Total tool ids available */ #define PY_MONITORING_TOOL_IDS 8 -/* Count of all local monitoring events */ -#define _PY_MONITORING_LOCAL_EVENTS 11 -/* Count of all "real" monitoring events (not derived from other events) */ +/* Count of all "real" monitoring events (not derived from other events). + * "Other" events can now be turned on/disabled per code object. */ #define _PY_MONITORING_UNGROUPED_EVENTS 16 /* Count of all monitoring events */ #define _PY_MONITORING_EVENTS 19 /* Tables of which tools are active for each monitored event. */ typedef struct _Py_LocalMonitors { - uint8_t tools[_PY_MONITORING_LOCAL_EVENTS]; + uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS]; } _Py_LocalMonitors; typedef struct _Py_GlobalMonitors { diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index bc7af6e1538..b8861d09e15 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -196,13 +196,10 @@ def test_c_return_count(self): (E.BRANCH, "branch"), ] -EXCEPT_EVENTS = [ +SIMPLE_EVENTS = INSTRUMENTED_EVENTS + [ (E.RAISE, "raise"), - (E.PY_UNWIND, "unwind"), (E.EXCEPTION_HANDLED, "exception_handled"), -] - -SIMPLE_EVENTS = INSTRUMENTED_EVENTS + EXCEPT_EVENTS + [ + (E.PY_UNWIND, "unwind"), (E.C_RAISE, "c_raise"), (E.C_RETURN, "c_return"), ] @@ -738,18 +735,6 @@ def test_disable_legal_events(self): sys.monitoring.register_callback(TEST_TOOL, event, None) - def test_disable_illegal_events(self): - for event, name in EXCEPT_EVENTS: - try: - counter = CounterWithDisable() - counter.disable = True - sys.monitoring.register_callback(TEST_TOOL, event, counter) - sys.monitoring.set_events(TEST_TOOL, event) - with self.assertRaises(ValueError): - self.raise_handle_reraise() - finally: - sys.monitoring.set_events(TEST_TOOL, 0) - sys.monitoring.register_callback(TEST_TOOL, event, None) class ExceptionRecorder: @@ -1481,8 +1466,334 @@ def func3(): ('line', 'func3', 6)]) def test_set_non_local_event(self): + # C_RETURN/C_RAISE are ancillary (derived) events — not settable as local with self.assertRaises(ValueError): - sys.monitoring.set_local_events(TEST_TOOL, just_call.__code__, E.RAISE) + sys.monitoring.set_local_events(TEST_TOOL, just_call.__code__, E.C_RETURN) + + def test_local_reraise(self): + """RERAISE fires as a local event only for the instrumented code object.""" + + def foo(): + try: + raise RuntimeError("test") + except RuntimeError: + raise + + def bar(): + try: + raise RuntimeError("test") + except RuntimeError: + raise + + events = set() + + def callback(code, offset, exc): + events.add(code.co_name) + + try: + sys.monitoring.register_callback(TEST_TOOL, E.RERAISE, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.RERAISE) + try: + foo() + except RuntimeError: + pass + try: + bar() # should NOT trigger the callback + except RuntimeError: + pass + self.assertEqual(events, {'foo'}) + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.RERAISE, None) + + def test_local_reraise_disable(self): + """Returning DISABLE from a RERAISE callback disables it for that code object.""" + + call_count = 0 + + def foo(): + try: + raise RuntimeError("test") + except RuntimeError: + raise + + def callback(code, offset, exc): + nonlocal call_count + call_count += 1 + return sys.monitoring.DISABLE + + try: + sys.monitoring.register_callback(TEST_TOOL, E.RERAISE, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.RERAISE) + try: + foo() + except RuntimeError: + pass + self.assertEqual(call_count, 1) + try: + foo() + except RuntimeError: + pass + self.assertEqual(call_count, 1) # not fired again — disabled + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.RERAISE, None) + + def test_local_py_throw(self): + """PY_THROW fires as a local event only for the instrumented code object.""" + + def gen_foo(): + yield 1 + yield 2 + + def gen_bar(): + yield 1 + yield 2 + + events = [] + + def callback(code, offset, exc): + events.append(code.co_name) + + try: + sys.monitoring.register_callback(TEST_TOOL, E.PY_THROW, callback) + sys.monitoring.set_local_events(TEST_TOOL, gen_foo.__code__, E.PY_THROW) + + g = gen_foo() + next(g) + try: + g.throw(RuntimeError("test")) + except RuntimeError: + pass + + h = gen_bar() + next(h) + try: + h.throw(RuntimeError("test")) # should NOT trigger the callback + except RuntimeError: + pass + + self.assertEqual(events, ['gen_foo']) + finally: + sys.monitoring.set_local_events(TEST_TOOL, gen_foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_THROW, None) + + def test_local_py_throw_disable(self): + """Returning DISABLE from a PY_THROW callback disables it for that code object.""" + + call_count = 0 + + def gen_foo(): + yield 1 + yield 2 + + def callback(code, offset, exc): + nonlocal call_count + call_count += 1 + return sys.monitoring.DISABLE + + try: + sys.monitoring.register_callback(TEST_TOOL, E.PY_THROW, callback) + sys.monitoring.set_local_events(TEST_TOOL, gen_foo.__code__, E.PY_THROW) + + g = gen_foo() + next(g) + try: + g.throw(RuntimeError("test")) + except RuntimeError: + pass + self.assertEqual(call_count, 1) + + g2 = gen_foo() + next(g2) + try: + g2.throw(RuntimeError("test")) + except RuntimeError: + pass + self.assertEqual(call_count, 1) # not fired again — disabled + finally: + sys.monitoring.set_local_events(TEST_TOOL, gen_foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_THROW, None) + + def test_local_raise(self): + """RAISE fires as a local event only for the instrumented code object.""" + + def foo(): + try: + raise RuntimeError("test") + except RuntimeError: + pass + + def bar(): + try: + raise RuntimeError("test") + except RuntimeError: + pass + + events = [] + + def callback(code, offset, exc): + events.append(code.co_name) + + try: + sys.monitoring.register_callback(TEST_TOOL, E.RAISE, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.RAISE) + foo() + bar() # should NOT trigger the callback + self.assertEqual(events, ['foo']) + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.RAISE, None) + + def test_local_raise_disable(self): + """Returning DISABLE from a RAISE callback disables it for that code object.""" + + call_count = 0 + + def foo(): + try: + raise RuntimeError("test") + except RuntimeError: + pass + + def callback(code, offset, exc): + nonlocal call_count + call_count += 1 + return sys.monitoring.DISABLE + + try: + sys.monitoring.register_callback(TEST_TOOL, E.RAISE, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.RAISE) + foo() + self.assertEqual(call_count, 1) + foo() + self.assertEqual(call_count, 1) # not fired again — disabled + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.RAISE, None) + + def test_local_exception_handled(self): + """EXCEPTION_HANDLED fires as a local event only for the instrumented code object.""" + + def foo(): + try: + raise RuntimeError("test") + except RuntimeError: + pass + + def bar(): + try: + raise RuntimeError("test") + except RuntimeError: + pass + + events = [] + + def callback(code, offset, exc): + events.append(code.co_name) + + try: + sys.monitoring.register_callback(TEST_TOOL, E.EXCEPTION_HANDLED, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.EXCEPTION_HANDLED) + foo() + bar() # should NOT trigger the callback + self.assertEqual(events, ['foo']) + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.EXCEPTION_HANDLED, None) + + def test_local_exception_handled_disable(self): + """Returning DISABLE from an EXCEPTION_HANDLED callback disables it for that code object.""" + + call_count = 0 + + def foo(): + try: + raise RuntimeError("test") + except RuntimeError: + pass + + def callback(code, offset, exc): + nonlocal call_count + call_count += 1 + return sys.monitoring.DISABLE + + try: + sys.monitoring.register_callback(TEST_TOOL, E.EXCEPTION_HANDLED, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.EXCEPTION_HANDLED) + foo() + self.assertEqual(call_count, 1) + foo() + self.assertEqual(call_count, 1) # not fired again — disabled + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.EXCEPTION_HANDLED, None) + + def test_local_py_unwind(self): + """PY_UNWIND fires as a local event only for the instrumented code object.""" + + def foo(): + raise RuntimeError("test") + + def bar(): + raise RuntimeError("test") + + events = [] + + def callback(code, offset, exc): + events.append(code.co_name) + + try: + sys.monitoring.register_callback(TEST_TOOL, E.PY_UNWIND, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.PY_UNWIND) + + try: + foo() + except RuntimeError: + pass + + try: + bar() # should NOT trigger the callback + except RuntimeError: + pass + + self.assertEqual(events, ['foo']) + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_UNWIND, None) + + def test_local_py_unwind_disable(self): + """Returning DISABLE from a PY_UNWIND callback disables it for that code object.""" + + call_count = 0 + + def foo(): + raise RuntimeError("test") + + def callback(code, offset, exc): + nonlocal call_count + call_count += 1 + return sys.monitoring.DISABLE + + try: + sys.monitoring.register_callback(TEST_TOOL, E.PY_UNWIND, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.PY_UNWIND) + + try: + foo() + except RuntimeError: + pass + self.assertEqual(call_count, 1) # fired once + + try: + foo() + except RuntimeError: + pass + self.assertEqual(call_count, 1) # not fired again — disabled by DISABLE return + + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_UNWIND, None) def line_from_offset(code, offset): for start, end, line in code.co_lines(): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-23-11-34-37.gh-issue-142186.v8Yp3W.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-23-11-34-37.gh-issue-142186.v8Yp3W.rst new file mode 100644 index 00000000000..4a04658551c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-23-11-34-37.gh-issue-142186.v8Yp3W.rst @@ -0,0 +1,3 @@ +Global :mod:`sys.monitoring` events can now be turned on and disabled on a +per code object basis. Returning ``DISABLE`` from a callback disables the +event for the entire code object (for the current tool). diff --git a/Objects/genobject.c b/Objects/genobject.c index 2bbe79c253d..d628889afc6 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -496,7 +496,7 @@ gen_close(PyObject *self, PyObject *args) } if (is_resume(frame->instr_ptr)) { - bool no_unwind_tools = _PyEval_NoToolsForUnwind(_PyThreadState_GET()); + bool no_unwind_tools = _PyEval_NoToolsForUnwind(_PyThreadState_GET(), frame); /* We can safely ignore the outermost try block * as it is automatically generated to handle * StopIteration. */ diff --git a/Python/ceval.c b/Python/ceval.c index 03bc5229565..967d92f4ea6 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2406,15 +2406,16 @@ void _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RAISE)) { + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_RAISE)) { return; } do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RAISE); } bool -_PyEval_NoToolsForUnwind(PyThreadState *tstate) { - return no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_UNWIND); +_PyEval_NoToolsForUnwind(PyThreadState *tstate, _PyInterpreterFrame *frame) +{ + return no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_PY_UNWIND); } diff --git a/Python/ceval.h b/Python/ceval.h index bb5f7ddb857..0437ab85c5a 100644 --- a/Python/ceval.h +++ b/Python/ceval.h @@ -367,7 +367,7 @@ no_tools_for_global_event(PyThreadState *tstate, int event) static inline bool no_tools_for_local_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event) { - assert(event < _PY_MONITORING_LOCAL_EVENTS); + assert(event < _PY_MONITORING_UNGROUPED_EVENTS); _PyCoMonitoringData *data = _PyFrame_GetCode(frame)->_co_monitoring; if (data) { return data->active_monitors.tools[event] == 0; @@ -382,7 +382,7 @@ monitor_handled(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *exc) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { return 0; } return _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); @@ -393,7 +393,7 @@ monitor_throw(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_THROW)) { + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_PY_THROW)) { return; } do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW); @@ -403,7 +403,7 @@ static void monitor_reraise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RERAISE)) { + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_RERAISE)) { return; } do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RERAISE); @@ -431,7 +431,7 @@ monitor_unwind(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_UNWIND)) { + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_PY_UNWIND)) { return; } do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_UNWIND); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 4041aa0d8ae..51bcbfdb3b6 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -203,7 +203,7 @@ is_instrumented(int opcode) static inline bool monitors_equals(_Py_LocalMonitors a, _Py_LocalMonitors b) { - for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { if (a.tools[i] != b.tools[i]) { return false; } @@ -216,7 +216,7 @@ static inline _Py_LocalMonitors monitors_sub(_Py_LocalMonitors a, _Py_LocalMonitors b) { _Py_LocalMonitors res; - for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { res.tools[i] = a.tools[i] & ~b.tools[i]; } return res; @@ -227,7 +227,7 @@ static inline _Py_LocalMonitors monitors_and(_Py_LocalMonitors a, _Py_LocalMonitors b) { _Py_LocalMonitors res; - for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { res.tools[i] = a.tools[i] & b.tools[i]; } return res; @@ -243,7 +243,7 @@ static inline _Py_LocalMonitors local_union(_Py_GlobalMonitors a, _Py_LocalMonitors b) { _Py_LocalMonitors res; - for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { res.tools[i] = a.tools[i] | b.tools[i]; } return res; @@ -252,7 +252,7 @@ local_union(_Py_GlobalMonitors a, _Py_LocalMonitors b) static inline bool monitors_are_empty(_Py_LocalMonitors m) { - for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { if (m.tools[i]) { return false; } @@ -263,7 +263,7 @@ monitors_are_empty(_Py_LocalMonitors m) static inline bool multiple_tools(_Py_LocalMonitors *m) { - for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { if (_Py_popcount32(m->tools[i]) > 1) { return true; } @@ -275,7 +275,7 @@ static inline _PyMonitoringEventSet get_local_events(_Py_LocalMonitors *m, int tool_id) { _PyMonitoringEventSet result = 0; - for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) { + for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) { if ((m->tools[e] >> tool_id) & 1) { result |= (1 << e); } @@ -453,7 +453,7 @@ static void dump_local_monitors(const char *prefix, _Py_LocalMonitors monitors, FILE*out) { fprintf(out, "%s monitors:\n", prefix); - for (int event = 0; event < _PY_MONITORING_LOCAL_EVENTS; event++) { + for (int event = 0; event < _PY_MONITORING_UNGROUPED_EVENTS; event++) { fprintf(out, " Event %d: Tools %x\n", event, monitors.tools[event]); } } @@ -1102,8 +1102,10 @@ get_tools_for_instruction(PyCodeObject *code, PyInterpreterState *interp, int i, event == PY_MONITORING_EVENT_C_RETURN); event = PY_MONITORING_EVENT_CALL; } + assert(_PY_MONITORING_IS_UNGROUPED_EVENT(event)); + CHECK(debug_check_sanity(interp, code)); if (PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) { - CHECK(debug_check_sanity(interp, code)); + /* Instrumented events use per-instruction tool bitmaps. */ if (code->_co_monitoring->tools) { tools = code->_co_monitoring->tools[i]; } @@ -1112,7 +1114,9 @@ get_tools_for_instruction(PyCodeObject *code, PyInterpreterState *interp, int i, } } else { - tools = interp->monitors.tools[event]; + /* Other (non-instrumented) events are not tied to specific instructions; + * use the code-object-level active_monitors bitmap instead. */ + tools = code->_co_monitoring->active_monitors.tools[event]; } return tools; } @@ -1139,6 +1143,25 @@ static const char *const event_names [] = { [PY_MONITORING_EVENT_STOP_ITERATION] = "STOP_ITERATION", }; +/* Disable an "other" (non-instrumented) event (e.g. PY_UNWIND) for a single + * tool on this code object. Must be called with the world stopped or the + * code lock held. */ +static void +remove_local_tool(PyCodeObject *code, PyInterpreterState *interp, + int event, int tool) +{ + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + assert(_PY_MONITORING_IS_UNGROUPED_EVENT(event)); + assert(!PY_MONITORING_IS_INSTRUMENTED_EVENT(event)); + assert(code->_co_monitoring); + code->_co_monitoring->local_monitors.tools[event] &= ~(1 << tool); + /* Recompute active_monitors for this event as the union of global and + * (now updated) local monitors. */ + code->_co_monitoring->active_monitors.tools[event] = + interp->monitors.tools[event] | + code->_co_monitoring->local_monitors.tools[event]; +} + static int call_instrumentation_vector( _Py_CODEUNIT *instr, PyThreadState *tstate, int event, @@ -1183,7 +1206,18 @@ call_instrumentation_vector( } else { /* DISABLE */ - if (!PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) { + if (PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) { + _PyEval_StopTheWorld(interp); + remove_tools(code, offset, event, 1 << tool); + _PyEval_StartTheWorld(interp); + } + else if (_PY_MONITORING_IS_UNGROUPED_EVENT(event)) { + /* Other (non-instrumented) event: disable for this code object. */ + _PyEval_StopTheWorld(interp); + remove_local_tool(code, interp, event, tool); + _PyEval_StartTheWorld(interp); + } + else { PyErr_Format(PyExc_ValueError, "Cannot disable %s events. Callback removed.", event_names[event]); @@ -1192,12 +1226,6 @@ call_instrumentation_vector( err = -1; break; } - else { - PyInterpreterState *interp = tstate->interp; - _PyEval_StopTheWorld(interp); - remove_tools(code, offset, event, 1 << tool); - _PyEval_StartTheWorld(interp); - } } } Py_DECREF(arg2_obj); @@ -1681,7 +1709,7 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) _Py_LocalMonitors *local_monitors = &code->_co_monitoring->local_monitors; for (int i = 0; i < PY_MONITORING_TOOL_IDS; i++) { if (code->_co_monitoring->tool_versions[i] != interp->monitoring_tool_versions[i]) { - for (int j = 0; j < _PY_MONITORING_LOCAL_EVENTS; j++) { + for (int j = 0; j < _PY_MONITORING_UNGROUPED_EVENTS; j++) { local_monitors->tools[j] &= ~(1 << i); } } @@ -1977,7 +2005,7 @@ static void set_local_events(_Py_LocalMonitors *m, int tool_id, _PyMonitoringEventSet events) { assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); - for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) { + for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) { uint8_t *tools = &m->tools[e]; int val = (events >> e) & 1; *tools &= ~(1 << tool_id); @@ -2037,7 +2065,7 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); PyInterpreterState *interp = _PyInterpreterState_GET(); - assert(events < (1 << _PY_MONITORING_LOCAL_EVENTS)); + assert(events < (1 << _PY_MONITORING_UNGROUPED_EVENTS)); if (code->_co_firsttraceable >= Py_SIZE(code)) { PyErr_Format(PyExc_SystemError, "cannot instrument shim code object '%U'", code->co_name); return -1; @@ -2373,7 +2401,7 @@ monitoring_get_local_events_impl(PyObject *module, int tool_id, _PyMonitoringEventSet event_set = 0; _PyCoMonitoringData *data = ((PyCodeObject *)code)->_co_monitoring; if (data != NULL) { - for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) { + for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) { if ((data->local_monitors.tools[e] >> tool_id) & 1) { event_set |= (1 << e); } @@ -2416,7 +2444,7 @@ monitoring_set_local_events_impl(PyObject *module, int tool_id, event_set &= ~(1 << PY_MONITORING_EVENT_BRANCH); event_set |= (1 << PY_MONITORING_EVENT_BRANCH_RIGHT) | (1 << PY_MONITORING_EVENT_BRANCH_LEFT); } - if (event_set < 0 || event_set >= (1 << _PY_MONITORING_LOCAL_EVENTS)) { + if (event_set < 0 || event_set >= (1 << _PY_MONITORING_UNGROUPED_EVENTS)) { PyErr_Format(PyExc_ValueError, "invalid local event set 0x%x", event_set); return NULL; } From f93834ff01128774532c101c574e47c6c0418540 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 Apr 2026 11:09:05 +0100 Subject: [PATCH 037/152] GH-146073: Add example script for dumping JIT traces (GH-148840) --- Tools/jit/README.md | 5 + Tools/jit/example_trace_dump.py | 191 ++++++++++++++++++++++++++++++++ 2 files changed, 196 insertions(+) create mode 100644 Tools/jit/example_trace_dump.py diff --git a/Tools/jit/README.md b/Tools/jit/README.md index fd7154d0e76..9361f39dcc6 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -86,3 +86,8 @@ ## Miscellaneous [^pep-744]: [PEP 744](https://peps.python.org/pep-0744/) [^why-llvm]: Clang is specifically needed because it's the only C compiler with support for guaranteed tail calls (`musttail`), which are required by CPython's continuation-passing-style approach to JIT compilation. Since LLVM also includes other functionalities we need (namely, object file parsing and disassembly), it's convenient to only support one toolchain at this time. + +### Understanding JIT behavior + +The [example_trace_dump.py](./example_trace_dump.py) script will (when configured as described in the script) dump out the +executors for a range of tiny programs to show the behavior of the JIT front-end. \ No newline at end of file diff --git a/Tools/jit/example_trace_dump.py b/Tools/jit/example_trace_dump.py new file mode 100644 index 00000000000..e3c3df94059 --- /dev/null +++ b/Tools/jit/example_trace_dump.py @@ -0,0 +1,191 @@ +# This script is best run with pystats enabled to help visualize the shape of the traces. +# ./configure --enable-experimental-jit=interpreter -C --with-pydebug --enable-pystats + +# The resulting images can be visualize on linux as follows: +# $ cd folder_with_gv_files +# $ dot -Tsvg -Osvg *.gv +# $ firefox *.gv.svg + +# type: ignore + +import sys +import os.path +from types import FunctionType + +# All functions declared in this module will be run to generate +# a .gv file of the executors, unless the name starts with an underscore. + + +def _gen(n): + for _ in range(n): + yield n + + +def gen_in_loop(n): + t = 0 + for n in _gen(n): + t += n + return n + + +def short_loop(n): + t = 0 + for _ in range(n): + t += 1 + t += 1 + t += 1 + t += 1 + t += 1 + return t + + +exec( + "\n".join( + ["def mid_loop(n):"] + + [" t = 0"] + + [" for _ in range(n):"] + + [" t += 1"] * 20 + + [" return t"] + ), + globals(), +) + +exec( + "\n".join( + ["def long_loop(n):"] + + [" t = 0"] + + [" for _ in range(n):"] + + [" t += 1"] * 100 + + [" return t"] + ), + globals(), +) + + +def _add(a, b): + return a + b + + +def short_loop_with_calls(n): + t = 0 + for _ in range(n): + t = _add(t, 1) + t = _add(t, 1) + t = _add(t, 1) + t = _add(t, 1) + t = _add(t, 1) + return t + + +exec( + "\n".join( + ["def mid_loop_with_calls(n):"] + + [" t = 0"] + + [" for _ in range(n):"] + + [" t = _add(t, 1)"] * 20 + + [" return t"] + ), + globals(), +) + +exec( + "\n".join( + ["def long_loop_with_calls(n):"] + + [" t = 0"] + + [" for _ in range(n):"] + + [" t = _add(t, 1)"] * 100 + + [" return t"] + ), + globals(), +) + + +def short_loop_with_side_exits(n): + t = 0 + for i in range(n): + if t < 0: + break + t += 1 + if t < 0: + break + t += 1 + if t < 0: + break + t += 1 + if t < 0: + break + t += 1 + if t < 0: + break + t += 1 + return t + + +exec( + "\n".join( + ["def mid_loop_with_side_exits(n):"] + + [" t = 0"] + + [" for _ in range(n):"] + + [" if t < 0:", " break", " t += 1"] * 20 + + [" return t"] + ), + globals(), +) + +exec( + "\n".join( + ["def long_loop_with_side_exits(n):"] + + [" t = 0"] + + [" for _ in range(n):"] + + [" if t < 0:", " break", " t += 1"] * 100 + + [" return t"] + ), + globals(), +) + + +def short_branchy_loop(n): + # Branches are correlated and exit 1 time in 4. + t = 0 + for i in range(n): + # Start with a few operations to form a viable trace + t += 1 + t += 1 + t += 1 + if not t & 6: + continue + t += 1 + if not t & 12: + continue + t += 1 + if not t & 24: + continue + t += 1 + if not t & 48: + continue + t += 1 + return t + + +def _run_and_dump(func, n, outdir): + sys._clear_internal_caches() + func(n) + sys._dump_tracelets(os.path.join(outdir, f"{func.__name__}.gv")) + + +def _main(): + if len(sys.argv) < 2 or len(sys.argv) > 3: + print(f"Usage: {sys.argv[0] if sys.argv else " "} OUTDIR [loops]") + outdir = sys.argv[1] + n = int(sys.argv[2]) if len(sys.argv) > 2 else 5000 + functions = [ + func + for func in globals().values() + if isinstance(func, FunctionType) and not func.__name__.startswith("_") + ] + for func in functions: + _run_and_dump(func, n, outdir) + + +if __name__ == "__main__": + _main() From 04fd103713a3aa6052a9afbf4b3132d4e318d0ad Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+KotlinIsland@users.noreply.github.com> Date: Wed, 22 Apr 2026 23:28:12 +1000 Subject: [PATCH 038/152] gh-148207: add additional keywords to `typing.TypeVarTuple` (#148212) --- Doc/library/typing.rst | 44 ++++++++- Doc/whatsnew/3.15.rst | 5 + Lib/test/test_typing.py | 95 ++++++++++++++----- ...-04-07-12-37-53.gh-issue-148207.YhGem4.rst | 3 + Objects/clinic/typevarobject.c.h | 57 +++++++++-- Objects/typevarobject.c | 64 +++++++++++-- 6 files changed, 228 insertions(+), 40 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-04-07-12-37-53.gh-issue-148207.YhGem4.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 9150385bd58..9bc0a3caeee 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1980,7 +1980,7 @@ without the dedicated syntax, as documented below. .. _typevartuple: -.. class:: TypeVarTuple(name, *, default=typing.NoDefault) +.. class:: TypeVarTuple(name, *, bound=None, covariant=False, contravariant=False, infer_variance=False, default=typing.NoDefault) Type variable tuple. A specialized form of :ref:`type variable ` that enables *variadic* generics. @@ -2090,6 +2090,24 @@ without the dedicated syntax, as documented below. The name of the type variable tuple. + .. attribute:: __covariant__ + + Whether the type variable tuple has been explicitly marked as covariant. + + .. versionadded:: 3.15 + + .. attribute:: __contravariant__ + + Whether the type variable tuple has been explicitly marked as contravariant. + + .. versionadded:: 3.15 + + .. attribute:: __infer_variance__ + + Whether the type variable tuple's variance should be inferred by type checkers. + + .. versionadded:: 3.15 + .. attribute:: __default__ The default value of the type variable tuple, or :data:`typing.NoDefault` if it @@ -2116,6 +2134,11 @@ without the dedicated syntax, as documented below. .. versionadded:: 3.13 + Type variable tuples created with ``covariant=True`` or + ``contravariant=True`` can be used to declare covariant or contravariant + generic types. The ``bound`` argument is also accepted, similar to + :class:`TypeVar`, but its actual semantics are yet to be decided. + .. versionadded:: 3.11 .. versionchanged:: 3.12 @@ -2127,6 +2150,11 @@ without the dedicated syntax, as documented below. Support for default values was added. + .. versionchanged:: 3.15 + + Added support for the ``bound``, ``covariant``, ``contravariant``, and + ``infer_variance`` parameters. + .. class:: ParamSpec(name, *, bound=None, covariant=False, contravariant=False, default=typing.NoDefault) Parameter specification variable. A specialized version of @@ -2196,6 +2224,20 @@ without the dedicated syntax, as documented below. The name of the parameter specification. + .. attribute:: __covariant__ + + Whether the parameter specification has been explicitly marked as covariant. + + .. attribute:: __contravariant__ + + Whether the parameter specification has been explicitly marked as contravariant. + + .. attribute:: __infer_variance__ + + Whether the parameter specification's variance should be inferred by type checkers. + + .. versionadded:: 3.12 + .. attribute:: __default__ The default value of the parameter specification, or :data:`typing.NoDefault` if it diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 9630df9aad3..500797910ed 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1296,6 +1296,11 @@ typing child classes of that class cannot inherit from other disjoint bases that are not parent or child classes of ``C``. (Contributed by Jelle Zijlstra in :gh:`148639`.) +* :class:`~typing.TypeVarTuple` now accepts ``bound``, ``covariant``, + ``contravariant``, and ``infer_variance`` keyword arguments, matching the + interface of :class:`~typing.TypeVar` and :class:`~typing.ParamSpec`. + ``bound`` semantics remain undefined in the specification. + unicodedata ----------- diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 3fb974c517d..bfae83fdaf6 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -780,7 +780,7 @@ def test_typevartuple_none(self): self.assertIs(U_None.__default__, None) self.assertIs(U_None.has_default(), True) - class X[**Ts]: ... + class X[*Ts]: ... Ts, = X.__type_params__ self.assertIs(Ts.__default__, NoDefault) self.assertIs(Ts.has_default(), False) @@ -1288,6 +1288,57 @@ def test_cannot_call_instance(self): with self.assertRaises(TypeError): Ts() + def test_default_variance(self): + Ts = TypeVarTuple('Ts') + self.assertIs(Ts.__covariant__, False) + self.assertIs(Ts.__contravariant__, False) + self.assertIs(Ts.__infer_variance__, False) + self.assertIsNone(Ts.__bound__) + + def test_covariant(self): + Ts_co = TypeVarTuple('Ts_co', covariant=True) + self.assertIs(Ts_co.__covariant__, True) + self.assertIs(Ts_co.__contravariant__, False) + self.assertIs(Ts_co.__infer_variance__, False) + + def test_contravariant(self): + Ts_contra = TypeVarTuple('Ts_contra', contravariant=True) + self.assertIs(Ts_contra.__covariant__, False) + self.assertIs(Ts_contra.__contravariant__, True) + self.assertIs(Ts_contra.__infer_variance__, False) + + def test_infer_variance(self): + Ts = TypeVarTuple('Ts', infer_variance=True) + self.assertIs(Ts.__covariant__, False) + self.assertIs(Ts.__contravariant__, False) + self.assertIs(Ts.__infer_variance__, True) + + def test_bound(self): + Ts_bound = TypeVarTuple('Ts_bound', bound=int) + self.assertIs(Ts_bound.__bound__, int) + Ts_no_bound = TypeVarTuple('Ts_no_bound') + self.assertIsNone(Ts_no_bound.__bound__) + + def test_no_bivariant(self): + with self.assertRaises(ValueError): + TypeVarTuple('Ts', covariant=True, contravariant=True) + + def test_cannot_combine_explicit_and_infer(self): + with self.assertRaises(ValueError): + TypeVarTuple('Ts', covariant=True, infer_variance=True) + with self.assertRaises(ValueError): + TypeVarTuple('Ts', contravariant=True, infer_variance=True) + + def test_repr_with_variance(self): + Ts = TypeVarTuple('Ts') + self.assertEqual(repr(Ts), '~Ts') + Ts_co = TypeVarTuple('Ts_co', covariant=True) + self.assertEqual(repr(Ts_co), '+Ts_co') + Ts_contra = TypeVarTuple('Ts_contra', contravariant=True) + self.assertEqual(repr(Ts_contra), '-Ts_contra') + Ts_infer = TypeVarTuple('Ts_infer', infer_variance=True) + self.assertEqual(repr(Ts_infer), 'Ts_infer') + def test_unpacked_typevartuple_is_equal_to_itself(self): Ts = TypeVarTuple('Ts') self.assertEqual((*Ts,)[0], (*Ts,)[0]) @@ -1427,16 +1478,16 @@ def test_repr_is_correct(self): class G1(Generic[*Ts]): pass class G2(Generic[Unpack[Ts]]): pass - self.assertEqual(repr(Ts), 'Ts') + self.assertEqual(repr(Ts), '~Ts') - self.assertEqual(repr((*Ts,)[0]), 'typing.Unpack[Ts]') - self.assertEqual(repr(Unpack[Ts]), 'typing.Unpack[Ts]') + self.assertEqual(repr((*Ts,)[0]), 'typing.Unpack[~Ts]') + self.assertEqual(repr(Unpack[Ts]), 'typing.Unpack[~Ts]') - self.assertEqual(repr(tuple[*Ts]), 'tuple[typing.Unpack[Ts]]') - self.assertEqual(repr(Tuple[Unpack[Ts]]), 'typing.Tuple[typing.Unpack[Ts]]') + self.assertEqual(repr(tuple[*Ts]), 'tuple[typing.Unpack[~Ts]]') + self.assertEqual(repr(Tuple[Unpack[Ts]]), 'typing.Tuple[typing.Unpack[~Ts]]') - self.assertEqual(repr(*tuple[*Ts]), '*tuple[typing.Unpack[Ts]]') - self.assertEqual(repr(Unpack[Tuple[Unpack[Ts]]]), 'typing.Unpack[typing.Tuple[typing.Unpack[Ts]]]') + self.assertEqual(repr(*tuple[*Ts]), '*tuple[typing.Unpack[~Ts]]') + self.assertEqual(repr(Unpack[Tuple[Unpack[Ts]]]), 'typing.Unpack[typing.Tuple[typing.Unpack[~Ts]]]') def test_variadic_class_repr_is_correct(self): Ts = TypeVarTuple('Ts') @@ -1475,61 +1526,61 @@ def test_variadic_class_alias_repr_is_correct(self): class A(Generic[Unpack[Ts]]): pass B = A[*Ts] - self.assertEndsWith(repr(B), 'A[typing.Unpack[Ts]]') + self.assertEndsWith(repr(B), 'A[typing.Unpack[~Ts]]') self.assertEndsWith(repr(B[()]), 'A[()]') self.assertEndsWith(repr(B[float]), 'A[float]') self.assertEndsWith(repr(B[float, str]), 'A[float, str]') C = A[Unpack[Ts]] - self.assertEndsWith(repr(C), 'A[typing.Unpack[Ts]]') + self.assertEndsWith(repr(C), 'A[typing.Unpack[~Ts]]') self.assertEndsWith(repr(C[()]), 'A[()]') self.assertEndsWith(repr(C[float]), 'A[float]') self.assertEndsWith(repr(C[float, str]), 'A[float, str]') D = A[*Ts, int] - self.assertEndsWith(repr(D), 'A[typing.Unpack[Ts], int]') + self.assertEndsWith(repr(D), 'A[typing.Unpack[~Ts], int]') self.assertEndsWith(repr(D[()]), 'A[int]') self.assertEndsWith(repr(D[float]), 'A[float, int]') self.assertEndsWith(repr(D[float, str]), 'A[float, str, int]') E = A[Unpack[Ts], int] - self.assertEndsWith(repr(E), 'A[typing.Unpack[Ts], int]') + self.assertEndsWith(repr(E), 'A[typing.Unpack[~Ts], int]') self.assertEndsWith(repr(E[()]), 'A[int]') self.assertEndsWith(repr(E[float]), 'A[float, int]') self.assertEndsWith(repr(E[float, str]), 'A[float, str, int]') F = A[int, *Ts] - self.assertEndsWith(repr(F), 'A[int, typing.Unpack[Ts]]') + self.assertEndsWith(repr(F), 'A[int, typing.Unpack[~Ts]]') self.assertEndsWith(repr(F[()]), 'A[int]') self.assertEndsWith(repr(F[float]), 'A[int, float]') self.assertEndsWith(repr(F[float, str]), 'A[int, float, str]') G = A[int, Unpack[Ts]] - self.assertEndsWith(repr(G), 'A[int, typing.Unpack[Ts]]') + self.assertEndsWith(repr(G), 'A[int, typing.Unpack[~Ts]]') self.assertEndsWith(repr(G[()]), 'A[int]') self.assertEndsWith(repr(G[float]), 'A[int, float]') self.assertEndsWith(repr(G[float, str]), 'A[int, float, str]') H = A[int, *Ts, str] - self.assertEndsWith(repr(H), 'A[int, typing.Unpack[Ts], str]') + self.assertEndsWith(repr(H), 'A[int, typing.Unpack[~Ts], str]') self.assertEndsWith(repr(H[()]), 'A[int, str]') self.assertEndsWith(repr(H[float]), 'A[int, float, str]') self.assertEndsWith(repr(H[float, str]), 'A[int, float, str, str]') I = A[int, Unpack[Ts], str] - self.assertEndsWith(repr(I), 'A[int, typing.Unpack[Ts], str]') + self.assertEndsWith(repr(I), 'A[int, typing.Unpack[~Ts], str]') self.assertEndsWith(repr(I[()]), 'A[int, str]') self.assertEndsWith(repr(I[float]), 'A[int, float, str]') self.assertEndsWith(repr(I[float, str]), 'A[int, float, str, str]') J = A[*Ts, *tuple[str, ...]] - self.assertEndsWith(repr(J), 'A[typing.Unpack[Ts], *tuple[str, ...]]') + self.assertEndsWith(repr(J), 'A[typing.Unpack[~Ts], *tuple[str, ...]]') self.assertEndsWith(repr(J[()]), 'A[*tuple[str, ...]]') self.assertEndsWith(repr(J[float]), 'A[float, *tuple[str, ...]]') self.assertEndsWith(repr(J[float, str]), 'A[float, str, *tuple[str, ...]]') K = A[Unpack[Ts], Unpack[Tuple[str, ...]]] - self.assertEndsWith(repr(K), 'A[typing.Unpack[Ts], typing.Unpack[typing.Tuple[str, ...]]]') + self.assertEndsWith(repr(K), 'A[typing.Unpack[~Ts], typing.Unpack[typing.Tuple[str, ...]]]') self.assertEndsWith(repr(K[()]), 'A[typing.Unpack[typing.Tuple[str, ...]]]') self.assertEndsWith(repr(K[float]), 'A[float, typing.Unpack[typing.Tuple[str, ...]]]') self.assertEndsWith(repr(K[float, str]), 'A[float, str, typing.Unpack[typing.Tuple[str, ...]]]') @@ -1550,9 +1601,9 @@ class G(type(Unpack[Ts])): pass with self.assertRaisesRegex(TypeError, r'Cannot subclass typing\.Unpack'): class H(Unpack): pass - with self.assertRaisesRegex(TypeError, r'Cannot subclass typing.Unpack\[Ts\]'): + with self.assertRaisesRegex(TypeError, r'Cannot subclass typing.Unpack\[~Ts\]'): class I(*Ts): pass - with self.assertRaisesRegex(TypeError, r'Cannot subclass typing.Unpack\[Ts\]'): + with self.assertRaisesRegex(TypeError, r'Cannot subclass typing.Unpack\[~Ts\]'): class J(Unpack[Ts]): pass def test_variadic_class_args_are_correct(self): @@ -5596,13 +5647,13 @@ class TsP(Generic[*Ts, P]): MyCallable[[int], bool]: "MyCallable[[int], bool]", MyCallable[[int, str], bool]: "MyCallable[[int, str], bool]", MyCallable[[int, list[int]], bool]: "MyCallable[[int, list[int]], bool]", - MyCallable[Concatenate[*Ts, P], T]: "MyCallable[typing.Concatenate[typing.Unpack[Ts], ~P], ~T]", + MyCallable[Concatenate[*Ts, P], T]: "MyCallable[typing.Concatenate[typing.Unpack[~Ts], ~P], ~T]", DoubleSpec[P2, P, T]: "DoubleSpec[~P2, ~P, ~T]", DoubleSpec[[int], [str], bool]: "DoubleSpec[[int], [str], bool]", DoubleSpec[[int, int], [str, str], bool]: "DoubleSpec[[int, int], [str, str], bool]", - TsP[*Ts, P]: "TsP[typing.Unpack[Ts], ~P]", + TsP[*Ts, P]: "TsP[typing.Unpack[~Ts], ~P]", TsP[int, str, list[int], []]: "TsP[int, str, list[int], []]", TsP[int, [str, list[int]]]: "TsP[int, [str, list[int]]]", diff --git a/Misc/NEWS.d/next/Library/2026-04-07-12-37-53.gh-issue-148207.YhGem4.rst b/Misc/NEWS.d/next/Library/2026-04-07-12-37-53.gh-issue-148207.YhGem4.rst new file mode 100644 index 00000000000..dd88be0ad25 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-07-12-37-53.gh-issue-148207.YhGem4.rst @@ -0,0 +1,3 @@ +:class:`typing.TypeVarTuple` now accepts ``bound``, ``covariant``, +``contravariant``, and ``infer_variance`` parameters, matching the interface +of :class:`typing.TypeVar` and :class:`typing.ParamSpec`. diff --git a/Objects/clinic/typevarobject.c.h b/Objects/clinic/typevarobject.c.h index bd4c7a0e64f..d2f350a3487 100644 --- a/Objects/clinic/typevarobject.c.h +++ b/Objects/clinic/typevarobject.c.h @@ -517,13 +517,15 @@ paramspec_has_default(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(typevartuple__doc__, -"typevartuple(name, *, default=typing.NoDefault)\n" +"typevartuple(name, *, bound=None, covariant=False, contravariant=False,\n" +" infer_variance=False, default=typing.NoDefault)\n" "--\n" "\n" "Create a new TypeVarTuple with the given name."); static PyObject * -typevartuple_impl(PyTypeObject *type, PyObject *name, +typevartuple_impl(PyTypeObject *type, PyObject *name, PyObject *bound, + int covariant, int contravariant, int infer_variance, PyObject *default_value); static PyObject * @@ -532,7 +534,7 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 6 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -541,7 +543,7 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(name), &_Py_ID(default), }, + .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), &_Py_ID(default), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -550,18 +552,22 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "default", NULL}; + static const char * const _keywords[] = {"name", "bound", "covariant", "contravariant", "infer_variance", "default", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "typevartuple", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[6]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *name; + PyObject *bound = Py_None; + int covariant = 0; + int contravariant = 0; + int infer_variance = 0; PyObject *default_value = &_Py_NoDefaultStruct; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, @@ -577,9 +583,42 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (!noptargs) { goto skip_optional_kwonly; } - default_value = fastargs[1]; + if (fastargs[1]) { + bound = fastargs[1]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[2]) { + covariant = PyObject_IsTrue(fastargs[2]); + if (covariant < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[3]) { + contravariant = PyObject_IsTrue(fastargs[3]); + if (contravariant < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[4]) { + infer_variance = PyObject_IsTrue(fastargs[4]); + if (infer_variance < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + default_value = fastargs[5]; skip_optional_kwonly: - return_value = typevartuple_impl(type, name, default_value); + return_value = typevartuple_impl(type, name, bound, covariant, contravariant, infer_variance, default_value); exit: return return_value; @@ -764,4 +803,4 @@ skip_optional_kwonly: exit: return return_value; } -/*[clinic end generated code: output=67ab9a5d1869f2c9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2e7dd170924d92e5 input=a9049054013a1b77]*/ diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index c2b8ee43119..cdc0ea42eac 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -36,8 +36,12 @@ typedef struct { typedef struct { PyObject_HEAD PyObject *name; + PyObject *bound; PyObject *default_value; PyObject *evaluate_default; + bool covariant; + bool contravariant; + bool infer_variance; } typevartupleobject; typedef struct { @@ -1524,6 +1528,7 @@ typevartuple_dealloc(PyObject *self) typevartupleobject *tvt = typevartupleobject_CAST(self); Py_XDECREF(tvt->name); + Py_XDECREF(tvt->bound); Py_XDECREF(tvt->default_value); Py_XDECREF(tvt->evaluate_default); PyObject_ClearManagedDict(self); @@ -1555,16 +1560,28 @@ static PyObject * typevartuple_repr(PyObject *self) { typevartupleobject *tvt = typevartupleobject_CAST(self); - return Py_NewRef(tvt->name); + + if (tvt->infer_variance) { + return Py_NewRef(tvt->name); + } + + char variance = tvt->covariant ? '+' : tvt->contravariant ? '-' : '~'; + return PyUnicode_FromFormat("%c%U", variance, tvt->name); } static PyMemberDef typevartuple_members[] = { {"__name__", _Py_T_OBJECT, offsetof(typevartupleobject, name), Py_READONLY}, + {"__bound__", _Py_T_OBJECT, offsetof(typevartupleobject, bound), Py_READONLY}, + {"__covariant__", Py_T_BOOL, offsetof(typevartupleobject, covariant), Py_READONLY}, + {"__contravariant__", Py_T_BOOL, offsetof(typevartupleobject, contravariant), Py_READONLY}, + {"__infer_variance__", Py_T_BOOL, offsetof(typevartupleobject, infer_variance), Py_READONLY}, {0} }; static typevartupleobject * -typevartuple_alloc(PyObject *name, PyObject *module, PyObject *default_value) +typevartuple_alloc(PyObject *name, PyObject *bound, PyObject *default_value, + bool covariant, bool contravariant, bool infer_variance, + PyObject *module) { PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.typevartuple_type; typevartupleobject *tvt = PyObject_GC_New(typevartupleobject, tp); @@ -1572,6 +1589,10 @@ typevartuple_alloc(PyObject *name, PyObject *module, PyObject *default_value) return NULL; } tvt->name = Py_NewRef(name); + tvt->bound = Py_XNewRef(bound); + tvt->covariant = covariant; + tvt->contravariant = contravariant; + tvt->infer_variance = infer_variance; tvt->default_value = Py_XNewRef(default_value); tvt->evaluate_default = NULL; _PyObject_GC_TRACK(tvt); @@ -1590,21 +1611,46 @@ typevartuple.__new__ name: object(subclass_of="&PyUnicode_Type") * + bound: object = None + covariant: bool = False + contravariant: bool = False + infer_variance: bool = False default as default_value: object(c_default="&_Py_NoDefaultStruct") = typing.NoDefault Create a new TypeVarTuple with the given name. [clinic start generated code]*/ static PyObject * -typevartuple_impl(PyTypeObject *type, PyObject *name, +typevartuple_impl(PyTypeObject *type, PyObject *name, PyObject *bound, + int covariant, int contravariant, int infer_variance, PyObject *default_value) -/*[clinic end generated code: output=9d6b76dfe95aae51 input=e149739929a866d0]*/ +/*[clinic end generated code: output=40bc9ca10f64e392 input=56e28c725a8da40b]*/ { - PyObject *module = caller(); - if (module == NULL) { + if (covariant && contravariant) { + PyErr_SetString(PyExc_ValueError, "Bivariant types are not supported."); return NULL; } - PyObject *result = (PyObject *)typevartuple_alloc(name, module, default_value); + if (infer_variance && (covariant || contravariant)) { + PyErr_SetString(PyExc_ValueError, "Variance cannot be specified with infer_variance."); + return NULL; + } + if (Py_IsNone(bound)) { + bound = NULL; + } + if (bound != NULL) { + bound = type_check(bound, "Bound must be a type."); + if (bound == NULL) { + return NULL; + } + } + PyObject *module = caller(); + if (module == NULL) { + Py_XDECREF(bound); + return NULL; + } + PyObject *result = (PyObject *)typevartuple_alloc( + name, bound, default_value, covariant, contravariant, infer_variance, module); + Py_XDECREF(bound); Py_DECREF(module); return result; } @@ -1688,6 +1734,7 @@ typevartuple_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(Py_TYPE(self)); typevartupleobject *tvt = typevartupleobject_CAST(self); Py_VISIT(tvt->name); + Py_VISIT(tvt->bound); Py_VISIT(tvt->default_value); Py_VISIT(tvt->evaluate_default); return PyObject_VisitManagedDict(self, visit, arg); @@ -1698,6 +1745,7 @@ typevartuple_clear(PyObject *self) { typevartupleobject *tvt = typevartupleobject_CAST(self); Py_CLEAR(tvt->name); + Py_CLEAR(tvt->bound); Py_CLEAR(tvt->default_value); Py_CLEAR(tvt->evaluate_default); PyObject_ClearManagedDict(self); @@ -1829,7 +1877,7 @@ PyObject * _Py_make_typevartuple(PyThreadState *Py_UNUSED(ignored), PyObject *v) { assert(PyUnicode_Check(v)); - return (PyObject *)typevartuple_alloc(v, NULL, NULL); + return (PyObject *)typevartuple_alloc(v, NULL, NULL, false, false, true, NULL); } static PyObject * From b16886528ee43ad8618e81b833be022710c4a8d2 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 22 Apr 2026 11:52:41 -0500 Subject: [PATCH 039/152] Additional itertool recipes for running statistics (gh-148879) --- Doc/library/itertools.rst | 75 ++++++++++++++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 9 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 5a0ac60ab7d..06f8bf2a8b6 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -833,6 +833,7 @@ and :term:`generators ` which incur interpreter overhead. from collections import Counter, deque from contextlib import suppress from functools import reduce + from heapq import heappush, heappushpop, heappush_max, heappushpop_max from math import comb, isqrt, prod, sumprod from operator import getitem, is_not, itemgetter, mul, neg, truediv @@ -848,11 +849,6 @@ and :term:`generators ` which incur interpreter overhead. # prepend(1, [2, 3, 4]) → 1 2 3 4 return chain([value], iterable) - def running_mean(iterable): - "Yield the average of all values seen so far." - # running_mean([8.5, 9.5, 7.5, 6.5]) → 8.5 9.0 8.5 8.0 - return map(truediv, accumulate(iterable), count(1)) - def repeatfunc(function, times=None, *args): "Repeat calls to a function with specified arguments." if times is None: @@ -1150,6 +1146,49 @@ and :term:`generators ` which incur interpreter overhead. return n + # ==== Running statistics ==== + + def running_mean(iterable): + "Average of values seen so far." + # running_mean([37, 33, 38, 28]) → 37 35 36 34 + return map(truediv, accumulate(iterable), count(1)) + + def running_min(iterable): + "Smallest of values seen so far." + # running_min([37, 33, 38, 28]) → 37 33 33 28 + return accumulate(iterable, func=min) + + def running_max(iterable): + "Largest of values seen so far." + # running_max([37, 33, 38, 28]) → 37 37 38 38 + return accumulate(iterable, func=max) + + def running_median(iterable): + "Median of values seen so far." + # running_median([37, 33, 38, 28]) → 37 35 37 35 + read = iter(iterable).__next__ + lo = [] # max-heap + hi = [] # min-heap the same size as or one smaller than lo + with suppress(StopIteration): + while True: + heappush_max(lo, heappushpop(hi, read())) + yield lo[0] + heappush(hi, heappushpop_max(lo, read())) + yield (lo[0] + hi[0]) / 2 + + def running_statistics(iterable): + "Aggregate statistics for values seen so far." + # Generate tuples: (size, minimum, median, maximum, mean) + t0, t1, t2, t3 = tee(iterable, 4) + return zip( + count(1), + running_min(t0), + running_median(t1), + running_max(t2), + running_mean(t3), + ) + + .. doctest:: :hide: @@ -1226,10 +1265,6 @@ and :term:`generators ` which incur interpreter overhead. [(0, 'a'), (1, 'b'), (2, 'c')] - >>> list(running_mean([8.5, 9.5, 7.5, 6.5])) - [8.5, 9.0, 8.5, 8.0] - - >>> for _ in loops(5): ... print('hi') ... @@ -1789,6 +1824,28 @@ and :term:`generators ` which incur interpreter overhead. True + >>> list(running_mean([8.5, 9.5, 7.5, 6.5])) + [8.5, 9.0, 8.5, 8.0] + >>> list(running_mean([37, 33, 38, 28])) + [37.0, 35.0, 36.0, 34.0] + + + >>> list(running_min([37, 33, 38, 28])) + [37, 33, 33, 28] + + + >>> list(running_max([37, 33, 38, 28])) + [37, 37, 38, 38] + + + >>> list(running_median([37, 33, 38, 28])) + [37, 35.0, 37, 35.0] + + + >>> list(running_statistics([37, 33, 38, 28])) + [(1, 37, 37, 37, 37.0), (2, 33, 35.0, 37, 35.0), (3, 33, 37, 38, 36.0), (4, 28, 35.0, 38, 34.0)] + + .. testcode:: :hide: From 59b41c8c3ba3251f15e6b58d9793d72499b298c0 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Wed, 22 Apr 2026 10:50:30 -0700 Subject: [PATCH 040/152] gh-148858: Remove duplicated recipe.yaml files in Tools/pixi-packages (#148859) --- Tools/pixi-packages/README.md | 7 +- Tools/pixi-packages/asan/pixi.toml | 4 + Tools/pixi-packages/asan/recipe.yaml | 94 ------------------- Tools/pixi-packages/clone-recipe.sh | 2 +- Tools/pixi-packages/default/pixi.toml | 4 + Tools/pixi-packages/freethreading/pixi.toml | 4 + Tools/pixi-packages/freethreading/recipe.yaml | 94 ------------------- .../tsan-freethreading/pixi.toml | 4 + .../tsan-freethreading/recipe.yaml | 94 ------------------- 9 files changed, 20 insertions(+), 287 deletions(-) delete mode 100644 Tools/pixi-packages/asan/recipe.yaml delete mode 100644 Tools/pixi-packages/freethreading/recipe.yaml delete mode 100644 Tools/pixi-packages/tsan-freethreading/recipe.yaml diff --git a/Tools/pixi-packages/README.md b/Tools/pixi-packages/README.md index 4b44fd12150..d818fddaac6 100644 --- a/Tools/pixi-packages/README.md +++ b/Tools/pixi-packages/README.md @@ -36,9 +36,8 @@ ## Opportunities for future improvement - More package variants (such as UBSan) - Support for Windows -- Using a single `pixi.toml` and `recipe.yaml` for all package variants is blocked on - [pixi#5364](https://github.com/prefix-dev/pixi/pull/5364) - and [pixi#5248](https://github.com/prefix-dev/pixi/issues/5248) +- Using a single `pixi.toml` for all package variants is blocked on + [pixi#5248](https://github.com/prefix-dev/pixi/issues/5248) ## Troubleshooting @@ -48,7 +47,7 @@ ## Troubleshooting ``` To fix it, try reducing `mmap_rnd_bits`: -```bash +```console $ sudo sysctl vm.mmap_rnd_bits vm.mmap_rnd_bits = 32 # too high for TSan $ sudo sysctl vm.mmap_rnd_bits=28 # reduce it diff --git a/Tools/pixi-packages/asan/pixi.toml b/Tools/pixi-packages/asan/pixi.toml index e3b5673d962..bf9841e1867 100644 --- a/Tools/pixi-packages/asan/pixi.toml +++ b/Tools/pixi-packages/asan/pixi.toml @@ -5,7 +5,11 @@ channels = ["https://prefix.dev/conda-forge"] platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"] preview = ["pixi-build"] +requires-pixi = ">=0.66.0" [package.build.backend] name = "pixi-build-rattler-build" version = "*" + +[package.build.config] +recipe = "../default/recipe.yaml" diff --git a/Tools/pixi-packages/asan/recipe.yaml b/Tools/pixi-packages/asan/recipe.yaml deleted file mode 100644 index 30d0d5a2ed2..00000000000 --- a/Tools/pixi-packages/asan/recipe.yaml +++ /dev/null @@ -1,94 +0,0 @@ -# NOTE: Please always only modify default/recipe.yaml and then run clone-recipe.sh to -# propagate the changes to the other variants. - -context: - # Keep up to date - freethreading_tag: ${{ "t" if "freethreading" in variant else "" }} - -recipe: - name: python - -source: - - path: ../../.. - -outputs: -- package: - name: python_abi - version: ${{ version }} - build: - string: "0_${{ abi_tag }}" - requirements: - run_constraints: - - python ${{ version }}.* *_${{ abi_tag }} - -- package: - name: python - version: ${{ version }} - build: - string: "0_${{ abi_tag }}" - files: - exclude: - - "*.o" - script: - file: ../build.sh - env: - PYTHON_VARIANT: ${{ variant }} - python: - site_packages_path: "lib/python${{ version }}${{ freethreading_tag }}/site-packages" - - # derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml - requirements: - build: - - ${{ compiler('c') }} - - ${{ compiler('cxx') }} - # Note that we are not using stdlib arguments which means the packages - # are built for the build settings and are not relocatable to a different - # machine that has a older system version. (eg: macOS/glibc version) - - make - - pkg-config - # configure script looks for llvm-ar for lto - - if: osx - then: - - llvm-tools - - host: - - bzip2 - - sqlite - - liblzma-devel - - zlib - - zstd - - openssl - - readline - - tk - # These two are just to get the headers needed for tk.h, but is unused - - xorg-libx11 - - xorg-xorgproto - - ncurses - - libffi - - if: linux - then: - - libuuid - - libmpdec-devel - - expat - - if: linux and "san" in variant - then: - - libsanitizer - - if: osx and "san" in variant - then: - - libcompiler-rt - - ignore_run_exports: - from_package: - - xorg-libx11 - - xorg-xorgproto - - run_exports: - noarch: - - python - weak: - - python_abi ${{ version }}.* *_${{ abi_tag }} - -about: - homepage: https://www.python.org/ - license: Python-2.0 - license_file: LICENSE diff --git a/Tools/pixi-packages/clone-recipe.sh b/Tools/pixi-packages/clone-recipe.sh index 52b2568837c..25ceaf85c35 100755 --- a/Tools/pixi-packages/clone-recipe.sh +++ b/Tools/pixi-packages/clone-recipe.sh @@ -6,5 +6,5 @@ set -o errexit cd "$(dirname "$0")" for variant in asan freethreading tsan-freethreading; do - cp -av default/recipe.yaml default/pixi.toml ${variant}/ + cp -av default/pixi.toml ${variant}/ done diff --git a/Tools/pixi-packages/default/pixi.toml b/Tools/pixi-packages/default/pixi.toml index e3b5673d962..bf9841e1867 100644 --- a/Tools/pixi-packages/default/pixi.toml +++ b/Tools/pixi-packages/default/pixi.toml @@ -5,7 +5,11 @@ channels = ["https://prefix.dev/conda-forge"] platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"] preview = ["pixi-build"] +requires-pixi = ">=0.66.0" [package.build.backend] name = "pixi-build-rattler-build" version = "*" + +[package.build.config] +recipe = "../default/recipe.yaml" diff --git a/Tools/pixi-packages/freethreading/pixi.toml b/Tools/pixi-packages/freethreading/pixi.toml index e3b5673d962..bf9841e1867 100644 --- a/Tools/pixi-packages/freethreading/pixi.toml +++ b/Tools/pixi-packages/freethreading/pixi.toml @@ -5,7 +5,11 @@ channels = ["https://prefix.dev/conda-forge"] platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"] preview = ["pixi-build"] +requires-pixi = ">=0.66.0" [package.build.backend] name = "pixi-build-rattler-build" version = "*" + +[package.build.config] +recipe = "../default/recipe.yaml" diff --git a/Tools/pixi-packages/freethreading/recipe.yaml b/Tools/pixi-packages/freethreading/recipe.yaml deleted file mode 100644 index 30d0d5a2ed2..00000000000 --- a/Tools/pixi-packages/freethreading/recipe.yaml +++ /dev/null @@ -1,94 +0,0 @@ -# NOTE: Please always only modify default/recipe.yaml and then run clone-recipe.sh to -# propagate the changes to the other variants. - -context: - # Keep up to date - freethreading_tag: ${{ "t" if "freethreading" in variant else "" }} - -recipe: - name: python - -source: - - path: ../../.. - -outputs: -- package: - name: python_abi - version: ${{ version }} - build: - string: "0_${{ abi_tag }}" - requirements: - run_constraints: - - python ${{ version }}.* *_${{ abi_tag }} - -- package: - name: python - version: ${{ version }} - build: - string: "0_${{ abi_tag }}" - files: - exclude: - - "*.o" - script: - file: ../build.sh - env: - PYTHON_VARIANT: ${{ variant }} - python: - site_packages_path: "lib/python${{ version }}${{ freethreading_tag }}/site-packages" - - # derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml - requirements: - build: - - ${{ compiler('c') }} - - ${{ compiler('cxx') }} - # Note that we are not using stdlib arguments which means the packages - # are built for the build settings and are not relocatable to a different - # machine that has a older system version. (eg: macOS/glibc version) - - make - - pkg-config - # configure script looks for llvm-ar for lto - - if: osx - then: - - llvm-tools - - host: - - bzip2 - - sqlite - - liblzma-devel - - zlib - - zstd - - openssl - - readline - - tk - # These two are just to get the headers needed for tk.h, but is unused - - xorg-libx11 - - xorg-xorgproto - - ncurses - - libffi - - if: linux - then: - - libuuid - - libmpdec-devel - - expat - - if: linux and "san" in variant - then: - - libsanitizer - - if: osx and "san" in variant - then: - - libcompiler-rt - - ignore_run_exports: - from_package: - - xorg-libx11 - - xorg-xorgproto - - run_exports: - noarch: - - python - weak: - - python_abi ${{ version }}.* *_${{ abi_tag }} - -about: - homepage: https://www.python.org/ - license: Python-2.0 - license_file: LICENSE diff --git a/Tools/pixi-packages/tsan-freethreading/pixi.toml b/Tools/pixi-packages/tsan-freethreading/pixi.toml index e3b5673d962..bf9841e1867 100644 --- a/Tools/pixi-packages/tsan-freethreading/pixi.toml +++ b/Tools/pixi-packages/tsan-freethreading/pixi.toml @@ -5,7 +5,11 @@ channels = ["https://prefix.dev/conda-forge"] platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"] preview = ["pixi-build"] +requires-pixi = ">=0.66.0" [package.build.backend] name = "pixi-build-rattler-build" version = "*" + +[package.build.config] +recipe = "../default/recipe.yaml" diff --git a/Tools/pixi-packages/tsan-freethreading/recipe.yaml b/Tools/pixi-packages/tsan-freethreading/recipe.yaml deleted file mode 100644 index 30d0d5a2ed2..00000000000 --- a/Tools/pixi-packages/tsan-freethreading/recipe.yaml +++ /dev/null @@ -1,94 +0,0 @@ -# NOTE: Please always only modify default/recipe.yaml and then run clone-recipe.sh to -# propagate the changes to the other variants. - -context: - # Keep up to date - freethreading_tag: ${{ "t" if "freethreading" in variant else "" }} - -recipe: - name: python - -source: - - path: ../../.. - -outputs: -- package: - name: python_abi - version: ${{ version }} - build: - string: "0_${{ abi_tag }}" - requirements: - run_constraints: - - python ${{ version }}.* *_${{ abi_tag }} - -- package: - name: python - version: ${{ version }} - build: - string: "0_${{ abi_tag }}" - files: - exclude: - - "*.o" - script: - file: ../build.sh - env: - PYTHON_VARIANT: ${{ variant }} - python: - site_packages_path: "lib/python${{ version }}${{ freethreading_tag }}/site-packages" - - # derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml - requirements: - build: - - ${{ compiler('c') }} - - ${{ compiler('cxx') }} - # Note that we are not using stdlib arguments which means the packages - # are built for the build settings and are not relocatable to a different - # machine that has a older system version. (eg: macOS/glibc version) - - make - - pkg-config - # configure script looks for llvm-ar for lto - - if: osx - then: - - llvm-tools - - host: - - bzip2 - - sqlite - - liblzma-devel - - zlib - - zstd - - openssl - - readline - - tk - # These two are just to get the headers needed for tk.h, but is unused - - xorg-libx11 - - xorg-xorgproto - - ncurses - - libffi - - if: linux - then: - - libuuid - - libmpdec-devel - - expat - - if: linux and "san" in variant - then: - - libsanitizer - - if: osx and "san" in variant - then: - - libcompiler-rt - - ignore_run_exports: - from_package: - - xorg-libx11 - - xorg-xorgproto - - run_exports: - noarch: - - python - weak: - - python_abi ${{ version }}.* *_${{ abi_tag }} - -about: - homepage: https://www.python.org/ - license: Python-2.0 - license_file: LICENSE From ad3c5b7958b890382f431a53349320cb7c84d405 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 22 Apr 2026 14:31:19 -0400 Subject: [PATCH 041/152] gh-148820: Fix _PyRawMutex use-after-free on spurious semaphore wakeup (gh-148852) _PyRawMutex_UnlockSlow CAS-removes the waiter from the list and then calls _PySemaphore_Wakeup, with no handshake. If _PySemaphore_Wait returns Py_PARK_INTR, the waiter can destroy its stack-allocated semaphore before the unlocker's Wakeup runs, causing a fatal error from ReleaseSemaphore / sem_post. Loop in _PyRawMutex_LockSlow until _PySemaphore_Wait returns Py_PARK_OK, which is only signalled when a matching Wakeup has been observed. Also include GetLastError() and the handle in the Windows fatal messages in _PySemaphore_Init, _PySemaphore_Wait, and _PySemaphore_Wakeup to make similar races easier to diagnose in the future. --- .../2026-04-21-14-36-44.gh-issue-148820.XhOGhA.rst | 5 +++++ Python/lock.c | 11 ++++++++++- Python/parking_lot.c | 12 ++++++++---- 3 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-14-36-44.gh-issue-148820.XhOGhA.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-14-36-44.gh-issue-148820.XhOGhA.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-14-36-44.gh-issue-148820.XhOGhA.rst new file mode 100644 index 00000000000..392becaffb7 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-14-36-44.gh-issue-148820.XhOGhA.rst @@ -0,0 +1,5 @@ +Fix a race in :c:type:`!_PyRawMutex` on the free-threaded build where a +``Py_PARK_INTR`` return from ``_PySemaphore_Wait`` could let the waiter +destroy its semaphore before the unlocking thread's +``_PySemaphore_Wakeup`` completed, causing a fatal ``ReleaseSemaphore`` +error. diff --git a/Python/lock.c b/Python/lock.c index 752a5899e08..af136fefd29 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -248,7 +248,16 @@ _PyRawMutex_LockSlow(_PyRawMutex *m) // Wait for us to be woken up. Note that we still have to lock the // mutex ourselves: it is NOT handed off to us. - _PySemaphore_Wait(&waiter.sema, -1); + // + // Loop until we observe an actual wakeup. A return of Py_PARK_INTR + // could otherwise let us exit _PySemaphore_Wait and destroy + // `waiter.sema` while _PyRawMutex_UnlockSlow's matching + // _PySemaphore_Wakeup is still pending, since the unlocker has + // already CAS-removed us from the waiter list without any handshake. + int res; + do { + res = _PySemaphore_Wait(&waiter.sema, -1); + } while (res != Py_PARK_OK); } _PySemaphore_Destroy(&waiter.sema); diff --git a/Python/parking_lot.c b/Python/parking_lot.c index 99c1ad848be..8823d77719c 100644 --- a/Python/parking_lot.c +++ b/Python/parking_lot.c @@ -61,7 +61,9 @@ _PySemaphore_Init(_PySemaphore *sema) NULL // unnamed ); if (!sema->platform_sem) { - Py_FatalError("parking_lot: CreateSemaphore failed"); + _Py_FatalErrorFormat(__func__, + "parking_lot: CreateSemaphore failed (error: %u)", + GetLastError()); } #elif defined(_Py_USE_SEMAPHORES) if (sem_init(&sema->platform_sem, /*pshared=*/0, /*value=*/0) < 0) { @@ -141,8 +143,8 @@ _PySemaphore_Wait(_PySemaphore *sema, PyTime_t timeout) } else { _Py_FatalErrorFormat(__func__, - "unexpected error from semaphore: %u (error: %u)", - wait, GetLastError()); + "unexpected error from semaphore: %u (error: %u, handle: %p)", + wait, GetLastError(), sema->platform_sem); } #elif defined(_Py_USE_SEMAPHORES) int err; @@ -230,7 +232,9 @@ _PySemaphore_Wakeup(_PySemaphore *sema) { #if defined(MS_WINDOWS) if (!ReleaseSemaphore(sema->platform_sem, 1, NULL)) { - Py_FatalError("parking_lot: ReleaseSemaphore failed"); + _Py_FatalErrorFormat(__func__, + "parking_lot: ReleaseSemaphore failed (error: %u, handle: %p)", + GetLastError(), sema->platform_sem); } #elif defined(_Py_USE_SEMAPHORES) int err = sem_post(&sema->platform_sem); From 76b3923d688c0efc580658476c5f525ec8735104 Mon Sep 17 00:00:00 2001 From: Seth Larson Date: Wed, 22 Apr 2026 14:22:31 -0500 Subject: [PATCH 042/152] gh-90309: Base64-encode cookie values embedded in JS --- Lib/http/cookies.py | 8 +++-- Lib/test/test_http_cookies.py | 29 ++++++++++++------- ...6-04-21-13-46-30.gh-issue-90309.srvj9q.rst | 3 ++ 3 files changed, 27 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-04-21-13-46-30.gh-issue-90309.srvj9q.rst diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index 76954111699..660fec4f1be 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -391,17 +391,21 @@ def __repr__(self): return '<%s: %s>' % (self.__class__.__name__, self.OutputString()) def js_output(self, attrs=None): + import base64 # Print javascript output_string = self.OutputString(attrs) if _has_control_character(output_string): raise CookieError("Control characters are not allowed in cookies") + # Base64-encode value to avoid template + # injection in cookie values. + output_encoded = base64.b64encode(output_string.encode('utf-8')).decode("ascii") return """ - """ % (output_string.replace('"', r'\"')) + """ % (output_encoded,) def OutputString(self, attrs=None): # Build up our result diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index e2c7551c0b3..cfcbc17bd6d 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -1,5 +1,5 @@ # Simple test suite for http/cookies.py - +import base64 import copy import unittest import doctest @@ -175,17 +175,19 @@ def test_load(self): self.assertEqual(C.output(['path']), 'Set-Cookie: Customer="WILE_E_COYOTE"; Path=/acme') - self.assertEqual(C.js_output(), r""" + cookie_encoded = base64.b64encode(b'Customer="WILE_E_COYOTE"; Path=/acme; Version=1').decode('ascii') + self.assertEqual(C.js_output(), fr""" """) - self.assertEqual(C.js_output(['path']), r""" + cookie_encoded = base64.b64encode(b'Customer="WILE_E_COYOTE"; Path=/acme').decode('ascii') + self.assertEqual(C.js_output(['path']), fr""" """) @@ -290,17 +292,19 @@ def test_quoted_meta(self): self.assertEqual(C.output(['path']), 'Set-Cookie: Customer="WILE_E_COYOTE"; Path=/acme') - self.assertEqual(C.js_output(), r""" + expected_encoded_cookie = base64.b64encode(b'Customer=\"WILE_E_COYOTE\"; Path=/acme; Version=1').decode('ascii') + self.assertEqual(C.js_output(), fr""" """) - self.assertEqual(C.js_output(['path']), r""" + expected_encoded_cookie = base64.b64encode(b'Customer=\"WILE_E_COYOTE\"; Path=/acme').decode('ascii') + self.assertEqual(C.js_output(['path']), fr""" """) @@ -391,13 +395,16 @@ def test_setter(self): self.assertEqual( M.output(), "Set-Cookie: %s=%s; Path=/foo" % (i, "%s_coded_val" % i)) + expected_encoded_cookie = base64.b64encode( + ("%s=%s; Path=/foo" % (i, "%s_coded_val" % i)).encode("ascii") + ).decode('ascii') expected_js_output = """ - """ % (i, "%s_coded_val" % i) + """ % (expected_encoded_cookie,) self.assertEqual(M.js_output(), expected_js_output) for i in ["foo bar", "foo@bar"]: # Try some illegal characters diff --git a/Misc/NEWS.d/next/Security/2026-04-21-13-46-30.gh-issue-90309.srvj9q.rst b/Misc/NEWS.d/next/Security/2026-04-21-13-46-30.gh-issue-90309.srvj9q.rst new file mode 100644 index 00000000000..d7d376737e4 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-04-21-13-46-30.gh-issue-90309.srvj9q.rst @@ -0,0 +1,3 @@ +Base64-encode values when embedding cookies to JavaScript using the +:meth:`http.cookies.BaseCookie.js_output` method to avoid injection +and escaping. From 79321fdce3227cf09bb8a2894d856753f1ba098e Mon Sep 17 00:00:00 2001 From: Sanjay Janardhan <21janardhansanjay@gmail.com> Date: Wed, 22 Apr 2026 15:56:14 -0700 Subject: [PATCH 043/152] gh-148883: Docs: clarify grammar in Counter dictionary methods note (gh-148882) --- Doc/library/collections.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index cb9300f072b..e42bdc06be0 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -326,7 +326,7 @@ For example:: .. versionadded:: 3.10 The usual dictionary methods are available for :class:`Counter` objects - except for two which work differently for counters. + except for these two which work differently for counters: .. method:: fromkeys(iterable) From be833e658aaf6703b0dd0c0dadb893d72cbe4c77 Mon Sep 17 00:00:00 2001 From: Shamil Date: Thu, 23 Apr 2026 05:31:58 +0300 Subject: [PATCH 044/152] gh-146553: Fix infinite loop in typing.get_type_hints() on circular __wrapped__ (#148595) --- Lib/test/test_typing.py | 18 ++++++++++++++++++ Lib/typing.py | 4 ++++ ...6-04-15-11-00-39.gh-issue-146553.VGOsoP.rst | 2 ++ 3 files changed, 24 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-04-15-11-00-39.gh-issue-146553.VGOsoP.rst diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index bfae83fdaf6..6c3d67fb6b7 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -6888,6 +6888,24 @@ def test_get_type_hints_wrapped_decoratored_func(self): self.assertEqual(gth(ForRefExample.func), expects) self.assertEqual(gth(ForRefExample.nested), expects) + def test_get_type_hints_wrapped_cycle_self(self): + # gh-146553: __wrapped__ self-reference must raise ValueError, + # not loop forever. + def f(x: int) -> str: ... + f.__wrapped__ = f + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + get_type_hints(f) + + def test_get_type_hints_wrapped_cycle_mutual(self): + # gh-146553: mutual __wrapped__ cycle (a -> b -> a) must raise + # ValueError, not loop forever. + def a(): ... + def b(): ... + a.__wrapped__ = b + b.__wrapped__ = a + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + get_type_hints(a) + def test_get_type_hints_annotated(self): def foobar(x: List['X']): ... X = Annotated[int, (1, 10)] diff --git a/Lib/typing.py b/Lib/typing.py index 3e7661dd2f8..46e7122b6c9 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2486,8 +2486,12 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, else: nsobj = obj # Find globalns for the unwrapped object. + seen = {id(nsobj)} while hasattr(nsobj, '__wrapped__'): nsobj = nsobj.__wrapped__ + if id(nsobj) in seen: + raise ValueError(f'wrapper loop when unwrapping {obj!r}') + seen.add(id(nsobj)) globalns = getattr(nsobj, '__globals__', {}) if localns is None: localns = globalns diff --git a/Misc/NEWS.d/next/Library/2026-04-15-11-00-39.gh-issue-146553.VGOsoP.rst b/Misc/NEWS.d/next/Library/2026-04-15-11-00-39.gh-issue-146553.VGOsoP.rst new file mode 100644 index 00000000000..44216318d47 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-15-11-00-39.gh-issue-146553.VGOsoP.rst @@ -0,0 +1,2 @@ +Fix infinite loop in :func:`typing.get_type_hints` when ``__wrapped__`` +forms a cycle. Patch by Shamil Abdulaev. From 8e43f3d1177f22c95f5fc66349a3b748a36470c9 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 23 Apr 2026 04:39:08 +0200 Subject: [PATCH 045/152] gh-145056: Add support for frozendict in dataclass asdict and astuple (#145125) --- Doc/library/dataclasses.rst | 8 ++++---- Lib/dataclasses.py | 10 ++++++---- Lib/test/test_dataclasses/__init__.py | 20 ++++++++++++++++--- ...-02-22-19-36-00.gh-issue-145056.TH8nX4.rst | 1 + 4 files changed, 28 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-02-22-19-36-00.gh-issue-145056.TH8nX4.rst diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index fd8e0c0bea1..0bce3e5b762 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -371,8 +371,8 @@ Module contents Converts the dataclass *obj* to a dict (by using the factory function *dict_factory*). Each dataclass is converted to a dict of its fields, as ``name: value`` pairs. dataclasses, dicts, - lists, and tuples are recursed into. Other objects are copied with - :func:`copy.deepcopy`. + frozendicts, lists, and tuples are recursed into. Other objects are copied + with :func:`copy.deepcopy`. Example of using :func:`!asdict` on nested dataclasses:: @@ -402,8 +402,8 @@ Module contents Converts the dataclass *obj* to a tuple (by using the factory function *tuple_factory*). Each dataclass is converted - to a tuple of its field values. dataclasses, dicts, lists, and - tuples are recursed into. Other objects are copied with + to a tuple of its field values. dataclasses, dicts, frozendicts, lists, + and tuples are recursed into. Other objects are copied with :func:`copy.deepcopy`. Continuing from the previous example:: diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 0c7e01cb16b..9d5bed6b96f 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1496,7 +1496,8 @@ class C: If given, 'dict_factory' will be used instead of built-in dict. The function applies recursively to field values that are dataclass instances. This will also look into built-in containers: - tuples, lists, and dicts. Other objects are copied with 'copy.deepcopy()'. + tuples, lists, dicts, and frozendicts. Other objects are copied + with 'copy.deepcopy()'. """ if not _is_dataclass_instance(obj): raise TypeError("asdict() should be called on dataclass instances") @@ -1552,7 +1553,7 @@ def _asdict_inner(obj, dict_factory): return obj_type(*[_asdict_inner(v, dict_factory) for v in obj]) else: return obj_type(_asdict_inner(v, dict_factory) for v in obj) - elif issubclass(obj_type, dict): + elif issubclass(obj_type, (dict, frozendict)): if hasattr(obj_type, 'default_factory'): # obj is a defaultdict, which has a different constructor from # dict as it requires the default_factory as its first arg. @@ -1587,7 +1588,8 @@ class C: If given, 'tuple_factory' will be used instead of built-in tuple. The function applies recursively to field values that are dataclass instances. This will also look into built-in containers: - tuples, lists, and dicts. Other objects are copied with 'copy.deepcopy()'. + tuples, lists, dicts, and frozendicts. Other objects are copied + with 'copy.deepcopy()'. """ if not _is_dataclass_instance(obj): @@ -1616,7 +1618,7 @@ def _astuple_inner(obj, tuple_factory): # generator (which is not true for namedtuples, handled # above). return type(obj)(_astuple_inner(v, tuple_factory) for v in obj) - elif isinstance(obj, dict): + elif isinstance(obj, (dict, frozendict)): obj_type = type(obj) if hasattr(obj_type, 'default_factory'): # obj is a defaultdict, which has a different constructor from diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index b44b1da0336..e0cfe3df3e6 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -1693,17 +1693,24 @@ class GroupTuple: class GroupDict: id: int users: Dict[str, User] + @dataclass + class GroupFrozenDict: + id: int + users: frozendict[str, User] a = User('Alice', 1) b = User('Bob', 2) gl = GroupList(0, [a, b]) gt = GroupTuple(0, (a, b)) gd = GroupDict(0, {'first': a, 'second': b}) + gfd = GroupFrozenDict(0, frozendict({'first': a, 'second': b})) self.assertEqual(asdict(gl), {'id': 0, 'users': [{'name': 'Alice', 'id': 1}, {'name': 'Bob', 'id': 2}]}) self.assertEqual(asdict(gt), {'id': 0, 'users': ({'name': 'Alice', 'id': 1}, {'name': 'Bob', 'id': 2})}) - self.assertEqual(asdict(gd), {'id': 0, 'users': {'first': {'name': 'Alice', 'id': 1}, - 'second': {'name': 'Bob', 'id': 2}}}) + expected_dict = {'id': 0, 'users': {'first': {'name': 'Alice', 'id': 1}, + 'second': {'name': 'Bob', 'id': 2}}} + self.assertEqual(asdict(gd), expected_dict) + self.assertEqual(asdict(gfd), expected_dict) def test_helper_asdict_builtin_object_containers(self): @dataclass @@ -1884,14 +1891,21 @@ class GroupTuple: class GroupDict: id: int users: Dict[str, User] + @dataclass + class GroupFrozenDict: + id: int + users: frozendict[str, User] a = User('Alice', 1) b = User('Bob', 2) gl = GroupList(0, [a, b]) gt = GroupTuple(0, (a, b)) gd = GroupDict(0, {'first': a, 'second': b}) + gfd = GroupFrozenDict(0, frozendict({'first': a, 'second': b})) self.assertEqual(astuple(gl), (0, [('Alice', 1), ('Bob', 2)])) self.assertEqual(astuple(gt), (0, (('Alice', 1), ('Bob', 2)))) - self.assertEqual(astuple(gd), (0, {'first': ('Alice', 1), 'second': ('Bob', 2)})) + d = {'first': ('Alice', 1), 'second': ('Bob', 2)} + self.assertEqual(astuple(gd), (0, d)) + self.assertEqual(astuple(gfd), (0, frozendict(d))) def test_helper_astuple_builtin_object_containers(self): @dataclass diff --git a/Misc/NEWS.d/next/Library/2026-02-22-19-36-00.gh-issue-145056.TH8nX4.rst b/Misc/NEWS.d/next/Library/2026-02-22-19-36-00.gh-issue-145056.TH8nX4.rst new file mode 100644 index 00000000000..45be0109677 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-02-22-19-36-00.gh-issue-145056.TH8nX4.rst @@ -0,0 +1 @@ +Add support for :class:`frozendict` in :meth:`dataclasses.asdict` and :meth:`dataclasses.astuple`. From bd7352d8071dc00531f2c527977602729f2d3ec6 Mon Sep 17 00:00:00 2001 From: Vikash Kumar <163628932+Vikash-Kumar-23@users.noreply.github.com> Date: Thu, 23 Apr 2026 08:10:10 +0530 Subject: [PATCH 046/152] gh-145194: Fix typing in re tokenizer example (#145198) --- Doc/library/re.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 6ed285c4b11..a46fd424581 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -1953,7 +1953,7 @@ successive matches:: class Token(NamedTuple): type: str - value: str + value: int | float | str line: int column: int From 75ff1afcb6a1bb2b3d54899e9b222a61798fa491 Mon Sep 17 00:00:00 2001 From: John Seong <39040639+sandole@users.noreply.github.com> Date: Thu, 23 Apr 2026 10:46:04 +0800 Subject: [PATCH 047/152] gh-142965: Fix Concatenate documentation to reflect valid use cases (#143316) The documentation previously stated that Concatenate is only valid when used as the first argument to Callable, but according to PEP 612, it can also be used when instantiating user-defined generic classes with ParamSpec parameters. --- Doc/library/typing.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 9bc0a3caeee..04acf2c16d1 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1174,7 +1174,8 @@ These can be used as types in annotations. They all support subscription using or transforms parameters of another callable. Usage is in the form ``Concatenate[Arg1Type, Arg2Type, ..., ParamSpecVariable]``. ``Concatenate`` - is currently only valid when used as the first argument to a :ref:`Callable `. + is valid when used in :ref:`Callable ` type hints + and when instantiating user-defined generic classes with :class:`ParamSpec` parameters. The last parameter to ``Concatenate`` must be a :class:`ParamSpec` or ellipsis (``...``). From 8bf99ae3a9f12d105a70d6fda93dddde4adeee8f Mon Sep 17 00:00:00 2001 From: Victorien <65306057+Viicos@users.noreply.github.com> Date: Thu, 23 Apr 2026 04:50:15 +0200 Subject: [PATCH 048/152] gh-119180: Document the `format` parameter in `typing.get_type_hints()` (#143758) Do not mention `__annotations__` dictionaries, as this is slightly outdated since 3.14. Rewrite the note about possible exceptions for clarity. Also do not mention imported type aliases, as since 3.12 aliases with the `type` statement do not suffer from this limitation anymore. --- Doc/library/typing.rst | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 04acf2c16d1..1957cadcbb1 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -3453,13 +3453,13 @@ Functions and decorators Introspection helpers --------------------- -.. function:: get_type_hints(obj, globalns=None, localns=None, include_extras=False) +.. function:: get_type_hints(obj, globalns=None, localns=None, include_extras=False, *, format=Format.VALUE) Return a dictionary containing type hints for a function, method, module, class object, or other callable object. - This is often the same as ``obj.__annotations__``, but this function makes - the following changes to the annotations dictionary: + This is often the same as :func:`annotationlib.get_annotations`, but this + function makes the following changes to the annotations dictionary: * Forward references encoded as string literals or :class:`ForwardRef` objects are handled by evaluating them in *globalns*, *localns*, and @@ -3473,17 +3473,15 @@ Introspection helpers annotations from ``C``'s base classes with those on ``C`` directly. This is done by traversing :attr:`C.__mro__ ` and iteratively combining - ``__annotations__`` dictionaries. Annotations on classes appearing - earlier in the :term:`method resolution order` always take precedence over - annotations on classes appearing later in the method resolution order. + :term:`annotations ` of each base class. Annotations + on classes appearing earlier in the :term:`method resolution order` always + take precedence over annotations on classes appearing later in the method + resolution order. * The function recursively replaces all occurrences of ``Annotated[T, ...]``, ``Required[T]``, ``NotRequired[T]``, and ``ReadOnly[T]`` with ``T``, unless *include_extras* is set to ``True`` (see :class:`Annotated` for more information). - See also :func:`annotationlib.get_annotations`, a lower-level function that - returns annotations more directly. - .. caution:: This function may execute arbitrary code contained in annotations. @@ -3491,11 +3489,12 @@ Introspection helpers .. note:: - If any forward references in the annotations of *obj* are not resolvable - or are not valid Python code, this function will raise an exception - such as :exc:`NameError`. For example, this can happen with imported - :ref:`type aliases ` that include forward references, - or with names imported under :data:`if TYPE_CHECKING `. + If :attr:`Format.VALUE ` is used and any + forward references in the annotations of *obj* are not resolvable, a + :exc:`NameError` exception is raised. For example, this can happen + with names imported under :data:`if TYPE_CHECKING `. + More generally, any kind of exception can be raised if an annotation + contains invalid Python code. .. note:: @@ -3513,6 +3512,10 @@ Introspection helpers if a default value equal to ``None`` was set. Now the annotation is returned unchanged. + .. versionchanged:: 3.14 + Added the ``format`` parameter. See the documentation on + :func:`annotationlib.get_annotations` for more information. + .. versionchanged:: 3.14 Calling :func:`get_type_hints` on instances is no longer supported. Some instances were accepted in earlier versions as an undocumented From fbc7676df6256071682f4179818b74ba29f162cd Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 22 Apr 2026 22:06:56 -0500 Subject: [PATCH 049/152] Speed up counting in statistics.fmean() (gh-148875) --- Lib/statistics.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/statistics.py b/Lib/statistics.py index e635b99f958..32fcf2313a8 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -136,7 +136,7 @@ from fractions import Fraction from decimal import Decimal -from itertools import count, groupby, repeat +from itertools import compress, count, groupby, repeat from bisect import bisect_left, bisect_right from math import hypot, sqrt, fabs, exp, erfc, tau, log, fsum, sumprod from math import isfinite, isinf, pi, cos, sin, tan, cosh, asin, atan, acos @@ -195,9 +195,9 @@ def fmean(data, weights=None): n = len(data) except TypeError: # Handle iterators that do not define __len__(). - counter = count() - total = fsum(map(itemgetter(0), zip(data, counter))) - n = next(counter) + counter = count(1) + total = fsum(compress(data, counter)) + n = next(counter) - 1 else: total = fsum(data) From 3b9397988d1f83740e7d73d17d56767976a583b4 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Wed, 22 Apr 2026 22:00:35 -0600 Subject: [PATCH 050/152] gh-148892: Drop mention of deprecated cibuildwheel option (#148893) --- Doc/howto/free-threading-extensions.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index 2f089a3d896..b21ed1c8f37 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -416,11 +416,9 @@ C API extensions need to be built specifically for the free-threaded build. The wheels, shared libraries, and binaries are indicated by a ``t`` suffix. * `pypa/manylinux `_ supports the - free-threaded build, with the ``t`` suffix, such as ``python3.13t``. -* `pypa/cibuildwheel `_ supports the - free-threaded build on Python 3.13 and 3.14. On Python 3.14, free-threaded - wheels will be built by default. On Python 3.13, you will need to set - `CIBW_ENABLE to cpython-freethreading `_. + free-threaded build, with the ``t`` suffix, such as ``python3.14t``. +* `pypa/cibuildwheel `_ supports + building wheels for the free-threaded build of Python 3.14 and newer. Limited C API and Stable ABI ............................ From ab41a347ebc7e6c3e3f5795c4a24545bfbf92a6e Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 23 Apr 2026 11:52:13 +0200 Subject: [PATCH 051/152] gh-146636: Improve ABI/feature selection, add new header for it (GH-148302) Improve ABI/feature selection, add new header for it. Add a test that Python headers themselves don't use Py_GIL_DISABLED in abi3t: abi3 and abi3t ought to be the same except the _Py_OPAQUE_PYOBJECT differences. This is done using the GCC-only poison pragma. Co-authored-by: Victor Stinner --- Include/Python.h | 22 ++-- Include/exports.h | 8 +- Include/patchlevel.h | 28 ---- Include/pyabi.h | 121 ++++++++++++++++++ Include/pyport.h | 39 ------ Lib/test/test_cext/setup.py | 5 + Makefile.pre.in | 1 + ...-04-09-14-45-44.gh-issue-148267.p84kG_.rst | 2 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 + 10 files changed, 148 insertions(+), 82 deletions(-) create mode 100644 Include/pyabi.h create mode 100644 Misc/NEWS.d/next/C_API/2026-04-09-14-45-44.gh-issue-148267.p84kG_.rst diff --git a/Include/Python.h b/Include/Python.h index e6e5cab67e2..8b76195b320 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -9,10 +9,11 @@ // is not needed. -// Include Python header files -#include "patchlevel.h" -#include "pyconfig.h" -#include "pymacconfig.h" +// Include Python configuration headers +#include "patchlevel.h" // the Python version +#include "pyconfig.h" // information from configure +#include "pymacconfig.h" // overrides for pyconfig +#include "pyabi.h" // feature/ABI selection // Include standard header files @@ -46,13 +47,11 @@ # endif #endif -#if defined(Py_GIL_DISABLED) -# if defined(_MSC_VER) -# include // __readgsqword() -# endif - -# if defined(__MINGW32__) -# include // __readgsqword() +#if !defined(Py_LIMITED_API) +# if defined(Py_GIL_DISABLED) +# if defined(_MSC_VER) || defined(__MINGW32__) +# include // __readgsqword() +# endif # endif #endif // Py_GIL_DISABLED @@ -67,6 +66,7 @@ __pragma(warning(disable: 4201)) // Include Python header files #include "pyport.h" +#include "exports.h" #include "pymacro.h" #include "pymath.h" #include "pymem.h" diff --git a/Include/exports.h b/Include/exports.h index 97a674ec240..a863ecb3307 100644 --- a/Include/exports.h +++ b/Include/exports.h @@ -36,7 +36,7 @@ #define Py_LOCAL_SYMBOL #endif /* module init functions outside the core must be exported */ - #if defined(Py_BUILD_CORE) + #if defined(_PyEXPORTS_CORE) #define _PyINIT_EXPORTED_SYMBOL Py_EXPORTED_SYMBOL #else #define _PyINIT_EXPORTED_SYMBOL __declspec(dllexport) @@ -64,13 +64,13 @@ /* only get special linkage if built as shared or platform is Cygwin */ #if defined(Py_ENABLE_SHARED) || defined(__CYGWIN__) # if defined(HAVE_DECLSPEC_DLL) -# if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# if defined(_PyEXPORTS_CORE) && !defined(_PyEXPORTS_CORE_MODULE) /* module init functions inside the core need no external linkage */ /* except for Cygwin to handle embedding */ # if !defined(__CYGWIN__) # define _PyINIT_FUNC_DECLSPEC # endif /* __CYGWIN__ */ -# else /* Py_BUILD_CORE */ +# else /* _PyEXPORTS_CORE */ /* Building an extension module, or an embedded situation */ /* public Python functions and data are imported */ /* Under Cygwin, auto-import functions to prevent compilation */ @@ -80,7 +80,7 @@ # define PyAPI_FUNC(RTYPE) Py_IMPORTED_SYMBOL RTYPE # endif /* !__CYGWIN__ */ # define PyAPI_DATA(RTYPE) extern Py_IMPORTED_SYMBOL RTYPE -# endif /* Py_BUILD_CORE */ +# endif /* _PyEXPORTS_CORE */ # endif /* HAVE_DECLSPEC_DLL */ #endif /* Py_ENABLE_SHARED */ diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 9f5c36230a7..974246f896e 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -61,32 +61,4 @@ #define PYTHON_ABI_VERSION 3 #define PYTHON_ABI_STRING "3" - -/* Stable ABI for free-threaded builds (introduced in PEP 803) - is enabled by one of: - - Py_TARGET_ABI3T, or - - Py_LIMITED_API and Py_GIL_DISABLED. - "Output" macros to be used internally: - - Py_LIMITED_API (defines the subset of API we expose) - - _Py_OPAQUE_PYOBJECT (additionally hides what's ABI-incompatible between - free-threaded & GIL) - (Don't use Py_TARGET_ABI3T directly: it's currently only used to set these - 2 macros. It's also available for users' convenience.) - */ -#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) \ - && !defined(Py_TARGET_ABI3T) -# define Py_TARGET_ABI3T Py_LIMITED_API -#endif -#if defined(Py_TARGET_ABI3T) -# define _Py_OPAQUE_PYOBJECT -# if !defined(Py_LIMITED_API) -# define Py_LIMITED_API Py_TARGET_ABI3T -# elif Py_LIMITED_API > Py_TARGET_ABI3T - // if both are defined, use the *lower* version, - // i.e. maximum compatibility -# undef Py_LIMITED_API -# define Py_LIMITED_API Py_TARGET_ABI3T -# endif -#endif - #endif //_Py_PATCHLEVEL_H diff --git a/Include/pyabi.h b/Include/pyabi.h new file mode 100644 index 00000000000..8c4ae281a43 --- /dev/null +++ b/Include/pyabi.h @@ -0,0 +1,121 @@ +/* Macros that restrict available definitions and select implementations + * to match an ABI stability promise: + * + * - internal API/ABI (may change at any time) -- Py_BUILD_CORE* + * - general CPython API/ABI (may change in 3.x.0) -- default + * - Stable ABI: abi3, abi3t (long-term stable) -- Py_LIMITED_API, + * Py_TARGET_ABI3T, _Py_OPAQUE_PYOBJECT + * - Free-threading (incompatible with non-free-threading builds) + * -- Py_GIL_DISABLED + */ + +#ifndef _Py_PYABI_H +#define _Py_PYABI_H + +/* Defines to build Python and its standard library: + * + * - Py_BUILD_CORE: Build Python core. Gives access to Python internals; should + * not be used by third-party modules. + * - Py_BUILD_CORE_BUILTIN: Build a Python stdlib module as a built-in module. + * - Py_BUILD_CORE_MODULE: Build a Python stdlib module as a dynamic library. + * + * Py_BUILD_CORE_BUILTIN and Py_BUILD_CORE_MODULE imply Py_BUILD_CORE. + * + * On Windows, Py_BUILD_CORE_MODULE exports "PyInit_xxx" symbol, whereas + * Py_BUILD_CORE_BUILTIN does not. + */ +#if defined(Py_BUILD_CORE_BUILTIN) && !defined(Py_BUILD_CORE) +# define Py_BUILD_CORE +#endif +#if defined(Py_BUILD_CORE_MODULE) && !defined(Py_BUILD_CORE) +# define Py_BUILD_CORE +#endif + +/* Check valid values for target ABI macros. + */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 3 + // Empty Py_LIMITED_API used to work; redefine to + // Python 3.2 to be explicit. +# undef Py_LIMITED_API +# define Py_LIMITED_API 0x03020000 +#endif +#if defined(Py_TARGET_ABI3T) && Py_TARGET_ABI3T+0 < 0x030f0000 +# error "Py_TARGET_ABI3T must be 0x030f0000 (3.15) or above" +#endif + +/* Stable ABI for free-threaded builds (abi3t, introduced in PEP 803) + * is enabled by one of: + * - Py_TARGET_ABI3T, or + * - Py_LIMITED_API and Py_GIL_DISABLED. + * + * These affect set the following, which Python.h should use internally: + * - Py_LIMITED_API (defines the subset of API we expose) + * - _Py_OPAQUE_PYOBJECT (additionally hides what's ABI-incompatible between + * free-threaded & GIL) + * + * (Don't use Py_TARGET_ABI3T directly. It's currently only used to set these + * 2 macros, and defined for users' convenience.) + */ +#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) \ + && !defined(Py_TARGET_ABI3T) +# define Py_TARGET_ABI3T Py_LIMITED_API +#endif +#if defined(Py_TARGET_ABI3T) +# define _Py_OPAQUE_PYOBJECT +# if !defined(Py_LIMITED_API) +# define Py_LIMITED_API Py_TARGET_ABI3T +# elif Py_LIMITED_API > Py_TARGET_ABI3T + // if both are defined, use the *lower* version, + // i.e. maximum compatibility +# undef Py_LIMITED_API +# define Py_LIMITED_API Py_TARGET_ABI3T +# endif +#else +# ifdef _Py_OPAQUE_PYOBJECT + // _Py_OPAQUE_PYOBJECT is a private macro; do not define it directly. +# error "Define Py_TARGET_ABI3T to target abi3t." +# endif +#endif + +#if defined(Py_TARGET_ABI3T) +# if !defined(Py_GIL_DISABLED) + // Define Py_GIL_DISABLED for users' needs. Users check this macro to see + // whether they need extra synchronization. +# define Py_GIL_DISABLED +# endif +# if defined(_Py_IS_TESTCEXT) + // When compiling for abi3t, contents of Python.h should not depend + // on Py_GIL_DISABLED. + // We ask GCC to error if it sees the macro from this point on. + // Since users are free to the macro, and there's no way to undo the + // poisoning at the end of Python.h, we only do this in a test module + // (test_cext). + // + // Clang's poisoning is stricter than GCC's: it looks in `#elif` + // expressions after matching `#if`s. We disable it for now. + // We also provide an undocumented, unsupported opt-out macro to help + // porting to other compilers. Consider reaching out if you use it. +# if defined(__GNUC__) && !defined(__clang__) && !defined(_Py_NO_GCC_POISON) +# undef Py_GIL_DISABLED +# pragma GCC poison Py_GIL_DISABLED +# endif +# endif +#endif + +/* The internal C API must not be used with the limited C API: make sure + * that Py_BUILD_CORE* macros are not defined in this case. + * But, keep the "original" values, under different names, for "exports.h" + */ +#ifdef Py_BUILD_CORE +# define _PyEXPORTS_CORE +#endif +#ifdef Py_BUILD_CORE_MODULE +# define _PyEXPORTS_CORE_MODULE +#endif +#ifdef Py_LIMITED_API +# undef Py_BUILD_CORE +# undef Py_BUILD_CORE_BUILTIN +# undef Py_BUILD_CORE_MODULE +#endif + +#endif // _Py_PYABI_H diff --git a/Include/pyport.h b/Include/pyport.h index 62cba4c1421..c975921beaf 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -58,34 +58,6 @@ #endif -/* Defines to build Python and its standard library: - * - * - Py_BUILD_CORE: Build Python core. Give access to Python internals, but - * should not be used by third-party modules. - * - Py_BUILD_CORE_BUILTIN: Build a Python stdlib module as a built-in module. - * - Py_BUILD_CORE_MODULE: Build a Python stdlib module as a dynamic library. - * - * Py_BUILD_CORE_BUILTIN and Py_BUILD_CORE_MODULE imply Py_BUILD_CORE. - * - * On Windows, Py_BUILD_CORE_MODULE exports "PyInit_xxx" symbol, whereas - * Py_BUILD_CORE_BUILTIN does not. - */ -#if defined(Py_BUILD_CORE_BUILTIN) && !defined(Py_BUILD_CORE) -# define Py_BUILD_CORE -#endif -#if defined(Py_BUILD_CORE_MODULE) && !defined(Py_BUILD_CORE) -# define Py_BUILD_CORE -#endif - -#if defined(Py_TARGET_ABI3T) -# if !defined(Py_GIL_DISABLED) -// Define Py_GIL_DISABLED for users' needs. This macro is used to enable -// locking needed in for free-threaded interpreters builds. -# define Py_GIL_DISABLED -# endif -#endif - - /************************************************************************** Symbols and macros to supply platform-independent interfaces to basic C language & library operations whose spellings vary across platforms. @@ -393,17 +365,6 @@ extern "C" { # define Py_NO_INLINE #endif -#include "exports.h" - -#ifdef Py_LIMITED_API - // The internal C API must not be used with the limited C API: make sure - // that Py_BUILD_CORE macro is not defined in this case. These 3 macros are - // used by exports.h, so only undefine them afterwards. -# undef Py_BUILD_CORE -# undef Py_BUILD_CORE_BUILTIN -# undef Py_BUILD_CORE_MODULE -#endif - /* limits.h constants that may be missing */ #ifndef INT_MAX diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index 7262a110d83..25fe50df603 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -18,6 +18,11 @@ # The purpose of test_cext extension is to check that building a C # extension using the Python C API does not emit C compiler warnings. '-Werror', + # Enable extra checks for header files, which: + # - need to be enabled somewhere inside Python headers (rather than + # before including Python.h) + # - should not be checked for user code + '-D_Py_IS_TESTCEXT', ] # C compiler flags for GCC and clang diff --git a/Makefile.pre.in b/Makefile.pre.in index f869c1f7c93..57fce05d476 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1214,6 +1214,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/osdefs.h \ $(srcdir)/Include/osmodule.h \ $(srcdir)/Include/patchlevel.h \ + $(srcdir)/Include/pyabi.h \ $(srcdir)/Include/pyatomic.h \ $(srcdir)/Include/pybuffer.h \ $(srcdir)/Include/pycapsule.h \ diff --git a/Misc/NEWS.d/next/C_API/2026-04-09-14-45-44.gh-issue-148267.p84kG_.rst b/Misc/NEWS.d/next/C_API/2026-04-09-14-45-44.gh-issue-148267.p84kG_.rst new file mode 100644 index 00000000000..1ec1afd2cbf --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-04-09-14-45-44.gh-issue-148267.p84kG_.rst @@ -0,0 +1,2 @@ +Using :c:macro:`Py_LIMITED_API` on a non-Windows free-threaded build no +longer needs an extra :c:macro:`Py_GIL_DISABLED`. diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 61bee29c0af..fe70e02536b 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -359,6 +359,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 664788e69af..629f063861d 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -156,6 +156,9 @@ Include + + Include + Include From 29917d51ab41df9a00f1bb35fa9d3e1392ac48e8 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Thu, 23 Apr 2026 16:42:57 +0530 Subject: [PATCH 052/152] gh-148907: fix performance regression in `PyType_GetModuleByDef` on free-threading (#148908) --- Objects/typeobject.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 08b95cfbc6c..fb3c7101410 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5878,7 +5878,13 @@ PyType_GetModuleByToken_DuringGC(PyTypeObject *type, const void *token) PyObject * PyType_GetModuleByToken(PyTypeObject *type, const void *token) { - PyObject *mod = PyType_GetModuleByToken_DuringGC(type, token); + return Py_XNewRef(PyType_GetModuleByDef(type, (PyModuleDef *)token)); +} + +PyObject * +PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) +{ + PyObject *mod = PyType_GetModuleByToken_DuringGC(type, def); if (!mod) { PyErr_Format( PyExc_TypeError, @@ -5886,14 +5892,6 @@ PyType_GetModuleByToken(PyTypeObject *type, const void *token) type->tp_name); return NULL; } - return Py_NewRef(mod); -} - -PyObject * -PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) -{ - PyObject *mod = PyType_GetModuleByToken(type, def); - Py_XDECREF(mod); // return borrowed ref return mod; } From 9633c5239daae3a180f8ce263ce77e4e522e6aa4 Mon Sep 17 00:00:00 2001 From: Diego Russo Date: Thu, 23 Apr 2026 12:23:18 +0100 Subject: [PATCH 053/152] GH-126910: Build/link the JIT shim in the Python interpreter (#148872) --- Include/internal/pycore_ceval.h | 7 --- Include/internal/pycore_jit.h | 6 +- Makefile.pre.in | 26 ++++++-- PCbuild/pyproject.props | 5 +- PCbuild/pythoncore.vcxproj | 3 + PCbuild/regen.targets | 5 +- Python/ceval.c | 2 +- Python/jit.c | 96 ----------------------------- Python/pylifecycle.c | 8 --- Python/pystate.c | 5 -- Tools/jit/_targets.py | 106 ++++++++++++++++++++++---------- Tools/jit/_writer.py | 4 -- Tools/jit/build.py | 1 + Tools/jit/shim.c | 2 +- configure | 68 ++++++++++++-------- configure.ac | 67 ++++++++++++-------- 16 files changed, 202 insertions(+), 209 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index ee8eb1095fe..f9507fda160 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -121,18 +121,11 @@ _PyEval_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwfl } #ifdef _Py_TIER2 -#ifdef _Py_JIT -_Py_CODEUNIT *_Py_LazyJitShim( - struct _PyExecutorObject *current_executor, _PyInterpreterFrame *frame, - _PyStackRef *stack_pointer, PyThreadState *tstate -); -#else _Py_CODEUNIT *_PyTier2Interpreter( struct _PyExecutorObject *current_executor, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate ); #endif -#endif extern _PyJitEntryFuncPtr _Py_jit_entry; diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h index 70bccce4166..b3cadcce824 100644 --- a/Include/internal/pycore_jit.h +++ b/Include/internal/pycore_jit.h @@ -23,9 +23,13 @@ typedef _Py_CODEUNIT *(*jit_func)( _PyStackRef _tos_cache0, _PyStackRef _tos_cache1, _PyStackRef _tos_cache2 ); +_Py_CODEUNIT *_PyJIT( + _PyExecutorObject *executor, _PyInterpreterFrame *frame, + _PyStackRef *stack_pointer, PyThreadState *tstate +); + int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length); void _PyJIT_Free(_PyExecutorObject *executor); -void _PyJIT_Fini(void); PyAPI_FUNC(int) _PyJIT_AddressInJitCode(PyInterpreterState *interp, uintptr_t addr); #endif // _Py_JIT diff --git a/Makefile.pre.in b/Makefile.pre.in index 57fce05d476..8b46db33a2a 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -290,6 +290,7 @@ LDLIBRARYDIR= @LDLIBRARYDIR@ INSTSONAME= @INSTSONAME@ LIBRARY_DEPS= @LIBRARY_DEPS@ LINK_PYTHON_DEPS=@LINK_PYTHON_DEPS@ +JIT_OBJS= @JIT_SHIM_O@ PY_ENABLE_SHARED= @PY_ENABLE_SHARED@ STATIC_LIBPYTHON= @STATIC_LIBPYTHON@ @@ -469,6 +470,7 @@ PYTHON_OBJS= \ Python/instruction_sequence.o \ Python/intrinsics.o \ Python/jit.o \ + $(JIT_OBJS) \ Python/legacy_tracing.o \ Python/lock.o \ Python/marshal.o \ @@ -3204,21 +3206,37 @@ Python/emscripten_trampoline_inner.wasm: $(srcdir)/Python/emscripten_trampoline_ Python/emscripten_trampoline_wasm.c: Python/emscripten_trampoline_inner.wasm $(PYTHON_FOR_REGEN) $(srcdir)/Platforms/emscripten/prepare_external_wasm.py $< $@ getWasmTrampolineModule +JIT_SHIM_BUILD_OBJS= @JIT_SHIM_BUILD_O@ +JIT_BUILD_TARGETS= jit_stencils.h @JIT_STENCILS_H@ $(JIT_SHIM_BUILD_OBJS) +JIT_TARGETS= $(JIT_BUILD_TARGETS) $(filter-out $(JIT_SHIM_BUILD_OBJS),$(JIT_OBJS)) +JIT_GENERATED_STAMP= .jit-stamp + JIT_DEPS = \ $(srcdir)/Tools/jit/*.c \ + $(srcdir)/Tools/jit/*.h \ $(srcdir)/Tools/jit/*.py \ $(srcdir)/Python/executor_cases.c.h \ pyconfig.h -jit_stencils.h @JIT_STENCILS_H@: $(JIT_DEPS) +$(JIT_GENERATED_STAMP): $(JIT_DEPS) @REGEN_JIT_COMMAND@ + @touch $@ + +$(JIT_BUILD_TARGETS): $(JIT_GENERATED_STAMP) + @if test ! -f "$@"; then \ + rm -f $(JIT_GENERATED_STAMP); \ + $(MAKE) $(JIT_GENERATED_STAMP); \ + test -f "$@"; \ + fi + +jit_shim-universal2-apple-darwin.o: jit_shim-aarch64-apple-darwin.o jit_shim-x86_64-apple-darwin.o + lipo -create -output $@ jit_shim-aarch64-apple-darwin.o jit_shim-x86_64-apple-darwin.o Python/jit.o: $(srcdir)/Python/jit.c @JIT_STENCILS_H@ $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< .PHONY: regen-jit -regen-jit: - @REGEN_JIT_COMMAND@ +regen-jit: $(JIT_TARGETS) # Some make's put the object file in the current directory .c.o: @@ -3342,7 +3360,7 @@ clean-profile: clean-retain-profile clean-bolt # gh-141808: The JIT stencils are deliberately kept in clean-profile .PHONY: clean-jit-stencils clean-jit-stencils: - -rm -f jit_stencils*.h + -rm -f $(JIT_TARGETS) $(JIT_GENERATED_STAMP) jit_stencils*.h jit_shim*.o .PHONY: clean clean: clean-profile clean-jit-stencils diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 94ae718d58c..f79608e1d58 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -12,8 +12,9 @@ $(IntDir.Replace(`\\`, `\`)) $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_frozen\ $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\zlib-ng\ - $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_$(Configuration) - $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_PGInstrument + $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_$(Configuration)\ + $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_PGInstrument\ + $(GeneratedJitStencilsDir.Replace(`\\`, `\`)) $(ProjectName) $(TargetName)$(PyDebugExt) false diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index fe70e02536b..07305add81d 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -115,6 +115,9 @@ version.lib;ws2_32.lib;pathcch.lib;bcrypt.lib;%(AdditionalDependencies) zlib-ng$(PyDebugExt).lib;%(AdditionalDependencies) + $(GeneratedJitStencilsDir)jit_shim-aarch64-pc-windows-msvc.o;%(AdditionalDependencies) + $(GeneratedJitStencilsDir)jit_shim-i686-pc-windows-msvc.o;%(AdditionalDependencies) + $(GeneratedJitStencilsDir)jit_shim-x86_64-pc-windows-msvc.o;%(AdditionalDependencies) diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index bb059f382eb..9552e73ef6a 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -35,6 +35,9 @@ <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-aarch64-pc-windows-msvc.h" Condition="$(Platform) == 'ARM64'"/> <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-i686-pc-windows-msvc.h" Condition="$(Platform) == 'Win32'"/> <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-x86_64-pc-windows-msvc.h" Condition="$(Platform) == 'x64'"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_shim-aarch64-pc-windows-msvc.o" Condition="$(Platform) == 'ARM64'"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_shim-i686-pc-windows-msvc.o" Condition="$(Platform) == 'Win32'"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_shim-x86_64-pc-windows-msvc.o" Condition="$(Platform) == 'x64'"/> <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/> <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/> <_SbomSources Include="$(PySourcePath)PCbuild\get_externals.bat" /> @@ -129,7 +132,7 @@ x86_64-pc-windows-msvc $(JITArgs) --debug - + diff --git a/Python/ceval.c b/Python/ceval.c index 967d92f4ea6..506ea591c38 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1305,7 +1305,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } #ifdef _Py_TIER2 #ifdef _Py_JIT -_PyJitEntryFuncPtr _Py_jit_entry = _Py_LazyJitShim; +_PyJitEntryFuncPtr _Py_jit_entry = _PyJIT; #else _PyJitEntryFuncPtr _Py_jit_entry = _PyTier2Interpreter; #endif diff --git a/Python/jit.c b/Python/jit.c index af75acf1ff2..26e01b25d48 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -60,8 +60,6 @@ jit_error(const char *message) PyErr_Format(PyExc_RuntimeWarning, "JIT %s (%d)", message, hint); } -static size_t _Py_jit_shim_size = 0; - static int address_in_executor_array(_PyExecutorObject **ptrs, size_t count, uintptr_t addr) { @@ -104,13 +102,6 @@ _PyJIT_AddressInJitCode(PyInterpreterState *interp, uintptr_t addr) if (interp == NULL) { return 0; } - if (_Py_jit_entry != _Py_LazyJitShim && _Py_jit_shim_size != 0) { - uintptr_t start = (uintptr_t)_Py_jit_entry; - uintptr_t end = start + _Py_jit_shim_size; - if (addr >= start && addr < end) { - return 1; - } - } if (address_in_executor_array(interp->executor_ptrs, interp->executor_count, addr)) { return 1; } @@ -727,75 +718,6 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz return 0; } -/* One-off compilation of the jit entry shim - * We compile this once only as it effectively a normal - * function, but we need to use the JIT because it needs - * to understand the jit-specific calling convention. - * Don't forget to call _PyJIT_Fini later! - */ -static _PyJitEntryFuncPtr -compile_shim(void) -{ - _PyExecutorObject dummy; - const StencilGroup *group; - size_t code_size = 0; - size_t data_size = 0; - jit_state state = {0}; - group = &shim; - code_size += group->code_size; - data_size += group->data_size; - combine_symbol_mask(group->trampoline_mask, state.trampolines.mask); - combine_symbol_mask(group->got_mask, state.got_symbols.mask); - // Round up to the nearest page: - size_t page_size = get_page_size(); - assert((page_size & (page_size - 1)) == 0); - size_t code_padding = DATA_ALIGN - ((code_size + state.trampolines.size) & (DATA_ALIGN - 1)); - size_t padding = page_size - ((code_size + state.trampolines.size + code_padding + data_size + state.got_symbols.size) & (page_size - 1)); - size_t total_size = code_size + state.trampolines.size + code_padding + data_size + state.got_symbols.size + padding; - unsigned char *memory = jit_alloc(total_size); - if (memory == NULL) { - return NULL; - } - unsigned char *code = memory; - state.trampolines.mem = memory + code_size; - unsigned char *data = memory + code_size + state.trampolines.size + code_padding; - state.got_symbols.mem = data + data_size; - // Compile the shim, which handles converting between the native - // calling convention and the calling convention used by jitted code - // (which may be different for efficiency reasons). - group = &shim; - group->emit(code, data, &dummy, NULL, &state); - code += group->code_size; - data += group->data_size; - assert(code == memory + code_size); - assert(data == memory + code_size + state.trampolines.size + code_padding + data_size); - if (mark_executable(memory, total_size)) { - jit_free(memory, total_size); - return NULL; - } - _Py_jit_shim_size = total_size; - return (_PyJitEntryFuncPtr)memory; -} - -static PyMutex lazy_jit_mutex = { 0 }; - -_Py_CODEUNIT * -_Py_LazyJitShim( - _PyExecutorObject *executor, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate -) { - PyMutex_Lock(&lazy_jit_mutex); - if (_Py_jit_entry == _Py_LazyJitShim) { - _PyJitEntryFuncPtr shim = compile_shim(); - if (shim == NULL) { - PyMutex_Unlock(&lazy_jit_mutex); - Py_FatalError("Cannot allocate core JIT code"); - } - _Py_jit_entry = shim; - } - PyMutex_Unlock(&lazy_jit_mutex); - return _Py_jit_entry(executor, frame, stack_pointer, tstate); -} - // Free executor's memory allocated with _PyJIT_Compile void _PyJIT_Free(_PyExecutorObject *executor) @@ -812,22 +734,4 @@ _PyJIT_Free(_PyExecutorObject *executor) } } -// Free shim memory allocated with compile_shim -void -_PyJIT_Fini(void) -{ - PyMutex_Lock(&lazy_jit_mutex); - unsigned char *memory = (unsigned char *)_Py_jit_entry; - size_t size = _Py_jit_shim_size; - if (size) { - _Py_jit_entry = _Py_LazyJitShim; - _Py_jit_shim_size = 0; - if (jit_free(memory, size)) { - PyErr_FormatUnraisable("Exception ignored while " - "freeing JIT entry code"); - } - } - PyMutex_Unlock(&lazy_jit_mutex); -} - #endif // _Py_JIT diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 0232ed6c382..0a88e32bb6b 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -37,9 +37,6 @@ #include "pycore_uniqueid.h" // _PyObject_FinalizeUniqueIdPool() #include "pycore_warnings.h" // _PyWarnings_InitState() #include "pycore_weakref.h" // _PyWeakref_GET_REF() -#ifdef _Py_JIT -#include "pycore_jit.h" // _PyJIT_Fini() -#endif #if defined(PYMALLOC_USE_HUGEPAGES) && defined(MS_WINDOWS) #include @@ -2531,11 +2528,6 @@ _Py_Finalize(_PyRuntimeState *runtime) finalize_interp_clear(tstate); -#ifdef _Py_JIT - /* Free JIT shim memory */ - _PyJIT_Fini(); -#endif - #ifdef Py_TRACE_REFS /* Display addresses (& refcnts) of all objects still alive. * An address can be used to find the repr of the object, printed diff --git a/Python/pystate.c b/Python/pystate.c index d6a26f3339b..b7c838a1c15 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -489,11 +489,6 @@ free_interpreter(PyInterpreterState *interp) static inline int check_interpreter_whence(long); #endif -extern _Py_CODEUNIT * -_Py_LazyJitShim( - struct _PyExecutorObject *exec, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate -); - /* Get the interpreter state to a minimal consistent state. Further init happens in pylifecycle.c before it can be used. All fields not initialized here are expected to be zeroed out, diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index f78e80db165..15cac3de3fe 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -57,6 +57,12 @@ class _Target(typing.Generic[_S, _R]): known_symbols: dict[str, int] = dataclasses.field(default_factory=dict) pyconfig_dir: pathlib.Path = pathlib.Path.cwd().resolve() + def _compile_args(self) -> list[str]: + return list(self.args) + + def _shim_compile_args(self) -> list[str]: + return [] + def _get_nop(self) -> bytes: if re.fullmatch(r"aarch64-.*", self.triple): nop = b"\x1f\x20\x03\xd5" @@ -139,12 +145,8 @@ def _handle_relocation( ) -> _stencils.Hole: raise NotImplementedError(type(self)) - async def _compile( - self, opname: str, c: pathlib.Path, tempdir: pathlib.Path - ) -> _stencils.StencilGroup: - s = tempdir / f"{opname}.s" - o = tempdir / f"{opname}.o" - args_s = [ + def _base_clang_args(self, opname: str, tempdir: pathlib.Path) -> list[str]: + return [ f"--target={self.triple}", "-DPy_BUILD_CORE_MODULE", "-D_DEBUG" if self.debug else "-DNDEBUG", @@ -167,29 +169,38 @@ async def _compile( # generates better code than -O2 (and -O2 usually generates better # code than -O3). As a nice benefit, it uses less memory too: "-Os", - "-S", # Shorten full absolute file paths in the generated code (like the # __FILE__ macro and assert failure messages) for reproducibility: f"-ffile-prefix-map={CPYTHON}=.", f"-ffile-prefix-map={tempdir}=.", - # This debug info isn't necessary, and bloats out the JIT'ed code. - # We *may* be able to re-enable this, process it, and JIT it for a - # nicer debugging experience... but that needs a lot more research: - "-fno-asynchronous-unwind-tables", # Don't call built-in functions that we can't find or patch: "-fno-builtin", # Don't call stack-smashing canaries that we can't find or patch: "-fno-stack-protector", "-std=c11", + ] + + async def _build_stencil_group( + self, opname: str, c: pathlib.Path, tempdir: pathlib.Path + ) -> _stencils.StencilGroup: + s = tempdir / f"{opname}.s" + o = tempdir / f"{opname}.o" + args_s = self._base_clang_args(opname, tempdir) + args_s += [ + "-S", + # Stencils do not need unwind info, and the optimizer does not + # preserve .cfi_* directives correctly. On Darwin, + # -fno-asynchronous-unwind-tables alone still leaves synchronous + # unwind directives in the assembly, so disable both forms here. + "-fno-unwind-tables", + "-fno-asynchronous-unwind-tables", "-o", f"{s}", f"{c}", ] - is_shim = opname == "shim" if self.frame_pointers: - frame_pointer = "all" if is_shim else "reserved" - args_s += ["-Xclang", f"-mframe-pointer={frame_pointer}"] - args_s += self.args + args_s += ["-Xclang", "-mframe-pointer=reserved"] + args_s += self._compile_args() # Allow user-provided CFLAGS to override any defaults args_s += shlex.split(self.cflags) await _llvm.run( @@ -199,14 +210,13 @@ async def _compile( llvm_version=self.llvm_version, llvm_tools_install_dir=self.llvm_tools_install_dir, ) - if not is_shim: - self.optimizer( - s, - label_prefix=self.label_prefix, - symbol_prefix=self.symbol_prefix, - re_global=self.re_global, - frame_pointers=self.frame_pointers, - ).run() + self.optimizer( + s, + label_prefix=self.label_prefix, + symbol_prefix=self.symbol_prefix, + re_global=self.re_global, + frame_pointers=self.frame_pointers, + ).run() args_o = [f"--target={self.triple}", "-c", "-o", f"{o}", f"{s}"] await _llvm.run( "clang", @@ -217,6 +227,30 @@ async def _compile( ) return await self._parse(o) + async def _build_shim_object(self, output: pathlib.Path) -> None: + with tempfile.TemporaryDirectory() as tempdir: + work = pathlib.Path(tempdir).resolve() + args_o = self._base_clang_args("shim", work) + args_o += self._shim_compile_args() + args_o += [ + "-c", + # The linked shim is a real function in the final binary, so + # keep unwind info for debuggers and stack walkers. + "-fasynchronous-unwind-tables", + ] + if self.frame_pointers: + args_o += ["-Xclang", "-mframe-pointer=all"] + args_o += self._compile_args() + args_o += shlex.split(self.cflags) + args_o += ["-o", f"{output}", f"{TOOLS_JIT / 'shim.c'}"] + await _llvm.run( + "clang", + args_o, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, + ) + async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: generated_cases = PYTHON_EXECUTOR_CASES_C_H.read_text() cases_and_opnames = sorted( @@ -231,8 +265,6 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: with tempfile.TemporaryDirectory() as tempdir: work = pathlib.Path(tempdir).resolve() async with asyncio.TaskGroup() as group: - coro = self._compile("shim", TOOLS_JIT / "shim.c", work) - tasks.append(group.create_task(coro, name="shim")) template = TOOLS_JIT_TEMPLATE_C.read_text() for case, opname in cases_and_opnames: # Write out a copy of the template with *only* this case @@ -242,7 +274,7 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: # all of the other cases): c = work / f"{opname}.c" c.write_text(template.replace("CASE", case)) - coro = self._compile(opname, c, work) + coro = self._build_stencil_group(opname, c, work) tasks.append(group.create_task(coro, name=opname)) stencil_groups = {task.get_name(): task.result() for task in tasks} for stencil_group in stencil_groups.values(): @@ -256,8 +288,9 @@ def build( comment: str = "", force: bool = False, jit_stencils: pathlib.Path, + jit_shim_object: pathlib.Path, ) -> None: - """Build jit_stencils.h in the given directory.""" + """Build jit_stencils.h and the shim object in the given directory.""" jit_stencils.parent.mkdir(parents=True, exist_ok=True) if not self.stable: warning = f"JIT support for {self.triple} is still experimental!" @@ -271,8 +304,10 @@ def build( not force and jit_stencils.exists() and jit_stencils.read_text().startswith(digest) + and jit_shim_object.exists() ): return + ASYNCIO_RUNNER.run(self._build_shim_object(jit_shim_object)) stencil_groups = ASYNCIO_RUNNER.run(self._build_stencils()) jit_stencils_new = jit_stencils.parent / "jit_stencils.h.new" try: @@ -296,6 +331,13 @@ def build( class _COFF( _Target[_schema.COFFSection, _schema.COFFRelocation] ): # pylint: disable = too-few-public-methods + def _shim_compile_args(self) -> list[str]: + # The linked shim is part of pythoncore, not a shared extension. + # On Windows, Py_BUILD_CORE_MODULE makes public APIs import from + # pythonXY.lib, which creates a self-dependency when linking + # pythoncore.dll. Build the shim with builtin/core semantics. + return ["-UPy_BUILD_CORE_MODULE", "-DPy_BUILD_CORE_BUILTIN"] + def _handle_section( self, section: _schema.COFFSection, group: _stencils.StencilGroup ) -> None: @@ -396,6 +438,10 @@ class _COFF64(_COFF): symbol_prefix = "" re_global = re.compile(r'\s*\.def\s+(?P