From 9e5628ad68b768e2a8f0bcd3204b0027e238c45c Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Sun, 2 Nov 2025 22:56:59 +0100 Subject: [PATCH 001/313] gh-140808: Remove __class_getitem__ from mailbox._ProxyFile (#140838) Co-authored-by: Emma Smith --- Lib/mailbox.py | 2 -- Lib/test/test_genericalias.py | 4 ++-- .../Library/2025-10-31-16-25-13.gh-issue-140808.XBiQ4j.rst | 1 + 3 files changed, 3 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-31-16-25-13.gh-issue-140808.XBiQ4j.rst diff --git a/Lib/mailbox.py b/Lib/mailbox.py index b00d9e8634c..4a44642765c 100644 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -2090,8 +2090,6 @@ def closed(self): return False return self._file.closed - __class_getitem__ = classmethod(GenericAlias) - class _PartialFile(_ProxyFile): """A read-only wrapper of part of a file.""" diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 4e08adaca05..9df9296e26a 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -17,7 +17,7 @@ from functools import partial, partialmethod, cached_property from graphlib import TopologicalSorter from logging import LoggerAdapter, StreamHandler -from mailbox import Mailbox, _PartialFile +from mailbox import Mailbox try: import ctypes except ImportError: @@ -117,7 +117,7 @@ class BaseTest(unittest.TestCase): Iterable, Iterator, Reversible, Container, Collection, - Mailbox, _PartialFile, + Mailbox, ContextVar, Token, Field, Set, MutableSet, diff --git a/Misc/NEWS.d/next/Library/2025-10-31-16-25-13.gh-issue-140808.XBiQ4j.rst b/Misc/NEWS.d/next/Library/2025-10-31-16-25-13.gh-issue-140808.XBiQ4j.rst new file mode 100644 index 00000000000..090f39c6e25 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-31-16-25-13.gh-issue-140808.XBiQ4j.rst @@ -0,0 +1 @@ +The internal class ``mailbox._ProxyFile`` is no longer a parameterized generic. From 31de83d5e2e17f4e9a37e08b384bab916e1da7c1 Mon Sep 17 00:00:00 2001 From: Krishna Chaitanya <141550576+XChaitanyaX@users.noreply.github.com> Date: Mon, 3 Nov 2025 03:28:16 +0530 Subject: [PATCH 002/313] gh-140693: Improve `argparse` documentation about controlling color (#140737) --- Doc/library/argparse.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 418f514995d..9655db4f301 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -638,6 +638,11 @@ by setting ``color`` to ``False``:: ... help='an integer for the accumulator') >>> parser.parse_args(['--help']) +Note that when ``color=True``, colored output depends on both environment +variables and terminal capabilities. However, if ``color=False``, colored +output is always disabled, even if environment variables like ``FORCE_COLOR`` +are set. + .. versionadded:: 3.14 From e66f87ca73516efb4aec1f2f056d2a4efd73248a Mon Sep 17 00:00:00 2001 From: dr-carlos <77367421+dr-carlos@users.noreply.github.com> Date: Mon, 3 Nov 2025 09:45:47 +1030 Subject: [PATCH 003/313] gh-138425: Correctly partially evaluate global generics with undefined params in `ref.evaluate(format=Format.FORWARDREF)` (#138430) Co-authored-by: sobolevn --- Lib/annotationlib.py | 5 +++- Lib/test/test_annotationlib.py | 26 +++++++++++++++++++ ...-09-03-18-26-07.gh-issue-138425.cVE9Ho.rst | 2 ++ 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-09-03-18-26-07.gh-issue-138425.cVE9Ho.rst diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 81886a0467d..16dbb128bc9 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -187,8 +187,11 @@ def evaluate( except Exception: if not is_forwardref_format: raise + + # All variables, in scoping order, should be checked before + # triggering __missing__ to create a _Stringifier. new_locals = _StringifierDict( - {**builtins.__dict__, **locals}, + {**builtins.__dict__, **globals, **locals}, globals=globals, owner=owner, is_class=self.__forward_is_class__, diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 8da4ff096e7..7b08f58bfb8 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1877,6 +1877,32 @@ def test_name_lookup_without_eval(self): self.assertEqual(exc.exception.name, "doesntexist") + def test_evaluate_undefined_generic(self): + # Test the codepath where have to eval() with undefined variables. + class C: + x: alias[int, undef] + + generic = get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + format=Format.FORWARDREF, + globals={"alias": dict} + ) + self.assertNotIsInstance(generic, ForwardRef) + self.assertIs(generic.__origin__, dict) + self.assertEqual(len(generic.__args__), 2) + self.assertIs(generic.__args__[0], int) + self.assertIsInstance(generic.__args__[1], ForwardRef) + + generic = get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + format=Format.FORWARDREF, + globals={"alias": Union}, + locals={"alias": dict} + ) + self.assertNotIsInstance(generic, ForwardRef) + self.assertIs(generic.__origin__, dict) + self.assertEqual(len(generic.__args__), 2) + self.assertIs(generic.__args__[0], int) + self.assertIsInstance(generic.__args__[1], ForwardRef) + def test_fwdref_invalid_syntax(self): fr = ForwardRef("if") with self.assertRaises(SyntaxError): diff --git a/Misc/NEWS.d/next/Library/2025-09-03-18-26-07.gh-issue-138425.cVE9Ho.rst b/Misc/NEWS.d/next/Library/2025-09-03-18-26-07.gh-issue-138425.cVE9Ho.rst new file mode 100644 index 00000000000..328e5988cb0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-03-18-26-07.gh-issue-138425.cVE9Ho.rst @@ -0,0 +1,2 @@ +Fix partial evaluation of :class:`annotationlib.ForwardRef` objects which rely +on names defined as globals. From 63e01d6bae9ddc9ff35aca2134945670eacef163 Mon Sep 17 00:00:00 2001 From: dr-carlos <77367421+dr-carlos@users.noreply.github.com> Date: Mon, 3 Nov 2025 11:50:30 +1030 Subject: [PATCH 004/313] gh-137969: Fix evaluation of `ref.evaluate(format=Format.FORWARDREF)` objects (#138075) Co-authored-by: Jelle Zijlstra --- Lib/annotationlib.py | 14 ++++++++------ Lib/test/test_annotationlib.py | 9 +++++++++ .../2025-08-22-23-50-38.gh-issue-137969.Fkvis3.rst | 2 ++ 3 files changed, 19 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-08-22-23-50-38.gh-issue-137969.Fkvis3.rst diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 16dbb128bc9..26e7c200248 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -159,12 +159,12 @@ def evaluate( type_params = getattr(owner, "__type_params__", None) # Type parameters exist in their own scope, which is logically - # between the locals and the globals. We simulate this by adding - # them to the globals. + # between the locals and the globals. + type_param_scope = {} if type_params is not None: - globals = dict(globals) for param in type_params: - globals[param.__name__] = param + type_param_scope[param.__name__] = param + if self.__extra_names__: locals = {**locals, **self.__extra_names__} @@ -172,6 +172,8 @@ def evaluate( if arg.isidentifier() and not keyword.iskeyword(arg): if arg in locals: return locals[arg] + elif arg in type_param_scope: + return type_param_scope[arg] elif arg in globals: return globals[arg] elif hasattr(builtins, arg): @@ -183,7 +185,7 @@ def evaluate( else: code = self.__forward_code__ try: - return eval(code, globals=globals, locals=locals) + return eval(code, globals=globals, locals={**type_param_scope, **locals}) except Exception: if not is_forwardref_format: raise @@ -191,7 +193,7 @@ def evaluate( # All variables, in scoping order, should be checked before # triggering __missing__ to create a _Stringifier. new_locals = _StringifierDict( - {**builtins.__dict__, **globals, **locals}, + {**builtins.__dict__, **globals, **type_param_scope, **locals}, globals=globals, owner=owner, is_class=self.__forward_is_class__, diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 7b08f58bfb8..08f7161a273 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1911,6 +1911,15 @@ def test_fwdref_invalid_syntax(self): with self.assertRaises(SyntaxError): fr.evaluate() + def test_re_evaluate_generics(self): + global alias + class C: + x: alias[int] + + evaluated = get_annotations(C, format=Format.FORWARDREF)["x"].evaluate(format=Format.FORWARDREF) + alias = list + self.assertEqual(evaluated.evaluate(), list[int]) + class TestAnnotationLib(unittest.TestCase): def test__all__(self): diff --git a/Misc/NEWS.d/next/Library/2025-08-22-23-50-38.gh-issue-137969.Fkvis3.rst b/Misc/NEWS.d/next/Library/2025-08-22-23-50-38.gh-issue-137969.Fkvis3.rst new file mode 100644 index 00000000000..59f9e6e3d33 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-22-23-50-38.gh-issue-137969.Fkvis3.rst @@ -0,0 +1,2 @@ +Fix :meth:`annotationlib.ForwardRef.evaluate` returning :class:`annotationlib.ForwardRef` +objects which do not update in new contexts. From 121c219e302365e63eef0aef81b94a3de7a37fec Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Mon, 3 Nov 2025 10:47:18 +0800 Subject: [PATCH 005/313] Remove redundant `sys.exit(2)` call in `pdb` CLI (#139948) --- Lib/pdb.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/pdb.py b/Lib/pdb.py index 4ee12d17a61..fdc74198582 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -3603,7 +3603,6 @@ def main(): invalid_args = list(itertools.takewhile(lambda a: a.startswith('-'), args)) if invalid_args: parser.error(f"unrecognized arguments: {' '.join(invalid_args)}") - sys.exit(2) if opts.module: file = opts.module From 349de57839afcd1a1813a0cb53ba9cf1610ba7a5 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 2 Nov 2025 21:35:15 -0800 Subject: [PATCH 006/313] Revert "gh-137969: Fix evaluation of `ref.evaluate(format=Format.FORWARDREF)` objects (#138075)" (#140930) This reverts commit 63e01d6bae9ddc9ff35aca2134945670eacef163. --- Lib/annotationlib.py | 14 ++++++-------- Lib/test/test_annotationlib.py | 9 --------- .../2025-08-22-23-50-38.gh-issue-137969.Fkvis3.rst | 2 -- 3 files changed, 6 insertions(+), 19 deletions(-) delete mode 100644 Misc/NEWS.d/next/Library/2025-08-22-23-50-38.gh-issue-137969.Fkvis3.rst diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 26e7c200248..16dbb128bc9 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -159,12 +159,12 @@ def evaluate( type_params = getattr(owner, "__type_params__", None) # Type parameters exist in their own scope, which is logically - # between the locals and the globals. - type_param_scope = {} + # between the locals and the globals. We simulate this by adding + # them to the globals. if type_params is not None: + globals = dict(globals) for param in type_params: - type_param_scope[param.__name__] = param - + globals[param.__name__] = param if self.__extra_names__: locals = {**locals, **self.__extra_names__} @@ -172,8 +172,6 @@ def evaluate( if arg.isidentifier() and not keyword.iskeyword(arg): if arg in locals: return locals[arg] - elif arg in type_param_scope: - return type_param_scope[arg] elif arg in globals: return globals[arg] elif hasattr(builtins, arg): @@ -185,7 +183,7 @@ def evaluate( else: code = self.__forward_code__ try: - return eval(code, globals=globals, locals={**type_param_scope, **locals}) + return eval(code, globals=globals, locals=locals) except Exception: if not is_forwardref_format: raise @@ -193,7 +191,7 @@ def evaluate( # All variables, in scoping order, should be checked before # triggering __missing__ to create a _Stringifier. new_locals = _StringifierDict( - {**builtins.__dict__, **globals, **type_param_scope, **locals}, + {**builtins.__dict__, **globals, **locals}, globals=globals, owner=owner, is_class=self.__forward_is_class__, diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 08f7161a273..7b08f58bfb8 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1911,15 +1911,6 @@ def test_fwdref_invalid_syntax(self): with self.assertRaises(SyntaxError): fr.evaluate() - def test_re_evaluate_generics(self): - global alias - class C: - x: alias[int] - - evaluated = get_annotations(C, format=Format.FORWARDREF)["x"].evaluate(format=Format.FORWARDREF) - alias = list - self.assertEqual(evaluated.evaluate(), list[int]) - class TestAnnotationLib(unittest.TestCase): def test__all__(self): diff --git a/Misc/NEWS.d/next/Library/2025-08-22-23-50-38.gh-issue-137969.Fkvis3.rst b/Misc/NEWS.d/next/Library/2025-08-22-23-50-38.gh-issue-137969.Fkvis3.rst deleted file mode 100644 index 59f9e6e3d33..00000000000 --- a/Misc/NEWS.d/next/Library/2025-08-22-23-50-38.gh-issue-137969.Fkvis3.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :meth:`annotationlib.ForwardRef.evaluate` returning :class:`annotationlib.ForwardRef` -objects which do not update in new contexts. From 248ce9fa8c7a1ed1912e0db31172c8cd2c298b37 Mon Sep 17 00:00:00 2001 From: AN Long Date: Mon, 3 Nov 2025 17:14:22 +0900 Subject: [PATCH 007/313] gh-140826: Compare winreg.HKEYType by the internal handle value (GH-140843) --- Doc/library/winreg.rst | 8 +++-- Lib/test/test_winreg.py | 27 +++++++++++++++ ...-11-01-00-34-53.gh-issue-140826.JEDd7U.rst | 2 ++ PC/winreg.c | 34 ++++++++++++++++--- 4 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-01-00-34-53.gh-issue-140826.JEDd7U.rst diff --git a/Doc/library/winreg.rst b/Doc/library/winreg.rst index 83c49876d26..df8fb83a018 100644 --- a/Doc/library/winreg.rst +++ b/Doc/library/winreg.rst @@ -771,8 +771,9 @@ Handle objects provide semantics for :meth:`~object.__bool__` -- thus :: will print ``Yes`` if the handle is currently valid (has not been closed or detached). -The object also support comparison semantics, so handle objects will compare -true if they both reference the same underlying Windows handle value. +The object also support equality comparison semantics, so handle objects will +compare equal if they both reference the same underlying Windows handle value. +Closed handle objects (those with a handle value of zero) always compare equal. Handle objects can be converted to an integer (e.g., using the built-in :func:`int` function), in which case the underlying Windows handle value is @@ -815,3 +816,6 @@ integer handle, and also disconnect the Windows handle from the handle object. will automatically close *key* when control leaves the :keyword:`with` block. +.. versionchanged:: next + Handle objects are now compared by their underlying Windows handle value + instead of object identity for equality comparisons. diff --git a/Lib/test/test_winreg.py b/Lib/test/test_winreg.py index 6f2a6ac900b..733d30b3922 100644 --- a/Lib/test/test_winreg.py +++ b/Lib/test/test_winreg.py @@ -209,6 +209,33 @@ def _test_named_args(self, key, sub_key): access=KEY_ALL_ACCESS) as okey: self.assertTrue(okey.handle != 0) + def test_hkey_comparison(self): + """Test HKEY comparison by handle value rather than object identity.""" + key1 = OpenKey(HKEY_CURRENT_USER, None) + key2 = OpenKey(HKEY_CURRENT_USER, None) + key3 = OpenKey(HKEY_LOCAL_MACHINE, None) + + self.addCleanup(CloseKey, key1) + self.addCleanup(CloseKey, key2) + self.addCleanup(CloseKey, key3) + + self.assertEqual(key1.handle, key2.handle) + self.assertTrue(key1 == key2) + self.assertFalse(key1 != key2) + + self.assertTrue(key1 != key3) + self.assertFalse(key1 == key3) + + # Closed keys should be equal (all have handle=0) + CloseKey(key1) + CloseKey(key2) + CloseKey(key3) + + self.assertEqual(key1.handle, 0) + self.assertEqual(key2.handle, 0) + self.assertEqual(key3.handle, 0) + self.assertEqual(key2, key3) + class LocalWinregTests(BaseWinregTests): diff --git a/Misc/NEWS.d/next/Library/2025-11-01-00-34-53.gh-issue-140826.JEDd7U.rst b/Misc/NEWS.d/next/Library/2025-11-01-00-34-53.gh-issue-140826.JEDd7U.rst new file mode 100644 index 00000000000..875d15f2f89 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-01-00-34-53.gh-issue-140826.JEDd7U.rst @@ -0,0 +1,2 @@ +Now :class:`!winreg.HKEYType` objects are compared by their underlying Windows +registry handle value instead of their object identity. diff --git a/PC/winreg.c b/PC/winreg.c index 8633f29670e..c7bc74728f1 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -181,13 +181,38 @@ PyHKEY_strFunc(PyObject *ob) return PyUnicode_FromFormat("", pyhkey->hkey); } -static int -PyHKEY_compareFunc(PyObject *ob1, PyObject *ob2) +static PyObject * +PyHKEY_richcompare(PyObject *ob1, PyObject *ob2, int op) { + /* Both objects must be PyHKEY objects from the same module */ + if (Py_TYPE(ob1) != Py_TYPE(ob2)) { + Py_RETURN_NOTIMPLEMENTED; + } + PyHKEYObject *pyhkey1 = (PyHKEYObject *)ob1; PyHKEYObject *pyhkey2 = (PyHKEYObject *)ob2; - return pyhkey1 == pyhkey2 ? 0 : - (pyhkey1 < pyhkey2 ? -1 : 1); + HKEY hkey1 = pyhkey1->hkey; + HKEY hkey2 = pyhkey2->hkey; + int result; + + switch (op) { + case Py_EQ: + result = (hkey1 == hkey2); + break; + case Py_NE: + result = (hkey1 != hkey2); + break; + default: + /* Only support equality comparisons, not ordering */ + Py_RETURN_NOTIMPLEMENTED; + } + + if (result) { + Py_RETURN_TRUE; + } + else { + Py_RETURN_FALSE; + } } static Py_hash_t @@ -365,6 +390,7 @@ static PyType_Slot pyhkey_type_slots[] = { {Py_tp_traverse, _PyObject_VisitType}, {Py_tp_hash, PyHKEY_hashFunc}, {Py_tp_str, PyHKEY_strFunc}, + {Py_tp_richcompare, PyHKEY_richcompare}, // Number protocol {Py_nb_add, PyHKEY_binaryFailureFunc}, From 7a9437d98641e3c3749ab2fd9fb54eac7614f9af Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 3 Nov 2025 06:50:37 -0800 Subject: [PATCH 008/313] gh-140348: Fix using | on unusual objects plus Unions (#140383) --- Lib/test/test_typing.py | 9 +++++++++ ...25-10-20-12-33-49.gh-issue-140348.SAKnQZ.rst | 3 +++ Objects/unionobject.c | 17 ++++++++++++++++- 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-20-12-33-49.gh-issue-140348.SAKnQZ.rst diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 4076004bc13..e896df51844 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -2283,6 +2283,15 @@ class Ints(enum.IntEnum): self.assertEqual(Union[Literal[1], Literal[Ints.B], Literal[True]].__args__, (Literal[1], Literal[Ints.B], Literal[True])) + def test_allow_non_types_in_or(self): + # gh-140348: Test that using | with a Union object allows things that are + # not allowed by is_unionable(). + U1 = Union[int, str] + self.assertEqual(U1 | float, Union[int, str, float]) + self.assertEqual(U1 | "float", Union[int, str, "float"]) + self.assertEqual(float | U1, Union[float, int, str]) + self.assertEqual("float" | U1, Union["float", int, str]) + class TupleTests(BaseTestCase): diff --git a/Misc/NEWS.d/next/Library/2025-10-20-12-33-49.gh-issue-140348.SAKnQZ.rst b/Misc/NEWS.d/next/Library/2025-10-20-12-33-49.gh-issue-140348.SAKnQZ.rst new file mode 100644 index 00000000000..16d5b2a8bf0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-20-12-33-49.gh-issue-140348.SAKnQZ.rst @@ -0,0 +1,3 @@ +Fix regression in Python 3.14.0 where using the ``|`` operator on a +:class:`typing.Union` object combined with an object that is not a type +would raise an error. diff --git a/Objects/unionobject.c b/Objects/unionobject.c index c4ece0fe09f..a47d6193d70 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -393,8 +393,23 @@ static PyGetSetDef union_properties[] = { {0} }; +static PyObject * +union_nb_or(PyObject *a, PyObject *b) +{ + unionbuilder ub; + if (!unionbuilder_init(&ub, true)) { + return NULL; + } + if (!unionbuilder_add_single(&ub, a) || + !unionbuilder_add_single(&ub, b)) { + unionbuilder_finalize(&ub); + return NULL; + } + return make_union(&ub); +} + static PyNumberMethods union_as_number = { - .nb_or = _Py_union_type_or, // Add __or__ function + .nb_or = union_nb_or, // Add __or__ function }; static const char* const cls_attrs[] = { From d590685297165d16df41003c3566b75a0a4ced61 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 3 Nov 2025 06:54:23 -0800 Subject: [PATCH 009/313] gh-133879: Clean up What's New for 3.15 (#140435) Clean up What's New for 3.15 A bit early but I was reading through it and noticed some issues: - A few improvements were listed in the removals section - The "Porting to 3.15" section in the C API chapter had some changes that aren't about the C API - Some other typos and wording fixes --- Doc/deprecations/pending-removal-in-3.16.rst | 6 +- Doc/whatsnew/3.15.rst | 86 ++++++++++---------- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst index cdd76ee693f..b00c7002b03 100644 --- a/Doc/deprecations/pending-removal-in-3.16.rst +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -63,9 +63,9 @@ Pending removal in Python 3.16 * :mod:`logging`: - Support for custom logging handlers with the *strm* argument is deprecated - and scheduled for removal in Python 3.16. Define handlers with the *stream* - argument instead. (Contributed by Mariusz Felisiak in :gh:`115032`.) + * Support for custom logging handlers with the *strm* argument is deprecated + and scheduled for removal in Python 3.16. Define handlers with the *stream* + argument instead. (Contributed by Mariusz Felisiak in :gh:`115032`.) * :mod:`mimetypes`: diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 7338fb51964..5379ac3abba 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -440,6 +440,14 @@ math (Contributed by Bénédikt Tran in :gh:`135853`.) +mimetypes +--------- + +* Add ``application/toml``. (Contributed by Gil Forcada in :gh:`139959`.) +* Rename ``application/x-texinfo`` to ``application/texinfo``. + (Contributed by Charlie Lin in :gh:`140165`) + + mmap ---- @@ -612,6 +620,17 @@ types as described in :pep:`667`. +unicodedata +----------- + +* The Unicode database has been updated to Unicode 17.0.0. + +* Add :func:`unicodedata.isxidstart` and :func:`unicodedata.isxidcontinue` + functions to check whether a character can start or continue a + `Unicode Standard Annex #31 `_ identifier. + (Contributed by Stan Ulbrych in :gh:`129117`.) + + unittest -------- @@ -720,14 +739,6 @@ importlib.resources (Contributed by Semyon Moroz in :gh:`138044`) -mimetypes ---------- - -* Add ``application/toml``. (Contributed by Gil Forcada in :gh:`139959`.) -* Rename ``application/x-texinfo`` to ``application/texinfo``. - (Contributed by Charlie Lin in :gh:`140165`) - - pathlib ------- @@ -777,7 +788,7 @@ typing (Contributed by Bénédikt Tran in :gh:`133817`.) * Using ``TD = TypedDict("TD")`` or ``TD = TypedDict("TD", None)`` to - construct a :class:`~typing.TypedDict` type with zero field is no + construct a :class:`~typing.TypedDict` type with zero fields is no longer supported. Use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})`` instead. (Contributed by Bénédikt Tran in :gh:`133823`.) @@ -810,17 +821,6 @@ typing (Contributed by Nikita Sobolev in :gh:`133601`.) -unicodedata ------------ - -* The Unicode database has been updated to Unicode 17.0.0. - -* Add :func:`unicodedata.isxidstart` and :func:`unicodedata.isxidcontinue` - functions to check whether a character can start or continue a - `Unicode Standard Annex #31 `_ identifier. - (Contributed by Stan Ulbrych in :gh:`129117`.) - - wave ---- @@ -847,7 +847,7 @@ New deprecations * CLI: * Deprecate :option:`-b` and :option:`!-bb` command-line options - and schedule them to become no-op in Python 3.17. + and schedule them to become no-ops in Python 3.17. These were primarily helpers for the Python 2 -> 3 transition. Starting with Python 3.17, no :exc:`BytesWarning` will be raised for these cases; use a type checker instead. @@ -858,8 +858,8 @@ New deprecations * In hash function constructors such as :func:`~hashlib.new` or the direct hash-named constructors such as :func:`~hashlib.md5` and - :func:`~hashlib.sha256`, their optional initial data parameter could - also be passed a keyword argument named ``data=`` or ``string=`` in + :func:`~hashlib.sha256`, the optional initial data parameter could + also be passed as a keyword argument named ``data=`` or ``string=`` in various :mod:`hashlib` implementations. Support for the ``string`` keyword argument name is now deprecated and @@ -962,31 +962,11 @@ Changed C APIs Porting to Python 3.15 ---------------------- -* :class:`sqlite3.Connection` APIs has been cleaned up. - - * All parameters of :func:`sqlite3.connect` except *database* are now keyword-only. - * The first three parameters of methods :meth:`~sqlite3.Connection.create_function` - and :meth:`~sqlite3.Connection.create_aggregate` are now positional-only. - * The first parameter of methods :meth:`~sqlite3.Connection.set_authorizer`, - :meth:`~sqlite3.Connection.set_progress_handler` and - :meth:`~sqlite3.Connection.set_trace_callback` is now positional-only. - - (Contributed by Serhiy Storchaka in :gh:`133595`.) - * Private functions promoted to public C APIs: The |pythoncapi_compat_project| can be used to get most of these new functions on Python 3.14 and older. -* :data:`resource.RLIM_INFINITY` is now always positive. - Passing a negative integer value that corresponded to its old value - (such as ``-1`` or ``-3``, depending on platform) to - :func:`resource.setrlimit` and :func:`resource.prlimit` is now deprecated. - (Contributed by Serhiy Storchaka in :gh:`137044`.) - -* :meth:`~mmap.mmap.resize` has been removed on platforms that don't support the - underlying syscall, instead of raising a :exc:`SystemError`. - Removed C APIs -------------- @@ -1106,3 +1086,23 @@ Porting to Python 3.15 This section lists previously described changes and other bugfixes that may require changes to your code. + +* :class:`sqlite3.Connection` APIs has been cleaned up. + + * All parameters of :func:`sqlite3.connect` except *database* are now keyword-only. + * The first three parameters of methods :meth:`~sqlite3.Connection.create_function` + and :meth:`~sqlite3.Connection.create_aggregate` are now positional-only. + * The first parameter of methods :meth:`~sqlite3.Connection.set_authorizer`, + :meth:`~sqlite3.Connection.set_progress_handler` and + :meth:`~sqlite3.Connection.set_trace_callback` is now positional-only. + + (Contributed by Serhiy Storchaka in :gh:`133595`.) + +* :data:`resource.RLIM_INFINITY` is now always positive. + Passing a negative integer value that corresponded to its old value + (such as ``-1`` or ``-3``, depending on platform) to + :func:`resource.setrlimit` and :func:`resource.prlimit` is now deprecated. + (Contributed by Serhiy Storchaka in :gh:`137044`.) + +* :meth:`~mmap.mmap.resize` has been removed on platforms that don't support the + underlying syscall, instead of raising a :exc:`SystemError`. From b1027d4762435b97546c122dd94290d707b3ff39 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 3 Nov 2025 07:22:32 -0800 Subject: [PATCH 010/313] gh-138151: Fix annotationlib handling of multiple nonlocals (#138164) --- Lib/annotationlib.py | 40 +++++++++++++------ Lib/test/test_annotationlib.py | 15 +++++++ ...-08-26-08-17-56.gh-issue-138151.I6CdAk.rst | 3 ++ 3 files changed, 45 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-08-26-08-17-56.gh-issue-138151.I6CdAk.rst diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 16dbb128bc9..2166dbff0ee 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -85,6 +85,9 @@ def __init__( # These are always set to None here but may be non-None if a ForwardRef # is created through __class__ assignment on a _Stringifier object. self.__globals__ = None + # This may be either a cell object (for a ForwardRef referring to a single name) + # or a dict mapping cell names to cell objects (for a ForwardRef containing references + # to multiple names). self.__cell__ = None self.__extra_names__ = None # These are initially None but serve as a cache and may be set to a non-None @@ -117,7 +120,7 @@ def evaluate( is_forwardref_format = True case _: raise NotImplementedError(format) - if self.__cell__ is not None: + if isinstance(self.__cell__, types.CellType): try: return self.__cell__.cell_contents except ValueError: @@ -160,11 +163,18 @@ def evaluate( # Type parameters exist in their own scope, which is logically # between the locals and the globals. We simulate this by adding - # them to the globals. - if type_params is not None: + # them to the globals. Similar reasoning applies to nonlocals stored in cells. + if type_params is not None or isinstance(self.__cell__, dict): globals = dict(globals) + if type_params is not None: for param in type_params: globals[param.__name__] = param + if isinstance(self.__cell__, dict): + for cell_name, cell_value in self.__cell__.items(): + try: + globals[cell_name] = cell_value.cell_contents + except ValueError: + pass if self.__extra_names__: locals = {**locals, **self.__extra_names__} @@ -202,7 +212,7 @@ def evaluate( except Exception: return self else: - new_locals.transmogrify() + new_locals.transmogrify(self.__cell__) return result def _evaluate(self, globalns, localns, type_params=_sentinel, *, recursive_guard): @@ -274,7 +284,7 @@ def __hash__(self): self.__forward_module__, id(self.__globals__), # dictionaries are not hashable, so hash by identity self.__forward_is_class__, - self.__cell__, + tuple(sorted(self.__cell__.items())) if isinstance(self.__cell__, dict) else self.__cell__, self.__owner__, tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None, )) @@ -642,13 +652,15 @@ def __missing__(self, key): self.stringifiers.append(fwdref) return fwdref - def transmogrify(self): + def transmogrify(self, cell_dict): for obj in self.stringifiers: obj.__class__ = ForwardRef obj.__stringifier_dict__ = None # not needed for ForwardRef if isinstance(obj.__ast_node__, str): obj.__arg__ = obj.__ast_node__ obj.__ast_node__ = None + if cell_dict is not None and obj.__cell__ is None: + obj.__cell__ = cell_dict def create_unique_name(self): name = f"__annotationlib_name_{self.next_id}__" @@ -712,7 +724,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): globals = _StringifierDict({}, format=format) is_class = isinstance(owner, type) - closure = _build_closure( + closure, _ = _build_closure( annotate, owner, is_class, globals, allow_evaluation=False ) func = types.FunctionType( @@ -756,7 +768,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): is_class=is_class, format=format, ) - closure = _build_closure( + closure, cell_dict = _build_closure( annotate, owner, is_class, globals, allow_evaluation=True ) func = types.FunctionType( @@ -774,7 +786,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): except Exception: pass else: - globals.transmogrify() + globals.transmogrify(cell_dict) return result # Try again, but do not provide any globals. This allows us to return @@ -786,7 +798,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): is_class=is_class, format=format, ) - closure = _build_closure( + closure, cell_dict = _build_closure( annotate, owner, is_class, globals, allow_evaluation=False ) func = types.FunctionType( @@ -797,7 +809,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): kwdefaults=annotate.__kwdefaults__, ) result = func(Format.VALUE_WITH_FAKE_GLOBALS) - globals.transmogrify() + globals.transmogrify(cell_dict) if _is_evaluate: if isinstance(result, ForwardRef): return result.evaluate(format=Format.FORWARDREF) @@ -822,14 +834,16 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): def _build_closure(annotate, owner, is_class, stringifier_dict, *, allow_evaluation): if not annotate.__closure__: - return None + return None, None freevars = annotate.__code__.co_freevars new_closure = [] + cell_dict = {} for i, cell in enumerate(annotate.__closure__): if i < len(freevars): name = freevars[i] else: name = "__cell__" + cell_dict[name] = cell new_cell = None if allow_evaluation: try: @@ -850,7 +864,7 @@ def _build_closure(annotate, owner, is_class, stringifier_dict, *, allow_evaluat stringifier_dict.stringifiers.append(fwdref) new_cell = types.CellType(fwdref) new_closure.append(new_cell) - return tuple(new_closure) + return tuple(new_closure), cell_dict def _stringify_single(anno): diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 7b08f58bfb8..fd5d43b09b9 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1194,6 +1194,21 @@ class RaisesAttributeError: }, ) + def test_nonlocal_in_annotation_scope(self): + class Demo: + nonlocal sequence_b + x: sequence_b + y: sequence_b[int] + + fwdrefs = get_annotations(Demo, format=Format.FORWARDREF) + + self.assertIsInstance(fwdrefs["x"], ForwardRef) + self.assertIsInstance(fwdrefs["y"], ForwardRef) + + sequence_b = list + self.assertIs(fwdrefs["x"].evaluate(), list) + self.assertEqual(fwdrefs["y"].evaluate(), list[int]) + def test_raises_error_from_value(self): # test that if VALUE is the only supported format, but raises an error # that error is propagated from get_annotations diff --git a/Misc/NEWS.d/next/Library/2025-08-26-08-17-56.gh-issue-138151.I6CdAk.rst b/Misc/NEWS.d/next/Library/2025-08-26-08-17-56.gh-issue-138151.I6CdAk.rst new file mode 100644 index 00000000000..de29f536afc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-26-08-17-56.gh-issue-138151.I6CdAk.rst @@ -0,0 +1,3 @@ +In :mod:`annotationlib`, improve evaluation of forward references to +nonlocal variables that are not yet defined when the annotations are +initially evaluated. From 478b8dab0b40f08c3ded40bf988ec49a50f4c8fb Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Mon, 3 Nov 2025 15:47:52 +0000 Subject: [PATCH 011/313] Docs: Fix typo in codecs documentation (GH-140883) --- Doc/library/codecs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 2a5994b11d8..305e5d07a35 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1088,7 +1088,7 @@ alias for the ``'utf_8'`` codec. refer to the source :source:`aliases.py ` file. On Windows, ``cpXXX`` codecs are available for all code pages. -But only codecs listed in the following table are guarantead to exist on +But only codecs listed in the following table are guaranteed to exist on other platforms. .. impl-detail:: From b373d3494c587cf27b31f3dff89a8d96f7d29b9d Mon Sep 17 00:00:00 2001 From: Yongzi Li <204532581+Yzi-Li@users.noreply.github.com> Date: Mon, 3 Nov 2025 23:48:10 +0800 Subject: [PATCH 012/313] Docs: Fix a typo in `idle.rst` (Chitespace -> Whitespace) (GH-140946) Fix typo in idle.rst --- Doc/library/idle.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index fabea611e0e..e547c96b580 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -204,7 +204,7 @@ New Indent Width Open a dialog to change indent width. The accepted default by the Python community is 4 spaces. -Strip Trailing Chitespace +Strip Trailing Whitespace Remove trailing space and other whitespace characters after the last non-whitespace character of a line by applying str.rstrip to each line, including lines within multiline strings. Except for Shell windows, From 4e2ff4ac4cc725572e1458a91418deeadd03d8aa Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 3 Nov 2025 10:01:44 -0800 Subject: [PATCH 013/313] GH-136895: Update JIT builds to use LLVM 20 (#140329) Co-authored-by: Emma Harper Smith --- .github/workflows/jit.yml | 6 +- ...-10-19-10-32-28.gh-issue-136895.HfsEh0.rst | 1 + PCbuild/get_external.py | 63 ++++++++++++---- PCbuild/get_externals.bat | 8 ++- Python/jit.c | 72 +++++++++++++++---- Tools/jit/README.md | 20 +++--- Tools/jit/_llvm.py | 4 +- Tools/jit/_stencils.py | 17 +++++ Tools/jit/_targets.py | 10 +-- 9 files changed, 151 insertions(+), 50 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-19-10-32-28.gh-issue-136895.HfsEh0.rst diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index c32bf4fd63c..151b17e8442 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -68,7 +68,7 @@ jobs: - true - false llvm: - - 19 + - 20 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 @@ -138,7 +138,7 @@ jobs: fail-fast: false matrix: llvm: - - 19 + - 20 steps: - uses: actions/checkout@v4 with: @@ -166,7 +166,7 @@ jobs: fail-fast: false matrix: llvm: - - 19 + - 20 steps: - uses: actions/checkout@v4 with: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-19-10-32-28.gh-issue-136895.HfsEh0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-19-10-32-28.gh-issue-136895.HfsEh0.rst new file mode 100644 index 00000000000..fffc264a865 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-19-10-32-28.gh-issue-136895.HfsEh0.rst @@ -0,0 +1 @@ +Update JIT compilation to use LLVM 20 at build time. diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index a78aa6a2304..07970624e86 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -3,6 +3,7 @@ import argparse import os import pathlib +import shutil import sys import time import urllib.error @@ -22,15 +23,13 @@ def retrieve_with_retries(download_location, output_path, reporthook, ) except (urllib.error.URLError, ConnectionError) as ex: if attempt == max_retries: - msg = f"Download from {download_location} failed." - raise OSError(msg) from ex + raise OSError(f'Download from {download_location} failed.') from ex time.sleep(2.25**attempt) else: return resp - def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose): - repo = f'cpython-{"bin" if binary else "source"}-deps' + repo = 'cpython-bin-deps' if binary else 'cpython-source-deps' url = f'https://github.com/{org}/{repo}/archive/{commit_hash}.zip' reporthook = None if verbose: @@ -44,6 +43,23 @@ def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose): return filename +def fetch_release(tag, tarball_dir, *, org='python', verbose=False): + url = f'https://github.com/{org}/cpython-bin-deps/releases/download/{tag}/{tag}.tar.xz' + reporthook = None + if verbose: + reporthook = print + tarball_dir.mkdir(parents=True, exist_ok=True) + output_path = tarball_dir / f'{tag}.tar.xz' + retrieve_with_retries(url, output_path, reporthook) + return output_path + + +def extract_tarball(externals_dir, tarball_path, tag): + output_path = externals_dir / tag + shutil.unpack_archive(os.fspath(tarball_path), os.fspath(output_path)) + return output_path + + def extract_zip(externals_dir, zip_path): with zipfile.ZipFile(os.fspath(zip_path)) as zf: zf.extractall(os.fspath(externals_dir)) @@ -55,6 +71,8 @@ def parse_args(): p.add_argument('-v', '--verbose', action='store_true') p.add_argument('-b', '--binary', action='store_true', help='Is the dependency in the binary repo?') + p.add_argument('-r', '--release', action='store_true', + help='Download from GitHub release assets instead of branch') p.add_argument('-O', '--organization', help='Organization owning the deps repos', default='python') p.add_argument('-e', '--externals-dir', type=pathlib.Path, @@ -67,15 +85,36 @@ def parse_args(): def main(): args = parse_args() - zip_path = fetch_zip( - args.tag, - args.externals_dir / 'zips', - org=args.organization, - binary=args.binary, - verbose=args.verbose, - ) final_name = args.externals_dir / args.tag - extracted = extract_zip(args.externals_dir, zip_path) + + # Check if the dependency already exists in externals/ directory + # (either already downloaded/extracted, or checked into the git tree) + if final_name.exists(): + if args.verbose: + print(f'{args.tag} already exists at {final_name}, skipping download.') + return + + # Determine download method: release artifacts for large deps (like LLVM), + # otherwise zip download from GitHub branches + if args.release: + tarball_path = fetch_release( + args.tag, + args.externals_dir / 'tarballs', + org=args.organization, + verbose=args.verbose, + ) + extracted = extract_tarball(args.externals_dir, tarball_path, args.tag) + else: + # Use zip download from GitHub branches + # (cpython-bin-deps if --binary, cpython-source-deps otherwise) + zip_path = fetch_zip( + args.tag, + args.externals_dir / 'zips', + org=args.organization, + binary=args.binary, + verbose=args.verbose, + ) + extracted = extract_zip(args.externals_dir, zip_path) for wait in [1, 2, 3, 5, 8, 0]: try: extracted.replace(final_name) diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 50a227b563a..319024e0f50 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -82,7 +82,7 @@ if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.18 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 -if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-19.1.7.0 +if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-20.1.8.0 for %%b in (%binaries%) do ( if exist "%EXTERNALS_DIR%\%%b" ( @@ -92,7 +92,11 @@ for %%b in (%binaries%) do ( git clone --depth 1 https://github.com/%ORG%/cpython-bin-deps --branch %%b "%EXTERNALS_DIR%\%%b" ) else ( echo.Fetching %%b... - %PYTHON% -E "%PCBUILD%\get_external.py" -b -O %ORG% -e "%EXTERNALS_DIR%" %%b + if "%%b"=="llvm-20.1.8.0" ( + %PYTHON% -E "%PCBUILD%\get_external.py" --release --organization %ORG% --externals-dir "%EXTERNALS_DIR%" %%b + ) else ( + %PYTHON% -E "%PCBUILD%\get_external.py" --binary --organization %ORG% --externals-dir "%EXTERNALS_DIR%" %%b + ) ) ) diff --git a/Python/jit.c b/Python/jit.c index c3f3d686013..279e1ce6a0d 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -444,17 +444,42 @@ patch_x86_64_32rx(unsigned char *location, uint64_t value) } void patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state); +void patch_x86_64_trampoline(unsigned char *location, int ordinal, jit_state *state); #include "jit_stencils.h" #if defined(__aarch64__) || defined(_M_ARM64) #define TRAMPOLINE_SIZE 16 #define DATA_ALIGN 8 +#elif defined(__x86_64__) && defined(__APPLE__) + // LLVM 20 on macOS x86_64 debug builds: GOT entries may exceed ±2GB PC-relative + // range. + #define TRAMPOLINE_SIZE 16 // 14 bytes + 2 bytes padding for alignment + #define DATA_ALIGN 8 #else #define TRAMPOLINE_SIZE 0 #define DATA_ALIGN 1 #endif +// Get the trampoline memory location for a given symbol ordinal. +static unsigned char * +get_trampoline_slot(int ordinal, jit_state *state) +{ + const uint32_t symbol_mask = 1 << (ordinal % 32); + const uint32_t trampoline_mask = state->trampolines.mask[ordinal / 32]; + assert(symbol_mask & trampoline_mask); + + // Count the number of set bits in the trampoline mask lower than ordinal + int index = _Py_popcount32(trampoline_mask & (symbol_mask - 1)); + for (int i = 0; i < ordinal / 32; i++) { + index += _Py_popcount32(state->trampolines.mask[i]); + } + + unsigned char *trampoline = state->trampolines.mem + index * TRAMPOLINE_SIZE; + assert((size_t)(index + 1) * TRAMPOLINE_SIZE <= state->trampolines.size); + return trampoline; +} + // Generate and patch AArch64 trampolines. The symbols to jump to are stored // in the jit_stencils.h in the symbols_map. void @@ -471,20 +496,8 @@ patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state) return; } - // Masking is done modulo 32 as the mask is stored as an array of uint32_t - const uint32_t symbol_mask = 1 << (ordinal % 32); - const uint32_t trampoline_mask = state->trampolines.mask[ordinal / 32]; - assert(symbol_mask & trampoline_mask); - - // Count the number of set bits in the trampoline mask lower than ordinal, - // this gives the index into the array of trampolines. - int index = _Py_popcount32(trampoline_mask & (symbol_mask - 1)); - for (int i = 0; i < ordinal / 32; i++) { - index += _Py_popcount32(state->trampolines.mask[i]); - } - - uint32_t *p = (uint32_t*)(state->trampolines.mem + index * TRAMPOLINE_SIZE); - assert((size_t)(index + 1) * TRAMPOLINE_SIZE <= state->trampolines.size); + // Out of range - need a trampoline + uint32_t *p = (uint32_t *)get_trampoline_slot(ordinal, state); /* Generate the trampoline @@ -501,6 +514,37 @@ patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state) patch_aarch64_26r(location, (uintptr_t)p); } +// Generate and patch x86_64 trampolines. +void +patch_x86_64_trampoline(unsigned char *location, int ordinal, jit_state *state) +{ + uint64_t value = (uintptr_t)symbols_map[ordinal]; + int64_t range = (int64_t)value - 4 - (int64_t)location; + + // If we are in range of 32 signed bits, we can patch directly + if (range >= -(1LL << 31) && range < (1LL << 31)) { + patch_32r(location, value - 4); + return; + } + + // Out of range - need a trampoline + unsigned char *trampoline = get_trampoline_slot(ordinal, state); + + /* Generate the trampoline (14 bytes, padded to 16): + 0: ff 25 00 00 00 00 jmp *(%rip) + 6: XX XX XX XX XX XX XX XX (64-bit target address) + + Reference: https://wiki.osdev.org/X86-64_Instruction_Encoding#FF (JMP r/m64) + */ + trampoline[0] = 0xFF; + trampoline[1] = 0x25; + memset(trampoline + 2, 0, 4); + memcpy(trampoline + 6, &value, 8); + + // Patch the call site to call the trampoline instead + patch_32r(location, (uintptr_t)trampoline - 4); +} + static void combine_symbol_mask(const symbol_mask src, symbol_mask dest) { diff --git a/Tools/jit/README.md b/Tools/jit/README.md index 35c7ffd7a28..d83b09aab59 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -9,32 +9,32 @@ ## Installing LLVM The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). -LLVM version 19 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. +LLVM version 20 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. It's easy to install all of the required tools: ### Linux -Install LLVM 19 on Ubuntu/Debian: +Install LLVM 20 on Ubuntu/Debian: ```sh wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh -sudo ./llvm.sh 19 +sudo ./llvm.sh 20 ``` -Install LLVM 19 on Fedora Linux 40 or newer: +Install LLVM 20 on Fedora Linux 40 or newer: ```sh -sudo dnf install 'clang(major) = 19' 'llvm(major) = 19' +sudo dnf install 'clang(major) = 20' 'llvm(major) = 20' ``` ### macOS -Install LLVM 19 with [Homebrew](https://brew.sh): +Install LLVM 20 with [Homebrew](https://brew.sh): ```sh -brew install llvm@19 +brew install llvm@20 ``` Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them. @@ -43,18 +43,18 @@ ### Windows LLVM is downloaded automatically (along with other external binary dependencies) by `PCbuild\build.bat`. -Otherwise, you can install LLVM 19 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=19), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** +Otherwise, you can install LLVM 20 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=20), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** Alternatively, you can use [chocolatey](https://chocolatey.org): ```sh -choco install llvm --version=19.1.0 +choco install llvm --version=20.1.8 ``` ### Dev Containers If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no -need to install LLVM as the Fedora 41 base image includes LLVM 19 out of the box. +need to install LLVM as the Fedora 42 base image includes LLVM 20 out of the box. ## Building diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index bc3b50ffe61..54c2bf86a36 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -11,8 +11,8 @@ import _targets -_LLVM_VERSION = "19" -_EXTERNALS_LLVM_TAG = "llvm-19.1.7.0" +_LLVM_VERSION = "20" +_EXTERNALS_LLVM_TAG = "llvm-20.1.8.0" _P = typing.ParamSpec("_P") _R = typing.TypeVar("_R") diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 777db7366b1..e717365b6b9 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -253,6 +253,23 @@ def process_relocations(self, known_symbols: dict[str, int]) -> None: self._trampolines.add(ordinal) hole.addend = ordinal hole.symbol = None + # x86_64 Darwin trampolines for external symbols + elif ( + hole.kind == "X86_64_RELOC_BRANCH" + and hole.value is HoleValue.ZERO + and hole.symbol not in self.symbols + ): + hole.func = "patch_x86_64_trampoline" + hole.need_state = True + assert hole.symbol is not None + if hole.symbol in known_symbols: + ordinal = known_symbols[hole.symbol] + else: + ordinal = len(known_symbols) + known_symbols[hole.symbol] = ordinal + self._trampolines.add(ordinal) + hole.addend = ordinal + hole.symbol = None self.data.pad(8) for stencil in [self.code, self.data]: for hole in stencil.holes: diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index dcc0abaf23f..a76d8ff2792 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -166,10 +166,6 @@ async def _compile( "-fno-asynchronous-unwind-tables", # Don't call built-in functions that we can't find or patch: "-fno-builtin", - # Emit relaxable 64-bit calls/jumps, so we don't have to worry about - # about emitting in-range trampolines for out-of-range targets. - # We can probably remove this and emit trampolines in the future: - "-fno-plt", # Don't call stack-smashing canaries that we can't find or patch: "-fno-stack-protector", "-std=c11", @@ -571,14 +567,14 @@ def get_target(host: str) -> _COFF32 | _COFF64 | _ELF | _MachO: elif re.fullmatch(r"aarch64-pc-windows-msvc", host): host = "aarch64-pc-windows-msvc" condition = "defined(_M_ARM64)" - args = ["-fms-runtime-lib=dll", "-fplt"] + args = ["-fms-runtime-lib=dll"] optimizer = _optimizers.OptimizerAArch64 target = _COFF64(host, condition, args=args, optimizer=optimizer) elif re.fullmatch(r"aarch64-.*-linux-gnu", host): host = "aarch64-unknown-linux-gnu" condition = "defined(__aarch64__) && defined(__linux__)" # -mno-outline-atomics: Keep intrinsics from being emitted. - args = ["-fpic", "-mno-outline-atomics"] + args = ["-fpic", "-mno-outline-atomics", "-fno-plt"] optimizer = _optimizers.OptimizerAArch64 target = _ELF(host, condition, args=args, optimizer=optimizer) elif re.fullmatch(r"i686-pc-windows-msvc", host): @@ -602,7 +598,7 @@ def get_target(host: str) -> _COFF32 | _COFF64 | _ELF | _MachO: elif re.fullmatch(r"x86_64-.*-linux-gnu", host): host = "x86_64-unknown-linux-gnu" condition = "defined(__x86_64__) && defined(__linux__)" - args = ["-fno-pic", "-mcmodel=medium", "-mlarge-data-threshold=0"] + args = ["-fno-pic", "-mcmodel=medium", "-mlarge-data-threshold=0", "-fno-plt"] optimizer = _optimizers.OptimizerX86 target = _ELF(host, condition, args=args, optimizer=optimizer) else: From 9791a506c22b2e1b52dc9fb333b6f010d57f9eda Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Tue, 4 Nov 2025 03:07:55 +0800 Subject: [PATCH 014/313] gh-140889: Test tailcall and JIT in CI (GH-140891) --- .github/workflows/jit.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 151b17e8442..40d8b74e982 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -183,3 +183,27 @@ jobs: - name: Run tests without optimizations run: | PYTHON_UOPS_OPTIMIZE=0 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + tail-call-jit: + name: JIT with tail calling interpreter + needs: interpreter + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + llvm: + - 19 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Build with JIT and tailcall + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + CC=clang-${{ matrix.llvm }} ./configure --enable-experimental-jit --with-tail-call-interp --with-pydebug + make all --jobs 4 From cf1a2c1ee46b62fb3a5938fdfe90b7a9df312c3a Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 3 Nov 2025 21:26:09 +0200 Subject: [PATCH 015/313] gh-133600: Run `Tools/wasm/wasi` on CI instead of deprecated `Tools/wasm/wasi.py` (#140907) --- .github/workflows/reusable-wasi.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 7c40e566b2a..18feb564822 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -60,24 +60,24 @@ jobs: with: path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python. - # Include the hash of `Tools/wasm/wasi.py` as it may change the environment variables. + # Include the hash of `Tools/wasm/wasi/__main__.py` as it may change the environment variables. # (Make sure to keep the key in sync with the other config.cache step below.) - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi/__main__.py') }}-${{ env.pythonLocation }} - name: "Configure build Python" - run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug + run: python3 Tools/wasm/wasi configure-build-python -- --config-cache --with-pydebug - name: "Make build Python" - run: python3 Tools/wasm/wasi.py make-build-python + run: python3 Tools/wasm/wasi make-build-python - name: "Restore host config.cache" uses: actions/cache@v4 with: path: ${{ env.CROSS_BUILD_WASI }}/config.cache # Should be kept in sync with the other config.cache step above. - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi/__main__.py') }}-${{ env.pythonLocation }} - name: "Configure host" # `--with-pydebug` inferred from configure-build-python - run: python3 Tools/wasm/wasi.py configure-host -- --config-cache + run: python3 Tools/wasm/wasi configure-host -- --config-cache - name: "Make host" - run: python3 Tools/wasm/wasi.py make-host + run: python3 Tools/wasm/wasi make-host - name: "Display build info" run: make --directory "${CROSS_BUILD_WASI}" pythoninfo - name: "Test" From c98c5b344941c45e50dded708177f7ec2e225b2b Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Mon, 3 Nov 2025 11:36:37 -0800 Subject: [PATCH 016/313] gh-131253: free-threaded build support for pystats (gh-137189) Allow the --enable-pystats build option to be used with free-threading. The stats are now stored on a per-interpreter basis, rather than process global. For free-threaded builds, the stats structure is allocated per-thread and then periodically merged into the per-interpreter stats structure (on thread exit or when the reporting function is called). Most of the pystats related code has be moved into the file Python/pystats.c. --- Include/cpython/pystate.h | 24 + Include/cpython/pystats.h | 64 +- Include/internal/pycore_interp_structs.h | 14 +- Include/internal/pycore_pystats.h | 2 +- Include/internal/pycore_stats.h | 85 +- Include/internal/pycore_tstate.h | 6 + Lib/test/test_pystats.py | 215 +++++ Makefile.pre.in | 1 + ...-07-29-17-51-14.gh-issue-131253.GpRjWy.rst | 1 + Modules/_xxtestfuzz/fuzzer.c | 4 +- PCbuild/_freeze_module.vcxproj | 1 + PCbuild/_freeze_module.vcxproj.filters | 3 + PCbuild/pythoncore.vcxproj | 1 + Python/ceval_macros.h | 3 +- Python/gc.c | 7 +- Python/gc_free_threading.c | 16 +- Python/initconfig.c | 6 - Python/lock.c | 3 + Python/pylifecycle.c | 17 + Python/pystate.c | 29 + Python/pystats.c | 819 ++++++++++++++++++ Python/qsbr.c | 3 +- Python/specialize.c | 426 +-------- Python/sysmodule.c | 4 +- 24 files changed, 1269 insertions(+), 485 deletions(-) create mode 100644 Lib/test/test_pystats.py create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-29-17-51-14.gh-issue-131253.GpRjWy.rst create mode 100644 Python/pystats.c diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index ac8798ff612..dd2ea1202b3 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -217,6 +217,15 @@ struct _ts { */ PyObject *threading_local_sentinel; _PyRemoteDebuggerSupport remote_debugger_support; + +#ifdef Py_STATS + // Pointer to PyStats structure, NULL if recording is off. For the + // free-threaded build, the structure is per-thread (stored as a pointer + // in _PyThreadStateImpl). For the default build, the structure is stored + // in the PyInterpreterState structure (threads do not have their own + // structure and all share the same per-interpreter structure). + PyStats *pystats; +#endif }; /* other API */ @@ -239,6 +248,21 @@ PyAPI_FUNC(void) PyThreadState_EnterTracing(PyThreadState *tstate); // function is set, otherwise disable them. PyAPI_FUNC(void) PyThreadState_LeaveTracing(PyThreadState *tstate); +#ifdef Py_STATS +#if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) +extern _Py_thread_local PyThreadState *_Py_tss_tstate; + +static inline PyStats* +_PyThreadState_GetStatsFast(void) +{ + if (_Py_tss_tstate == NULL) { + return NULL; // no attached thread state + } + return _Py_tss_tstate->pystats; +} +#endif +#endif // Py_STATS + /* PyGILState */ /* Helper/diagnostic function - return 1 if the current thread diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index cf830b6066f..d0a925a3055 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -4,7 +4,7 @@ // // - _Py_INCREF_STAT_INC() and _Py_DECREF_STAT_INC() used by Py_INCREF() // and Py_DECREF(). -// - _Py_stats variable +// - _PyStats_GET() // // Functions of the sys module: // @@ -14,7 +14,7 @@ // - sys._stats_dump() // // Python must be built with ./configure --enable-pystats to define the -// Py_STATS macro. +// _PyStats_GET() macro. // // Define _PY_INTERPRETER macro to increment interpreter_increfs and // interpreter_decrefs. Otherwise, increment increfs and decrefs. @@ -109,6 +109,18 @@ typedef struct _gc_stats { uint64_t objects_not_transitively_reachable; } GCStats; +#ifdef Py_GIL_DISABLED +// stats specific to free-threaded build +typedef struct _ft_stats { + // number of times interpreter had to spin or park when trying to acquire a mutex + uint64_t mutex_sleeps; + // number of times that the QSBR mechanism polled (compute read sequence value) + uint64_t qsbr_polls; + // number of times stop-the-world mechanism was used + uint64_t world_stops; +} FTStats; +#endif + typedef struct _uop_stats { uint64_t execution_count; uint64_t miss; @@ -173,22 +185,48 @@ typedef struct _stats { CallStats call_stats; ObjectStats object_stats; OptimizationStats optimization_stats; +#ifdef Py_GIL_DISABLED + FTStats ft_stats; +#endif RareEventStats rare_event_stats; - GCStats *gc_stats; + GCStats gc_stats[3]; // must match NUM_GENERATIONS } PyStats; +// Export for most shared extensions +PyAPI_FUNC(PyStats *) _PyStats_GetLocal(void); -// Export for shared extensions like 'math' -PyAPI_DATA(PyStats*) _Py_stats; +#if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) +// use inline function version defined in cpython/pystate.h +static inline PyStats *_PyThreadState_GetStatsFast(void); +#define _PyStats_GET _PyThreadState_GetStatsFast +#else +#define _PyStats_GET _PyStats_GetLocal +#endif + +#define _Py_STATS_EXPR(expr) \ + do { \ + PyStats *s = _PyStats_GET(); \ + if (s != NULL) { \ + s->expr; \ + } \ + } while (0) + +#define _Py_STATS_COND_EXPR(cond, expr) \ + do { \ + PyStats *s = _PyStats_GET(); \ + if (s != NULL && (cond)) { \ + s->expr; \ + } \ + } while (0) #ifdef _PY_INTERPRETER -# define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_increfs++; } while (0) -# define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_decrefs++; } while (0) -# define _Py_INCREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_immortal_increfs++; } while (0) -# define _Py_DECREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_immortal_decrefs++; } while (0) +# define _Py_INCREF_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_increfs++) +# define _Py_DECREF_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_decrefs++) +# define _Py_INCREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_immortal_increfs++) +# define _Py_DECREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_immortal_decrefs++) #else -# define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.increfs++; } while (0) -# define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.decrefs++; } while (0) -# define _Py_INCREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.immortal_increfs++; } while (0) -# define _Py_DECREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.immortal_decrefs++; } while (0) +# define _Py_INCREF_STAT_INC() _Py_STATS_EXPR(object_stats.increfs++) +# define _Py_DECREF_STAT_INC() _Py_STATS_EXPR(object_stats.decrefs++) +# define _Py_INCREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.immortal_increfs++) +# define _Py_DECREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.immortal_decrefs++) #endif diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 9cdaa950e34..e8cbe9d894e 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -199,7 +199,7 @@ enum _GCPhase { }; /* If we change this, we need to change the default value in the - signature of gc.collect. */ + signature of gc.collect and change the size of PyStats.gc_stats */ #define NUM_GENERATIONS 3 struct _gc_runtime_state { @@ -963,6 +963,18 @@ struct _is { # ifdef Py_STACKREF_CLOSE_DEBUG _Py_hashtable_t *closed_stackrefs_table; # endif +#endif + +#ifdef Py_STATS + // true if recording of pystats is on, this is used when new threads + // are created to decide if recording should be on for them + int pystats_enabled; + // allocated when (and if) stats are first enabled + PyStats *pystats_struct; +#ifdef Py_GIL_DISABLED + // held when pystats related interpreter state is being updated + PyMutex pystats_mutex; +#endif #endif /* the initial PyInterpreterState.threads.head */ diff --git a/Include/internal/pycore_pystats.h b/Include/internal/pycore_pystats.h index f8af398a560..50ab21aa0f1 100644 --- a/Include/internal/pycore_pystats.h +++ b/Include/internal/pycore_pystats.h @@ -9,7 +9,7 @@ extern "C" { #endif #ifdef Py_STATS -extern void _Py_StatsOn(void); +extern int _Py_StatsOn(void); extern void _Py_StatsOff(void); extern void _Py_StatsClear(void); extern int _Py_PrintSpecializationStats(int to_file); diff --git a/Include/internal/pycore_stats.h b/Include/internal/pycore_stats.h index 24f239a2135..850e6ea4552 100644 --- a/Include/internal/pycore_stats.h +++ b/Include/internal/pycore_stats.h @@ -15,39 +15,56 @@ extern "C" { #include "pycore_bitutils.h" // _Py_bit_length -#define STAT_INC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name++; } while (0) -#define STAT_DEC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name--; } while (0) -#define OPCODE_EXE_INC(opname) do { if (_Py_stats) _Py_stats->opcode_stats[opname].execution_count++; } while (0) -#define CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.name++; } while (0) -#define OBJECT_STAT_INC(name) do { if (_Py_stats) _Py_stats->object_stats.name++; } while (0) -#define OBJECT_STAT_INC_COND(name, cond) \ - do { if (_Py_stats && cond) _Py_stats->object_stats.name++; } while (0) -#define EVAL_CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.eval_calls[name]++; } while (0) -#define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) \ - do { if (_Py_stats && PyFunction_Check(callable)) _Py_stats->call_stats.eval_calls[name]++; } while (0) -#define GC_STAT_ADD(gen, name, n) do { if (_Py_stats) _Py_stats->gc_stats[(gen)].name += (n); } while (0) -#define OPT_STAT_INC(name) do { if (_Py_stats) _Py_stats->optimization_stats.name++; } while (0) -#define OPT_STAT_ADD(name, n) do { if (_Py_stats) _Py_stats->optimization_stats.name += (n); } while (0) -#define UOP_STAT_INC(opname, name) do { if (_Py_stats) { assert(opname < 512); _Py_stats->optimization_stats.opcode[opname].name++; } } while (0) -#define UOP_PAIR_INC(uopcode, lastuop) \ - do { \ - if (lastuop && _Py_stats) { \ - _Py_stats->optimization_stats.opcode[lastuop].pair_count[uopcode]++; \ - } \ - lastuop = uopcode; \ - } while (0) -#define OPT_UNSUPPORTED_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.unsupported_opcode[opname]++; } while (0) -#define OPT_ERROR_IN_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.error_in_opcode[opname]++; } while (0) -#define OPT_HIST(length, name) \ +#define STAT_INC(opname, name) _Py_STATS_EXPR(opcode_stats[opname].specialization.name++) +#define STAT_DEC(opname, name) _Py_STATS_EXPR(opcode_stats[opname].specialization.name--) +#define OPCODE_EXE_INC(opname) _Py_STATS_EXPR(opcode_stats[opname].execution_count++) +#define CALL_STAT_INC(name) _Py_STATS_EXPR(call_stats.name++) +#define OBJECT_STAT_INC(name) _Py_STATS_EXPR(object_stats.name++) +#define OBJECT_STAT_INC_COND(name, cond) _Py_STATS_COND_EXPR(cond, object_stats.name++) +#define EVAL_CALL_STAT_INC(name) _Py_STATS_EXPR(call_stats.eval_calls[name]++) +#define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) _Py_STATS_COND_EXPR(PyFunction_Check(callable), call_stats.eval_calls[name]++) +#define GC_STAT_ADD(gen, name, n) _Py_STATS_EXPR(gc_stats[(gen)].name += (n)) +#define OPT_STAT_INC(name) _Py_STATS_EXPR(optimization_stats.name++) +#define OPT_STAT_ADD(name, n) _Py_STATS_EXPR(optimization_stats.name += (n)) +#define UOP_STAT_INC(opname, name) \ do { \ - if (_Py_stats) { \ - int bucket = _Py_bit_length(length >= 1 ? length - 1 : 0); \ - bucket = (bucket >= _Py_UOP_HIST_SIZE) ? _Py_UOP_HIST_SIZE - 1 : bucket; \ - _Py_stats->optimization_stats.name[bucket]++; \ + PyStats *s = _PyStats_GET(); \ + if (s) { \ + assert(opname < 512); \ + s->optimization_stats.opcode[opname].name++; \ } \ } while (0) -#define RARE_EVENT_STAT_INC(name) do { if (_Py_stats) _Py_stats->rare_event_stats.name++; } while (0) -#define OPCODE_DEFERRED_INC(opname) do { if (_Py_stats && opcode == opname) _Py_stats->opcode_stats[opname].specialization.deferred++; } while (0) +#define UOP_PAIR_INC(uopcode, lastuop) \ + do { \ + PyStats *s = _PyStats_GET(); \ + if (lastuop && s) { \ + s->optimization_stats.opcode[lastuop].pair_count[uopcode]++; \ + } \ + lastuop = uopcode; \ + } while (0) +#define OPT_UNSUPPORTED_OPCODE(opname) _Py_STATS_EXPR(optimization_stats.unsupported_opcode[opname]++) +#define OPT_ERROR_IN_OPCODE(opname) _Py_STATS_EXPR(optimization_stats.error_in_opcode[opname]++) +#define OPT_HIST(length, name) \ + do { \ + PyStats *s = _PyStats_GET(); \ + if (s) { \ + int bucket = _Py_bit_length(length >= 1 ? length - 1 : 0); \ + bucket = (bucket >= _Py_UOP_HIST_SIZE) ? _Py_UOP_HIST_SIZE - 1 : bucket; \ + s->optimization_stats.name[bucket]++; \ + } \ + } while (0) +#define RARE_EVENT_STAT_INC(name) _Py_STATS_EXPR(rare_event_stats.name++) +#define OPCODE_DEFERRED_INC(opname) _Py_STATS_COND_EXPR(opcode==opname, opcode_stats[opname].specialization.deferred++) + +#ifdef Py_GIL_DISABLED +#define FT_STAT_MUTEX_SLEEP_INC() _Py_STATS_EXPR(ft_stats.mutex_sleeps++) +#define FT_STAT_QSBR_POLL_INC() _Py_STATS_EXPR(ft_stats.qsbr_polls++) +#define FT_STAT_WORLD_STOP_INC() _Py_STATS_EXPR(ft_stats.world_stops++) +#else +#define FT_STAT_MUTEX_SLEEP_INC() +#define FT_STAT_QSBR_POLL_INC() +#define FT_STAT_WORLD_STOP_INC() +#endif // Export for '_opcode' shared extension PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); @@ -71,6 +88,9 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define OPT_HIST(length, name) ((void)0) #define RARE_EVENT_STAT_INC(name) ((void)0) #define OPCODE_DEFERRED_INC(opname) ((void)0) +#define FT_STAT_MUTEX_SLEEP_INC() +#define FT_STAT_QSBR_POLL_INC() +#define FT_STAT_WORLD_STOP_INC() #endif // !Py_STATS @@ -90,6 +110,11 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); RARE_EVENT_INTERP_INC(interp, name); \ } while (0); \ +PyStatus _PyStats_InterpInit(PyInterpreterState *); +bool _PyStats_ThreadInit(PyInterpreterState *, _PyThreadStateImpl *); +void _PyStats_ThreadFini(_PyThreadStateImpl *); +void _PyStats_Attach(_PyThreadStateImpl *); +void _PyStats_Detach(_PyThreadStateImpl *); #ifdef __cplusplus } diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index bad968428c7..29ebdfd7e01 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -70,8 +70,14 @@ typedef struct _PyThreadStateImpl { // When >1, code objects do not immortalize their non-string constants. int suppress_co_const_immortalization; + +#ifdef Py_STATS + // per-thread stats, will be merged into interp->pystats_struct + PyStats *pystats_struct; // allocated by _PyStats_ThreadInit() #endif +#endif // Py_GIL_DISABLED + #if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) Py_ssize_t reftotal; // this thread's total refcount operations #endif diff --git a/Lib/test/test_pystats.py b/Lib/test/test_pystats.py new file mode 100644 index 00000000000..c50cecfcfdd --- /dev/null +++ b/Lib/test/test_pystats.py @@ -0,0 +1,215 @@ +import sys +import textwrap +import unittest +from test.support import script_helper + +# This function is available for the --enable-pystats config. +HAVE_PYSTATS = hasattr(sys, '_stats_on') + +TEST_TEMPLATE = """ + import sys + import threading + import time + + THREADS = 2 + + class A: + pass + + class B: + pass + + def modify_class(): + # This is used as a rare event we can assume doesn't happen unless we do it. + # It increments the "Rare event (set_class)" count. + a = A() + a.__class__ = B + + TURNED_ON = False + def stats_on(): + global TURNED_ON + sys._stats_on() + TURNED_ON = True + + TURNED_OFF = False + def stats_off(): + global TURNED_OFF + sys._stats_off() + TURNED_OFF = True + + CLEARED = False + def stats_clear(): + global CLEARED + sys._stats_clear() + CLEARED = True + + def func_start(): + pass + + def func_end(): + pass + + def func_test(thread_id): + pass + + _TEST_CODE_ + + func_start() + threads = [] + for i in range(THREADS): + t = threading.Thread(target=func_test, args=(i,)) + threads.append(t) + t.start() + for t in threads: + t.join() + func_end() + """ + + +def run_test_code( + test_code, + args=[], + env_vars=None, +): + """Run test code and return the value of the "set_class" stats counter. + """ + code = textwrap.dedent(TEST_TEMPLATE) + code = code.replace('_TEST_CODE_', textwrap.dedent(test_code)) + script_args = args + ['-c', code] + env_vars = env_vars or {} + res, _ = script_helper.run_python_until_end(*script_args, **env_vars) + stderr = res.err.decode("ascii", "backslashreplace") + for line in stderr.split('\n'): + if 'Rare event (set_class)' in line: + label, _, value = line.partition(':') + return value.strip() + return '' + + +@unittest.skipUnless(HAVE_PYSTATS, "requires pystats build option") +class TestPyStats(unittest.TestCase): + """Tests for pystats functionality (requires --enable-pystats build + option). + """ + + def test_stats_toggle_on(self): + """Check the toggle on functionality. + """ + code = """ + def func_start(): + modify_class() + """ + + # If turned on with command line flag, should get one count. + stat_count = run_test_code(code, args=['-X', 'pystats']) + self.assertEqual(stat_count, '1') + + # If turned on with env var, should get one count. + stat_count = run_test_code(code, env_vars={'PYTHONSTATS': '1'}) + self.assertEqual(stat_count, '1') + + # If not turned on, should be no counts. + stat_count = run_test_code(code) + self.assertEqual(stat_count, '') + + code = """ + def func_start(): + modify_class() + sys._stats_on() + modify_class() + """ + # Not initially turned on but enabled by sys._stats_on(), should get + # one count. + stat_count = run_test_code(code) + self.assertEqual(stat_count, '1') + + def test_stats_toggle_on_thread(self): + """Check the toggle on functionality when threads are used. + """ + code = """ + def func_test(thread_id): + if thread_id == 0: + modify_class() + stats_on() + modify_class() + else: + while not TURNED_ON: + pass + modify_class() + """ + # Turning on in one thread will count in other thread. + stat_count = run_test_code(code) + self.assertEqual(stat_count, '2') + + code = """ + def func_test(thread_id): + if thread_id == 0: + modify_class() + stats_off() + modify_class() + else: + while not TURNED_OFF: + pass + modify_class() + """ + # Turning off in one thread will not count in other threads. + stat_count = run_test_code(code, args=['-X', 'pystats']) + self.assertEqual(stat_count, '1') + + def test_thread_exit_merge(self): + """Check that per-thread stats (when free-threading enabled) are merged. + """ + code = """ + def func_test(thread_id): + modify_class() + if thread_id == 0: + raise SystemExit + """ + # Stats from a thread exiting early should still be counted. + stat_count = run_test_code(code, args=['-X', 'pystats']) + self.assertEqual(stat_count, '2') + + def test_stats_dump(self): + """Check that sys._stats_dump() works. + """ + code = """ + def func_test(thread_id): + if thread_id == 0: + stats_on() + else: + while not TURNED_ON: + pass + modify_class() + sys._stats_dump() + stats_off() + """ + # Stats from a thread exiting early should still be counted. + stat_count = run_test_code(code) + self.assertEqual(stat_count, '1') + + def test_stats_clear(self): + """Check that sys._stats_clear() works. + """ + code = """ + ready = False + def func_test(thread_id): + global ready + if thread_id == 0: + stats_on() + modify_class() + while not ready: + pass # wait until other thread has called modify_class() + stats_clear() # clears stats for all threads + else: + while not TURNED_ON: + pass + modify_class() + ready = True + """ + # Clearing stats will clear for all threads + stat_count = run_test_code(code) + self.assertEqual(stat_count, '0') + + +if __name__ == "__main__": + unittest.main() diff --git a/Makefile.pre.in b/Makefile.pre.in index 656d9dacd96..dd28ff5d2a3 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -483,6 +483,7 @@ PYTHON_OBJS= \ Python/pylifecycle.o \ Python/pymath.o \ Python/pystate.o \ + Python/pystats.o \ Python/pythonrun.o \ Python/pytime.o \ Python/qsbr.o \ diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-29-17-51-14.gh-issue-131253.GpRjWy.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-29-17-51-14.gh-issue-131253.GpRjWy.rst new file mode 100644 index 00000000000..2826fad2330 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-29-17-51-14.gh-issue-131253.GpRjWy.rst @@ -0,0 +1 @@ +Support the ``--enable-pystats`` build option for the free-threaded build. diff --git a/Modules/_xxtestfuzz/fuzzer.c b/Modules/_xxtestfuzz/fuzzer.c index a04f1412eef..0cbe10c79ab 100644 --- a/Modules/_xxtestfuzz/fuzzer.c +++ b/Modules/_xxtestfuzz/fuzzer.c @@ -10,8 +10,8 @@ See the source code for LLVMFuzzerTestOneInput for details. */ -#ifndef Py_BUILD_CORE -# define Py_BUILD_CORE 1 +#ifndef Py_BUILD_CORE_MODULE +# define Py_BUILD_CORE_MODULE 1 #endif #include diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index e65f201623f..605861ad3fd 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -257,6 +257,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index a9fb6f2328a..c67fe53363e 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -367,6 +367,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index a50ffb120bc..359a47fbfc4 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -658,6 +658,7 @@ + diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 868ab6f7558..afdcbc563b2 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -62,8 +62,9 @@ #ifdef Py_STATS #define INSTRUCTION_STATS(op) \ do { \ + PyStats *s = _PyStats_GET(); \ OPCODE_EXE_INC(op); \ - if (_Py_stats) _Py_stats->opcode_stats[lastopcode].pair_count[op]++; \ + if (s) s->opcode_stats[lastopcode].pair_count[op]++; \ lastopcode = op; \ } while (0) #else diff --git a/Python/gc.c b/Python/gc.c index a1f3d86d910..03a5d7366ea 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -2111,10 +2111,11 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) _PyErr_SetRaisedException(tstate, exc); GC_STAT_ADD(generation, objects_collected, stats.collected); #ifdef Py_STATS - if (_Py_stats) { + PyStats *s = _PyStats_GET(); + if (s) { GC_STAT_ADD(generation, object_visits, - _Py_stats->object_stats.object_visits); - _Py_stats->object_stats.object_visits = 0; + s->object_stats.object_visits); + s->object_stats.object_visits = 0; } #endif validate_spaces(gcstate); diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 842aa340154..f39793c3eeb 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -2362,8 +2362,9 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) assert(generation >= 0 && generation < NUM_GENERATIONS); #ifdef Py_STATS - if (_Py_stats) { - _Py_stats->object_stats.object_visits = 0; + PyStats *s = _PyStats_GET(); + if (s) { + s->object_stats.object_visits = 0; } #endif GC_STAT_ADD(generation, collections, 1); @@ -2426,10 +2427,13 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) GC_STAT_ADD(generation, objects_collected, m); #ifdef Py_STATS - if (_Py_stats) { - GC_STAT_ADD(generation, object_visits, - _Py_stats->object_stats.object_visits); - _Py_stats->object_stats.object_visits = 0; + { + PyStats *s = _PyStats_GET(); + if (s) { + GC_STAT_ADD(generation, object_visits, + s->object_stats.object_visits); + s->object_stats.object_visits = 0; + } } #endif diff --git a/Python/initconfig.c b/Python/initconfig.c index 5dc68eb4ec2..7176670c110 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -2810,12 +2810,6 @@ _PyConfig_Write(const PyConfig *config, _PyRuntimeState *runtime) return _PyStatus_NO_MEMORY(); } -#ifdef Py_STATS - if (config->_pystats) { - _Py_StatsOn(); - } -#endif - return _PyStatus_OK(); } diff --git a/Python/lock.c b/Python/lock.c index 98f3f89c201..789065d8162 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -6,6 +6,7 @@ #include "pycore_parking_lot.h" #include "pycore_semaphore.h" #include "pycore_time.h" // _PyTime_Add() +#include "pycore_stats.h" // FT_STAT_MUTEX_SLEEP_INC() #ifdef MS_WINDOWS # ifndef WIN32_LEAN_AND_MEAN @@ -62,6 +63,8 @@ _PyMutex_LockTimed(PyMutex *m, PyTime_t timeout, _PyLockFlags flags) return PY_LOCK_FAILURE; } + FT_STAT_MUTEX_SLEEP_INC(); + PyTime_t now; // silently ignore error: cannot report error to the caller (void)PyTime_MonotonicRaw(&now); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 8fcb31cfd12..805805ef188 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -26,6 +26,7 @@ #include "pycore_runtime.h" // _Py_ID() #include "pycore_runtime_init.h" // _PyRuntimeState_INIT #include "pycore_setobject.h" // _PySet_NextEntry() +#include "pycore_stats.h" // _PyStats_InterpInit() #include "pycore_sysmodule.h" // _PySys_ClearAttrString() #include "pycore_traceback.h" // _Py_DumpTracebackThreads() #include "pycore_typeobject.h" // _PyTypes_InitTypes() @@ -656,6 +657,14 @@ pycore_create_interpreter(_PyRuntimeState *runtime, return status; } +#ifdef Py_STATS + // initialize pystats. This must be done after the settings are loaded. + status = _PyStats_InterpInit(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } +#endif + // initialize the interp->obmalloc state. This must be done after // the settings are loaded (so that feature_flags are set) but before // any calls are made to obmalloc functions. @@ -2469,6 +2478,14 @@ new_interpreter(PyThreadState **tstate_p, return status; } +#ifdef Py_STATS + // initialize pystats. This must be done after the settings are loaded. + status = _PyStats_InterpInit(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } +#endif + // initialize the interp->obmalloc state. This must be done after // the settings are loaded (so that feature_flags are set) but before // any calls are made to obmalloc functions. diff --git a/Python/pystate.c b/Python/pystate.c index 24681536797..cf251c120d7 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -21,6 +21,7 @@ #include "pycore_runtime.h" // _PyRuntime #include "pycore_runtime_init.h" // _PyRuntimeState_INIT #include "pycore_stackref.h" // Py_STACKREF_DEBUG +#include "pycore_stats.h" // FT_STAT_WORLD_STOP_INC() #include "pycore_time.h" // _PyTime_Init() #include "pycore_uop.h" // UOP_BUFFER_SIZE #include "pycore_uniqueid.h" // _PyObject_FinalizePerThreadRefcounts() @@ -465,6 +466,12 @@ alloc_interpreter(void) static void free_interpreter(PyInterpreterState *interp) { +#ifdef Py_STATS + if (interp->pystats_struct) { + PyMem_RawFree(interp->pystats_struct); + interp->pystats_struct = NULL; + } +#endif // The main interpreter is statically allocated so // should not be freed. if (interp != &_PyRuntime._main_interpreter) { @@ -1407,6 +1414,9 @@ static void free_threadstate(_PyThreadStateImpl *tstate) { PyInterpreterState *interp = tstate->base.interp; +#ifdef Py_STATS + _PyStats_ThreadFini(tstate); +#endif // The initial thread state of the interpreter is allocated // as part of the interpreter state so should not be freed. if (tstate == &interp->_initial_thread) { @@ -1535,6 +1545,13 @@ new_threadstate(PyInterpreterState *interp, int whence) return NULL; } #endif +#ifdef Py_STATS + // The PyStats structure is quite large and is allocated separated from tstate. + if (!_PyStats_ThreadInit(interp, tstate)) { + free_threadstate(tstate); + return NULL; + } +#endif /* We serialize concurrent creation to protect global state. */ HEAD_LOCK(interp->runtime); @@ -1846,6 +1863,9 @@ _PyThreadState_DeleteCurrent(PyThreadState *tstate) _Py_EnsureTstateNotNULL(tstate); #ifdef Py_GIL_DISABLED _Py_qsbr_detach(((_PyThreadStateImpl *)tstate)->qsbr); +#endif +#ifdef Py_STATS + _PyStats_Detach((_PyThreadStateImpl *)tstate); #endif current_fast_clear(tstate->interp->runtime); tstate_delete_common(tstate, 1); // release GIL as part of call @@ -2020,6 +2040,10 @@ tstate_deactivate(PyThreadState *tstate) assert(tstate_is_bound(tstate)); assert(tstate->_status.active); +#if Py_STATS + _PyStats_Detach((_PyThreadStateImpl *)tstate); +#endif + tstate->_status.active = 0; // We do not unbind the gilstate tstate here. @@ -2123,6 +2147,10 @@ _PyThreadState_Attach(PyThreadState *tstate) _PyCriticalSection_Resume(tstate); } +#ifdef Py_STATS + _PyStats_Attach((_PyThreadStateImpl *)tstate); +#endif + #if defined(Py_DEBUG) errno = err; #endif @@ -2272,6 +2300,7 @@ stop_the_world(struct _stoptheworld_state *stw) stw->thread_countdown = 0; stw->stop_event = (PyEvent){0}; // zero-initialize (unset) stw->requester = _PyThreadState_GET(); // may be NULL + FT_STAT_WORLD_STOP_INC(); _Py_FOR_EACH_STW_INTERP(stw, i) { _Py_FOR_EACH_TSTATE_UNLOCKED(i, t) { diff --git a/Python/pystats.c b/Python/pystats.c new file mode 100644 index 00000000000..2e377b8e6da --- /dev/null +++ b/Python/pystats.c @@ -0,0 +1,819 @@ +#include "Python.h" + +#include "pycore_opcode_metadata.h" // _PyOpcode_Caches +#include "pycore_pyatomic_ft_wrappers.h" +#include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() +#include "pycore_tstate.h" +#include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_uop_metadata.h" // _PyOpcode_uop_name +#include "pycore_uop_ids.h" // MAX_UOP_ID +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_runtime.h" // NUM_GENERATIONS + +#include // rand() + +#ifdef Py_STATS + +PyStats * +_PyStats_GetLocal(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate) { + return tstate->pystats; + } + return NULL; +} + +#ifdef Py_GIL_DISABLED +#define STATS_LOCK(interp) PyMutex_Lock(&interp->pystats_mutex) +#define STATS_UNLOCK(interp) PyMutex_Unlock(&interp->pystats_mutex) +#else +#define STATS_LOCK(interp) +#define STATS_UNLOCK(interp) +#endif + + +#if PYSTATS_MAX_UOP_ID < MAX_UOP_ID +#error "Not enough space allocated for pystats. Increase PYSTATS_MAX_UOP_ID to at least MAX_UOP_ID" +#endif + +#define ADD_STAT_TO_DICT(res, field) \ + do { \ + PyObject *val = PyLong_FromUnsignedLongLong(stats->field); \ + if (val == NULL) { \ + Py_DECREF(res); \ + return NULL; \ + } \ + if (PyDict_SetItemString(res, #field, val) == -1) { \ + Py_DECREF(res); \ + Py_DECREF(val); \ + return NULL; \ + } \ + Py_DECREF(val); \ + } while(0); + +static PyObject* +stats_to_dict(SpecializationStats *stats) +{ + PyObject *res = PyDict_New(); + if (res == NULL) { + return NULL; + } + ADD_STAT_TO_DICT(res, success); + ADD_STAT_TO_DICT(res, failure); + ADD_STAT_TO_DICT(res, hit); + ADD_STAT_TO_DICT(res, deferred); + ADD_STAT_TO_DICT(res, miss); + ADD_STAT_TO_DICT(res, deopt); + PyObject *failure_kinds = PyTuple_New(SPECIALIZATION_FAILURE_KINDS); + if (failure_kinds == NULL) { + Py_DECREF(res); + return NULL; + } + for (int i = 0; i < SPECIALIZATION_FAILURE_KINDS; i++) { + PyObject *stat = PyLong_FromUnsignedLongLong(stats->failure_kinds[i]); + if (stat == NULL) { + Py_DECREF(res); + Py_DECREF(failure_kinds); + return NULL; + } + PyTuple_SET_ITEM(failure_kinds, i, stat); + } + if (PyDict_SetItemString(res, "failure_kinds", failure_kinds)) { + Py_DECREF(res); + Py_DECREF(failure_kinds); + return NULL; + } + Py_DECREF(failure_kinds); + return res; +} +#undef ADD_STAT_TO_DICT + +static int +add_stat_dict( + PyStats *src, + PyObject *res, + int opcode, + const char *name) { + + SpecializationStats *stats = &src->opcode_stats[opcode].specialization; + PyObject *d = stats_to_dict(stats); + if (d == NULL) { + return -1; + } + int err = PyDict_SetItemString(res, name, d); + Py_DECREF(d); + return err; +} + +PyObject* +_Py_GetSpecializationStats(void) { + PyThreadState *tstate = _PyThreadState_GET(); + PyStats *src = FT_ATOMIC_LOAD_PTR_RELAXED(tstate->interp->pystats_struct); + if (src == NULL) { + Py_RETURN_NONE; + } + PyObject *stats = PyDict_New(); + if (stats == NULL) { + return NULL; + } + int err = 0; + err += add_stat_dict(src, stats, CONTAINS_OP, "contains_op"); + err += add_stat_dict(src, stats, LOAD_SUPER_ATTR, "load_super_attr"); + err += add_stat_dict(src, stats, LOAD_ATTR, "load_attr"); + err += add_stat_dict(src, stats, LOAD_GLOBAL, "load_global"); + err += add_stat_dict(src, stats, STORE_SUBSCR, "store_subscr"); + err += add_stat_dict(src, stats, STORE_ATTR, "store_attr"); + err += add_stat_dict(src, stats, JUMP_BACKWARD, "jump_backward"); + err += add_stat_dict(src, stats, CALL, "call"); + err += add_stat_dict(src, stats, CALL_KW, "call_kw"); + err += add_stat_dict(src, stats, BINARY_OP, "binary_op"); + err += add_stat_dict(src, stats, COMPARE_OP, "compare_op"); + err += add_stat_dict(src, stats, UNPACK_SEQUENCE, "unpack_sequence"); + err += add_stat_dict(src, stats, FOR_ITER, "for_iter"); + err += add_stat_dict(src, stats, TO_BOOL, "to_bool"); + err += add_stat_dict(src, stats, SEND, "send"); + if (err < 0) { + Py_DECREF(stats); + return NULL; + } + return stats; +} + + +#define PRINT_STAT(i, field) \ + if (stats[i].field) { \ + fprintf(out, " opcode[%s]." #field " : %" PRIu64 "\n", _PyOpcode_OpName[i], stats[i].field); \ + } + +static void +print_spec_stats(FILE *out, OpcodeStats *stats) +{ + /* Mark some opcodes as specializable for stats, + * even though we don't specialize them yet. */ + fprintf(out, "opcode[BINARY_SLICE].specializable : 1\n"); + fprintf(out, "opcode[STORE_SLICE].specializable : 1\n"); + fprintf(out, "opcode[GET_ITER].specializable : 1\n"); + for (int i = 0; i < 256; i++) { + if (_PyOpcode_Caches[i]) { + /* Ignore jumps as they cannot be specialized */ + switch (i) { + case POP_JUMP_IF_FALSE: + case POP_JUMP_IF_TRUE: + case POP_JUMP_IF_NONE: + case POP_JUMP_IF_NOT_NONE: + case JUMP_BACKWARD: + break; + default: + fprintf(out, "opcode[%s].specializable : 1\n", _PyOpcode_OpName[i]); + } + } + PRINT_STAT(i, specialization.success); + PRINT_STAT(i, specialization.failure); + PRINT_STAT(i, specialization.hit); + PRINT_STAT(i, specialization.deferred); + PRINT_STAT(i, specialization.miss); + PRINT_STAT(i, specialization.deopt); + PRINT_STAT(i, execution_count); + for (int j = 0; j < SPECIALIZATION_FAILURE_KINDS; j++) { + uint64_t val = stats[i].specialization.failure_kinds[j]; + if (val) { + fprintf(out, " opcode[%s].specialization.failure_kinds[%d] : %" + PRIu64 "\n", _PyOpcode_OpName[i], j, val); + } + } + for (int j = 0; j < 256; j++) { + if (stats[i].pair_count[j]) { + fprintf(out, "opcode[%s].pair_count[%s] : %" PRIu64 "\n", + _PyOpcode_OpName[i], _PyOpcode_OpName[j], stats[i].pair_count[j]); + } + } + } +} +#undef PRINT_STAT + + +static void +print_call_stats(FILE *out, CallStats *stats) +{ + fprintf(out, "Calls to PyEval_EvalDefault: %" PRIu64 "\n", stats->pyeval_calls); + fprintf(out, "Calls to Python functions inlined: %" PRIu64 "\n", stats->inlined_py_calls); + fprintf(out, "Frames pushed: %" PRIu64 "\n", stats->frames_pushed); + fprintf(out, "Frame objects created: %" PRIu64 "\n", stats->frame_objects_created); + for (int i = 0; i < EVAL_CALL_KINDS; i++) { + fprintf(out, "Calls via PyEval_EvalFrame[%d] : %" PRIu64 "\n", i, stats->eval_calls[i]); + } +} + +static void +print_object_stats(FILE *out, ObjectStats *stats) +{ + fprintf(out, "Object allocations from freelist: %" PRIu64 "\n", stats->from_freelist); + fprintf(out, "Object frees to freelist: %" PRIu64 "\n", stats->to_freelist); + fprintf(out, "Object allocations: %" PRIu64 "\n", stats->allocations); + fprintf(out, "Object allocations to 512 bytes: %" PRIu64 "\n", stats->allocations512); + fprintf(out, "Object allocations to 4 kbytes: %" PRIu64 "\n", stats->allocations4k); + fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big); + fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees); + fprintf(out, "Object inline values: %" PRIu64 "\n", stats->inline_values); + fprintf(out, "Object interpreter mortal increfs: %" PRIu64 "\n", stats->interpreter_increfs); + fprintf(out, "Object interpreter mortal decrefs: %" PRIu64 "\n", stats->interpreter_decrefs); + fprintf(out, "Object mortal increfs: %" PRIu64 "\n", stats->increfs); + fprintf(out, "Object mortal decrefs: %" PRIu64 "\n", stats->decrefs); + fprintf(out, "Object interpreter immortal increfs: %" PRIu64 "\n", stats->interpreter_immortal_increfs); + fprintf(out, "Object interpreter immortal decrefs: %" PRIu64 "\n", stats->interpreter_immortal_decrefs); + fprintf(out, "Object immortal increfs: %" PRIu64 "\n", stats->immortal_increfs); + fprintf(out, "Object immortal decrefs: %" PRIu64 "\n", stats->immortal_decrefs); + fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request); + fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key); + fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big); + fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass); + fprintf(out, "Object method cache hits: %" PRIu64 "\n", stats->type_cache_hits); + fprintf(out, "Object method cache misses: %" PRIu64 "\n", stats->type_cache_misses); + fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions); + fprintf(out, "Object method cache dunder hits: %" PRIu64 "\n", stats->type_cache_dunder_hits); + fprintf(out, "Object method cache dunder misses: %" PRIu64 "\n", stats->type_cache_dunder_misses); +} + +static void +print_gc_stats(FILE *out, GCStats *stats) +{ + for (int i = 0; i < NUM_GENERATIONS; i++) { + fprintf(out, "GC[%d] collections: %" PRIu64 "\n", i, stats[i].collections); + fprintf(out, "GC[%d] object visits: %" PRIu64 "\n", i, stats[i].object_visits); + fprintf(out, "GC[%d] objects collected: %" PRIu64 "\n", i, stats[i].objects_collected); + fprintf(out, "GC[%d] objects reachable from roots: %" PRIu64 "\n", i, stats[i].objects_transitively_reachable); + fprintf(out, "GC[%d] objects not reachable from roots: %" PRIu64 "\n", i, stats[i].objects_not_transitively_reachable); + } +} + +#ifdef _Py_TIER2 +static void +print_histogram(FILE *out, const char *name, uint64_t hist[_Py_UOP_HIST_SIZE]) +{ + for (int i = 0; i < _Py_UOP_HIST_SIZE; i++) { + fprintf(out, "%s[%" PRIu64"]: %" PRIu64 "\n", name, (uint64_t)1 << i, hist[i]); + } +} + +extern const char *_PyUOpName(int index); + +static void +print_optimization_stats(FILE *out, OptimizationStats *stats) +{ + fprintf(out, "Optimization attempts: %" PRIu64 "\n", stats->attempts); + fprintf(out, "Optimization traces created: %" PRIu64 "\n", stats->traces_created); + fprintf(out, "Optimization traces executed: %" PRIu64 "\n", stats->traces_executed); + fprintf(out, "Optimization uops executed: %" PRIu64 "\n", stats->uops_executed); + fprintf(out, "Optimization trace stack overflow: %" PRIu64 "\n", stats->trace_stack_overflow); + fprintf(out, "Optimization trace stack underflow: %" PRIu64 "\n", stats->trace_stack_underflow); + fprintf(out, "Optimization trace too long: %" PRIu64 "\n", stats->trace_too_long); + fprintf(out, "Optimization trace too short: %" PRIu64 "\n", stats->trace_too_short); + fprintf(out, "Optimization inner loop: %" PRIu64 "\n", stats->inner_loop); + fprintf(out, "Optimization recursive call: %" PRIu64 "\n", stats->recursive_call); + fprintf(out, "Optimization low confidence: %" PRIu64 "\n", stats->low_confidence); + fprintf(out, "Optimization unknown callee: %" PRIu64 "\n", stats->unknown_callee); + fprintf(out, "Executors invalidated: %" PRIu64 "\n", stats->executors_invalidated); + + print_histogram(out, "Trace length", stats->trace_length_hist); + print_histogram(out, "Trace run length", stats->trace_run_length_hist); + print_histogram(out, "Optimized trace length", stats->optimized_trace_length_hist); + + fprintf(out, "Optimization optimizer attempts: %" PRIu64 "\n", stats->optimizer_attempts); + fprintf(out, "Optimization optimizer successes: %" PRIu64 "\n", stats->optimizer_successes); + fprintf(out, "Optimization optimizer failure no memory: %" PRIu64 "\n", + stats->optimizer_failure_reason_no_memory); + fprintf(out, "Optimizer remove globals builtins changed: %" PRIu64 "\n", stats->remove_globals_builtins_changed); + fprintf(out, "Optimizer remove globals incorrect keys: %" PRIu64 "\n", stats->remove_globals_incorrect_keys); + for (int i = 0; i <= MAX_UOP_ID; i++) { + if (stats->opcode[i].execution_count) { + fprintf(out, "uops[%s].execution_count : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].execution_count); + } + if (stats->opcode[i].miss) { + fprintf(out, "uops[%s].specialization.miss : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].miss); + } + } + for (int i = 0; i < 256; i++) { + if (stats->unsupported_opcode[i]) { + fprintf( + out, + "unsupported_opcode[%s].count : %" PRIu64 "\n", + _PyOpcode_OpName[i], + stats->unsupported_opcode[i] + ); + } + } + + for (int i = 1; i <= MAX_UOP_ID; i++){ + for (int j = 1; j <= MAX_UOP_ID; j++) { + if (stats->opcode[i].pair_count[j]) { + fprintf(out, "uop[%s].pair_count[%s] : %" PRIu64 "\n", + _PyOpcode_uop_name[i], _PyOpcode_uop_name[j], stats->opcode[i].pair_count[j]); + } + } + } + for (int i = 0; i < MAX_UOP_ID; i++) { + if (stats->error_in_opcode[i]) { + fprintf( + out, + "error_in_opcode[%s].count : %" PRIu64 "\n", + _PyUOpName(i), + stats->error_in_opcode[i] + ); + } + } + fprintf(out, "JIT total memory size: %" PRIu64 "\n", stats->jit_total_memory_size); + fprintf(out, "JIT code size: %" PRIu64 "\n", stats->jit_code_size); + fprintf(out, "JIT trampoline size: %" PRIu64 "\n", stats->jit_trampoline_size); + fprintf(out, "JIT data size: %" PRIu64 "\n", stats->jit_data_size); + fprintf(out, "JIT padding size: %" PRIu64 "\n", stats->jit_padding_size); + fprintf(out, "JIT freed memory size: %" PRIu64 "\n", stats->jit_freed_memory_size); + + print_histogram(out, "Trace total memory size", stats->trace_total_memory_hist); +} +#endif + +#ifdef Py_GIL_DISABLED +static void +print_ft_stats(FILE *out, FTStats *stats) +{ + fprintf(out, "Mutex sleeps (mutex_sleeps): %" PRIu64 "\n", stats->mutex_sleeps); + fprintf(out, "QSBR polls (qsbr_polls): %" PRIu64 "\n", stats->qsbr_polls); + fprintf(out, "World stops (world_stops): %" PRIu64 "\n", stats->world_stops); +} +#endif + +static void +print_rare_event_stats(FILE *out, RareEventStats *stats) +{ + fprintf(out, "Rare event (set_class): %" PRIu64 "\n", stats->set_class); + fprintf(out, "Rare event (set_bases): %" PRIu64 "\n", stats->set_bases); + fprintf(out, "Rare event (set_eval_frame_func): %" PRIu64 "\n", stats->set_eval_frame_func); + fprintf(out, "Rare event (builtin_dict): %" PRIu64 "\n", stats->builtin_dict); + fprintf(out, "Rare event (func_modification): %" PRIu64 "\n", stats->func_modification); + fprintf(out, "Rare event (watched_dict_modification): %" PRIu64 "\n", stats->watched_dict_modification); + fprintf(out, "Rare event (watched_globals_modification): %" PRIu64 "\n", stats->watched_globals_modification); +} + +static void +print_stats(FILE *out, PyStats *stats) +{ + print_spec_stats(out, stats->opcode_stats); + print_call_stats(out, &stats->call_stats); + print_object_stats(out, &stats->object_stats); + print_gc_stats(out, stats->gc_stats); +#ifdef _Py_TIER2 + print_optimization_stats(out, &stats->optimization_stats); +#endif +#ifdef Py_GIL_DISABLED + print_ft_stats(out, &stats->ft_stats); +#endif + print_rare_event_stats(out, &stats->rare_event_stats); +} + +#ifdef Py_GIL_DISABLED + +static void +merge_specialization_stats(SpecializationStats *dest, const SpecializationStats *src) +{ + dest->success += src->success; + dest->failure += src->failure; + dest->hit += src->hit; + dest->deferred += src->deferred; + dest->miss += src->miss; + dest->deopt += src->deopt; + for (int i = 0; i < SPECIALIZATION_FAILURE_KINDS; i++) { + dest->failure_kinds[i] += src->failure_kinds[i]; + } +} + +static void +merge_opcode_stats_array(OpcodeStats *dest, const OpcodeStats *src) +{ + for (int i = 0; i < 256; i++) { + merge_specialization_stats(&dest[i].specialization, &src[i].specialization); + dest[i].execution_count += src[i].execution_count; + for (int j = 0; j < 256; j++) { + dest[i].pair_count[j] += src[i].pair_count[j]; + } + } +} + +static void +merge_call_stats(CallStats *dest, const CallStats *src) +{ + dest->inlined_py_calls += src->inlined_py_calls; + dest->pyeval_calls += src->pyeval_calls; + dest->frames_pushed += src->frames_pushed; + dest->frame_objects_created += src->frame_objects_created; + for (int i = 0; i < EVAL_CALL_KINDS; i++) { + dest->eval_calls[i] += src->eval_calls[i]; + } +} + +static void +merge_object_stats(ObjectStats *dest, const ObjectStats *src) +{ + dest->increfs += src->increfs; + dest->decrefs += src->decrefs; + dest->interpreter_increfs += src->interpreter_increfs; + dest->interpreter_decrefs += src->interpreter_decrefs; + dest->immortal_increfs += src->immortal_increfs; + dest->immortal_decrefs += src->immortal_decrefs; + dest->interpreter_immortal_increfs += src->interpreter_immortal_increfs; + dest->interpreter_immortal_decrefs += src->interpreter_immortal_decrefs; + dest->allocations += src->allocations; + dest->allocations512 += src->allocations512; + dest->allocations4k += src->allocations4k; + dest->allocations_big += src->allocations_big; + dest->frees += src->frees; + dest->to_freelist += src->to_freelist; + dest->from_freelist += src->from_freelist; + dest->inline_values += src->inline_values; + dest->dict_materialized_on_request += src->dict_materialized_on_request; + dest->dict_materialized_new_key += src->dict_materialized_new_key; + dest->dict_materialized_too_big += src->dict_materialized_too_big; + dest->dict_materialized_str_subclass += src->dict_materialized_str_subclass; + dest->type_cache_hits += src->type_cache_hits; + dest->type_cache_misses += src->type_cache_misses; + dest->type_cache_dunder_hits += src->type_cache_dunder_hits; + dest->type_cache_dunder_misses += src->type_cache_dunder_misses; + dest->type_cache_collisions += src->type_cache_collisions; + dest->object_visits += src->object_visits; +} + +static void +merge_uop_stats_array(UOpStats *dest, const UOpStats *src) +{ + for (int i = 0; i <= PYSTATS_MAX_UOP_ID; i++) { + dest[i].execution_count += src[i].execution_count; + dest[i].miss += src[i].miss; + for (int j = 0; j <= PYSTATS_MAX_UOP_ID; j++) { + dest[i].pair_count[j] += src[i].pair_count[j]; + } + } +} + +static void +merge_optimization_stats(OptimizationStats *dest, const OptimizationStats *src) +{ + dest->attempts += src->attempts; + dest->traces_created += src->traces_created; + dest->traces_executed += src->traces_executed; + dest->uops_executed += src->uops_executed; + dest->trace_stack_overflow += src->trace_stack_overflow; + dest->trace_stack_underflow += src->trace_stack_underflow; + dest->trace_too_long += src->trace_too_long; + dest->trace_too_short += src->trace_too_short; + dest->inner_loop += src->inner_loop; + dest->recursive_call += src->recursive_call; + dest->low_confidence += src->low_confidence; + dest->unknown_callee += src->unknown_callee; + dest->executors_invalidated += src->executors_invalidated; + dest->optimizer_attempts += src->optimizer_attempts; + dest->optimizer_successes += src->optimizer_successes; + dest->optimizer_failure_reason_no_memory += src->optimizer_failure_reason_no_memory; + dest->remove_globals_builtins_changed += src->remove_globals_builtins_changed; + dest->remove_globals_incorrect_keys += src->remove_globals_incorrect_keys; + dest->jit_total_memory_size += src->jit_total_memory_size; + dest->jit_code_size += src->jit_code_size; + dest->jit_trampoline_size += src->jit_trampoline_size; + dest->jit_data_size += src->jit_data_size; + dest->jit_padding_size += src->jit_padding_size; + dest->jit_freed_memory_size += src->jit_freed_memory_size; + + merge_uop_stats_array(dest->opcode, src->opcode); + + for (int i = 0; i < 256; i++) { + dest->unsupported_opcode[i] += src->unsupported_opcode[i]; + } + for (int i = 0; i < _Py_UOP_HIST_SIZE; i++) { + dest->trace_length_hist[i] += src->trace_length_hist[i]; + dest->trace_run_length_hist[i] += src->trace_run_length_hist[i]; + dest->optimized_trace_length_hist[i] += src->optimized_trace_length_hist[i]; + dest->trace_total_memory_hist[i] += src->trace_total_memory_hist[i]; + } + for (int i = 0; i <= PYSTATS_MAX_UOP_ID; i++) { + dest->error_in_opcode[i] += src->error_in_opcode[i]; + } +} + +static void +merge_ft_stats(FTStats *dest, const FTStats *src) +{ + dest->mutex_sleeps = src->mutex_sleeps; + dest->qsbr_polls = src->qsbr_polls; + dest->world_stops = src->world_stops; +} + +static void +merge_rare_event_stats(RareEventStats *dest, const RareEventStats *src) +{ + dest->set_class += src->set_class; + dest->set_bases += src->set_bases; + dest->set_eval_frame_func += src->set_eval_frame_func; + dest->builtin_dict += src->builtin_dict; + dest->func_modification += src->func_modification; + dest->watched_dict_modification += src->watched_dict_modification; + dest->watched_globals_modification += src->watched_globals_modification; +} + +static void +merge_gc_stats_array(GCStats *dest, const GCStats *src) +{ + for (int i = 0; i < NUM_GENERATIONS; i++) { + dest[i].collections += src[i].collections; + dest[i].object_visits += src[i].object_visits; + dest[i].objects_collected += src[i].objects_collected; + dest[i].objects_transitively_reachable += src[i].objects_transitively_reachable; + dest[i].objects_not_transitively_reachable += src[i].objects_not_transitively_reachable; + } +} + +void +stats_zero_thread(_PyThreadStateImpl *tstate) +{ + // Zero the thread local stat counters + if (tstate->pystats_struct) { + memset(tstate->pystats_struct, 0, sizeof(PyStats)); + } +} + +// merge stats for a single thread into the global structure +void +stats_merge_thread(_PyThreadStateImpl *tstate) +{ + PyStats *src = tstate->pystats_struct; + PyStats *dest = ((PyThreadState *)tstate)->interp->pystats_struct; + + if (src == NULL || dest == NULL) { + return; + } + + // Merge each category of stats using the helper functions. + merge_opcode_stats_array(dest->opcode_stats, src->opcode_stats); + merge_call_stats(&dest->call_stats, &src->call_stats); + merge_object_stats(&dest->object_stats, &src->object_stats); + merge_optimization_stats(&dest->optimization_stats, &src->optimization_stats); + merge_ft_stats(&dest->ft_stats, &src->ft_stats); + merge_rare_event_stats(&dest->rare_event_stats, &src->rare_event_stats); + merge_gc_stats_array(dest->gc_stats, src->gc_stats); +} +#endif // Py_GIL_DISABLED + +// toggle stats collection on or off for all threads +static int +stats_toggle_on_off(PyThreadState *tstate, int on) +{ + bool changed = false; + PyInterpreterState *interp = tstate->interp; + STATS_LOCK(interp); + if (on && interp->pystats_struct == NULL) { + PyStats *s = PyMem_RawCalloc(1, sizeof(PyStats)); + if (s == NULL) { + STATS_UNLOCK(interp); + return -1; + } + FT_ATOMIC_STORE_PTR_RELAXED(interp->pystats_struct, s); + } + if (tstate->interp->pystats_enabled != on) { + FT_ATOMIC_STORE_INT_RELAXED(interp->pystats_enabled, on); + changed = true; + } + STATS_UNLOCK(interp); + if (!changed) { + return 0; + } + _PyEval_StopTheWorld(interp); + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, ts) { + PyStats *s = NULL; + if (interp->pystats_enabled) { +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *ts_impl = (_PyThreadStateImpl *)ts; + if (ts_impl->pystats_struct == NULL) { + // first activation for this thread, allocate structure + ts_impl->pystats_struct = PyMem_RawCalloc(1, sizeof(PyStats)); + } + s = ts_impl->pystats_struct; +#else + s = ts->interp->pystats_struct; +#endif + } + ts->pystats = s; + } + _PyEval_StartTheWorld(interp); + return 0; +} + +// zero stats for all threads and for the interpreter +static void +stats_zero_all(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate == NULL) { + return; + } + if (FT_ATOMIC_LOAD_PTR_RELAXED(tstate->interp->pystats_struct) == NULL) { + return; + } + PyInterpreterState *interp = tstate->interp; + _PyEval_StopTheWorld(interp); +#ifdef Py_GIL_DISABLED + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, ts) { + stats_zero_thread((_PyThreadStateImpl *)ts); + } +#endif + if (interp->pystats_struct) { + memset(interp->pystats_struct, 0, sizeof(PyStats)); + } + _PyEval_StartTheWorld(interp); +} + +// merge stats for all threads into the per-interpreter structure +static void +stats_merge_all(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate == NULL) { + return; + } + if (FT_ATOMIC_LOAD_PTR_RELAXED(tstate->interp->pystats_struct) == NULL) { + return; + } + PyInterpreterState *interp = tstate->interp; + _PyEval_StopTheWorld(interp); +#ifdef Py_GIL_DISABLED + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, ts) { + stats_merge_thread((_PyThreadStateImpl *)ts); + stats_zero_thread((_PyThreadStateImpl *)ts); + } +#endif + _PyEval_StartTheWorld(interp); +} + +int +_Py_StatsOn(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + return stats_toggle_on_off(tstate, 1); +} + +void +_Py_StatsOff(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + stats_toggle_on_off(tstate, 0); +} + +void +_Py_StatsClear(void) +{ + stats_zero_all(); +} + +static int +mem_is_zero(unsigned char *ptr, size_t size) +{ + for (size_t i=0; i < size; i++) { + if (*ptr != 0) { + return 0; + } + ptr++; + } + return 1; +} + +int +_Py_PrintSpecializationStats(int to_file) +{ + assert(to_file); + stats_merge_all(); + PyThreadState *tstate = _PyThreadState_GET(); + STATS_LOCK(tstate->interp); + PyStats *stats = tstate->interp->pystats_struct; + if (stats == NULL) { + STATS_UNLOCK(tstate->interp); + return 0; + } +#define MEM_IS_ZERO(DATA) mem_is_zero((unsigned char*)DATA, sizeof(*(DATA))) + int is_zero = ( + MEM_IS_ZERO(stats->gc_stats) // is a pointer + && MEM_IS_ZERO(&stats->opcode_stats) + && MEM_IS_ZERO(&stats->call_stats) + && MEM_IS_ZERO(&stats->object_stats) + ); +#undef MEM_IS_ZERO + STATS_UNLOCK(tstate->interp); + if (is_zero) { + // gh-108753: -X pystats command line was used, but then _stats_off() + // and _stats_clear() have been called: in this case, avoid printing + // useless "all zeros" statistics. + return 0; + } + + FILE *out = stderr; + if (to_file) { + /* Write to a file instead of stderr. */ +# ifdef MS_WINDOWS + const char *dirname = "c:\\temp\\py_stats\\"; +# else + const char *dirname = "/tmp/py_stats/"; +# endif + /* Use random 160 bit number as file name, + * to avoid both accidental collisions and + * symlink attacks. */ + unsigned char rand[20]; + char hex_name[41]; + _PyOS_URandomNonblock(rand, 20); + for (int i = 0; i < 20; i++) { + hex_name[2*i] = Py_hexdigits[rand[i]&15]; + hex_name[2*i+1] = Py_hexdigits[(rand[i]>>4)&15]; + } + hex_name[40] = '\0'; + char buf[64]; + assert(strlen(dirname) + 40 + strlen(".txt") < 64); + sprintf(buf, "%s%s.txt", dirname, hex_name); + FILE *fout = fopen(buf, "w"); + if (fout) { + out = fout; + } + } + else { + fprintf(out, "Specialization stats:\n"); + } + STATS_LOCK(tstate->interp); + print_stats(out, stats); + STATS_UNLOCK(tstate->interp); + if (out != stderr) { + fclose(out); + } + return 1; +} + +PyStatus +_PyStats_InterpInit(PyInterpreterState *interp) +{ + if (interp->config._pystats) { + // start with pystats enabled, can be disabled via sys._stats_off() + // this needs to be set before the first tstate is created + interp->pystats_enabled = 1; + interp->pystats_struct = PyMem_RawCalloc(1, sizeof(PyStats)); + if (interp->pystats_struct == NULL) { + return _PyStatus_ERR("out-of-memory while initializing interpreter"); + } + } + return _PyStatus_OK(); +} + +bool +_PyStats_ThreadInit(PyInterpreterState *interp, _PyThreadStateImpl *tstate) +{ +#ifdef Py_GIL_DISABLED + if (FT_ATOMIC_LOAD_INT_RELAXED(interp->pystats_enabled)) { + assert(interp->pystats_struct != NULL); + tstate->pystats_struct = PyMem_RawCalloc(1, sizeof(PyStats)); + if (tstate->pystats_struct == NULL) { + return false; + } + } +#endif + return true; +} + +void +_PyStats_ThreadFini(_PyThreadStateImpl *tstate) +{ +#ifdef Py_GIL_DISABLED + STATS_LOCK(((PyThreadState *)tstate)->interp); + stats_merge_thread(tstate); + STATS_UNLOCK(((PyThreadState *)tstate)->interp); + PyMem_RawFree(tstate->pystats_struct); +#endif +} + +void +_PyStats_Attach(_PyThreadStateImpl *tstate_impl) +{ + PyStats *s; + PyThreadState *tstate = (PyThreadState *)tstate_impl; + PyInterpreterState *interp = tstate->interp; + if (FT_ATOMIC_LOAD_INT_RELAXED(interp->pystats_enabled)) { +#ifdef Py_GIL_DISABLED + s = ((_PyThreadStateImpl *)tstate)->pystats_struct; +#else + s = tstate->interp->pystats_struct; +#endif + } + else { + s = NULL; + } + tstate->pystats = s; +} + +void +_PyStats_Detach(_PyThreadStateImpl *tstate_impl) +{ + ((PyThreadState *)tstate_impl)->pystats = NULL; +} + +#endif // Py_STATS diff --git a/Python/qsbr.c b/Python/qsbr.c index c992c285cb1..b2153bf9d67 100644 --- a/Python/qsbr.c +++ b/Python/qsbr.c @@ -36,6 +36,7 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_qsbr.h" #include "pycore_tstate.h" // _PyThreadStateImpl +#include "pycore_stats.h" // FT_STAT_QSBR_POLL_INC() // Starting size of the array of qsbr thread states @@ -158,7 +159,7 @@ _Py_qsbr_poll(struct _qsbr_thread_state *qsbr, uint64_t goal) if (_Py_qbsr_goal_reached(qsbr, goal)) { return true; } - + FT_STAT_QSBR_POLL_INC(); uint64_t rd_seq = qsbr_poll_scan(qsbr->shared); return QSBR_LEQ(goal, rd_seq); } diff --git a/Python/specialize.c b/Python/specialize.c index a1c5dedd615..2193596a331 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -22,437 +22,23 @@ #include // rand() -extern const char *_PyUOpName(int index); - /* For guidance on adding or extending families of instructions see * InternalDocs/interpreter.md `Specialization` section. */ -#ifdef Py_STATS -GCStats _py_gc_stats[NUM_GENERATIONS] = { 0 }; -static PyStats _Py_stats_struct = { .gc_stats = _py_gc_stats }; -PyStats *_Py_stats = NULL; - -#if PYSTATS_MAX_UOP_ID < MAX_UOP_ID -#error "Not enough space allocated for pystats. Increase PYSTATS_MAX_UOP_ID to at least MAX_UOP_ID" -#endif - -#define ADD_STAT_TO_DICT(res, field) \ - do { \ - PyObject *val = PyLong_FromUnsignedLongLong(stats->field); \ - if (val == NULL) { \ - Py_DECREF(res); \ - return NULL; \ - } \ - if (PyDict_SetItemString(res, #field, val) == -1) { \ - Py_DECREF(res); \ - Py_DECREF(val); \ - return NULL; \ - } \ - Py_DECREF(val); \ - } while(0); - -static PyObject* -stats_to_dict(SpecializationStats *stats) -{ - PyObject *res = PyDict_New(); - if (res == NULL) { - return NULL; - } - ADD_STAT_TO_DICT(res, success); - ADD_STAT_TO_DICT(res, failure); - ADD_STAT_TO_DICT(res, hit); - ADD_STAT_TO_DICT(res, deferred); - ADD_STAT_TO_DICT(res, miss); - ADD_STAT_TO_DICT(res, deopt); - PyObject *failure_kinds = PyTuple_New(SPECIALIZATION_FAILURE_KINDS); - if (failure_kinds == NULL) { - Py_DECREF(res); - return NULL; - } - for (int i = 0; i < SPECIALIZATION_FAILURE_KINDS; i++) { - PyObject *stat = PyLong_FromUnsignedLongLong(stats->failure_kinds[i]); - if (stat == NULL) { - Py_DECREF(res); - Py_DECREF(failure_kinds); - return NULL; - } - PyTuple_SET_ITEM(failure_kinds, i, stat); - } - if (PyDict_SetItemString(res, "failure_kinds", failure_kinds)) { - Py_DECREF(res); - Py_DECREF(failure_kinds); - return NULL; - } - Py_DECREF(failure_kinds); - return res; -} -#undef ADD_STAT_TO_DICT - -static int -add_stat_dict( - PyObject *res, - int opcode, - const char *name) { - - SpecializationStats *stats = &_Py_stats_struct.opcode_stats[opcode].specialization; - PyObject *d = stats_to_dict(stats); - if (d == NULL) { - return -1; - } - int err = PyDict_SetItemString(res, name, d); - Py_DECREF(d); - return err; -} - -PyObject* -_Py_GetSpecializationStats(void) { - PyObject *stats = PyDict_New(); - if (stats == NULL) { - return NULL; - } - int err = 0; - err += add_stat_dict(stats, CONTAINS_OP, "contains_op"); - err += add_stat_dict(stats, LOAD_SUPER_ATTR, "load_super_attr"); - err += add_stat_dict(stats, LOAD_ATTR, "load_attr"); - err += add_stat_dict(stats, LOAD_GLOBAL, "load_global"); - err += add_stat_dict(stats, STORE_SUBSCR, "store_subscr"); - err += add_stat_dict(stats, STORE_ATTR, "store_attr"); - err += add_stat_dict(stats, JUMP_BACKWARD, "jump_backward"); - err += add_stat_dict(stats, CALL, "call"); - err += add_stat_dict(stats, CALL_KW, "call_kw"); - err += add_stat_dict(stats, BINARY_OP, "binary_op"); - err += add_stat_dict(stats, COMPARE_OP, "compare_op"); - err += add_stat_dict(stats, UNPACK_SEQUENCE, "unpack_sequence"); - err += add_stat_dict(stats, FOR_ITER, "for_iter"); - err += add_stat_dict(stats, TO_BOOL, "to_bool"); - err += add_stat_dict(stats, SEND, "send"); - if (err < 0) { - Py_DECREF(stats); - return NULL; - } - return stats; -} - - -#define PRINT_STAT(i, field) \ - if (stats[i].field) { \ - fprintf(out, " opcode[%s]." #field " : %" PRIu64 "\n", _PyOpcode_OpName[i], stats[i].field); \ - } - -static void -print_spec_stats(FILE *out, OpcodeStats *stats) -{ - /* Mark some opcodes as specializable for stats, - * even though we don't specialize them yet. */ - fprintf(out, "opcode[BINARY_SLICE].specializable : 1\n"); - fprintf(out, "opcode[STORE_SLICE].specializable : 1\n"); - fprintf(out, "opcode[GET_ITER].specializable : 1\n"); - for (int i = 0; i < 256; i++) { - if (_PyOpcode_Caches[i]) { - /* Ignore jumps as they cannot be specialized */ - switch (i) { - case POP_JUMP_IF_FALSE: - case POP_JUMP_IF_TRUE: - case POP_JUMP_IF_NONE: - case POP_JUMP_IF_NOT_NONE: - case JUMP_BACKWARD: - break; - default: - fprintf(out, "opcode[%s].specializable : 1\n", _PyOpcode_OpName[i]); - } - } - PRINT_STAT(i, specialization.success); - PRINT_STAT(i, specialization.failure); - PRINT_STAT(i, specialization.hit); - PRINT_STAT(i, specialization.deferred); - PRINT_STAT(i, specialization.miss); - PRINT_STAT(i, specialization.deopt); - PRINT_STAT(i, execution_count); - for (int j = 0; j < SPECIALIZATION_FAILURE_KINDS; j++) { - uint64_t val = stats[i].specialization.failure_kinds[j]; - if (val) { - fprintf(out, " opcode[%s].specialization.failure_kinds[%d] : %" - PRIu64 "\n", _PyOpcode_OpName[i], j, val); - } - } - for (int j = 0; j < 256; j++) { - if (stats[i].pair_count[j]) { - fprintf(out, "opcode[%s].pair_count[%s] : %" PRIu64 "\n", - _PyOpcode_OpName[i], _PyOpcode_OpName[j], stats[i].pair_count[j]); - } - } - } -} -#undef PRINT_STAT - - -static void -print_call_stats(FILE *out, CallStats *stats) -{ - fprintf(out, "Calls to PyEval_EvalDefault: %" PRIu64 "\n", stats->pyeval_calls); - fprintf(out, "Calls to Python functions inlined: %" PRIu64 "\n", stats->inlined_py_calls); - fprintf(out, "Frames pushed: %" PRIu64 "\n", stats->frames_pushed); - fprintf(out, "Frame objects created: %" PRIu64 "\n", stats->frame_objects_created); - for (int i = 0; i < EVAL_CALL_KINDS; i++) { - fprintf(out, "Calls via PyEval_EvalFrame[%d] : %" PRIu64 "\n", i, stats->eval_calls[i]); - } -} - -static void -print_object_stats(FILE *out, ObjectStats *stats) -{ - fprintf(out, "Object allocations from freelist: %" PRIu64 "\n", stats->from_freelist); - fprintf(out, "Object frees to freelist: %" PRIu64 "\n", stats->to_freelist); - fprintf(out, "Object allocations: %" PRIu64 "\n", stats->allocations); - fprintf(out, "Object allocations to 512 bytes: %" PRIu64 "\n", stats->allocations512); - fprintf(out, "Object allocations to 4 kbytes: %" PRIu64 "\n", stats->allocations4k); - fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big); - fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees); - fprintf(out, "Object inline values: %" PRIu64 "\n", stats->inline_values); - fprintf(out, "Object interpreter mortal increfs: %" PRIu64 "\n", stats->interpreter_increfs); - fprintf(out, "Object interpreter mortal decrefs: %" PRIu64 "\n", stats->interpreter_decrefs); - fprintf(out, "Object mortal increfs: %" PRIu64 "\n", stats->increfs); - fprintf(out, "Object mortal decrefs: %" PRIu64 "\n", stats->decrefs); - fprintf(out, "Object interpreter immortal increfs: %" PRIu64 "\n", stats->interpreter_immortal_increfs); - fprintf(out, "Object interpreter immortal decrefs: %" PRIu64 "\n", stats->interpreter_immortal_decrefs); - fprintf(out, "Object immortal increfs: %" PRIu64 "\n", stats->immortal_increfs); - fprintf(out, "Object immortal decrefs: %" PRIu64 "\n", stats->immortal_decrefs); - fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request); - fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key); - fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big); - fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass); - fprintf(out, "Object method cache hits: %" PRIu64 "\n", stats->type_cache_hits); - fprintf(out, "Object method cache misses: %" PRIu64 "\n", stats->type_cache_misses); - fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions); - fprintf(out, "Object method cache dunder hits: %" PRIu64 "\n", stats->type_cache_dunder_hits); - fprintf(out, "Object method cache dunder misses: %" PRIu64 "\n", stats->type_cache_dunder_misses); -} - -static void -print_gc_stats(FILE *out, GCStats *stats) -{ - for (int i = 0; i < NUM_GENERATIONS; i++) { - fprintf(out, "GC[%d] collections: %" PRIu64 "\n", i, stats[i].collections); - fprintf(out, "GC[%d] object visits: %" PRIu64 "\n", i, stats[i].object_visits); - fprintf(out, "GC[%d] objects collected: %" PRIu64 "\n", i, stats[i].objects_collected); - fprintf(out, "GC[%d] objects reachable from roots: %" PRIu64 "\n", i, stats[i].objects_transitively_reachable); - fprintf(out, "GC[%d] objects not reachable from roots: %" PRIu64 "\n", i, stats[i].objects_not_transitively_reachable); - } -} - -#ifdef _Py_TIER2 -static void -print_histogram(FILE *out, const char *name, uint64_t hist[_Py_UOP_HIST_SIZE]) -{ - for (int i = 0; i < _Py_UOP_HIST_SIZE; i++) { - fprintf(out, "%s[%" PRIu64"]: %" PRIu64 "\n", name, (uint64_t)1 << i, hist[i]); - } -} - -static void -print_optimization_stats(FILE *out, OptimizationStats *stats) -{ - fprintf(out, "Optimization attempts: %" PRIu64 "\n", stats->attempts); - fprintf(out, "Optimization traces created: %" PRIu64 "\n", stats->traces_created); - fprintf(out, "Optimization traces executed: %" PRIu64 "\n", stats->traces_executed); - fprintf(out, "Optimization uops executed: %" PRIu64 "\n", stats->uops_executed); - fprintf(out, "Optimization trace stack overflow: %" PRIu64 "\n", stats->trace_stack_overflow); - fprintf(out, "Optimization trace stack underflow: %" PRIu64 "\n", stats->trace_stack_underflow); - fprintf(out, "Optimization trace too long: %" PRIu64 "\n", stats->trace_too_long); - fprintf(out, "Optimization trace too short: %" PRIu64 "\n", stats->trace_too_short); - fprintf(out, "Optimization inner loop: %" PRIu64 "\n", stats->inner_loop); - fprintf(out, "Optimization recursive call: %" PRIu64 "\n", stats->recursive_call); - fprintf(out, "Optimization low confidence: %" PRIu64 "\n", stats->low_confidence); - fprintf(out, "Optimization unknown callee: %" PRIu64 "\n", stats->unknown_callee); - fprintf(out, "Executors invalidated: %" PRIu64 "\n", stats->executors_invalidated); - - print_histogram(out, "Trace length", stats->trace_length_hist); - print_histogram(out, "Trace run length", stats->trace_run_length_hist); - print_histogram(out, "Optimized trace length", stats->optimized_trace_length_hist); - - fprintf(out, "Optimization optimizer attempts: %" PRIu64 "\n", stats->optimizer_attempts); - fprintf(out, "Optimization optimizer successes: %" PRIu64 "\n", stats->optimizer_successes); - fprintf(out, "Optimization optimizer failure no memory: %" PRIu64 "\n", - stats->optimizer_failure_reason_no_memory); - fprintf(out, "Optimizer remove globals builtins changed: %" PRIu64 "\n", stats->remove_globals_builtins_changed); - fprintf(out, "Optimizer remove globals incorrect keys: %" PRIu64 "\n", stats->remove_globals_incorrect_keys); - for (int i = 0; i <= MAX_UOP_ID; i++) { - if (stats->opcode[i].execution_count) { - fprintf(out, "uops[%s].execution_count : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].execution_count); - } - if (stats->opcode[i].miss) { - fprintf(out, "uops[%s].specialization.miss : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].miss); - } - } - for (int i = 0; i < 256; i++) { - if (stats->unsupported_opcode[i]) { - fprintf( - out, - "unsupported_opcode[%s].count : %" PRIu64 "\n", - _PyOpcode_OpName[i], - stats->unsupported_opcode[i] - ); - } - } - - for (int i = 1; i <= MAX_UOP_ID; i++){ - for (int j = 1; j <= MAX_UOP_ID; j++) { - if (stats->opcode[i].pair_count[j]) { - fprintf(out, "uop[%s].pair_count[%s] : %" PRIu64 "\n", - _PyOpcode_uop_name[i], _PyOpcode_uop_name[j], stats->opcode[i].pair_count[j]); - } - } - } - for (int i = 0; i < MAX_UOP_ID; i++) { - if (stats->error_in_opcode[i]) { - fprintf( - out, - "error_in_opcode[%s].count : %" PRIu64 "\n", - _PyUOpName(i), - stats->error_in_opcode[i] - ); - } - } - fprintf(out, "JIT total memory size: %" PRIu64 "\n", stats->jit_total_memory_size); - fprintf(out, "JIT code size: %" PRIu64 "\n", stats->jit_code_size); - fprintf(out, "JIT trampoline size: %" PRIu64 "\n", stats->jit_trampoline_size); - fprintf(out, "JIT data size: %" PRIu64 "\n", stats->jit_data_size); - fprintf(out, "JIT padding size: %" PRIu64 "\n", stats->jit_padding_size); - fprintf(out, "JIT freed memory size: %" PRIu64 "\n", stats->jit_freed_memory_size); - - print_histogram(out, "Trace total memory size", stats->trace_total_memory_hist); -} -#endif - -static void -print_rare_event_stats(FILE *out, RareEventStats *stats) -{ - fprintf(out, "Rare event (set_class): %" PRIu64 "\n", stats->set_class); - fprintf(out, "Rare event (set_bases): %" PRIu64 "\n", stats->set_bases); - fprintf(out, "Rare event (set_eval_frame_func): %" PRIu64 "\n", stats->set_eval_frame_func); - fprintf(out, "Rare event (builtin_dict): %" PRIu64 "\n", stats->builtin_dict); - fprintf(out, "Rare event (func_modification): %" PRIu64 "\n", stats->func_modification); - fprintf(out, "Rare event (watched_dict_modification): %" PRIu64 "\n", stats->watched_dict_modification); - fprintf(out, "Rare event (watched_globals_modification): %" PRIu64 "\n", stats->watched_globals_modification); -} - -static void -print_stats(FILE *out, PyStats *stats) -{ - print_spec_stats(out, stats->opcode_stats); - print_call_stats(out, &stats->call_stats); - print_object_stats(out, &stats->object_stats); - print_gc_stats(out, stats->gc_stats); -#ifdef _Py_TIER2 - print_optimization_stats(out, &stats->optimization_stats); -#endif - print_rare_event_stats(out, &stats->rare_event_stats); -} - -void -_Py_StatsOn(void) -{ - _Py_stats = &_Py_stats_struct; -} - -void -_Py_StatsOff(void) -{ - _Py_stats = NULL; -} - -void -_Py_StatsClear(void) -{ - memset(&_py_gc_stats, 0, sizeof(_py_gc_stats)); - memset(&_Py_stats_struct, 0, sizeof(_Py_stats_struct)); - _Py_stats_struct.gc_stats = _py_gc_stats; -} - -static int -mem_is_zero(unsigned char *ptr, size_t size) -{ - for (size_t i=0; i < size; i++) { - if (*ptr != 0) { - return 0; - } - ptr++; - } - return 1; -} - -int -_Py_PrintSpecializationStats(int to_file) -{ - PyStats *stats = &_Py_stats_struct; -#define MEM_IS_ZERO(DATA) mem_is_zero((unsigned char*)DATA, sizeof(*(DATA))) - int is_zero = ( - MEM_IS_ZERO(stats->gc_stats) // is a pointer - && MEM_IS_ZERO(&stats->opcode_stats) - && MEM_IS_ZERO(&stats->call_stats) - && MEM_IS_ZERO(&stats->object_stats) - ); -#undef MEM_IS_ZERO - if (is_zero) { - // gh-108753: -X pystats command line was used, but then _stats_off() - // and _stats_clear() have been called: in this case, avoid printing - // useless "all zeros" statistics. - return 0; - } - - FILE *out = stderr; - if (to_file) { - /* Write to a file instead of stderr. */ -# ifdef MS_WINDOWS - const char *dirname = "c:\\temp\\py_stats\\"; -# else - const char *dirname = "/tmp/py_stats/"; -# endif - /* Use random 160 bit number as file name, - * to avoid both accidental collisions and - * symlink attacks. */ - unsigned char rand[20]; - char hex_name[41]; - _PyOS_URandomNonblock(rand, 20); - for (int i = 0; i < 20; i++) { - hex_name[2*i] = Py_hexdigits[rand[i]&15]; - hex_name[2*i+1] = Py_hexdigits[(rand[i]>>4)&15]; - } - hex_name[40] = '\0'; - char buf[64]; - assert(strlen(dirname) + 40 + strlen(".txt") < 64); - sprintf(buf, "%s%s.txt", dirname, hex_name); - FILE *fout = fopen(buf, "w"); - if (fout) { - out = fout; - } - } - else { - fprintf(out, "Specialization stats:\n"); - } - print_stats(out, stats); - if (out != stderr) { - fclose(out); - } - return 1; -} - +#if Py_STATS #define SPECIALIZATION_FAIL(opcode, kind) \ do { \ - if (_Py_stats) { \ + PyStats *s = _PyStats_GET(); \ + if (s) { \ int _kind = (kind); \ assert(_kind < SPECIALIZATION_FAILURE_KINDS); \ - _Py_stats->opcode_stats[opcode].specialization.failure_kinds[_kind]++; \ + s->opcode_stats[opcode].specialization.failure_kinds[_kind]++; \ } \ } while (0) - -#endif // Py_STATS - - -#ifndef SPECIALIZATION_FAIL +#else # define SPECIALIZATION_FAIL(opcode, kind) ((void)0) -#endif +#endif // Py_STATS // Initialize warmup counters and optimize instructions. This cannot fail. void diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 59baca26793..86dd1395cae 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2281,7 +2281,9 @@ static PyObject * sys__stats_on_impl(PyObject *module) /*[clinic end generated code: output=aca53eafcbb4d9fe input=43b5bfe145299e55]*/ { - _Py_StatsOn(); + if (_Py_StatsOn() < 0) { + return NULL; + } Py_RETURN_NONE; } From 57f4d09a6fcfec229603f644c4254abdded00429 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 3 Nov 2025 11:46:53 -0800 Subject: [PATCH 017/313] JIT: Fix compiler warning from visibility attribute in typedef (#139981) --- Tools/jit/trampoline.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/jit/trampoline.c b/Tools/jit/trampoline.c index 79d6af97961..fdc63b7fc40 100644 --- a/Tools/jit/trampoline.c +++ b/Tools/jit/trampoline.c @@ -10,7 +10,7 @@ _Py_CODEUNIT * _JIT_ENTRY( _PyExecutorObject *exec, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate ) { - typedef DECLARE_TARGET((*jit_func)); - jit_func jitted = (jit_func)exec->jit_code; + // Note that this is *not* a tail call + jit_func_preserve_none jitted = (jit_func_preserve_none)exec->jit_code; return jitted(frame, stack_pointer, tstate); } From 3f2b83e9595ff2436a646e6bbd335198c4bc06db Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 3 Nov 2025 21:04:46 +0100 Subject: [PATCH 018/313] Fix minor typos and wording in C API docs (#140955) --- Doc/c-api/cell.rst | 2 +- Doc/c-api/complex.rst | 2 +- Doc/c-api/datetime.rst | 2 +- Doc/c-api/descriptor.rst | 2 +- Doc/c-api/init_config.rst | 2 +- Doc/c-api/mapping.rst | 4 ++-- Doc/c-api/marshal.rst | 2 +- Doc/c-api/memory.rst | 2 +- Doc/c-api/module.rst | 2 +- Doc/c-api/monitoring.rst | 2 +- Doc/c-api/object.rst | 2 +- Doc/c-api/stable.rst | 2 +- Doc/c-api/tuple.rst | 2 +- Doc/c-api/veryhigh.rst | 2 +- 14 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst index 61eb994c370..2501ed9580d 100644 --- a/Doc/c-api/cell.rst +++ b/Doc/c-api/cell.rst @@ -7,7 +7,7 @@ Cell Objects "Cell" objects are used to implement variables referenced by multiple scopes. For each such variable, a cell object is created to store the value; the local -variables of each stack frame that references the value contains a reference to +variables of each stack frame that references the value contain a reference to the cells from outer scopes which also use that variable. When the value is accessed, the value contained in the cell is used instead of the cell object itself. This de-referencing of the cell object requires support from the diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index d135637a741..629312bd771 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -82,7 +82,7 @@ Complex Number Objects .. c:type:: Py_complex - This C structure defines export format for a Python complex + This C structure defines an export format for a Python complex number object. .. c:member:: double real diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index d2d4d5309c7..f311aad5f15 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -46,7 +46,7 @@ macros. .. c:var:: PyTypeObject PyDateTime_DeltaType - This instance of :c:type:`PyTypeObject` represents Python type for + This instance of :c:type:`PyTypeObject` represents the Python type for the difference between two datetime values; it is the same object as :class:`datetime.timedelta` in the Python layer. diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index b32c113e5f0..ff0df575279 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -32,7 +32,7 @@ found in the dictionary of type objects. .. c:function:: int PyDescr_IsData(PyObject *descr) - Return non-zero if the descriptor objects *descr* describes a data attribute, or + Return non-zero if the descriptor object *descr* describes a data attribute, or ``0`` if it describes a method. *descr* must be a descriptor object; there is no error checking. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index b20495e672d..c345029e4ac 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -102,7 +102,7 @@ Error Handling * Set *\*err_msg* and return ``1`` if an error is set. * Set *\*err_msg* to ``NULL`` and return ``0`` otherwise. - An error message is an UTF-8 encoded string. + An error message is a UTF-8 encoded string. If *config* has an exit code, format the exit code as an error message. diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst index 1f55c0aa955..2476ebb9b69 100644 --- a/Doc/c-api/mapping.rst +++ b/Doc/c-api/mapping.rst @@ -102,7 +102,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. note:: - Exceptions which occur when this calls :meth:`~object.__getitem__` + Exceptions which occur when this calls the :meth:`~object.__getitem__` method are silently ignored. For proper error handling, use :c:func:`PyMapping_HasKeyWithError`, :c:func:`PyMapping_GetOptionalItem` or :c:func:`PyObject_GetItem()` instead. @@ -116,7 +116,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. note:: - Exceptions that occur when this calls :meth:`~object.__getitem__` + Exceptions that occur when this calls the :meth:`~object.__getitem__` method or while creating the temporary :class:`str` object are silently ignored. For proper error handling, use :c:func:`PyMapping_HasKeyStringWithError`, diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index 61218a1bf6f..668a163b2df 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -82,7 +82,7 @@ The following functions allow marshalled values to be read back in. assumes that no further objects will be read from the file, allowing it to aggressively load file data into memory so that the de-serialization can operate from data in memory rather than reading a byte at a time from the - file. Only use these variant if you are certain that you won't be reading + file. Only use this variant if you are certain that you won't be reading anything else from the file. On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index df1bb0ce370..23958980102 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -102,7 +102,7 @@ All allocating functions belong to one of three different "domains" (see also strategies and are optimized for different purposes. The specific details on how every domain allocates memory or what internal functions each domain calls is considered an implementation detail, but for debugging purposes a simplified -table can be found at :ref:`here `. +table can be found at :ref:`default-memory-allocators`. The APIs used to allocate and free a block of memory must be from the same domain. For example, :c:func:`PyMem_Free` must be used to free memory allocated using :c:func:`PyMem_Malloc`. diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 6626f628fcc..ed2a7663375 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -103,7 +103,7 @@ Module Objects created, or ``NULL`` if the module wasn't created from a definition. On error, return ``NULL`` with an exception set. - Use :c:func:`PyErr_Occurred` to tell this case apart from a mising + Use :c:func:`PyErr_Occurred` to tell this case apart from a missing :c:type:`!PyModuleDef`. diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst index 7926148302a..b0227c2f4fa 100644 --- a/Doc/c-api/monitoring.rst +++ b/Doc/c-api/monitoring.rst @@ -136,7 +136,7 @@ Managing the Monitoring State ----------------------------- Monitoring states can be managed with the help of monitoring scopes. A scope -would typically correspond to a python function. +would typically correspond to a Python function. .. c:function:: int PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, const uint8_t *event_types, Py_ssize_t length) diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 8629b768a29..96353266ac7 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -73,7 +73,7 @@ Object Protocol Flag to be used with multiple functions that print the object (like :c:func:`PyObject_Print` and :c:func:`PyFile_WriteObject`). - If passed, these function would use the :func:`str` of the object + If passed, these functions use the :func:`str` of the object instead of the :func:`repr`. diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index b08a7bf1b2f..f5e6b7ad157 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -279,7 +279,7 @@ The full API is described below for advanced use cases. .. c:member:: uint8_t abiinfo_minor_version - The major version of :c:struct:`PyABIInfo`. + The minor version of :c:struct:`PyABIInfo`. Must be set to ``0``; larger values are reserved for backwards-compatible future versions of :c:struct:`!PyABIInfo`. diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index 14a7c05efea..d0add48d7e8 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -61,7 +61,7 @@ Tuple Objects .. c:function:: Py_ssize_t PyTuple_Size(PyObject *p) Take a pointer to a tuple object, and return the size of that tuple. - On error, return ``-1`` and with an exception set. + On error, return ``-1`` with an exception set. .. c:function:: Py_ssize_t PyTuple_GET_SIZE(PyObject *p) diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index ee0595a9e08..0b2b55b6387 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -140,7 +140,7 @@ the same library that the Python runtime is using. interpreter prompt is about to become idle and wait for user input from the terminal. The return value is ignored. Overriding this hook can be used to integrate the interpreter's prompt with other - event loops, as done in the :file:`Modules/_tkinter.c` in the + event loops, as done in :file:`Modules/_tkinter.c` in the Python source code. .. versionchanged:: 3.12 From 947bb4642c612b66de7cae815aaf9c570a93882a Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Tue, 4 Nov 2025 04:37:29 +0800 Subject: [PATCH 019/313] gh-140889: Bump tailcall and JIT CI to llvm 20 (#140963) --- .github/workflows/jit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 40d8b74e982..69d900091a3 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -193,7 +193,7 @@ jobs: fail-fast: false matrix: llvm: - - 19 + - 20 steps: - uses: actions/checkout@v4 with: From 08115d241a724a4769599993f654f77abcdebf5a Mon Sep 17 00:00:00 2001 From: commitWithTisha Date: Tue, 4 Nov 2025 03:23:49 -0600 Subject: [PATCH 020/313] Fix minor typo: 'web site' -> 'website' (GH-140561) --- Doc/library/http.cookiejar.rst | 2 +- Doc/library/urllib.robotparser.rst | 2 +- Doc/tutorial/index.rst | 2 +- Doc/tutorial/whatnow.rst | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index 251aea891c3..fcb0069b760 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -12,7 +12,7 @@ -------------- The :mod:`http.cookiejar` module defines classes for automatic handling of HTTP -cookies. It is useful for accessing web sites that require small pieces of data +cookies. It is useful for accessing websites that require small pieces of data -- :dfn:`cookies` -- to be set on the client machine by an HTTP response from a web server, and then returned to the server in later HTTP requests. diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index 016fcdc75da..674f646c633 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -19,7 +19,7 @@ This module provides a single class, :class:`RobotFileParser`, which answers questions about whether or not a particular user agent can fetch a URL on the -web site that published the :file:`robots.txt` file. For more details on the +website that published the :file:`robots.txt` file. For more details on the structure of :file:`robots.txt` files, see http://www.robotstxt.org/orig.html. diff --git a/Doc/tutorial/index.rst b/Doc/tutorial/index.rst index d0bf77dc40d..20fe161be4a 100644 --- a/Doc/tutorial/index.rst +++ b/Doc/tutorial/index.rst @@ -15,7 +15,7 @@ together with its interpreted nature, make it an ideal language for scripting and rapid application development in many areas on most platforms. The Python interpreter and the extensive standard library are freely available -in source or binary form for all major platforms from the Python web site, +in source or binary form for all major platforms from the Python website, https://www.python.org/, and may be freely distributed. The same site also contains distributions of and pointers to many free third party Python modules, programs and tools, and additional documentation. diff --git a/Doc/tutorial/whatnow.rst b/Doc/tutorial/whatnow.rst index dbe2d7fc099..359cf80a7b2 100644 --- a/Doc/tutorial/whatnow.rst +++ b/Doc/tutorial/whatnow.rst @@ -30,7 +30,7 @@ the set are: More Python resources: -* https://www.python.org: The major Python web site. It contains code, +* https://www.python.org: The major Python website. It contains code, documentation, and pointers to Python-related pages around the web. * https://docs.python.org: Fast access to Python's documentation. From a84181c31bfc45a1d6bcb1296bd298ad612c54d0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 4 Nov 2025 11:48:28 +0100 Subject: [PATCH 021/313] gh-140815: Fix faulthandler for invalid/freed frame (#140921) faulthandler now detects if a frame or a code object is invalid or freed. Add helper functions: * _PyCode_SafeAddr2Line() * _PyFrame_SafeGetCode() * _PyFrame_SafeGetLasti() _PyMem_IsPtrFreed() now detects pointers in [-0xff, 0xff] range as freed. --- Include/internal/pycore_code.h | 9 ++- Include/internal/pycore_interpframe.h | 55 ++++++++++++++++ Include/internal/pycore_pymem.h | 10 +-- ...-11-02-19-23-32.gh-issue-140815.McEG-T.rst | 2 + Objects/codeobject.c | 23 ++++++- Python/traceback.c | 62 ++++++++++++------- 6 files changed, 128 insertions(+), 33 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-02-19-23-32.gh-issue-140815.McEG-T.rst diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 2d7d81d491c..9748e036bf2 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -274,8 +274,13 @@ extern void _PyLineTable_InitAddressRange( /** API for traversing the line number table. */ PyAPI_FUNC(int) _PyLineTable_NextAddressRange(PyCodeAddressRange *range); extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); -// This is used in dump_frame() in traceback.c without an attached tstate. -extern int _PyCode_Addr2LineNoTstate(PyCodeObject *co, int addr); + +// Similar to PyCode_Addr2Line(), but return -1 if the code object is invalid +// and can be called without an attached tstate. Used by dump_frame() in +// Python/traceback.c. The function uses heuristics to detect freed memory, +// it's not 100% reliable. +extern int _PyCode_SafeAddr2Line(PyCodeObject *co, int addr); + /** API for executors */ extern void _PyCode_Clear_Executors(PyCodeObject *code); diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index 2ee3696317c..8949d6cc2fc 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -24,6 +24,36 @@ static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) { return (PyCodeObject *)executable; } +// Similar to _PyFrame_GetCode(), but return NULL if the frame is invalid or +// freed. Used by dump_frame() in Python/traceback.c. The function uses +// heuristics to detect freed memory, it's not 100% reliable. +static inline PyCodeObject* +_PyFrame_SafeGetCode(_PyInterpreterFrame *f) +{ + // globals and builtins may be NULL on a legit frame, but it's unlikely. + // It's more likely that it's a sign of an invalid frame. + if (f->f_globals == NULL || f->f_builtins == NULL) { + return NULL; + } + + if (PyStackRef_IsNull(f->f_executable)) { + return NULL; + } + void *ptr; + memcpy(&ptr, &f->f_executable, sizeof(f->f_executable)); + if (_PyMem_IsPtrFreed(ptr)) { + return NULL; + } + PyObject *executable = PyStackRef_AsPyObjectBorrow(f->f_executable); + if (_PyObject_IsFreed(executable)) { + return NULL; + } + if (!PyCode_Check(executable)) { + return NULL; + } + return (PyCodeObject *)executable; +} + static inline _Py_CODEUNIT * _PyFrame_GetBytecode(_PyInterpreterFrame *f) { @@ -37,6 +67,31 @@ _PyFrame_GetBytecode(_PyInterpreterFrame *f) #endif } +// Similar to PyUnstable_InterpreterFrame_GetLasti(), but return NULL if the +// frame is invalid or freed. Used by dump_frame() in Python/traceback.c. The +// function uses heuristics to detect freed memory, it's not 100% reliable. +static inline int +_PyFrame_SafeGetLasti(struct _PyInterpreterFrame *f) +{ + // Code based on _PyFrame_GetBytecode() but replace _PyFrame_GetCode() + // with _PyFrame_SafeGetCode(). + PyCodeObject *co = _PyFrame_SafeGetCode(f); + if (co == NULL) { + return -1; + } + + _Py_CODEUNIT *bytecode; +#ifdef Py_GIL_DISABLED + _PyCodeArray *tlbc = _PyCode_GetTLBCArray(co); + assert(f->tlbc_index >= 0 && f->tlbc_index < tlbc->size); + bytecode = (_Py_CODEUNIT *)tlbc->entries[f->tlbc_index]; +#else + bytecode = _PyCode_CODE(co); +#endif + + return (int)(f->instr_ptr - bytecode) * sizeof(_Py_CODEUNIT); +} + static inline PyFunctionObject *_PyFrame_GetFunction(_PyInterpreterFrame *f) { PyObject *func = PyStackRef_AsPyObjectBorrow(f->f_funcobj); assert(PyFunction_Check(func)); diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index f3f2ae0a140..ed943b51056 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -54,15 +54,17 @@ static inline int _PyMem_IsPtrFreed(const void *ptr) { uintptr_t value = (uintptr_t)ptr; #if SIZEOF_VOID_P == 8 - return (value == 0 + return (value <= 0xff // NULL, 0x1, 0x2, ..., 0xff || value == (uintptr_t)0xCDCDCDCDCDCDCDCD || value == (uintptr_t)0xDDDDDDDDDDDDDDDD - || value == (uintptr_t)0xFDFDFDFDFDFDFDFD); + || value == (uintptr_t)0xFDFDFDFDFDFDFDFD + || value >= (uintptr_t)0xFFFFFFFFFFFFFF00); // -0xff, ..., -2, -1 #elif SIZEOF_VOID_P == 4 - return (value == 0 + return (value <= 0xff || value == (uintptr_t)0xCDCDCDCD || value == (uintptr_t)0xDDDDDDDD - || value == (uintptr_t)0xFDFDFDFD); + || value == (uintptr_t)0xFDFDFDFD + || value >= (uintptr_t)0xFFFFFF00); #else # error "unknown pointer size" #endif diff --git a/Misc/NEWS.d/next/Library/2025-11-02-19-23-32.gh-issue-140815.McEG-T.rst b/Misc/NEWS.d/next/Library/2025-11-02-19-23-32.gh-issue-140815.McEG-T.rst new file mode 100644 index 00000000000..18c4d3836ef --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-02-19-23-32.gh-issue-140815.McEG-T.rst @@ -0,0 +1,2 @@ +:mod:`faulthandler` now detects if a frame or a code object is invalid or +freed. Patch by Victor Stinner. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 0d264a6e346..fc3f5d9dde0 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1005,8 +1005,8 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) * source location tracking (co_lines/co_positions) ******************/ -int -_PyCode_Addr2LineNoTstate(PyCodeObject *co, int addrq) +static int +_PyCode_Addr2Line(PyCodeObject *co, int addrq) { if (addrq < 0) { return co->co_firstlineno; @@ -1020,12 +1020,29 @@ _PyCode_Addr2LineNoTstate(PyCodeObject *co, int addrq) return _PyCode_CheckLineNumber(addrq, &bounds); } +int +_PyCode_SafeAddr2Line(PyCodeObject *co, int addrq) +{ + if (addrq < 0) { + return co->co_firstlineno; + } + if (co->_co_monitoring && co->_co_monitoring->lines) { + return _Py_Instrumentation_GetLine(co, addrq/sizeof(_Py_CODEUNIT)); + } + if (!(addrq >= 0 && addrq < _PyCode_NBYTES(co))) { + return -1; + } + PyCodeAddressRange bounds; + _PyCode_InitAddressRange(co, &bounds); + return _PyCode_CheckLineNumber(addrq, &bounds); +} + int PyCode_Addr2Line(PyCodeObject *co, int addrq) { int lineno; Py_BEGIN_CRITICAL_SECTION(co); - lineno = _PyCode_Addr2LineNoTstate(co, addrq); + lineno = _PyCode_Addr2Line(co, addrq); Py_END_CRITICAL_SECTION(); return lineno; } diff --git a/Python/traceback.c b/Python/traceback.c index ef67368550a..521d6322a5c 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -1028,14 +1028,24 @@ _Py_DumpWideString(int fd, wchar_t *str) /* Write a frame into the file fd: "File "xxx", line xxx in xxx". - This function is signal safe. */ + This function is signal safe. -static void + Return 0 on success. Return -1 if the frame is invalid. */ + +static int dump_frame(int fd, _PyInterpreterFrame *frame) { - assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); + if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { + /* Ignore trampoline frame */ + return 0; + } - PyCodeObject *code =_PyFrame_GetCode(frame); + PyCodeObject *code = _PyFrame_SafeGetCode(frame); + if (code == NULL) { + return -1; + } + + int res = 0; PUTS(fd, " File "); if (code->co_filename != NULL && PyUnicode_Check(code->co_filename)) @@ -1043,29 +1053,36 @@ dump_frame(int fd, _PyInterpreterFrame *frame) PUTS(fd, "\""); _Py_DumpASCII(fd, code->co_filename); PUTS(fd, "\""); - } else { - PUTS(fd, "???"); } - int lasti = PyUnstable_InterpreterFrame_GetLasti(frame); - int lineno = _PyCode_Addr2LineNoTstate(code, lasti); + else { + PUTS(fd, "???"); + res = -1; + } + PUTS(fd, ", line "); + int lasti = _PyFrame_SafeGetLasti(frame); + int lineno = -1; + if (lasti >= 0) { + lineno = _PyCode_SafeAddr2Line(code, lasti); + } if (lineno >= 0) { _Py_DumpDecimal(fd, (size_t)lineno); } else { PUTS(fd, "???"); + res = -1; } - PUTS(fd, " in "); - if (code->co_name != NULL - && PyUnicode_Check(code->co_name)) { + PUTS(fd, " in "); + if (code->co_name != NULL && PyUnicode_Check(code->co_name)) { _Py_DumpASCII(fd, code->co_name); } else { PUTS(fd, "???"); + res = -1; } - PUTS(fd, "\n"); + return res; } static int @@ -1108,17 +1125,6 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) unsigned int depth = 0; while (1) { - if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { - /* Trampoline frame */ - frame = frame->previous; - if (frame == NULL) { - break; - } - - /* Can't have more than one shim frame in a row */ - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - } - if (MAX_FRAME_DEPTH <= depth) { if (MAX_FRAME_DEPTH < depth) { PUTS(fd, "plus "); @@ -1128,7 +1134,15 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) break; } - dump_frame(fd, frame); + if (_PyMem_IsPtrFreed(frame)) { + PUTS(fd, " \n"); + break; + } + if (dump_frame(fd, frame) < 0) { + PUTS(fd, " \n"); + break; + } + frame = frame->previous; if (frame == NULL) { break; From fa9c3eefd475f0647a69bf3f49db8100848fb6a9 Mon Sep 17 00:00:00 2001 From: Abhishek Tiwari Date: Tue, 4 Nov 2025 16:24:28 +0530 Subject: [PATCH 022/313] gh-140797: Forbid capturing groups in re.Scanner lexicon patterns (GH-140944) --- Lib/re/__init__.py | 5 ++++- Lib/test/test_re.py | 18 ++++++++++++++++++ ...5-11-03-16-23-54.gh-issue-140797.DuFEeR.rst | 2 ++ 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-03-16-23-54.gh-issue-140797.DuFEeR.rst diff --git a/Lib/re/__init__.py b/Lib/re/__init__.py index a5316391297..ecec16e9005 100644 --- a/Lib/re/__init__.py +++ b/Lib/re/__init__.py @@ -397,9 +397,12 @@ def __init__(self, lexicon, flags=0): s = _parser.State() s.flags = flags for phrase, action in lexicon: + sub_pattern = _parser.parse(phrase, flags) + if sub_pattern.state.groups != 1: + raise ValueError("Cannot use capturing groups in re.Scanner") gid = s.opengroup() p.append(_parser.SubPattern(s, [ - (SUBPATTERN, (gid, 0, 0, _parser.parse(phrase, flags))), + (SUBPATTERN, (gid, 0, 0, sub_pattern)), ])) s.closegroup(gid, p[-1]) p = _parser.SubPattern(s, [(BRANCH, (None, p))]) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 5fc95087f2b..9f6f04bf6b8 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1639,6 +1639,24 @@ def s_int(scanner, token): return int(token) (['sum', 'op=', 3, 'op*', 'foo', 'op+', 312.5, 'op+', 'bar'], '')) + def test_bug_gh140797(self): + # gh140797: Capturing groups are not allowed in re.Scanner + + msg = r"Cannot use capturing groups in re\.Scanner" + # Capturing group throws an error + with self.assertRaisesRegex(ValueError, msg): + Scanner([("(a)b", None)]) + + # Named Group + with self.assertRaisesRegex(ValueError, msg): + Scanner([("(?Pa)", None)]) + + # Non-capturing groups should pass normally + s = Scanner([("(?:a)b", lambda scanner, token: token)]) + result, rem = s.scan("ab") + self.assertEqual(result,['ab']) + self.assertEqual(rem,'') + def test_bug_448951(self): # bug 448951 (similar to 429357, but with single char match) # (Also test greedy matches.) diff --git a/Misc/NEWS.d/next/Library/2025-11-03-16-23-54.gh-issue-140797.DuFEeR.rst b/Misc/NEWS.d/next/Library/2025-11-03-16-23-54.gh-issue-140797.DuFEeR.rst new file mode 100644 index 00000000000..493b740261e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-03-16-23-54.gh-issue-140797.DuFEeR.rst @@ -0,0 +1,2 @@ +The undocumented :class:`!re.Scanner` class now forbids regular expressions containing capturing groups in its lexicon patterns. Patterns using capturing groups could +previously lead to crashes with segmentation fault. Use non-capturing groups (?:...) instead. From 1326d2a808245e5f2de9e515460bab30556e8f05 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 4 Nov 2025 17:49:44 +0200 Subject: [PATCH 023/313] gh-140979: Fix off-by-one error in the RE code validator (GH-140984) It was too lenient and allowed MARK opcodes with too large value. --- Modules/_sre/sre.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index fdf00e6499c..4e97101b699 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -1946,7 +1946,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) sre_match() code is robust even if they don't, and the worst you can get is nonsensical match results. */ GET_ARG; - if (arg > 2 * (size_t)groups + 1) { + if (arg >= 2 * (size_t)groups) { VTRACE(("arg=%d, groups=%d\n", (int)arg, (int)groups)); FAIL; } From 40096da95a592ac0b2ad6aa9c731631784c3b393 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Tue, 4 Nov 2025 08:31:35 -0800 Subject: [PATCH 024/313] GH-139946: Colorize error and warning messages in argparse (#140695) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Lib/_colorize.py | 3 ++ Lib/argparse.py | 22 ++++++++-- Lib/test/test_argparse.py | 41 +++++++++++++++++++ Lib/test/test_clinic.py | 2 + Lib/test/test_gzip.py | 3 +- Lib/test/test_uuid.py | 4 +- Lib/test/test_webbrowser.py | 2 + ...-10-28-02-46-56.gh-issue-139946.aN3_uY.rst | 1 + 8 files changed, 73 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-28-02-46-56.gh-issue-139946.aN3_uY.rst diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 63e951d6488..57b712bc068 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -170,6 +170,9 @@ class Argparse(ThemeSection): label: str = ANSIColors.BOLD_YELLOW action: str = ANSIColors.BOLD_GREEN reset: str = ANSIColors.RESET + error: str = ANSIColors.BOLD_MAGENTA + warning: str = ANSIColors.BOLD_YELLOW + message: str = ANSIColors.MAGENTA @dataclass(frozen=True, kw_only=True) diff --git a/Lib/argparse.py b/Lib/argparse.py index 1f4413a9897..6b79747572f 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -2749,6 +2749,14 @@ def _print_message(self, message, file=None): except (AttributeError, OSError): pass + def _get_theme(self, file=None): + from _colorize import can_colorize, get_theme + + if self.color and can_colorize(file=file): + return get_theme(force_color=True).argparse + else: + return get_theme(force_no_color=True).argparse + # =============== # Exiting methods # =============== @@ -2768,13 +2776,21 @@ def error(self, message): should either exit or raise an exception. """ self.print_usage(_sys.stderr) + theme = self._get_theme(file=_sys.stderr) + fmt = _('%(prog)s: error: %(message)s\n') + fmt = fmt.replace('error: %(message)s', + f'{theme.error}error:{theme.reset} {theme.message}%(message)s{theme.reset}') + args = {'prog': self.prog, 'message': message} - self.exit(2, _('%(prog)s: error: %(message)s\n') % args) + self.exit(2, fmt % args) def _warning(self, message): + theme = self._get_theme(file=_sys.stderr) + fmt = _('%(prog)s: warning: %(message)s\n') + fmt = fmt.replace('warning: %(message)s', + f'{theme.warning}warning:{theme.reset} {theme.message}%(message)s{theme.reset}') args = {'prog': self.prog, 'message': message} - self._print_message(_('%(prog)s: warning: %(message)s\n') % args, _sys.stderr) - + self._print_message(fmt % args, _sys.stderr) def __getattr__(name): if name == "__version__": diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index d6c9c1ef2c8..3a8be68a546 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -2283,6 +2283,7 @@ class TestNegativeNumber(ParserTestCase): ('--complex -1e-3j', NS(int=None, float=None, complex=-0.001j)), ] +@force_not_colorized_test_class class TestArgumentAndSubparserSuggestions(TestCase): """Test error handling and suggestion when a user makes a typo""" @@ -6147,6 +6148,7 @@ def spam(string_to_convert): # Check that deprecated arguments output warning # ============================================== +@force_not_colorized_test_class class TestDeprecatedArguments(TestCase): def test_deprecated_option(self): @@ -7370,6 +7372,45 @@ def test_subparser_prog_is_stored_without_color(self): help_text = demo_parser.format_help() self.assertNotIn('\x1b[', help_text) + def test_error_and_warning_keywords_colorized(self): + parser = argparse.ArgumentParser(prog='PROG') + parser.add_argument('foo') + + with self.assertRaises(SystemExit): + with captured_stderr() as stderr: + parser.parse_args([]) + + err = stderr.getvalue() + error_color = self.theme.error + reset = self.theme.reset + self.assertIn(f'{error_color}error:{reset}', err) + + with captured_stderr() as stderr: + parser._warning('test warning') + + warn = stderr.getvalue() + warning_color = self.theme.warning + self.assertIn(f'{warning_color}warning:{reset}', warn) + + def test_error_and_warning_not_colorized_when_disabled(self): + parser = argparse.ArgumentParser(prog='PROG', color=False) + parser.add_argument('foo') + + with self.assertRaises(SystemExit): + with captured_stderr() as stderr: + parser.parse_args([]) + + err = stderr.getvalue() + self.assertNotIn('\x1b[', err) + self.assertIn('error:', err) + + with captured_stderr() as stderr: + parser._warning('test warning') + + warn = stderr.getvalue() + self.assertNotIn('\x1b[', warn) + self.assertIn('warning:', warn) + class TestModule(unittest.TestCase): def test_deprecated__version__(self): diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index e0dbb062eb0..e71f9fc181b 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -4,6 +4,7 @@ from functools import partial from test import support, test_tools +from test.support import force_not_colorized_test_class from test.support import os_helper from test.support.os_helper import TESTFN, unlink, rmtree from textwrap import dedent @@ -2758,6 +2759,7 @@ def test_allow_negative_accepted_by_py_ssize_t_converter_only(self): with self.assertRaisesRegex((AssertionError, TypeError), errmsg): self.parse_function(block) +@force_not_colorized_test_class class ClinicExternalTest(TestCase): maxDiff = None diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index f14a882d386..442d30fc970 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -11,7 +11,7 @@ import unittest from subprocess import PIPE, Popen from test.support import catch_unraisable_exception -from test.support import import_helper +from test.support import force_not_colorized_test_class, import_helper from test.support import os_helper from test.support import _4G, bigmemtest, requires_subprocess from test.support.script_helper import assert_python_ok, assert_python_failure @@ -1057,6 +1057,7 @@ def wrapper(*args, **kwargs): return decorator +@force_not_colorized_test_class class TestCommandLine(unittest.TestCase): data = b'This is a simple test with gzip' diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 33045a78721..5f9ab048cde 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -13,7 +13,7 @@ from unittest import mock from test import support -from test.support import import_helper, warnings_helper +from test.support import force_not_colorized_test_class, import_helper, warnings_helper from test.support.script_helper import assert_python_ok py_uuid = import_helper.import_fresh_module('uuid', blocked=['_uuid']) @@ -1250,10 +1250,12 @@ def test_cli_uuid8(self): self.do_test_standalone_uuid(8) +@force_not_colorized_test_class class TestUUIDWithoutExtModule(CommandLineTestCases, BaseTestUUID, unittest.TestCase): uuid = py_uuid +@force_not_colorized_test_class @unittest.skipUnless(c_uuid, 'requires the C _uuid module') class TestUUIDWithExtModule(CommandLineTestCases, BaseTestUUID, unittest.TestCase): uuid = c_uuid diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index 6b577ae100e..20d347168b3 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -7,6 +7,7 @@ import unittest import webbrowser from test import support +from test.support import force_not_colorized_test_class from test.support import import_helper from test.support import is_apple_mobile from test.support import os_helper @@ -503,6 +504,7 @@ def test_environment_preferred(self): self.assertEqual(webbrowser.get().name, sys.executable) +@force_not_colorized_test_class class CliTest(unittest.TestCase): def test_parse_args(self): for command, url, new_win in [ diff --git a/Misc/NEWS.d/next/Library/2025-10-28-02-46-56.gh-issue-139946.aN3_uY.rst b/Misc/NEWS.d/next/Library/2025-10-28-02-46-56.gh-issue-139946.aN3_uY.rst new file mode 100644 index 00000000000..4c68d4cd94b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-28-02-46-56.gh-issue-139946.aN3_uY.rst @@ -0,0 +1 @@ +Error and warning keywords in ``argparse.ArgumentParser`` messages are now colorized when color output is enabled, fixing a visual inconsistency in which they remained plain text while other output was colorized. From 8a7dbb7a68b5da1f3f1805f564c028f1eea4ebc3 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 4 Nov 2025 10:28:17 -0800 Subject: [PATCH 025/313] Document that returning `sys.monitoring.DISABLE` in response to a global event raises `ValueError` (#140726) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/library/sys.monitoring.rst | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 0f986aa580b..303655fb128 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -216,14 +216,17 @@ by another event: The :monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events are controlled by the :monitoring-event:`CALL` event. -:monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events will only be seen if the -corresponding :monitoring-event:`CALL` event is being monitored. +:monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events will only be +seen if the corresponding :monitoring-event:`CALL` event is being monitored. + + +.. _monitoring-event-global: Other events '''''''''''' Other events are not necessarily tied to a specific location in the -program and cannot be individually disabled. +program and cannot be individually disabled via :data:`DISABLE`. The other events that can be monitored are: @@ -289,12 +292,13 @@ in Python (see :ref:`c-api-monitoring`). .. function:: get_local_events(tool_id: int, code: CodeType, /) -> int - Returns all the local events for *code* + Returns all the :ref:`local events ` for *code* .. function:: set_local_events(tool_id: int, code: CodeType, event_set: int, /) -> None - Activates all the local events for *code* which are set in *event_set*. - Raises a :exc:`ValueError` if *tool_id* is not in use. + Activates all the :ref:`local events ` for *code* + which are set in *event_set*. Raises a :exc:`ValueError` if *tool_id* is not + in use. Disabling events @@ -305,15 +309,21 @@ Disabling events A special value that can be returned from a callback function to disable events for the current code location. -Local events can be disabled for a specific code location by returning -:data:`sys.monitoring.DISABLE` from a callback function. This does not change -which events are set, or any other code locations for the same event. +:ref:`Local events ` can be disabled for a specific code +location by returning :data:`sys.monitoring.DISABLE` from a callback function. +This does not change which events are set, or any other code locations for the +same event. Disabling events for specific locations is very important for high performance monitoring. For example, a program can be run under a debugger with no overhead if the debugger disables all monitoring except for a few breakpoints. +If :data:`DISABLE` is returned by a callback for a +:ref:`global event `, :exc:`ValueError` will be raised +by the interpreter in a non-specific location (that is, no traceback will be +provided). + .. function:: restart_events() -> None Enable all the events that were disabled by :data:`sys.monitoring.DISABLE` From 66c86c65633047c0faffba85ce6b0b3a82373657 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Tue, 4 Nov 2025 18:29:44 +0000 Subject: [PATCH 026/313] gh-134817: Restore accidentally deleted line in documentation. (GH-141013) --- Doc/library/logging.handlers.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index d74ef73ee28..c9cfbdb4126 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -463,6 +463,7 @@ timed intervals. .. method:: getFilesToDelete() Returns a list of filenames which should be deleted as part of rollover. These + are the absolute paths of the oldest backup log files written by the handler. .. method:: shouldRollover(record) From 97d8dda980fcddf88b782be343118257f483a864 Mon Sep 17 00:00:00 2001 From: Guo Ci Date: Tue, 4 Nov 2025 14:29:13 -0500 Subject: [PATCH 027/313] Docs: Fix typo in `email.headerregistry.rst` (#140965) Fix missing 'Header' suffix on header class name in `email.headerregistry.rst` --- Doc/library/email.headerregistry.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index 7f8044932fa..ff8b601fe3d 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -294,7 +294,7 @@ variant, :attr:`~.BaseHeader.max_count` is set to 1. ``inline`` and ``attachment`` are the only valid values in common use. -.. class:: ContentTransferEncoding +.. class:: ContentTransferEncodingHeader Handles the :mailheader:`Content-Transfer-Encoding` header. From ce1bb85d286130f44b7e874430b0b12990d61dc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20=C5=81ajszczak?= Date: Tue, 4 Nov 2025 20:46:07 +0100 Subject: [PATCH 028/313] gh-139434: Update selected RFC 2822 references to RFC 5322 (#139435) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update selected RFC 2822 references to RFC 5322 RFC 2822 was obsoleted by RFC 5322 in 2008. This updates references to use the current standard in documentation, docstrings, and comments. It preserves RFC 2822 references in legacy API components to maintain their historical context. RFC 822 → RFC 2822 → RFC 5322 progression is explained where relevant. In some places specific sections of RFC are referenced where it seems helpful. Scout rule was applied in some places and RFC mentions format was normalized in doc strings and comments. --- Doc/library/http.client.rst | 2 +- Doc/library/http.server.rst | 2 +- Doc/library/mailbox.rst | 2 +- Doc/library/time.rst | 5 ++-- Doc/tutorial/stdlib.rst | 2 +- Lib/email/_parseaddr.py | 13 +++++++---- Lib/email/_policybase.py | 2 +- Lib/email/feedparser.py | 4 ++-- Lib/email/generator.py | 2 +- Lib/email/message.py | 2 +- Lib/email/parser.py | 10 ++++---- Lib/http/client.py | 4 ++-- Lib/smtplib.py | 6 ++--- Lib/test/test_email/data/msg_35.txt | 2 +- Lib/test/test_email/test_email.py | 36 ++++++++++++++--------------- 15 files changed, 49 insertions(+), 45 deletions(-) diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 589152f2968..7c258b324d9 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -133,7 +133,7 @@ This module provides the following function: Parse the headers from a file pointer *fp* representing a HTTP request/response. The file has to be a :class:`~io.BufferedIOBase` reader - (i.e. not text) and must provide a valid :rfc:`2822` style header. + (i.e. not text) and must provide a valid :rfc:`5322` style header. This function returns an instance of :class:`http.client.HTTPMessage` that holds the header fields, but no payload diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 063344e0284..58f09634f95 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -154,7 +154,7 @@ instantiation, of which this module provides three different variants: variable. This instance parses and manages the headers in the HTTP request. The :func:`~http.client.parse_headers` function from :mod:`http.client` is used to parse the headers and it requires that the - HTTP request provide a valid :rfc:`2822` style header. + HTTP request provide a valid :rfc:`5322` style header. .. attribute:: rfile diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index e8a96f29ea1..62e289573c0 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -917,7 +917,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. copied; furthermore, any format-specific information is converted insofar as possible if *message* is a :class:`!Message` instance. If *message* is a string, a byte string, - or a file, it should contain an :rfc:`2822`\ -compliant message, which is read + or a file, it should contain an :rfc:`5322`\ -compliant message, which is read and parsed. Files should be open in binary mode, but text mode files are accepted for backward compatibility. diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 350ffade7af..69e6433e898 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -584,7 +584,7 @@ Functions calculations when the day of the week and the year are specified. Here is an example, a format for dates compatible with that specified in the - :rfc:`2822` Internet email standard. [1]_ :: + :rfc:`5322` Internet email standard. [1]_ :: >>> from time import gmtime, strftime >>> strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()) @@ -1066,4 +1066,5 @@ Timezone Constants strict reading of the original 1982 :rfc:`822` standard calls for a two-digit year (``%y`` rather than ``%Y``), but practice moved to 4-digit years long before the year 2000. After that, :rfc:`822` became obsolete and the 4-digit year has - been first recommended by :rfc:`1123` and then mandated by :rfc:`2822`. + been first recommended by :rfc:`1123` and then mandated by :rfc:`2822`, + with :rfc:`5322` continuing this requirement. diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index d83ecca270b..49a3e370a4c 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -335,7 +335,7 @@ sophisticated and robust capabilities of its larger packages. For example: names, no direct knowledge or handling of XML is needed. * The :mod:`email` package is a library for managing email messages, including - MIME and other :rfc:`2822`-based message documents. Unlike :mod:`smtplib` and + MIME and other :rfc:`5322`-based message documents. Unlike :mod:`smtplib` and :mod:`poplib` which actually send and receive messages, the email package has a complete toolset for building or decoding complex message structures (including attachments) and for implementing internet encoding and header diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py index 84917038874..6a7c5fa06d2 100644 --- a/Lib/email/_parseaddr.py +++ b/Lib/email/_parseaddr.py @@ -146,8 +146,9 @@ def _parsedate_tz(data): return None # Check for a yy specified in two-digit format, then convert it to the # appropriate four-digit format, according to the POSIX standard. RFC 822 - # calls for a two-digit yy, but RFC 2822 (which obsoletes RFC 822) - # mandates a 4-digit yy. For more information, see the documentation for + # calls for a two-digit yy, but RFC 2822 (which obsoletes RFC 822) already + # mandated a 4-digit yy, and RFC 5322 (which obsoletes RFC 2822) continues + # this requirement. For more information, see the documentation for # the time module. if yy < 100: # The year is between 1969 and 1999 (inclusive). @@ -233,9 +234,11 @@ def __init__(self, field): self.CR = '\r\n' self.FWS = self.LWS + self.CR self.atomends = self.specials + self.LWS + self.CR - # Note that RFC 2822 now specifies '.' as obs-phrase, meaning that it - # is obsolete syntax. RFC 2822 requires that we recognize obsolete - # syntax, so allow dots in phrases. + # Note that RFC 2822 section 4.1 introduced '.' as obs-phrase to handle + # existing practice (periods in display names), even though it was not + # allowed in RFC 822. RFC 5322 section 4.1 (which obsoletes RFC 2822) + # continues this requirement. We must recognize obsolete syntax, so + # allow dots in phrases. self.phraseends = self.atomends.replace('.', '') self.field = field self.commentlist = [] diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py index 95e79b8938b..e23843df448 100644 --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -380,7 +380,7 @@ def _fold(self, name, value, sanitize): h = value if h is not None: # The Header class interprets a value of None for maxlinelen as the - # default value of 78, as recommended by RFC 2822. + # default value of 78, as recommended by RFC 5322 section 2.1.1. maxlinelen = 0 if self.max_line_length is not None: maxlinelen = self.max_line_length diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py index 9d80a5822af..6479b9bab7a 100644 --- a/Lib/email/feedparser.py +++ b/Lib/email/feedparser.py @@ -32,7 +32,7 @@ NLCRE_bol = re.compile(r'(\r\n|\r|\n)') NLCRE_eol = re.compile(r'(\r\n|\r|\n)\z') NLCRE_crack = re.compile(r'(\r\n|\r|\n)') -# RFC 2822 $3.6.8 Optional fields. ftext is %d33-57 / %d59-126, Any character +# RFC 5322 section 3.6.8 Optional fields. ftext is %d33-57 / %d59-126, Any character # except controls, SP, and ":". headerRE = re.compile(r'^(From |[\041-\071\073-\176]*:|[\t ])') EMPTYSTRING = '' @@ -294,7 +294,7 @@ def _parsegen(self): return if self._cur.get_content_maintype() == 'message': # The message claims to be a message/* type, then what follows is - # another RFC 2822 message. + # another RFC 5322 message. for retval in self._parsegen(): if retval is NeedMoreData: yield NeedMoreData diff --git a/Lib/email/generator.py b/Lib/email/generator.py index ab5bd0653e4..03524c96559 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -50,7 +50,7 @@ def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, *, expanded to 8 spaces) than maxheaderlen, the header will split as defined in the Header class. Set maxheaderlen to zero to disable header wrapping. The default is 78, as recommended (but not required) - by RFC 2822. + by RFC 5322 section 2.1.1. The policy keyword specifies a policy object that controls a number of aspects of the generator's operation. If no policy is specified, diff --git a/Lib/email/message.py b/Lib/email/message.py index 4380e0ec50b..641fb2e944d 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -141,7 +141,7 @@ def _decode_uu(encoded): class Message: """Basic message object. - A message object is defined as something that has a bunch of RFC 2822 + A message object is defined as something that has a bunch of RFC 5322 headers and a payload. It may optionally have an envelope header (a.k.a. Unix-From or From_ header). If the message is a container (i.e. a multipart or a message/rfc822), then the payload is a list of Message diff --git a/Lib/email/parser.py b/Lib/email/parser.py index 039f03cba74..c6a51dd8e37 100644 --- a/Lib/email/parser.py +++ b/Lib/email/parser.py @@ -2,7 +2,7 @@ # Author: Barry Warsaw, Thomas Wouters, Anthony Baxter # Contact: email-sig@python.org -"""A parser of RFC 2822 and MIME email messages.""" +"""A parser of RFC 5322 and MIME email messages.""" __all__ = ['Parser', 'HeaderParser', 'BytesParser', 'BytesHeaderParser', 'FeedParser', 'BytesFeedParser'] @@ -15,13 +15,13 @@ class Parser: def __init__(self, _class=None, *, policy=compat32): - """Parser of RFC 2822 and MIME email messages. + """Parser of RFC 5322 and MIME email messages. Creates an in-memory object tree representing the email message, which can then be manipulated and turned over to a Generator to return the textual representation of the message. - The string must be formatted as a block of RFC 2822 headers and header + The string must be formatted as a block of RFC 5322 headers and header continuation lines, optionally preceded by a 'Unix-from' header. The header block is terminated either by the end of the string or by a blank line. @@ -75,13 +75,13 @@ def parsestr(self, text, headersonly=True): class BytesParser: def __init__(self, *args, **kw): - """Parser of binary RFC 2822 and MIME email messages. + """Parser of binary RFC 5322 and MIME email messages. Creates an in-memory object tree representing the email message, which can then be manipulated and turned over to a Generator to return the textual representation of the message. - The input must be formatted as a block of RFC 2822 headers and header + The input must be formatted as a block of RFC 5322 headers and header continuation lines, optionally preceded by a 'Unix-from' header. The header block is terminated either by the end of the input or by a blank line. diff --git a/Lib/http/client.py b/Lib/http/client.py index 425d9bdad8c..4b9a61cfc11 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -231,7 +231,7 @@ def _read_headers(fp, max_headers): def _parse_header_lines(header_lines, _class=HTTPMessage): """ - Parses only RFC2822 headers from header lines. + Parses only RFC 5322 headers from header lines. email Parser wants to see strings rather than bytes. But a TextIOWrapper around self.rfile would buffer too many bytes @@ -244,7 +244,7 @@ def _parse_header_lines(header_lines, _class=HTTPMessage): return email.parser.Parser(_class=_class).parsestr(hstring) def parse_headers(fp, _class=HTTPMessage, *, _max_headers=None): - """Parses only RFC2822 headers from a file pointer.""" + """Parses only RFC 5322 headers from a file pointer.""" headers = _read_headers(fp, _max_headers) return _parse_header_lines(headers, _class) diff --git a/Lib/smtplib.py b/Lib/smtplib.py index 808f0fd47e8..72093f7f8b0 100644 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -917,7 +917,7 @@ def send_message(self, msg, from_addr=None, to_addrs=None, The arguments are as for sendmail, except that msg is an email.message.Message object. If from_addr is None or to_addrs is None, these arguments are taken from the headers of the Message as - described in RFC 2822 (a ValueError is raised if there is more than + described in RFC 5322 (a ValueError is raised if there is more than one set of 'Resent-' headers). Regardless of the values of from_addr and to_addr, any Bcc field (or Resent-Bcc field, when the Message is a resent) of the Message object won't be transmitted. The Message @@ -931,7 +931,7 @@ def send_message(self, msg, from_addr=None, to_addrs=None, policy. """ - # 'Resent-Date' is a mandatory field if the Message is resent (RFC 2822 + # 'Resent-Date' is a mandatory field if the Message is resent (RFC 5322 # Section 3.6.6). In such a case, we use the 'Resent-*' fields. However, # if there is more than one 'Resent-' block there's no way to # unambiguously determine which one is the most recent in all cases, @@ -950,7 +950,7 @@ def send_message(self, msg, from_addr=None, to_addrs=None, else: raise ValueError("message has more than one 'Resent-' header block") if from_addr is None: - # Prefer the sender field per RFC 2822:3.6.2. + # Prefer the sender field per RFC 5322 section 3.6.2. from_addr = (msg[header_prefix + 'Sender'] if (header_prefix + 'Sender') in msg else msg[header_prefix + 'From']) diff --git a/Lib/test/test_email/data/msg_35.txt b/Lib/test/test_email/data/msg_35.txt index be7d5a2f7b9..0e2bbcaf718 100644 --- a/Lib/test/test_email/data/msg_35.txt +++ b/Lib/test/test_email/data/msg_35.txt @@ -1,4 +1,4 @@ From: aperson@dom.ain To: bperson@dom.ain Subject: here's something interesting -counter to RFC 2822, there's no separating newline here +counter to RFC 5322, there's no separating newline here diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index b458d3f0efa..4cd587bcd76 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -2373,7 +2373,7 @@ def test_no_separating_blank_line(self): To: bperson@dom.ain Subject: here's something interesting -counter to RFC 2822, there's no separating newline here +counter to RFC 5322, there's no separating newline here """) # test_defect_handling @@ -2529,49 +2529,49 @@ def test_rfc2047_Q_invalid_digits(self): [(b'andr\xe9=zz', 'iso-8859-1')]) def test_rfc2047_rfc2047_1(self): - # 1st testcase at end of rfc2047 + # 1st testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'a', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_2(self): - # 2nd testcase at end of rfc2047 + # 2nd testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a?= b)' self.assertEqual(decode_header(s), [(b'(', None), (b'a', 'iso-8859-1'), (b' b)', None)]) def test_rfc2047_rfc2047_3(self): - # 3rd testcase at end of rfc2047 + # 3rd testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a?= =?ISO-8859-1?Q?b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_4(self): - # 4th testcase at end of rfc2047 + # 4th testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a?= =?ISO-8859-1?Q?b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_5a(self): - # 5th testcase at end of rfc2047 newline is \r\n + # 5th testcase at end of RFC 2047 newline is \r\n s = '(=?ISO-8859-1?Q?a?=\r\n =?ISO-8859-1?Q?b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_5b(self): - # 5th testcase at end of rfc2047 newline is \n + # 5th testcase at end of RFC 2047 newline is \n s = '(=?ISO-8859-1?Q?a?=\n =?ISO-8859-1?Q?b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_6(self): - # 6th testcase at end of rfc2047 + # 6th testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a_b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'a b', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_7(self): - # 7th testcase at end of rfc2047 + # 7th testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a?= =?ISO-8859-2?Q?_b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'a', 'iso-8859-1'), (b' b', 'iso-8859-2'), @@ -3273,8 +3273,8 @@ def test_parsedate_y2k(self): """Test for parsing a date with a two-digit year. Parsing a date with a two-digit year should return the correct - four-digit year. RFC822 allows two-digit years, but RFC2822 (which - obsoletes RFC822) requires four-digit years. + four-digit year. RFC 822 allows two-digit years, but RFC 5322 (which + obsoletes RFC 2822, which obsoletes RFC 822) requires four-digit years. """ self.assertEqual(utils.parsedate_tz('25 Feb 03 13:47:26 -0800'), @@ -3325,7 +3325,7 @@ def test_escape_backslashes(self): self.assertEqual(utils.parseaddr(utils.formataddr((a, b))), (a, b)) def test_quotes_unicode_names(self): - # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + # issue 1690608. email.utils.formataddr() should be RFC 2047 aware. name = "H\u00e4ns W\u00fcrst" addr = 'person@dom.ain' utf8_base64 = "=?utf-8?b?SMOkbnMgV8O8cnN0?= " @@ -3335,7 +3335,7 @@ def test_quotes_unicode_names(self): latin1_quopri) def test_accepts_any_charset_like_object(self): - # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + # issue 1690608. email.utils.formataddr() should be RFC 2047 aware. name = "H\u00e4ns W\u00fcrst" addr = 'person@dom.ain' utf8_base64 = "=?utf-8?b?SMOkbnMgV8O8cnN0?= " @@ -3350,7 +3350,7 @@ def header_encode(self, string): utf8_base64) def test_invalid_charset_like_object_raises_error(self): - # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + # issue 1690608. email.utils.formataddr() should be RFC 2047 aware. name = "H\u00e4ns W\u00fcrst" addr = 'person@dom.ain' # An object without a header_encode method: @@ -3359,7 +3359,7 @@ def test_invalid_charset_like_object_raises_error(self): bad_charset) def test_unicode_address_raises_error(self): - # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + # issue 1690608. email.utils.formataddr() should be RFC 2047 aware. addr = 'pers\u00f6n@dom.in' self.assertRaises(UnicodeError, utils.formataddr, (None, addr)) self.assertRaises(UnicodeError, utils.formataddr, ("Name", addr)) @@ -3380,7 +3380,7 @@ def test_parseaddr_preserves_quoted_pairs_in_addresses(self): # string containing a quoted backslash, followed by 'example' and two # backslashes, followed by another quoted string containing a space and # the word 'example'. parseaddr copies those two backslashes - # literally. Per rfc5322 this is not technically correct since a \ may + # literally. Per RFC 5322 this is not technically correct since a \ may # not appear in an address outside of a quoted string. It is probably # a sensible Postel interpretation, though. eq = self.assertEqual @@ -3392,12 +3392,12 @@ def test_parseaddr_preserves_quoted_pairs_in_addresses(self): ('', '"\\\\"example\\\\" example"@example.com')) def test_parseaddr_preserves_spaces_in_local_part(self): - # issue 9286. A normal RFC5322 local part should not contain any + # issue 9286. A normal RFC 5322 local part should not contain any # folding white space, but legacy local parts can (they are a sequence # of atoms, not dotatoms). On the other hand we strip whitespace from # before the @ and around dots, on the assumption that the whitespace # around the punctuation is a mistake in what would otherwise be - # an RFC5322 local part. Leading whitespace is, usual, stripped as well. + # an RFC 5322 local part. Leading whitespace is, usual, stripped as well. self.assertEqual(('', "merwok wok@xample.com"), utils.parseaddr("merwok wok@xample.com")) self.assertEqual(('', "merwok wok@xample.com"), From bfe54810c408ff066591d1af0411b1d9c10084b1 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Tue, 4 Nov 2025 21:19:06 +0000 Subject: [PATCH 029/313] gh-141004: Document `Py_UNICODE_{HIGH, LOW}_SURROGATE` functions (GH-141019) --- Doc/c-api/unicode.rst | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 22b0a6aff6e..ca7c8bb11a5 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -321,12 +321,22 @@ These APIs can be used to work with surrogates: Check if *ch* is a low surrogate (``0xDC00 <= ch <= 0xDFFF``). +.. c:function:: Py_UCS4 Py_UNICODE_HIGH_SURROGATE(Py_UCS4 ch) + + Return the high UTF-16 surrogate (``0xD800`` to ``0xDBFF``) for a Unicode + code point in the range ``[0x10000; 0x10FFFF]``. + +.. c:function:: Py_UCS4 Py_UNICODE_LOW_SURROGATE(Py_UCS4 ch) + + Return the low UTF-16 surrogate (``0xDC00`` to ``0xDFFF``) for a Unicode + code point in the range ``[0x10000; 0x10FFFF]``. + .. c:function:: Py_UCS4 Py_UNICODE_JOIN_SURROGATES(Py_UCS4 high, Py_UCS4 low) Join two surrogate code points and return a single :c:type:`Py_UCS4` value. *high* and *low* are respectively the leading and trailing surrogates in a - surrogate pair. *high* must be in the range [0xD800; 0xDBFF] and *low* must - be in the range [0xDC00; 0xDFFF]. + surrogate pair. *high* must be in the range ``[0xD800; 0xDBFF]`` and *low* must + be in the range ``[0xDC00; 0xDFFF]``. Creating and accessing Unicode strings From d81e1ef0f3a7c63f5d246e4cf918700016b72489 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 4 Nov 2025 22:58:53 +0100 Subject: [PATCH 030/313] gh-138189: Document type slots, and other constants, as part of Limited API (GH-138190) Co-authored-by: Peter Bierma --- Doc/c-api/typeobj.rst | 200 ++++++++++++++++++++++++++ Doc/data/stable_abi.dat | 153 ++++++++++++++++++++ Doc/tools/.nitignore | 1 - Doc/tools/extensions/c_annotations.py | 76 +++++++++- Misc/stable_abi.toml | 4 + Tools/build/stable_abi.py | 2 +- 6 files changed, 430 insertions(+), 6 deletions(-) diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 59c26a713e4..9d23aea5734 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -676,6 +676,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_dealloc + .. corresponding-type-slot:: Py_tp_dealloc + A pointer to the instance destructor function. The function signature is:: void tp_dealloc(PyObject *self); @@ -860,6 +862,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: getattrfunc PyTypeObject.tp_getattr + .. corresponding-type-slot:: Py_tp_getattr + An optional pointer to the get-attribute-string function. This field is deprecated. When it is defined, it should point to a function @@ -877,6 +881,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: setattrfunc PyTypeObject.tp_setattr + .. corresponding-type-slot:: Py_tp_setattr + An optional pointer to the function for setting and deleting attributes. This field is deprecated. When it is defined, it should point to a function @@ -909,6 +915,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: reprfunc PyTypeObject.tp_repr + .. corresponding-type-slot:: Py_tp_repr + .. index:: pair: built-in function; repr An optional pointer to a function that implements the built-in function @@ -974,6 +982,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: hashfunc PyTypeObject.tp_hash + .. corresponding-type-slot:: Py_tp_hash + .. index:: pair: built-in function; hash An optional pointer to a function that implements the built-in function @@ -1015,6 +1025,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: ternaryfunc PyTypeObject.tp_call + .. corresponding-type-slot:: Py_tp_call + An optional pointer to a function that implements calling the object. This should be ``NULL`` if the object is not callable. The signature is the same as for :c:func:`PyObject_Call`:: @@ -1028,6 +1040,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: reprfunc PyTypeObject.tp_str + .. corresponding-type-slot:: Py_tp_str + An optional pointer to a function that implements the built-in operation :func:`str`. (Note that :class:`str` is a type now, and :func:`str` calls the constructor for that type. This constructor calls :c:func:`PyObject_Str` to do @@ -1053,6 +1067,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: getattrofunc PyTypeObject.tp_getattro + .. corresponding-type-slot:: Py_tp_getattro + An optional pointer to the get-attribute function. The signature is the same as for :c:func:`PyObject_GetAttr`:: @@ -1077,6 +1093,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: setattrofunc PyTypeObject.tp_setattro + .. corresponding-type-slot:: Py_tp_setattro + An optional pointer to the function for setting and deleting attributes. The signature is the same as for :c:func:`PyObject_SetAttr`:: @@ -1475,6 +1493,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: const char* PyTypeObject.tp_doc + .. corresponding-type-slot:: Py_tp_doc + An optional pointer to a NUL-terminated C string giving the docstring for this type object. This is exposed as the :attr:`~type.__doc__` attribute on the type and instances of the type. @@ -1486,6 +1506,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: traverseproc PyTypeObject.tp_traverse + .. corresponding-type-slot:: Py_tp_traverse + An optional pointer to a traversal function for the garbage collector. This is only used if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is:: @@ -1582,6 +1604,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: inquiry PyTypeObject.tp_clear + .. corresponding-type-slot:: Py_tp_clear + An optional pointer to a clear function. The signature is:: int tp_clear(PyObject *); @@ -1730,6 +1754,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: richcmpfunc PyTypeObject.tp_richcompare + .. corresponding-type-slot:: Py_tp_richcompare + An optional pointer to the rich comparison function, whose signature is:: PyObject *tp_richcompare(PyObject *self, PyObject *other, int op); @@ -1832,6 +1858,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: getiterfunc PyTypeObject.tp_iter + .. corresponding-type-slot:: Py_tp_iter + An optional pointer to a function that returns an :term:`iterator` for the object. Its presence normally signals that the instances of this type are :term:`iterable` (although sequences may be iterable without this function). @@ -1847,6 +1875,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: iternextfunc PyTypeObject.tp_iternext + .. corresponding-type-slot:: Py_tp_iternext + An optional pointer to a function that returns the next item in an :term:`iterator`. The signature is:: @@ -1870,6 +1900,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyMethodDef* PyTypeObject.tp_methods + .. corresponding-type-slot:: Py_tp_methods + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyMethodDef` structures, declaring regular methods of this type. @@ -1884,6 +1916,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyMemberDef* PyTypeObject.tp_members + .. corresponding-type-slot:: Py_tp_members + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyMemberDef` structures, declaring regular data members (fields or slots) of instances of this type. @@ -1899,6 +1933,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyGetSetDef* PyTypeObject.tp_getset + .. corresponding-type-slot:: Py_tp_getset + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyGetSetDef` structures, declaring computed attributes of instances of this type. @@ -1913,6 +1949,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: PyTypeObject* PyTypeObject.tp_base + .. corresponding-type-slot:: Py_tp_base + An optional pointer to a base type from which type properties are inherited. At this level, only single inheritance is supported; multiple inheritance require dynamically creating a type object by calling the metatype. @@ -1985,6 +2023,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: descrgetfunc PyTypeObject.tp_descr_get + .. corresponding-type-slot:: Py_tp_descr_get + An optional pointer to a "descriptor get" function. The function signature is:: @@ -2000,6 +2040,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: descrsetfunc PyTypeObject.tp_descr_set + .. corresponding-type-slot:: Py_tp_descr_set + An optional pointer to a function for setting and deleting a descriptor's value. @@ -2060,6 +2102,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: initproc PyTypeObject.tp_init + .. corresponding-type-slot:: Py_tp_init + An optional pointer to an instance initialization function. This function corresponds to the :meth:`~object.__init__` method of classes. Like @@ -2095,6 +2139,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: allocfunc PyTypeObject.tp_alloc + .. corresponding-type-slot:: Py_tp_alloc + An optional pointer to an instance allocation function. The function signature is:: @@ -2118,6 +2164,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: newfunc PyTypeObject.tp_new + .. corresponding-type-slot:: Py_tp_new + An optional pointer to an instance creation function. The function signature is:: @@ -2157,6 +2205,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: freefunc PyTypeObject.tp_free + .. corresponding-type-slot:: Py_tp_free + An optional pointer to an instance deallocation function. Its signature is:: void tp_free(void *self); @@ -2186,6 +2236,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: inquiry PyTypeObject.tp_is_gc + .. corresponding-type-slot:: Py_tp_is_gc + An optional pointer to a function called by the garbage collector. The garbage collector needs to know whether a particular object is collectible @@ -2214,6 +2266,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: PyObject* PyTypeObject.tp_bases + .. corresponding-type-slot:: Py_tp_bases + Tuple of base types. This field should be set to ``NULL`` and treated as read-only. @@ -2294,6 +2348,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_del + .. corresponding-type-slot:: Py_tp_del + This field is deprecated. Use :c:member:`~PyTypeObject.tp_finalize` instead. @@ -2308,6 +2364,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_finalize + .. corresponding-type-slot:: Py_tp_finalize + An optional pointer to an instance finalization function. This is the C implementation of the :meth:`~object.__del__` special method. Its signature is:: @@ -2466,6 +2524,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: vectorcallfunc PyTypeObject.tp_vectorcall + .. corresponding-type-slot:: Py_tp_vectorcall + A :ref:`vectorcall function ` to use for calls of this type object (rather than instances). In other words, ``tp_vectorcall`` can be used to optimize ``type.__call__``, @@ -2631,42 +2691,148 @@ Number Object Structures Python 3.0.1. .. c:member:: binaryfunc PyNumberMethods.nb_add + + .. corresponding-type-slot:: Py_nb_add + .. c:member:: binaryfunc PyNumberMethods.nb_subtract + + .. corresponding-type-slot:: Py_nb_subtract + .. c:member:: binaryfunc PyNumberMethods.nb_multiply + + .. corresponding-type-slot:: Py_nb_multiply + .. c:member:: binaryfunc PyNumberMethods.nb_remainder + + .. corresponding-type-slot:: Py_nb_remainder + .. c:member:: binaryfunc PyNumberMethods.nb_divmod + + .. corresponding-type-slot:: Py_nb_divmod + .. c:member:: ternaryfunc PyNumberMethods.nb_power + + .. corresponding-type-slot:: Py_nb_power + .. c:member:: unaryfunc PyNumberMethods.nb_negative + + .. corresponding-type-slot:: Py_nb_negative + .. c:member:: unaryfunc PyNumberMethods.nb_positive + + .. corresponding-type-slot:: Py_nb_positive + .. c:member:: unaryfunc PyNumberMethods.nb_absolute + + .. corresponding-type-slot:: Py_nb_absolute + .. c:member:: inquiry PyNumberMethods.nb_bool + + .. corresponding-type-slot:: Py_nb_bool + .. c:member:: unaryfunc PyNumberMethods.nb_invert + + .. corresponding-type-slot:: Py_nb_invert + .. c:member:: binaryfunc PyNumberMethods.nb_lshift + + .. corresponding-type-slot:: Py_nb_lshift + .. c:member:: binaryfunc PyNumberMethods.nb_rshift + + .. corresponding-type-slot:: Py_nb_rshift + .. c:member:: binaryfunc PyNumberMethods.nb_and + + .. corresponding-type-slot:: Py_nb_and + .. c:member:: binaryfunc PyNumberMethods.nb_xor + + .. corresponding-type-slot:: Py_nb_xor + .. c:member:: binaryfunc PyNumberMethods.nb_or + + .. corresponding-type-slot:: Py_nb_or + .. c:member:: unaryfunc PyNumberMethods.nb_int + + .. corresponding-type-slot:: Py_nb_int + .. c:member:: void *PyNumberMethods.nb_reserved + .. c:member:: unaryfunc PyNumberMethods.nb_float + + .. corresponding-type-slot:: Py_nb_float + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_add + + .. corresponding-type-slot:: Py_nb_inplace_add + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_subtract + + .. corresponding-type-slot:: Py_nb_inplace_subtract + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_multiply + + .. corresponding-type-slot:: Py_nb_inplace_multiply + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_remainder + + .. corresponding-type-slot:: Py_nb_inplace_remainder + .. c:member:: ternaryfunc PyNumberMethods.nb_inplace_power + + .. corresponding-type-slot:: Py_nb_inplace_power + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_lshift + + .. corresponding-type-slot:: Py_nb_inplace_lshift + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_rshift + + .. corresponding-type-slot:: Py_nb_inplace_rshift + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_and + + .. corresponding-type-slot:: Py_nb_inplace_and + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_xor + + .. corresponding-type-slot:: Py_nb_inplace_xor + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_or + + .. corresponding-type-slot:: Py_nb_inplace_or + .. c:member:: binaryfunc PyNumberMethods.nb_floor_divide + + .. corresponding-type-slot:: Py_nb_floor_divide + .. c:member:: binaryfunc PyNumberMethods.nb_true_divide + + .. corresponding-type-slot:: Py_nb_true_divide + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_floor_divide + + .. corresponding-type-slot:: Py_nb_inplace_floor_divide + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_true_divide + + .. corresponding-type-slot:: Py_nb_inplace_true_divide + .. c:member:: unaryfunc PyNumberMethods.nb_index + + .. corresponding-type-slot:: Py_nb_index + .. c:member:: binaryfunc PyNumberMethods.nb_matrix_multiply + + .. corresponding-type-slot:: Py_nb_matrix_multiply + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_matrix_multiply + .. corresponding-type-slot:: Py_nb_inplace_matrix_multiply + + .. _mapping-structs: @@ -2683,12 +2849,16 @@ Mapping Object Structures .. c:member:: lenfunc PyMappingMethods.mp_length + .. corresponding-type-slot:: Py_mp_length + This function is used by :c:func:`PyMapping_Size` and :c:func:`PyObject_Size`, and has the same signature. This slot may be set to ``NULL`` if the object has no defined length. .. c:member:: binaryfunc PyMappingMethods.mp_subscript + .. corresponding-type-slot:: Py_mp_subscript + This function is used by :c:func:`PyObject_GetItem` and :c:func:`PySequence_GetSlice`, and has the same signature as :c:func:`!PyObject_GetItem`. This slot must be filled for the @@ -2697,6 +2867,8 @@ Mapping Object Structures .. c:member:: objobjargproc PyMappingMethods.mp_ass_subscript + .. corresponding-type-slot:: Py_mp_ass_subscript + This function is used by :c:func:`PyObject_SetItem`, :c:func:`PyObject_DelItem`, :c:func:`PySequence_SetSlice` and :c:func:`PySequence_DelSlice`. It has the same signature as @@ -2720,6 +2892,8 @@ Sequence Object Structures .. c:member:: lenfunc PySequenceMethods.sq_length + .. corresponding-type-slot:: Py_sq_length + This function is used by :c:func:`PySequence_Size` and :c:func:`PyObject_Size`, and has the same signature. It is also used for handling negative indices via the :c:member:`~PySequenceMethods.sq_item` @@ -2727,18 +2901,24 @@ Sequence Object Structures .. c:member:: binaryfunc PySequenceMethods.sq_concat + .. corresponding-type-slot:: Py_sq_concat + This function is used by :c:func:`PySequence_Concat` and has the same signature. It is also used by the ``+`` operator, after trying the numeric addition via the :c:member:`~PyNumberMethods.nb_add` slot. .. c:member:: ssizeargfunc PySequenceMethods.sq_repeat + .. corresponding-type-slot:: Py_sq_repeat + This function is used by :c:func:`PySequence_Repeat` and has the same signature. It is also used by the ``*`` operator, after trying numeric multiplication via the :c:member:`~PyNumberMethods.nb_multiply` slot. .. c:member:: ssizeargfunc PySequenceMethods.sq_item + .. corresponding-type-slot:: Py_sq_item + This function is used by :c:func:`PySequence_GetItem` and has the same signature. It is also used by :c:func:`PyObject_GetItem`, after trying the subscription via the :c:member:`~PyMappingMethods.mp_subscript` slot. @@ -2752,6 +2932,8 @@ Sequence Object Structures .. c:member:: ssizeobjargproc PySequenceMethods.sq_ass_item + .. corresponding-type-slot:: Py_sq_ass_item + This function is used by :c:func:`PySequence_SetItem` and has the same signature. It is also used by :c:func:`PyObject_SetItem` and :c:func:`PyObject_DelItem`, after trying the item assignment and deletion @@ -2761,6 +2943,8 @@ Sequence Object Structures .. c:member:: objobjproc PySequenceMethods.sq_contains + .. corresponding-type-slot:: Py_sq_contains + This function may be used by :c:func:`PySequence_Contains` and has the same signature. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_Contains` simply traverses the sequence until it @@ -2768,6 +2952,8 @@ Sequence Object Structures .. c:member:: binaryfunc PySequenceMethods.sq_inplace_concat + .. corresponding-type-slot:: Py_sq_inplace_concat + This function is used by :c:func:`PySequence_InPlaceConcat` and has the same signature. It should modify its first operand, and return it. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_InPlaceConcat` @@ -2777,6 +2963,8 @@ Sequence Object Structures .. c:member:: ssizeargfunc PySequenceMethods.sq_inplace_repeat + .. corresponding-type-slot:: Py_sq_inplace_repeat + This function is used by :c:func:`PySequence_InPlaceRepeat` and has the same signature. It should modify its first operand, and return it. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_InPlaceRepeat` @@ -2802,6 +2990,8 @@ Buffer Object Structures .. c:member:: getbufferproc PyBufferProcs.bf_getbuffer + .. corresponding-type-slot:: Py_bf_getbuffer + The signature of this function is:: int (PyObject *exporter, Py_buffer *view, int flags); @@ -2851,6 +3041,8 @@ Buffer Object Structures .. c:member:: releasebufferproc PyBufferProcs.bf_releasebuffer + .. corresponding-type-slot:: Py_bf_releasebuffer + The signature of this function is:: void (PyObject *exporter, Py_buffer *view); @@ -2905,6 +3097,8 @@ Async Object Structures .. c:member:: unaryfunc PyAsyncMethods.am_await + .. corresponding-type-slot:: Py_am_await + The signature of this function is:: PyObject *am_await(PyObject *self); @@ -2916,6 +3110,8 @@ Async Object Structures .. c:member:: unaryfunc PyAsyncMethods.am_aiter + .. corresponding-type-slot:: Py_am_aiter + The signature of this function is:: PyObject *am_aiter(PyObject *self); @@ -2928,6 +3124,8 @@ Async Object Structures .. c:member:: unaryfunc PyAsyncMethods.am_anext + .. corresponding-type-slot:: Py_am_anext + The signature of this function is:: PyObject *am_anext(PyObject *self); @@ -2938,6 +3136,8 @@ Async Object Structures .. c:member:: sendfunc PyAsyncMethods.am_send + .. corresponding-type-slot:: Py_am_send + The signature of this function is:: PySendResult am_send(PyObject *self, PyObject *arg, PyObject **result); diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 7ad5f3ecfab..67b498c4268 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -1,7 +1,21 @@ role,name,added,ifdef_note,struct_abi_kind +macro,METH_CLASS,3.2,, +macro,METH_COEXIST,3.2,, +macro,METH_FASTCALL,3.7,, +macro,METH_METHOD,3.7,, +macro,METH_NOARGS,3.2,, +macro,METH_O,3.2,, +macro,METH_STATIC,3.2,, +macro,METH_VARARGS,3.2,, macro,PY_VECTORCALL_ARGUMENTS_OFFSET,3.12,, type,PyABIInfo,3.15,,full-abi func,PyABIInfo_Check,3.15,, +macro,PyABIInfo_DEFAULT_ABI_VERSION,3.15,, +macro,PyABIInfo_DEFAULT_FLAGS,3.15,, +macro,PyABIInfo_FREETHREADED,3.15,, +macro,PyABIInfo_FREETHREADING_AGNOSTIC,3.15,, +macro,PyABIInfo_GIL,3.15,, +macro,PyABIInfo_STABLE,3.15,, macro,PyABIInfo_VAR,3.15,, func,PyAIter_Check,3.10,, func,PyArg_Parse,3.2,, @@ -11,6 +25,26 @@ func,PyArg_UnpackTuple,3.2,, func,PyArg_VaParse,3.2,, func,PyArg_VaParseTupleAndKeywords,3.2,, func,PyArg_ValidateKeywordArguments,3.2,, +macro,PyBUF_ANY_CONTIGUOUS,3.11,, +macro,PyBUF_CONTIG,3.11,, +macro,PyBUF_CONTIG_RO,3.11,, +macro,PyBUF_C_CONTIGUOUS,3.11,, +macro,PyBUF_FORMAT,3.11,, +macro,PyBUF_FULL,3.11,, +macro,PyBUF_FULL_RO,3.11,, +macro,PyBUF_F_CONTIGUOUS,3.11,, +macro,PyBUF_INDIRECT,3.11,, +macro,PyBUF_MAX_NDIM,3.11,, +macro,PyBUF_ND,3.11,, +macro,PyBUF_READ,3.11,, +macro,PyBUF_RECORDS,3.11,, +macro,PyBUF_RECORDS_RO,3.11,, +macro,PyBUF_SIMPLE,3.11,, +macro,PyBUF_STRIDED,3.11,, +macro,PyBUF_STRIDED_RO,3.11,, +macro,PyBUF_STRIDES,3.11,, +macro,PyBUF_WRITABLE,3.11,, +macro,PyBUF_WRITE,3.11,, data,PyBaseObject_Type,3.2,, func,PyBool_FromLong,3.2,, data,PyBool_Type,3.2,, @@ -836,6 +870,14 @@ func,PyWeakref_NewRef,3.2,, data,PyWrapperDescr_Type,3.2,, func,PyWrapper_New,3.2,, data,PyZip_Type,3.2,, +macro,Py_ASNATIVEBYTES_ALLOW_INDEX,3.14,, +macro,Py_ASNATIVEBYTES_BIG_ENDIAN,3.14,, +macro,Py_ASNATIVEBYTES_DEFAULTS,3.14,, +macro,Py_ASNATIVEBYTES_LITTLE_ENDIAN,3.14,, +macro,Py_ASNATIVEBYTES_NATIVE_ENDIAN,3.14,, +macro,Py_ASNATIVEBYTES_REJECT_NEGATIVE,3.14,, +macro,Py_ASNATIVEBYTES_UNSIGNED_BUFFER,3.14,, +macro,Py_AUDIT_READ,3.12,, func,Py_AddPendingCall,3.2,, func,Py_AtExit,3.2,, macro,Py_BEGIN_ALLOW_THREADS,3.2,, @@ -882,22 +924,133 @@ func,Py_NewInterpreter,3.2,, func,Py_NewRef,3.10,, func,Py_PACK_FULL_VERSION,3.14,, func,Py_PACK_VERSION,3.14,, +macro,Py_READONLY,3.12,, func,Py_REFCNT,3.14,, +macro,Py_RELATIVE_OFFSET,3.12,, func,Py_ReprEnter,3.2,, func,Py_ReprLeave,3.2,, func,Py_SetProgramName,3.2,, func,Py_SetPythonHome,3.2,, func,Py_SetRecursionLimit,3.2,, +macro,Py_TPFLAGS_BASETYPE,3.2,, +macro,Py_TPFLAGS_DEFAULT,3.2,, +macro,Py_TPFLAGS_HAVE_GC,3.2,, +macro,Py_TPFLAGS_HAVE_VECTORCALL,3.12,, +macro,Py_TPFLAGS_ITEMS_AT_END,3.12,, +macro,Py_TPFLAGS_METHOD_DESCRIPTOR,3.8,, +macro,Py_TP_USE_SPEC,3.14,, func,Py_TYPE,3.14,, +macro,Py_T_BOOL,3.12,, +macro,Py_T_BYTE,3.12,, +macro,Py_T_CHAR,3.12,, +macro,Py_T_DOUBLE,3.12,, +macro,Py_T_FLOAT,3.12,, +macro,Py_T_INT,3.12,, +macro,Py_T_LONG,3.12,, +macro,Py_T_LONGLONG,3.12,, +macro,Py_T_OBJECT_EX,3.12,, +macro,Py_T_PYSSIZET,3.12,, +macro,Py_T_SHORT,3.12,, +macro,Py_T_STRING,3.12,, +macro,Py_T_STRING_INPLACE,3.12,, +macro,Py_T_UBYTE,3.12,, +macro,Py_T_UINT,3.12,, +macro,Py_T_ULONG,3.12,, +macro,Py_T_ULONGLONG,3.12,, +macro,Py_T_USHORT,3.12,, type,Py_UCS4,3.2,, macro,Py_UNBLOCK_THREADS,3.2,, data,Py_UTF8Mode,3.8,, func,Py_VaBuildValue,3.2,, data,Py_Version,3.11,, func,Py_XNewRef,3.10,, +macro,Py_am_aiter,3.5,, +macro,Py_am_anext,3.5,, +macro,Py_am_await,3.5,, +macro,Py_am_send,3.10,, +macro,Py_bf_getbuffer,3.11,, +macro,Py_bf_releasebuffer,3.11,, type,Py_buffer,3.11,,full-abi type,Py_intptr_t,3.2,, +macro,Py_mod_abi,3.15,, +macro,Py_mp_ass_subscript,3.2,, +macro,Py_mp_length,3.2,, +macro,Py_mp_subscript,3.2,, +macro,Py_nb_absolute,3.2,, +macro,Py_nb_add,3.2,, +macro,Py_nb_and,3.2,, +macro,Py_nb_bool,3.2,, +macro,Py_nb_divmod,3.2,, +macro,Py_nb_float,3.2,, +macro,Py_nb_floor_divide,3.2,, +macro,Py_nb_index,3.2,, +macro,Py_nb_inplace_add,3.2,, +macro,Py_nb_inplace_and,3.2,, +macro,Py_nb_inplace_floor_divide,3.2,, +macro,Py_nb_inplace_lshift,3.2,, +macro,Py_nb_inplace_matrix_multiply,3.5,, +macro,Py_nb_inplace_multiply,3.2,, +macro,Py_nb_inplace_or,3.2,, +macro,Py_nb_inplace_power,3.2,, +macro,Py_nb_inplace_remainder,3.2,, +macro,Py_nb_inplace_rshift,3.2,, +macro,Py_nb_inplace_subtract,3.2,, +macro,Py_nb_inplace_true_divide,3.2,, +macro,Py_nb_inplace_xor,3.2,, +macro,Py_nb_int,3.2,, +macro,Py_nb_invert,3.2,, +macro,Py_nb_lshift,3.2,, +macro,Py_nb_matrix_multiply,3.5,, +macro,Py_nb_multiply,3.2,, +macro,Py_nb_negative,3.2,, +macro,Py_nb_or,3.2,, +macro,Py_nb_positive,3.2,, +macro,Py_nb_power,3.2,, +macro,Py_nb_remainder,3.2,, +macro,Py_nb_rshift,3.2,, +macro,Py_nb_subtract,3.2,, +macro,Py_nb_true_divide,3.2,, +macro,Py_nb_xor,3.2,, +macro,Py_sq_ass_item,3.2,, +macro,Py_sq_concat,3.2,, +macro,Py_sq_contains,3.2,, +macro,Py_sq_inplace_concat,3.2,, +macro,Py_sq_inplace_repeat,3.2,, +macro,Py_sq_item,3.2,, +macro,Py_sq_length,3.2,, +macro,Py_sq_repeat,3.2,, type,Py_ssize_t,3.2,, +macro,Py_tp_alloc,3.2,, +macro,Py_tp_base,3.2,, +macro,Py_tp_bases,3.2,, +macro,Py_tp_call,3.2,, +macro,Py_tp_clear,3.2,, +macro,Py_tp_dealloc,3.2,, +macro,Py_tp_del,3.2,, +macro,Py_tp_descr_get,3.2,, +macro,Py_tp_descr_set,3.2,, +macro,Py_tp_doc,3.2,, +macro,Py_tp_finalize,3.5,, +macro,Py_tp_free,3.2,, +macro,Py_tp_getattr,3.2,, +macro,Py_tp_getattro,3.2,, +macro,Py_tp_getset,3.2,, +macro,Py_tp_hash,3.2,, +macro,Py_tp_init,3.2,, +macro,Py_tp_is_gc,3.2,, +macro,Py_tp_iter,3.2,, +macro,Py_tp_iternext,3.2,, +macro,Py_tp_members,3.2,, +macro,Py_tp_methods,3.2,, +macro,Py_tp_new,3.2,, +macro,Py_tp_repr,3.2,, +macro,Py_tp_richcompare,3.2,, +macro,Py_tp_setattr,3.2,, +macro,Py_tp_setattro,3.2,, +macro,Py_tp_str,3.2,, +macro,Py_tp_token,3.14,, +macro,Py_tp_traverse,3.2,, +macro,Py_tp_vectorcall,3.14,, type,Py_uintptr_t,3.2,, type,allocfunc,3.2,, type,binaryfunc,3.2,, diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 6fee1c192c3..04e8e5580fc 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -8,7 +8,6 @@ Doc/c-api/init_config.rst Doc/c-api/intro.rst Doc/c-api/module.rst Doc/c-api/stable.rst -Doc/c-api/type.rst Doc/c-api/typeobj.rst Doc/library/ast.rst Doc/library/asyncio-extending.rst diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index 089614a1f6c..e04a5f144c4 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -154,7 +154,10 @@ def add_annotations(app: Sphinx, doctree: nodes.document) -> None: node.insert(0, annotation) -def _stable_abi_annotation(record: StableABIEntry) -> nodes.emphasis: +def _stable_abi_annotation( + record: StableABIEntry, + is_corresponding_slot: bool = False, +) -> nodes.emphasis: """Create the Stable ABI annotation. These have two forms: @@ -168,9 +171,28 @@ def _stable_abi_annotation(record: StableABIEntry) -> nodes.emphasis: ... all of which can have "since version X.Y" appended. """ stable_added = record.added - message = sphinx_gettext("Part of the") - message = message.center(len(message) + 2) - emph_node = nodes.emphasis(message, message, classes=["stableabi"]) + emph_node = nodes.emphasis('', '', classes=["stableabi"]) + if is_corresponding_slot: + # See "Type slot annotations" in add_annotations + ref_node = addnodes.pending_xref( + "slot ID", + refdomain="c", + reftarget="PyType_Slot", + reftype="type", + refexplicit="True", + ) + ref_node += nodes.Text(sphinx_gettext("slot ID")) + + message = sphinx_gettext("The corresponding") + emph_node += nodes.Text(" " + message + " ") + emph_node += ref_node + emph_node += nodes.Text(" ") + emph_node += nodes.literal(record.name, record.name) + message = sphinx_gettext("is part of the") + emph_node += nodes.Text(" " + message + " ") + else: + message = sphinx_gettext("Part of the") + emph_node += nodes.Text(" " + message + " ") ref_node = addnodes.pending_xref( "Stable ABI", refdomain="std", @@ -265,6 +287,51 @@ def run(self) -> list[nodes.Node]: return [node] +class CorrespondingTypeSlot(SphinxDirective): + """Type slot annotations + + Docs for these are with the corresponding field, for example, + "Py_tp_repr" is documented under "PyTypeObject.tp_repr", with + only a stable ABI note mentioning "Py_tp_repr" (and linking to + docs on how this works). + + If there is no corresponding field, these should be documented as normal + macros. + """ + + has_content = False + + required_arguments = 1 + optional_arguments = 0 + + def run(self) -> list[nodes.Node]: + name = self.arguments[0] + state = self.env.domaindata["c_annotations"] + stable_abi_data = state["stable_abi_data"] + + try: + record = stable_abi_data[name] + except LookupError as err: + raise LookupError( + f"{name} is not part of stable ABI. " + + "Document it as `c:macro::` rather than " + + "`corresponding-type-slot::`." + ) from err + + annotation = _stable_abi_annotation(record, is_corresponding_slot=True) + + node = nodes.paragraph() + content = [ + ".. c:namespace:: NULL", + "", + ".. c:macro:: " + name, + " :no-typesetting:", + ] + self.state.nested_parse(StringList(content), 0, node) + node.insert(0, annotation) + return [node] + + def init_annotations(app: Sphinx) -> None: # Using domaindata is a bit hack-ish, # but allows storing state without a global variable or closure. @@ -281,6 +348,7 @@ def setup(app: Sphinx) -> ExtensionMetadata: app.add_config_value("refcount_file", "", "env", types={str}) app.add_config_value("stable_abi_file", "", "env", types={str}) app.add_directive("limited-api-list", LimitedAPIList) + app.add_directive("corresponding-type-slot", CorrespondingTypeSlot) app.connect("builder-inited", init_annotations) app.connect("doctree-read", add_annotations) diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 4a03cc76f5e..ad0f3704599 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2306,6 +2306,10 @@ added = '3.11' [function.PyMemoryView_FromBuffer] added = '3.11' +[const.Py_bf_getbuffer] + added = '3.11' +[const.Py_bf_releasebuffer] + added = '3.11' # Constants for Py_buffer API added to this list in Python 3.11.1 (https://github.com/python/cpython/issues/98680) # (they were available with 3.11.0) diff --git a/Tools/build/stable_abi.py b/Tools/build/stable_abi.py index 1ddd76cdd9b..39115b331ba 100644 --- a/Tools/build/stable_abi.py +++ b/Tools/build/stable_abi.py @@ -232,7 +232,7 @@ def sort_key(item): 'data': 'data', 'struct': 'type', 'macro': 'macro', - # 'const': 'const', # all undocumented + 'const': 'macro', 'typedef': 'type', } From d5b00c74b30a159d8d8c03e152c6cf8e8b1431b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 4 Nov 2025 23:29:15 +0100 Subject: [PATCH 031/313] gh-140454: Normalize the JIT stencils filename on Linux to avoid mismatches between the Makefile and the generator (#140823) --- .../next/Build/2025-10-31-13-20-16.gh-issue-140454.gF6dCe.rst | 3 +++ configure | 4 ++-- configure.ac | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2025-10-31-13-20-16.gh-issue-140454.gF6dCe.rst diff --git a/Misc/NEWS.d/next/Build/2025-10-31-13-20-16.gh-issue-140454.gF6dCe.rst b/Misc/NEWS.d/next/Build/2025-10-31-13-20-16.gh-issue-140454.gF6dCe.rst new file mode 100644 index 00000000000..4bb132ce01e --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-10-31-13-20-16.gh-issue-140454.gF6dCe.rst @@ -0,0 +1,3 @@ +When building the JIT, match the jit_stencils filename expectations in +Makefile with the generator script. This avoid needless JIT recompilation +during ``make install``. diff --git a/configure b/configure index 60521492755..8463b5b5e4a 100755 --- a/configure +++ b/configure @@ -34327,10 +34327,10 @@ else case e in #( JIT_STENCILS_H="jit_stencils-x86_64-pc-windows-msvc.h" ;; aarch64-*-linux-gnu) - JIT_STENCILS_H="jit_stencils-$host.h" + JIT_STENCILS_H="jit_stencils-aarch64-unknown-linux-gnu.h" ;; x86_64-*-linux-gnu) - JIT_STENCILS_H="jit_stencils-$host.h" + JIT_STENCILS_H="jit_stencils-x86_64-unknown-linux-gnu.h" ;; esac ;; esac diff --git a/configure.ac b/configure.ac index 135492d82e0..df94ae25e63 100644 --- a/configure.ac +++ b/configure.ac @@ -8219,10 +8219,10 @@ AS_VAR_IF([enable_experimental_jit], [no], JIT_STENCILS_H="jit_stencils-x86_64-pc-windows-msvc.h" ;; aarch64-*-linux-gnu) - JIT_STENCILS_H="jit_stencils-$host.h" + JIT_STENCILS_H="jit_stencils-aarch64-unknown-linux-gnu.h" ;; x86_64-*-linux-gnu) - JIT_STENCILS_H="jit_stencils-$host.h" + JIT_STENCILS_H="jit_stencils-x86_64-unknown-linux-gnu.h" ;; esac]) From fa02422918ac3251cdf88a626f90af260bf5224a Mon Sep 17 00:00:00 2001 From: alex <30386655+alexomics@users.noreply.github.com> Date: Wed, 5 Nov 2025 00:05:49 +0000 Subject: [PATCH 032/313] gh-141007: update string module source code link (#141008) In 3.14, the former string.py became `__init__.py` within a new `string` directory that also contains a new submodule file, `templatelib.py`. --- Doc/library/string.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 6336a0ec47b..58c836c7382 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -4,7 +4,7 @@ .. module:: string :synopsis: Common string operations. -**Source code:** :source:`Lib/string.py` +**Source code:** :source:`Lib/string/__init__.py` -------------- From 1ae900424b3c888d2b2cc97e6ef780717813d658 Mon Sep 17 00:00:00 2001 From: Clifford Gama Date: Wed, 5 Nov 2025 02:29:25 +0200 Subject: [PATCH 033/313] Docs: Fix cached calls count in factorial example (gh-140882) --- Doc/library/functools.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 37d9f87e779..1d9ac328f32 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -42,11 +42,11 @@ The :mod:`functools` module defines the following functions: def factorial(n): return n * factorial(n-1) if n else 1 - >>> factorial(10) # no previously cached result, makes 11 recursive calls + >>> factorial(10) # no previously cached result, makes 11 recursive calls 3628800 - >>> factorial(5) # just looks up cached value result + >>> factorial(5) # no new calls, just returns the cached result 120 - >>> factorial(12) # makes two new recursive calls, the other 10 are cached + >>> factorial(12) # two new recursive calls, factorial(10) is cached 479001600 The cache is threadsafe so that the wrapped function can be used in From 335d83ec0492779b7fbf2293690f06971cc9d04a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=AA=20Nam=20Kh=C3=A1nh?= <55955273+khanhkhanhlele@users.noreply.github.com> Date: Wed, 5 Nov 2025 14:23:25 +0700 Subject: [PATCH 034/313] Fix typo in Apple/__main__.py (#141038) Corrected a typo in a return value docstring. --- Apple/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Apple/__main__.py b/Apple/__main__.py index 34744871f68..e76fc351798 100644 --- a/Apple/__main__.py +++ b/Apple/__main__.py @@ -507,7 +507,7 @@ def lib_non_platform_files(dirname, names): def create_xcframework(platform: str) -> str: """Build an XCframework from the component parts for the platform. - :return: The version number of the Python verion that was packaged. + :return: The version number of the Python version that was packaged. """ package_path = CROSS_BUILD_DIR / platform try: From f2bce51b984f52db14d90f7bbd0b7df00b7c5637 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 5 Nov 2025 11:52:11 +0100 Subject: [PATCH 035/313] gh-140691: urllib.request: Close FTP control socket if data socket can't connect (GH-140835) Co-authored-by: codenamenam --- Lib/_py_warnings.py | 3 + Lib/test/test_urllib2net.py | 59 +++++++++++++++---- Lib/urllib/request.py | 24 +++++--- ...-10-31-15-06-26.gh-issue-140691.JzHGtg.rst | 3 + 4 files changed, 70 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-31-15-06-26.gh-issue-140691.JzHGtg.rst diff --git a/Lib/_py_warnings.py b/Lib/_py_warnings.py index 91a9f44b201..67c74fdd2d0 100644 --- a/Lib/_py_warnings.py +++ b/Lib/_py_warnings.py @@ -646,6 +646,9 @@ def __str__(self): "line : %r}" % (self.message, self._category_name, self.filename, self.lineno, self.line)) + def __repr__(self): + return f'<{type(self).__qualname__} {self}>' + class catch_warnings(object): diff --git a/Lib/test/test_urllib2net.py b/Lib/test/test_urllib2net.py index 0c5f99ec18b..17db686942f 100644 --- a/Lib/test/test_urllib2net.py +++ b/Lib/test/test_urllib2net.py @@ -1,9 +1,13 @@ +import contextlib import errno +import sysconfig import unittest +from unittest import mock from test import support from test.support import os_helper from test.support import socket_helper from test.support import ResourceDenied +from test.support.warnings_helper import check_no_resource_warning import os import socket @@ -143,6 +147,43 @@ def test_ftp(self): ] self._test_urls(urls, self._extra_handlers()) + @support.requires_resource('walltime') + @unittest.skipIf(sysconfig.get_platform() == 'linux-ppc64le', + 'leaks on PPC64LE (gh-140691)') + def test_ftp_no_leak(self): + # gh-140691: When the data connection (but not control connection) + # cannot be made established, we shouldn't leave an open socket object. + + class MockError(OSError): + pass + + orig_create_connection = socket.create_connection + def patched_create_connection(address, *args, **kwargs): + """Simulate REJECTing connections to ports other than 21""" + host, port = address + if port != 21: + raise MockError() + return orig_create_connection(address, *args, **kwargs) + + url = 'ftp://www.pythontest.net/README' + entry = url, None, urllib.error.URLError + no_cache_handlers = [urllib.request.FTPHandler()] + cache_handlers = self._extra_handlers() + with mock.patch('socket.create_connection', patched_create_connection): + with check_no_resource_warning(self): + # Try without CacheFTPHandler + self._test_urls([entry], handlers=no_cache_handlers, + retry=False) + with check_no_resource_warning(self): + # Try with CacheFTPHandler (uncached) + self._test_urls([entry], cache_handlers, retry=False) + with check_no_resource_warning(self): + # Try with CacheFTPHandler (cached) + self._test_urls([entry], cache_handlers, retry=False) + # Try without the mock: the handler should not use a closed connection + with check_no_resource_warning(self): + self._test_urls([url], cache_handlers, retry=False) + def test_file(self): TESTFN = os_helper.TESTFN f = open(TESTFN, 'w') @@ -234,18 +275,16 @@ def _test_urls(self, urls, handlers, retry=True): else: req = expected_err = None + if expected_err: + context = self.assertRaises(expected_err) + else: + context = contextlib.nullcontext() + with socket_helper.transient_internet(url): - try: + f = None + with context: f = urlopen(url, req, support.INTERNET_TIMEOUT) - # urllib.error.URLError is a subclass of OSError - except OSError as err: - if expected_err: - msg = ("Didn't get expected error(s) %s for %s %s, got %s: %s" % - (expected_err, url, req, type(err), err)) - self.assertIsInstance(err, expected_err, msg) - else: - raise - else: + if f is not None: try: with time_out, \ socket_peer_reset, \ diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index af93d4cd75d..566b8087aec 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1535,6 +1535,7 @@ def ftp_open(self, req): dirs, file = dirs[:-1], dirs[-1] if dirs and not dirs[0]: dirs = dirs[1:] + fw = None try: fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout) type = file and 'I' or 'D' @@ -1552,8 +1553,12 @@ def ftp_open(self, req): headers += "Content-length: %d\n" % retrlen headers = email.message_from_string(headers) return addinfourl(fp, headers, req.full_url) - except ftplib.all_errors as exp: - raise URLError(f"ftp error: {exp}") from exp + except Exception as exp: + if fw is not None and not fw.keepalive: + fw.close() + if isinstance(exp, ftplib.all_errors): + raise URLError(f"ftp error: {exp}") from exp + raise def connect_ftp(self, user, passwd, host, port, dirs, timeout): return ftpwrapper(user, passwd, host, port, dirs, timeout, @@ -1577,14 +1582,15 @@ def setMaxConns(self, m): def connect_ftp(self, user, passwd, host, port, dirs, timeout): key = user, host, port, '/'.join(dirs), timeout - if key in self.cache: - self.timeout[key] = time.time() + self.delay - else: - self.cache[key] = ftpwrapper(user, passwd, host, port, - dirs, timeout) - self.timeout[key] = time.time() + self.delay + conn = self.cache.get(key) + if conn is None or not conn.keepalive: + if conn is not None: + conn.close() + conn = self.cache[key] = ftpwrapper(user, passwd, host, port, + dirs, timeout) + self.timeout[key] = time.time() + self.delay self.check_cache() - return self.cache[key] + return conn def check_cache(self): # first check for old ones diff --git a/Misc/NEWS.d/next/Library/2025-10-31-15-06-26.gh-issue-140691.JzHGtg.rst b/Misc/NEWS.d/next/Library/2025-10-31-15-06-26.gh-issue-140691.JzHGtg.rst new file mode 100644 index 00000000000..84b6195c926 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-31-15-06-26.gh-issue-140691.JzHGtg.rst @@ -0,0 +1,3 @@ +In :mod:`urllib.request`, when opening a FTP URL fails because a data +connection cannot be made, the control connection's socket is now closed to +avoid a :exc:`ResourceWarning`. From 589a03a8ce60cc65f91930f7d63367b03cfbbb12 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 5 Nov 2025 12:31:42 +0100 Subject: [PATCH 036/313] =?UTF-8?q?gh-140550:=20Initial=20implementation?= =?UTF-8?q?=20of=20PEP=20793=20=E2=80=93=20PyModExport=20(GH-140556)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Victor Stinner Co-authored-by: Kumar Aditya --- Doc/data/stable_abi.dat | 6 + Include/exports.h | 17 +- Include/internal/pycore_importdl.h | 16 +- Include/internal/pycore_moduleobject.h | 36 +- Include/moduleobject.h | 17 +- Include/object.h | 5 + Lib/test/test_capi/test_module.py | 185 +++++++++ Lib/test/test_capi/test_type.py | 18 + Lib/test/test_cext/__init__.py | 1 - Lib/test/test_cext/create_moduledef.c | 29 -- Lib/test/test_cext/extension.c | 54 +-- Lib/test/test_cext/setup.py | 1 - Lib/test/test_import/__init__.py | 92 +++++ Lib/test/test_stable_abi_ctypes.py | 5 + Lib/test/test_sys.py | 5 +- ...-10-26-16-45-28.gh-issue-140556.s__Dae.rst | 2 + Misc/stable_abi.toml | 28 ++ Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/heaptype.c | 16 + Modules/_testcapi/module.c | 378 ++++++++++++++++++ Modules/_testcapi/parts.h | 1 + Modules/_testcapimodule.c | 3 + Modules/_testinternalcapi.c | 29 ++ Modules/_testmultiphase.c | 175 ++++++++ Modules/_testsinglephase.c | 2 + Objects/moduleobject.c | 361 +++++++++++++---- Objects/typeobject.c | 24 +- PC/python3dll.c | 5 + PCbuild/_testcapi.vcxproj | 1 + PCbuild/_testcapi.vcxproj.filters | 3 + Python/import.c | 113 ++++-- Python/importdl.c | 100 +++-- 32 files changed, 1494 insertions(+), 236 deletions(-) create mode 100644 Lib/test/test_capi/test_module.py delete mode 100644 Lib/test/test_cext/create_moduledef.c create mode 100644 Misc/NEWS.d/next/C_API/2025-10-26-16-45-28.gh-issue-140556.s__Dae.rst create mode 100644 Modules/_testcapi/module.c diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 67b498c4268..1359cfa4fbf 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -425,6 +425,7 @@ func,PyLong_FromUnsignedNativeBytes,3.14,, func,PyLong_FromVoidPtr,3.2,, func,PyLong_GetInfo,3.2,, data,PyLong_Type,3.2,, +macro,PyMODEXPORT_FUNC,3.15,, data,PyMap_Type,3.2,, func,PyMapping_Check,3.2,, func,PyMapping_GetItemString,3.2,, @@ -471,8 +472,10 @@ func,PyModule_AddObjectRef,3.10,, func,PyModule_AddStringConstant,3.2,, func,PyModule_AddType,3.10,, func,PyModule_Create2,3.2,, +func,PyModule_Exec,3.15,, func,PyModule_ExecDef,3.7,, func,PyModule_FromDefAndSpec2,3.7,, +func,PyModule_FromSlotsAndSpec,3.15,, func,PyModule_GetDef,3.2,, func,PyModule_GetDict,3.2,, func,PyModule_GetFilename,3.2,, @@ -480,6 +483,8 @@ func,PyModule_GetFilenameObject,3.2,, func,PyModule_GetName,3.2,, func,PyModule_GetNameObject,3.7,, func,PyModule_GetState,3.2,, +func,PyModule_GetStateSize,3.15,, +func,PyModule_GetToken,3.15,, func,PyModule_New,3.2,, func,PyModule_NewObject,3.7,, func,PyModule_SetDocString,3.7,, @@ -738,6 +743,7 @@ func,PyType_GetFlags,3.2,, func,PyType_GetFullyQualifiedName,3.13,, func,PyType_GetModule,3.10,, func,PyType_GetModuleByDef,3.13,, +func,PyType_GetModuleByToken,3.15,, func,PyType_GetModuleName,3.13,, func,PyType_GetModuleState,3.10,, func,PyType_GetName,3.11,, diff --git a/Include/exports.h b/Include/exports.h index 0c646d5beb6..62feb09ed2b 100644 --- a/Include/exports.h +++ b/Include/exports.h @@ -9,6 +9,7 @@ inside the Python core, they are private to the core. If in an extension module, it may be declared with external linkage depending on the platform. + PyMODEXPORT_FUNC: Like PyMODINIT_FUNC, but for a slots array As a number of platforms support/require "__declspec(dllimport/dllexport)", we support a HAVE_DECLSPEC_DLL macro to save duplication. @@ -62,9 +63,9 @@ /* module init functions inside the core need no external linkage */ /* except for Cygwin to handle embedding */ # if defined(__CYGWIN__) -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC Py_EXPORTED_SYMBOL # else /* __CYGWIN__ */ -# define PyMODINIT_FUNC PyObject* +# define _PyINIT_FUNC_DECLSPEC # endif /* __CYGWIN__ */ # else /* Py_BUILD_CORE */ /* Building an extension module, or an embedded situation */ @@ -78,9 +79,9 @@ # define PyAPI_DATA(RTYPE) extern Py_IMPORTED_SYMBOL RTYPE /* module init functions outside the core must be exported */ # if defined(__cplusplus) -# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC extern "C" Py_EXPORTED_SYMBOL # else /* __cplusplus */ -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC Py_EXPORTED_SYMBOL # endif /* __cplusplus */ # endif /* Py_BUILD_CORE */ # endif /* HAVE_DECLSPEC_DLL */ @@ -93,13 +94,15 @@ #ifndef PyAPI_DATA # define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE #endif -#ifndef PyMODINIT_FUNC +#ifndef _PyINIT_FUNC_DECLSPEC # if defined(__cplusplus) -# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC extern "C" Py_EXPORTED_SYMBOL # else /* __cplusplus */ -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC Py_EXPORTED_SYMBOL # endif /* __cplusplus */ #endif +#define PyMODINIT_FUNC _PyINIT_FUNC_DECLSPEC PyObject* +#define PyMODEXPORT_FUNC _PyINIT_FUNC_DECLSPEC PyModuleDef_Slot* #endif /* Py_EXPORTS_H */ diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index 3ba9229cc21..12a32a5f70e 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -28,6 +28,11 @@ typedef enum ext_module_origin { _Py_ext_module_origin_DYNAMIC = 3, } _Py_ext_module_origin; +struct hook_prefixes { + const char *const init_prefix; + const char *const export_prefix; +}; + /* Input for loading an extension module. */ struct _Py_ext_module_loader_info { PyObject *filename; @@ -40,7 +45,7 @@ struct _Py_ext_module_loader_info { * depending on if it's builtin or not. */ PyObject *path; _Py_ext_module_origin origin; - const char *hook_prefix; + const struct hook_prefixes *hook_prefixes; const char *newcontext; }; extern void _Py_ext_module_loader_info_clear( @@ -62,7 +67,9 @@ extern int _Py_ext_module_loader_info_init_from_spec( PyObject *spec); #endif -/* The result from running an extension module's init function. */ +/* The result from running an extension module's init function. + * Not used for modules defined via PyModExport (slots array). + */ struct _Py_ext_module_loader_result { PyModuleDef *def; PyObject *module; @@ -89,10 +96,11 @@ extern void _Py_ext_module_loader_result_apply_error( /* The module init function. */ typedef PyObject *(*PyModInitFunction)(void); +typedef PyModuleDef_Slot *(*PyModExportFunction)(void); #ifdef HAVE_DYNAMIC_LOADING -extern PyModInitFunction _PyImport_GetModInitFunc( +extern int _PyImport_GetModuleExportHooks( struct _Py_ext_module_loader_info *info, - FILE *fp); + FILE *fp, PyModInitFunction *modinit, PyModExportFunction *modexport); #endif extern int _PyImport_RunModInitFunc( PyModInitFunction p0, diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index b170d7bce70..c34e42e826e 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -1,5 +1,8 @@ #ifndef Py_INTERNAL_MODULEOBJECT_H #define Py_INTERNAL_MODULEOBJECT_H + +#include + #ifdef __cplusplus extern "C" { #endif @@ -16,32 +19,49 @@ extern int _PyModule_IsPossiblyShadowing(PyObject *); extern int _PyModule_IsExtension(PyObject *obj); +typedef int (*_Py_modexecfunc)(PyObject *); + typedef struct { PyObject_HEAD PyObject *md_dict; - PyModuleDef *md_def; void *md_state; PyObject *md_weaklist; // for logging purposes after md_dict is cleared PyObject *md_name; + bool md_token_is_def; /* if true, `md_token` is the PyModuleDef */ #ifdef Py_GIL_DISABLED void *md_gil; #endif + Py_ssize_t md_state_size; + traverseproc md_state_traverse; + inquiry md_state_clear; + freefunc md_state_free; + void *md_token; + _Py_modexecfunc md_exec; /* only set if md_token_is_def is true */ } PyModuleObject; -static inline PyModuleDef* _PyModule_GetDef(PyObject *mod) { - assert(PyModule_Check(mod)); - return ((PyModuleObject *)mod)->md_def; +#define _PyModule_CAST(op) \ + (assert(PyModule_Check(op)), _Py_CAST(PyModuleObject*, (op))) + +static inline PyModuleDef *_PyModule_GetDefOrNull(PyObject *arg) { + PyModuleObject *mod = _PyModule_CAST(arg); + if (mod->md_token_is_def) { + return (PyModuleDef *)mod->md_token; + } + return NULL; +} + +static inline PyModuleDef *_PyModule_GetToken(PyObject *arg) { + PyModuleObject *mod = _PyModule_CAST(arg); + return mod->md_token; } static inline void* _PyModule_GetState(PyObject* mod) { - assert(PyModule_Check(mod)); - return ((PyModuleObject *)mod)->md_state; + return _PyModule_CAST(mod)->md_state; } static inline PyObject* _PyModule_GetDict(PyObject *mod) { - assert(PyModule_Check(mod)); - PyObject *dict = ((PyModuleObject *)mod) -> md_dict; + PyObject *dict = _PyModule_CAST(mod)->md_dict; // _PyModule_GetDict(mod) must not be used after calling module_clear(mod) assert(dict != NULL); return dict; // borrowed reference diff --git a/Include/moduleobject.h b/Include/moduleobject.h index e3afac0a343..e83bc395aa4 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -83,11 +83,19 @@ struct PyModuleDef_Slot { #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) # define Py_mod_abi 5 +# define Py_mod_name 6 +# define Py_mod_doc 7 +# define Py_mod_state_size 8 +# define Py_mod_methods 9 +# define Py_mod_state_traverse 10 +# define Py_mod_state_clear 11 +# define Py_mod_state_free 12 +# define Py_mod_token 13 #endif #ifndef Py_LIMITED_API -#define _Py_mod_LAST_SLOT 5 +#define _Py_mod_LAST_SLOT 13 #endif #endif /* New in 3.5 */ @@ -109,6 +117,13 @@ struct PyModuleDef_Slot { PyAPI_FUNC(int) PyUnstable_Module_SetGIL(PyObject *module, void *gil); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +PyAPI_FUNC(PyObject *) PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *, + PyObject *spec); +PyAPI_FUNC(int) PyModule_Exec(PyObject *mod); +PyAPI_FUNC(int) PyModule_GetStateSize(PyObject *mod, Py_ssize_t *result); +PyAPI_FUNC(int) PyModule_GetToken(PyObject *, void **result); +#endif #ifndef _Py_OPAQUE_PYOBJECT struct PyModuleDef { diff --git a/Include/object.h b/Include/object.h index 291e4f0a7ed..f17dcba4f47 100644 --- a/Include/object.h +++ b/Include/object.h @@ -839,6 +839,11 @@ PyAPI_FUNC(PyObject *) PyType_GetModuleByDef(PyTypeObject *, PyModuleDef *); PyAPI_FUNC(int) PyType_Freeze(PyTypeObject *type); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +PyAPI_FUNC(PyObject *) PyType_GetModuleByToken(PyTypeObject *type, + const void *token); +#endif + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_capi/test_module.py b/Lib/test/test_capi/test_module.py new file mode 100644 index 00000000000..7ec23e637d7 --- /dev/null +++ b/Lib/test/test_capi/test_module.py @@ -0,0 +1,185 @@ +# The C functions used by this module are in: +# Modules/_testcapi/module.c + +import unittest +import types +from test.support import import_helper, subTests + +# Skip this test if the _testcapi module isn't available. +_testcapi = import_helper.import_module('_testcapi') + + +class FakeSpec: + name = 'testmod' + +DEF_SLOTS = ( + 'Py_mod_name', 'Py_mod_doc', 'Py_mod_state_size', 'Py_mod_methods', + 'Py_mod_state_traverse', 'Py_mod_state_clear', 'Py_mod_state_free', + 'Py_mod_token', +) + +def def_and_token(mod): + return ( + _testcapi.pymodule_get_def(mod), + _testcapi.pymodule_get_token(mod), + ) + +class TestModFromSlotsAndSpec(unittest.TestCase): + def test_empty(self): + mod = _testcapi.module_from_slots_empty(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, 0)) + self.assertEqual(mod.__name__, 'testmod') + size = _testcapi.pymodule_get_state_size(mod) + self.assertEqual(size, 0) + + def test_null_slots(self): + with self.assertRaises(SystemError): + _testcapi.module_from_slots_null(FakeSpec()) + + def test_none_spec(self): + # The spec currently must contain a name + with self.assertRaises(AttributeError): + _testcapi.module_from_slots_empty(None) + with self.assertRaises(AttributeError): + _testcapi.module_from_slots_name(None) + + def test_name(self): + # Py_mod_name (and PyModuleDef.m_name) are currently ignored when + # spec is given. + # We still test that it's accepted. + mod = _testcapi.module_from_slots_name(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, 0)) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + + def test_doc(self): + mod = _testcapi.module_from_slots_doc(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, 0)) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, 'the docstring') + + def test_size(self): + mod = _testcapi.module_from_slots_size(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, 0)) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + size = _testcapi.pymodule_get_state_size(mod) + self.assertEqual(size, 123) + + def test_methods(self): + mod = _testcapi.module_from_slots_methods(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, 0)) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + self.assertEqual(mod.a_method(456), (mod, 456)) + + def test_gc(self): + mod = _testcapi.module_from_slots_gc(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, 0)) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + + # Check that the requested hook functions (which module_from_slots_gc + # stores as attributes) match what's in the module (as retrieved by + # _testinternalcapi.module_get_gc_hooks) + _testinternalcapi = import_helper.import_module('_testinternalcapi') + traverse, clear, free = _testinternalcapi.module_get_gc_hooks(mod) + self.assertEqual(traverse, mod.traverse) + self.assertEqual(clear, mod.clear) + self.assertEqual(free, mod.free) + + def test_token(self): + mod = _testcapi.module_from_slots_token(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, _testcapi.module_test_token)) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + + def test_exec(self): + mod = _testcapi.module_from_slots_exec(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, 0)) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + self.assertEqual(mod.a_number, 456) + + def test_create(self): + spec = FakeSpec() + spec._gimme_this = "not a module object" + mod = _testcapi.module_from_slots_create(spec) + self.assertIsInstance(mod, str) + self.assertEqual(mod, "not a module object") + with self.assertRaises(TypeError): + _testcapi.pymodule_get_def(mod), + with self.assertRaises(TypeError): + _testcapi.pymodule_get_token(mod) + + def test_def_slot(self): + """Slots that replace PyModuleDef fields can't be used with PyModuleDef + """ + for name in DEF_SLOTS: + with self.subTest(name): + spec = FakeSpec() + spec._test_slot_id = getattr(_testcapi, name) + with self.assertRaises(SystemError) as cm: + _testcapi.module_from_def_slot(spec) + self.assertIn(name, str(cm.exception)) + self.assertIn("PyModuleDef", str(cm.exception)) + + def test_repeated_def_slot(self): + """Slots that replace PyModuleDef fields can't be repeated""" + for name in (*DEF_SLOTS, 'Py_mod_exec'): + with self.subTest(name): + spec = FakeSpec() + spec._test_slot_id = getattr(_testcapi, name) + with self.assertRaises(SystemError) as cm: + _testcapi.module_from_slots_repeat_slot(spec) + self.assertIn(name, str(cm.exception)) + self.assertIn("more than one", str(cm.exception)) + + def test_null_def_slot(self): + """Slots that replace PyModuleDef fields can't be NULL""" + for name in (*DEF_SLOTS, 'Py_mod_exec'): + with self.subTest(name): + spec = FakeSpec() + spec._test_slot_id = getattr(_testcapi, name) + with self.assertRaises(SystemError) as cm: + _testcapi.module_from_slots_null_slot(spec) + self.assertIn(name, str(cm.exception)) + self.assertIn("NULL", str(cm.exception)) + + def test_def_multiple_exec(self): + """PyModule_Exec runs all exec slots of PyModuleDef-defined module""" + mod = _testcapi.module_from_def_multiple_exec(FakeSpec()) + self.assertFalse(hasattr(mod, 'a_number')) + _testcapi.pymodule_exec(mod) + self.assertEqual(mod.a_number, 456) + self.assertEqual(mod.another_number, 789) + _testcapi.pymodule_exec(mod) + self.assertEqual(mod.a_number, 456) + self.assertEqual(mod.another_number, -789) + def_ptr, token = def_and_token(mod) + self.assertEqual(def_ptr, token) + + def test_def_token(self): + """In PyModuleDef-defined modules, the def is the token""" + mod = _testcapi.module_from_def_multiple_exec(FakeSpec()) + def_ptr, token = def_and_token(mod) + self.assertEqual(def_ptr, token) + self.assertGreater(def_ptr, 0) + + @subTests('name, expected_size', [ + (__name__, 0), # Python module + ('_testsinglephase', -1), # single-phase init + ('sys', -1), + ]) + def test_get_state_size(self, name, expected_size): + mod = import_helper.import_module(name) + size = _testcapi.pymodule_get_state_size(mod) + self.assertEqual(size, expected_size) diff --git a/Lib/test/test_capi/test_type.py b/Lib/test/test_capi/test_type.py index 93874fbee32..e6a8ef9eed6 100644 --- a/Lib/test/test_capi/test_type.py +++ b/Lib/test/test_capi/test_type.py @@ -195,6 +195,24 @@ class H2(int): pass with self.assertRaises(TypeError): _testcapi.pytype_getmodulebydef(H2) + def test_get_module_by_token(self): + token = _testcapi.pymodule_get_token(_testcapi) + + heaptype = _testcapi.create_type_with_token('_testcapi.H', 0) + mod = _testcapi.pytype_getmodulebytoken(heaptype, token) + self.assertIs(mod, _testcapi) + + class H1(heaptype): pass + mod = _testcapi.pytype_getmodulebytoken(H1, token) + self.assertIs(mod, _testcapi) + + with self.assertRaises(TypeError): + _testcapi.pytype_getmodulebytoken(int, token) + + class H2(int): pass + with self.assertRaises(TypeError): + _testcapi.pytype_getmodulebytoken(H2, token) + def test_freeze(self): # test PyType_Freeze() type_freeze = _testcapi.type_freeze diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py index fb93c6ccbb6..a52c2241f5d 100644 --- a/Lib/test/test_cext/__init__.py +++ b/Lib/test/test_cext/__init__.py @@ -14,7 +14,6 @@ SOURCES = [ os.path.join(os.path.dirname(__file__), 'extension.c'), - os.path.join(os.path.dirname(__file__), 'create_moduledef.c'), ] SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') diff --git a/Lib/test/test_cext/create_moduledef.c b/Lib/test/test_cext/create_moduledef.c deleted file mode 100644 index 249c3163552..00000000000 --- a/Lib/test/test_cext/create_moduledef.c +++ /dev/null @@ -1,29 +0,0 @@ -// Workaround for testing _Py_OPAQUE_PYOBJECT. -// See end of 'extension.c' - - -#undef _Py_OPAQUE_PYOBJECT -#undef Py_LIMITED_API -#include "Python.h" - - -// (repeated definition to avoid creating a header) -extern PyObject *testcext_create_moduledef( - const char *name, const char *doc, - PyMethodDef *methods, PyModuleDef_Slot *slots); - -PyObject *testcext_create_moduledef( - const char *name, const char *doc, - PyMethodDef *methods, PyModuleDef_Slot *slots) { - - static struct PyModuleDef _testcext_module = { - PyModuleDef_HEAD_INIT, - }; - if (!_testcext_module.m_name) { - _testcext_module.m_name = name; - _testcext_module.m_doc = doc; - _testcext_module.m_methods = methods; - _testcext_module.m_slots = slots; - } - return PyModuleDef_Init(&_testcext_module); -} diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c index 73fc67ae59d..0f668c1da32 100644 --- a/Lib/test/test_cext/extension.c +++ b/Lib/test/test_cext/extension.c @@ -78,7 +78,7 @@ _testcext_exec( return 0; } -#define _FUNC_NAME(NAME) PyInit_ ## NAME +#define _FUNC_NAME(NAME) PyModExport_ ## NAME #define FUNC_NAME(NAME) _FUNC_NAME(NAME) // Converting from function pointer to void* has undefined behavior, but @@ -88,58 +88,40 @@ _testcext_exec( _Py_COMP_DIAG_PUSH #if defined(__GNUC__) #pragma GCC diagnostic ignored "-Wpedantic" +#pragma GCC diagnostic ignored "-Wcast-qual" #elif defined(__clang__) #pragma clang diagnostic ignored "-Wpedantic" +#pragma clang diagnostic ignored "-Wcast-qual" #endif +PyDoc_STRVAR(_testcext_doc, "C test extension."); + static PyModuleDef_Slot _testcext_slots[] = { + {Py_mod_name, STR(MODULE_NAME)}, + {Py_mod_doc, (void*)(char*)_testcext_doc}, {Py_mod_exec, (void*)_testcext_exec}, + {Py_mod_methods, _testcext_methods}, {0, NULL} }; _Py_COMP_DIAG_POP -PyDoc_STRVAR(_testcext_doc, "C test extension."); - -#ifndef _Py_OPAQUE_PYOBJECT - -static struct PyModuleDef _testcext_module = { - PyModuleDef_HEAD_INIT, // m_base - STR(MODULE_NAME), // m_name - _testcext_doc, // m_doc - 0, // m_size - _testcext_methods, // m_methods - _testcext_slots, // m_slots - NULL, // m_traverse - NULL, // m_clear - NULL, // m_free -}; - - -PyMODINIT_FUNC +PyMODEXPORT_FUNC FUNC_NAME(MODULE_NAME)(void) { - return PyModuleDef_Init(&_testcext_module); + return _testcext_slots; } -#else // _Py_OPAQUE_PYOBJECT - -// Opaque PyObject means that PyModuleDef is also opaque and cannot be -// declared statically. See PEP 793. -// So, this part of module creation is split into a separate source file -// which uses non-limited API. - -// (repeated definition to avoid creating a header) -extern PyObject *testcext_create_moduledef( - const char *name, const char *doc, - PyMethodDef *methods, PyModuleDef_Slot *slots); +// Also define the soft-deprecated entrypoint to ensure it isn't called +#define _INITFUNC_NAME(NAME) PyInit_ ## NAME +#define INITFUNC_NAME(NAME) _INITFUNC_NAME(NAME) PyMODINIT_FUNC -FUNC_NAME(MODULE_NAME)(void) +INITFUNC_NAME(MODULE_NAME)(void) { - return testcext_create_moduledef( - STR(MODULE_NAME), _testcext_doc, _testcext_methods, _testcext_slots); + PyErr_SetString( + PyExc_AssertionError, + "PyInit_* function called while a PyModExport_* one is available"); + return NULL; } - -#endif // _Py_OPAQUE_PYOBJECT diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index 4d71e4751f7..67dfddec751 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -99,7 +99,6 @@ def main(): # Define _Py_OPAQUE_PYOBJECT macro if opaque_pyobject: cflags.append(f'-D_Py_OPAQUE_PYOBJECT') - sources.append('create_moduledef.c') if internal: cflags.append('-DTEST_INTERNAL_C_API=1') diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 072021e5959..e87d8b7e7bb 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -2497,6 +2497,21 @@ def test_multi_init_extension_per_interpreter_gil_compat(self): self.check_compatible_here( modname, filename, strict=False, isolated=False) + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_testmultiphase_exec_multiple(self): + modname = '_testmultiphase_exec_multiple' + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename, + put_in_sys_modules=False) + # All three exec's were called. + self.assertEqual(module.a, 1) + self.assertEqual(module.b, 2) + self.assertEqual(module.c, 3) + # They were called in order. + keys = list(module.__dict__) + self.assertLess(keys.index('a'), keys.index('b')) + self.assertLess(keys.index('b'), keys.index('c')) + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") def test_python_compat(self): module = 'threading' @@ -3394,6 +3409,83 @@ def test_basic_multiple_interpreters_reset_each(self): # * module's global state was initialized, not reset +@unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") +class ModexportTests(unittest.TestCase): + def test_from_modexport(self): + modname = '_test_from_modexport' + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename, + put_in_sys_modules=False) + + self.assertEqual(module.__name__, modname) + + def test_from_modexport_null(self): + modname = '_test_from_modexport_null' + filename = _testmultiphase.__file__ + with self.assertRaises(SystemError): + import_extension_from_file(modname, filename, + put_in_sys_modules=False) + + def test_from_modexport_exception(self): + modname = '_test_from_modexport_exception' + filename = _testmultiphase.__file__ + with self.assertRaises(ValueError): + import_extension_from_file(modname, filename, + put_in_sys_modules=False) + + def test_from_modexport_create_nonmodule(self): + modname = '_test_from_modexport_create_nonmodule' + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename, + put_in_sys_modules=False) + self.assertIsInstance(module, str) + + def test_from_modexport_smoke(self): + # General positive test for sundry features + # (PyModule_FromSlotsAndSpec tests exercise these more carefully) + modname = '_test_from_modexport_smoke' + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename, + put_in_sys_modules=False) + self.assertEqual(module.__doc__, "the expected docstring") + self.assertEqual(module.number, 147) + self.assertEqual(module.get_state_int(), 258) + self.assertGreater(module.get_test_token(), 0) + + def test_from_modexport_smoke_token(self): + _testcapi = import_module("_testcapi") + + modname = '_test_from_modexport_smoke' + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename, + put_in_sys_modules=False) + token = module.get_test_token() + self.assertEqual(_testcapi.pymodule_get_token(module), token) + + tp = module.Example + self.assertEqual(_testcapi.pytype_getmodulebytoken(tp, token), module) + class Sub(tp): + pass + self.assertEqual(_testcapi.pytype_getmodulebytoken(Sub, token), module) + + def test_from_modexport_empty_slots(self): + # Module to test that: + # - no slots are mandatory for PyModExport + # - the slots array is used as the default token + modname = '_test_from_modexport_empty_slots' + filename = _testmultiphase.__file__ + module = import_extension_from_file( + modname, filename, put_in_sys_modules=False) + + self.assertEqual(module.__name__, modname) + self.assertEqual(module.__doc__, None) + + _testcapi = import_module("_testcapi") + smoke_mod = import_extension_from_file( + '_test_from_modexport_smoke', filename, put_in_sys_modules=False) + self.assertEqual(_testcapi.pymodule_get_token(module), + smoke_mod.get_modexport_empty_slots()) + @cpython_only class TestMagicNumber(unittest.TestCase): def test_magic_number_endianness(self): diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index cbec7e43a7c..7167646ecc6 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -469,8 +469,10 @@ SYMBOL_NAMES = ( "PyModule_AddStringConstant", "PyModule_AddType", "PyModule_Create2", + "PyModule_Exec", "PyModule_ExecDef", "PyModule_FromDefAndSpec2", + "PyModule_FromSlotsAndSpec", "PyModule_GetDef", "PyModule_GetDict", "PyModule_GetFilename", @@ -478,6 +480,8 @@ SYMBOL_NAMES = ( "PyModule_GetName", "PyModule_GetNameObject", "PyModule_GetState", + "PyModule_GetStateSize", + "PyModule_GetToken", "PyModule_New", "PyModule_NewObject", "PyModule_SetDocString", @@ -733,6 +737,7 @@ SYMBOL_NAMES = ( "PyType_GetFullyQualifiedName", "PyType_GetModule", "PyType_GetModuleByDef", + "PyType_GetModuleByToken", "PyType_GetModuleName", "PyType_GetModuleState", "PyType_GetName", diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 1198c6d3511..3ceed019ac4 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1725,9 +1725,10 @@ def get_gen(): yield 1 check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit) # module if support.Py_GIL_DISABLED: - check(unittest, size('PPPPPP')) + md_gil = 'P' else: - check(unittest, size('PPPPP')) + md_gil = '' + check(unittest, size('PPPP?' + md_gil + 'NPPPPP')) # None check(None, size('')) # NotImplementedType diff --git a/Misc/NEWS.d/next/C_API/2025-10-26-16-45-28.gh-issue-140556.s__Dae.rst b/Misc/NEWS.d/next/C_API/2025-10-26-16-45-28.gh-issue-140556.s__Dae.rst new file mode 100644 index 00000000000..61da60903ee --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-10-26-16-45-28.gh-issue-140556.s__Dae.rst @@ -0,0 +1,2 @@ +:pep:`793`: Add a new entry point for C extension modules, +``PyModExport_``. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index ad0f3704599..7ee6cf1dae5 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2611,3 +2611,31 @@ added = '3.15' [const.PyABIInfo_FREETHREADING_AGNOSTIC] added = '3.15' +[function.PyModule_FromSlotsAndSpec] + added = '3.15' +[function.PyModule_Exec] + added = '3.15' +[function.PyModule_GetToken] + added = '3.15' +[function.PyType_GetModuleByToken] + added = '3.15' +[function.PyModule_GetStateSize] + added = '3.15' +[macro.PyMODEXPORT_FUNC] + added = '3.15' +[const.Py_mod_name] + added = '3.15' +[const.Py_mod_doc] + added = '3.15' +[const.Py_mod_state_size] + added = '3.15' +[const.Py_mod_methods] + added = '3.15' +[const.Py_mod_state_traverse] + added = '3.15' +[const.Py_mod_state_clear] + added = '3.15' +[const.Py_mod_state_free] + added = '3.15' +[const.Py_mod_token] + added = '3.15' diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index b9ffdcc65d1..2c3013e3d0c 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -175,7 +175,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/modsupport.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/modsupport.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/module.c @MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/heaptype.c b/Modules/_testcapi/heaptype.c index 69dcf072da1..4fdcc850a33 100644 --- a/Modules/_testcapi/heaptype.c +++ b/Modules/_testcapi/heaptype.c @@ -528,6 +528,21 @@ pytype_getmodulebydef(PyObject *self, PyObject *type) return Py_XNewRef(mod); } +static PyObject * +pytype_getmodulebytoken(PyObject *self, PyObject *args) +{ + PyObject *type; + PyObject *py_token; + if (!PyArg_ParseTuple(args, "OO", &type, &py_token)) { + return NULL; + } + void *token = PyLong_AsVoidPtr(py_token); + if ((!token) && PyErr_Occurred()) { + return NULL; + } + return PyType_GetModuleByToken((PyTypeObject *)type, token); +} + static PyMethodDef TestMethods[] = { {"pytype_fromspec_meta", pytype_fromspec_meta, METH_O}, @@ -546,6 +561,7 @@ static PyMethodDef TestMethods[] = { {"get_tp_token", get_tp_token, METH_O}, {"pytype_getbasebytoken", pytype_getbasebytoken, METH_VARARGS}, {"pytype_getmodulebydef", pytype_getmodulebydef, METH_O}, + {"pytype_getmodulebytoken", pytype_getmodulebytoken, METH_VARARGS}, {NULL}, }; diff --git a/Modules/_testcapi/module.c b/Modules/_testcapi/module.c new file mode 100644 index 00000000000..9349445351e --- /dev/null +++ b/Modules/_testcapi/module.c @@ -0,0 +1,378 @@ +#include "parts.h" +#include "util.h" + +// Test PyModule_* API + +/* unittest Cases that use these functions are in: + * Lib/test/test_capi/test_module.py + */ + +static PyObject * +module_from_slots_empty(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static PyObject * +module_from_slots_null(PyObject *self, PyObject *spec) +{ + return PyModule_FromSlotsAndSpec(NULL, spec); +} + +static PyObject * +module_from_slots_name(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_name, "currently ignored..."}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static PyObject * +module_from_slots_doc(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_doc, "the docstring"}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static PyObject * +module_from_slots_size(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_state_size, (void*)123}, + {0}, + }; + PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); + if (!mod) { + return NULL; + } + return mod; +} + +static PyObject * +a_method(PyObject *self, PyObject *arg) +{ + return PyTuple_Pack(2, self, arg); +} + +static PyMethodDef a_methoddef_array[] = { + {"a_method", a_method, METH_O}, + {0}, +}; + +static PyObject * +module_from_slots_methods(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_methods, a_methoddef_array}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static int noop_traverse(PyObject *self, visitproc visit, void *arg) { + return 0; +} +static int noop_clear(PyObject *self) { return 0; } +static void noop_free(void *self) { } + +static PyObject * +module_from_slots_gc(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_state_traverse, noop_traverse}, + {Py_mod_state_clear, noop_clear}, + {Py_mod_state_free, noop_free}, + {0}, + }; + PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); + if (!mod) { + return NULL; + } + if (PyModule_Add(mod, "traverse", PyLong_FromVoidPtr(&noop_traverse)) < 0) { + Py_DECREF(mod); + return NULL; + } + if (PyModule_Add(mod, "clear", PyLong_FromVoidPtr(&noop_clear)) < 0) { + Py_DECREF(mod); + return NULL; + } + if (PyModule_Add(mod, "free", PyLong_FromVoidPtr(&noop_free)) < 0) { + Py_DECREF(mod); + return NULL; + } + return mod; +} + +static const char test_token; + +static PyObject * +module_from_slots_token(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_token, (void*)&test_token}, + {0}, + }; + PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); + if (!mod) { + return NULL; + } + void *got_token; + if (PyModule_GetToken(mod, &got_token) < 0) { + Py_DECREF(mod); + return NULL; + } + assert(got_token == &test_token); + return mod; +} + +static int +simple_exec(PyObject *module) +{ + return PyModule_AddIntConstant(module, "a_number", 456); +} + +static PyObject * +module_from_slots_exec(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_exec, simple_exec}, + {0}, + }; + PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); + if (!mod) { + return NULL; + } + int res = PyObject_HasAttrStringWithError(mod, "a_number"); + if (res < 0) { + Py_DECREF(mod); + return NULL; + } + assert(res == 0); + if (PyModule_Exec(mod) < 0) { + Py_DECREF(mod); + return NULL; + } + return mod; +} + +static PyObject * +create_attr_from_spec(PyObject *spec, PyObject *def) +{ + assert(!def); + return PyObject_GetAttrString(spec, "_gimme_this"); +} + +static PyObject * +module_from_slots_create(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_create, create_attr_from_spec}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + + +static int +slot_from_object(PyObject *obj) +{ + PyObject *slot_id_obj = PyObject_GetAttrString(obj, "_test_slot_id"); + if (slot_id_obj == NULL) { + return -1; + } + int slot_id = PyLong_AsInt(slot_id_obj); + if (PyErr_Occurred()) { + return -1; + } + return slot_id; +} + +static PyObject * +module_from_slots_repeat_slot(PyObject *self, PyObject *spec) +{ + int slot_id = slot_from_object(spec); + if (slot_id < 0) { + return NULL; + } + PyModuleDef_Slot slots[] = { + {slot_id, "anything"}, + {slot_id, "anything else"}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static PyObject * +module_from_slots_null_slot(PyObject *self, PyObject *spec) +{ + int slot_id = slot_from_object(spec); + if (slot_id < 0) { + return NULL; + } + PyModuleDef_Slot slots[] = { + {slot_id, NULL}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static PyObject * +module_from_def_slot(PyObject *self, PyObject *spec) +{ + int slot_id = slot_from_object(spec); + if (slot_id < 0) { + return NULL; + } + PyModuleDef_Slot slots[] = { + {slot_id, "anything"}, + {0}, + }; + PyModuleDef def = { + PyModuleDef_HEAD_INIT, + .m_name = "currently ignored", + .m_slots = slots, + }; + // PyModuleDef is normally static; the real requirement is that it + // must outlive its module. + // Here, module creation fails, so it's fine on the stack. + PyObject *result = PyModule_FromDefAndSpec(&def, spec); + assert(result == NULL); + return result; +} + +static int +another_exec(PyObject *module) +{ + /* Make sure simple_exec was called */ + assert(PyObject_HasAttrString(module, "a_number")); + + /* Add or negate a global called 'another_number' */ + PyObject *another_number; + if (PyObject_GetOptionalAttrString(module, "another_number", + &another_number) < 0) { + return -1; + } + if (!another_number) { + return PyModule_AddIntConstant(module, "another_number", 789); + } + PyObject *neg_number = PyNumber_Negative(another_number); + Py_DECREF(another_number); + if (!neg_number) { + return -1; + } + int result = PyObject_SetAttrString(module, "another_number", + neg_number); + Py_DECREF(neg_number); + return result; +} + +static PyObject * +module_from_def_multiple_exec(PyObject *self, PyObject *spec) +{ + static PyModuleDef_Slot slots[] = { + {Py_mod_exec, simple_exec}, + {Py_mod_exec, another_exec}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0}, + }; + static PyModuleDef def = { + PyModuleDef_HEAD_INIT, + .m_name = "currently ignored", + .m_slots = slots, + }; + return PyModule_FromDefAndSpec(&def, spec); +} + +static PyObject * +pymodule_exec(PyObject *self, PyObject *module) +{ + if (PyModule_Exec(module) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +pymodule_get_token(PyObject *self, PyObject *module) +{ + void *token; + if (PyModule_GetToken(module, &token) < 0) { + return NULL; + } + return PyLong_FromVoidPtr(token); +} + +static PyObject * +pymodule_get_def(PyObject *self, PyObject *module) +{ + PyModuleDef *def = PyModule_GetDef(module); + if (!def && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromVoidPtr(def); +} + +static PyObject * +pymodule_get_state_size(PyObject *self, PyObject *module) +{ + Py_ssize_t size; + if (PyModule_GetStateSize(module, &size) < 0) { + return NULL; + } + return PyLong_FromSsize_t(size); +} + +static PyMethodDef test_methods[] = { + {"module_from_slots_empty", module_from_slots_empty, METH_O}, + {"module_from_slots_null", module_from_slots_null, METH_O}, + {"module_from_slots_name", module_from_slots_name, METH_O}, + {"module_from_slots_doc", module_from_slots_doc, METH_O}, + {"module_from_slots_size", module_from_slots_size, METH_O}, + {"module_from_slots_methods", module_from_slots_methods, METH_O}, + {"module_from_slots_gc", module_from_slots_gc, METH_O}, + {"module_from_slots_token", module_from_slots_token, METH_O}, + {"module_from_slots_exec", module_from_slots_exec, METH_O}, + {"module_from_slots_create", module_from_slots_create, METH_O}, + {"module_from_slots_repeat_slot", module_from_slots_repeat_slot, METH_O}, + {"module_from_slots_null_slot", module_from_slots_null_slot, METH_O}, + {"module_from_def_multiple_exec", module_from_def_multiple_exec, METH_O}, + {"module_from_def_slot", module_from_def_slot, METH_O}, + {"pymodule_get_token", pymodule_get_token, METH_O}, + {"pymodule_get_def", pymodule_get_def, METH_O}, + {"pymodule_get_state_size", pymodule_get_state_size, METH_O}, + {"pymodule_exec", pymodule_exec, METH_O}, + {NULL}, +}; + +int +_PyTestCapi_Init_Module(PyObject *m) +{ +#define ADD_INT_MACRO(C) if (PyModule_AddIntConstant(m, #C, C) < 0) return -1; + ADD_INT_MACRO(Py_mod_create); + ADD_INT_MACRO(Py_mod_exec); + ADD_INT_MACRO(Py_mod_multiple_interpreters); + ADD_INT_MACRO(Py_mod_gil); + ADD_INT_MACRO(Py_mod_name); + ADD_INT_MACRO(Py_mod_doc); + ADD_INT_MACRO(Py_mod_state_size); + ADD_INT_MACRO(Py_mod_methods); + ADD_INT_MACRO(Py_mod_state_traverse); + ADD_INT_MACRO(Py_mod_state_clear); + ADD_INT_MACRO(Py_mod_state_free); + ADD_INT_MACRO(Py_mod_token); +#undef ADD_INT_MACRO + if (PyModule_Add(m, "module_test_token", + PyLong_FromVoidPtr((void*)&test_token)) < 0) + { + return -1; + } + return PyModule_AddFunctions(m, test_methods); +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 32915d04bd3..a7feca5bd96 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -66,5 +66,6 @@ int _PyTestCapi_Init_Import(PyObject *mod); int _PyTestCapi_Init_Frame(PyObject *mod); int _PyTestCapi_Init_Type(PyObject *mod); int _PyTestCapi_Init_Function(PyObject *mod); +int _PyTestCapi_Init_Module(PyObject *mod); #endif // Py_TESTCAPI_PARTS_H diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index e29b9ae354b..22cd731d410 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3512,6 +3512,9 @@ _testcapi_exec(PyObject *m) if (_PyTestCapi_Init_Function(m) < 0) { return -1; } + if (_PyTestCapi_Init_Module(m) < 0) { + return -1; + } return 0; } diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index c2647d405e2..dede05960d7 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2418,6 +2418,34 @@ set_vectorcall_nop(PyObject *self, PyObject *func) Py_RETURN_NONE; } +static PyObject * +module_get_gc_hooks(PyObject *self, PyObject *arg) +{ + PyModuleObject *mod = (PyModuleObject *)arg; + PyObject *traverse = NULL; + PyObject *clear = NULL; + PyObject *free = NULL; + PyObject *result = NULL; + traverse = PyLong_FromVoidPtr(mod->md_state_traverse); + if (!traverse) { + goto finally; + } + clear = PyLong_FromVoidPtr(mod->md_state_clear); + if (!clear) { + goto finally; + } + free = PyLong_FromVoidPtr(mod->md_state_free); + if (!free) { + goto finally; + } + result = PyTuple_FromArray((PyObject*[]){ traverse, clear, free }, 3); +finally: + Py_XDECREF(traverse); + Py_XDECREF(clear); + Py_XDECREF(free); + return result; +} + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -2527,6 +2555,7 @@ static PyMethodDef module_functions[] = { #endif {"simple_pending_call", simple_pending_call, METH_O}, {"set_vectorcall_nop", set_vectorcall_nop, METH_O}, + {"module_get_gc_hooks", module_get_gc_hooks, METH_O}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index bfec0678e2c..220fa888e49 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -850,6 +850,28 @@ PyInit__testmultiphase_exec_unreported_exception(void) return PyModuleDef_Init(&def_exec_unreported_exception); } +static int execfn_a1(PyObject*m) { return PyModule_AddIntConstant(m, "a", 1); } +static int execfn_b2(PyObject*m) { return PyModule_AddIntConstant(m, "b", 2); } +static int execfn_c3(PyObject*m) { return PyModule_AddIntConstant(m, "c", 3); } + +PyMODINIT_FUNC +PyInit__testmultiphase_exec_multiple(void) +{ + static PyModuleDef_Slot slots[] = { + {Py_mod_exec, execfn_a1}, + {Py_mod_exec, execfn_b2}, + {Py_mod_exec, execfn_c3}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0} + }; + static PyModuleDef def = { + PyModuleDef_HEAD_INIT, + .m_name="_testmultiphase_exec_multiple", + .m_slots=slots, + }; + return PyModuleDef_Init(&def); +} + static int meth_state_access_exec(PyObject *m) { @@ -993,3 +1015,156 @@ PyInit__test_no_multiple_interpreter_slot(void) { return PyModuleDef_Init(&no_multiple_interpreter_slot_def); } + + +/* PyModExport_* hooks */ + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport(void) +{ + static PyModuleDef_Slot slots[] = { + {Py_mod_name, "_test_from_modexport"}, + {0}, + }; + return slots; +} + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport_null(void) +{ + return NULL; +} + +PyMODINIT_FUNC +PyModInit__test_from_modexport_null(void) +{ + // This is not called as fallback for failed PyModExport_* + assert(0); + PyErr_SetString(PyExc_AssertionError, "PyInit_ fallback called"); + return NULL; +} + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport_exception(void) +{ + PyErr_SetString(PyExc_ValueError, "failed as requested"); + return NULL; +} + +PyMODINIT_FUNC +PyModInit__test_from_modexport_exception(void) +{ + // This is not called as fallback for failed PyModExport_* + assert(0); + PyErr_SetString(PyExc_AssertionError, "PyInit_ fallback called"); + return NULL; +} + +static PyObject * +modexport_create_string(PyObject *spec, PyObject *def) +{ + assert(def == NULL); + return PyUnicode_FromString("is this \xf0\x9f\xa6\x8b... a module?"); +} + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport_create_nonmodule(void) +{ + static PyModuleDef_Slot slots[] = { + {Py_mod_name, "_test_from_modexport_create_nonmodule"}, + {Py_mod_create, modexport_create_string}, + {0}, + }; + return slots; +} + +static PyModuleDef_Slot modexport_empty_slots[] = { + {0}, +}; + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport_empty_slots(void) +{ + return modexport_empty_slots; +} + +static int +modexport_smoke_exec(PyObject *mod) +{ + // "magic" values 147 & 258 are expected in the test + if (PyModule_AddIntConstant(mod, "number", 147) < 0) { + return 0; + } + int *state = PyModule_GetState(mod); + if (!state) { + return -1; + } + *state = 258; + + PyObject *tp = PyType_FromModuleAndSpec(mod, &StateAccessType_spec, NULL); + if (PyModule_Add(mod, "Example", tp) < 0) { + return -1; + } + + return 0; +} + +static PyObject * +modexport_smoke_get_state_int(PyObject *mod, PyObject *arg) +{ + int *state = PyModule_GetState(mod); + if (!state) { + return NULL; + } + return PyLong_FromLong(*state); +} + +static const char modexport_smoke_test_token; + +static PyObject * +modexport_smoke_get_test_token(PyObject *mod, PyObject *arg) +{ + return PyLong_FromVoidPtr((void*)&modexport_smoke_test_token); +} + +static PyObject * +modexport_get_empty_slots(PyObject *mod, PyObject *arg) +{ + /* Get the address of modexport_empty_slots. + * This method would be in the `_test_from_modexport_empty_slots` module, + * if it had a methods slot. + */ + return PyLong_FromVoidPtr(&modexport_empty_slots); +} + +static void +modexport_smoke_free(PyObject *mod) +{ + int *state = PyModule_GetState(mod); + if (!state) { + PyErr_FormatUnraisable("Exception ignored in module %R free", mod); + } + assert(*state == 258); +} + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport_smoke(void) +{ + static PyMethodDef methods[] = { + {"get_state_int", modexport_smoke_get_state_int, METH_NOARGS}, + {"get_test_token", modexport_smoke_get_test_token, METH_NOARGS}, + {"get_modexport_empty_slots", modexport_get_empty_slots, METH_NOARGS}, + {0}, + }; + static PyModuleDef_Slot slots[] = { + {Py_mod_name, "_test_from_modexport_smoke"}, + {Py_mod_doc, "the expected docstring"}, + {Py_mod_exec, modexport_smoke_exec}, + {Py_mod_state_size, (void*)sizeof(int)}, + {Py_mod_methods, methods}, + {Py_mod_state_free, modexport_smoke_free}, + {Py_mod_token, (void*)&modexport_smoke_test_token}, + {0}, + }; + return slots; +} diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index 2c59085d15b..ee38d61b43a 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -244,6 +244,8 @@ static inline module_state * get_module_state(PyObject *module) { PyModuleDef *def = PyModule_GetDef(module); + assert(def); + if (def->m_size == -1) { return &global_state.module; } diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 0d45c117168..9dee03bdb5e 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -8,7 +8,7 @@ #include "pycore_interp.h" // PyInterpreterState.importlib #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyModule_CreateInitialized() -#include "pycore_moduleobject.h" // _PyModule_GetDef() +#include "pycore_moduleobject.h" // _PyModule_GetDefOrNull() #include "pycore_object.h" // _PyType_AllocNoTrack #include "pycore_pyerrors.h" // _PyErr_FormatFromCause() #include "pycore_pystate.h" // _PyInterpreterState_GET() @@ -27,6 +27,27 @@ static PyMemberDef module_members[] = { {0} }; +static void +assert_def_missing_or_redundant(PyModuleObject *m) +{ + /* We copy all relevant info into the module object. + * Modules created using a def keep a reference to that (statically + * allocated) def; the info there should match what we have in the module. + */ +#ifndef NDEBUG + if (m->md_token_is_def) { + PyModuleDef *def = (PyModuleDef *)m->md_token; + assert(def); +#define DO_ASSERT(F) assert (def->m_ ## F == m->md_state_ ## F); + DO_ASSERT(size); + DO_ASSERT(traverse); + DO_ASSERT(clear); + DO_ASSERT(free); +#undef DO_ASSERT + } +#endif // NDEBUG +} + PyTypeObject PyModuleDef_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) @@ -44,8 +65,14 @@ _PyModule_IsExtension(PyObject *obj) } PyModuleObject *module = (PyModuleObject*)obj; - PyModuleDef *def = module->md_def; - return (def != NULL && def->m_methods != NULL); + if (module->md_exec) { + return 1; + } + if (module->md_token_is_def) { + PyModuleDef *def = (PyModuleDef *)module->md_token; + return (module->md_token_is_def && def->m_methods != NULL); + } + return 0; } @@ -146,10 +173,19 @@ new_module_notrack(PyTypeObject *mt) m = (PyModuleObject *)_PyType_AllocNoTrack(mt, 0); if (m == NULL) return NULL; - m->md_def = NULL; m->md_state = NULL; m->md_weaklist = NULL; m->md_name = NULL; + m->md_token_is_def = false; +#ifdef Py_GIL_DISABLED + m->md_gil = Py_MOD_GIL_USED; +#endif + m->md_state_size = 0; + m->md_state_traverse = NULL; + m->md_state_clear = NULL; + m->md_state_free = NULL; + m->md_exec = NULL; + m->md_token = NULL; m->md_dict = PyDict_New(); if (m->md_dict == NULL) { Py_DECREF(m); @@ -264,6 +300,17 @@ PyModule_Create2(PyModuleDef* module, int module_api_version) return _PyModule_CreateInitialized(module, module_api_version); } +static void +module_copy_members_from_deflike( + PyModuleObject *md, + PyModuleDef *def_like /* not necessarily a valid Python object */) +{ + md->md_state_size = def_like->m_size; + md->md_state_traverse = def_like->m_traverse; + md->md_state_clear = def_like->m_clear; + md->md_state_free = def_like->m_free; +} + PyObject * _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) { @@ -310,15 +357,21 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) return NULL; } } - m->md_def = module; + m->md_token = module; + m->md_token_is_def = true; + module_copy_members_from_deflike(m, module); #ifdef Py_GIL_DISABLED m->md_gil = Py_MOD_GIL_USED; #endif return (PyObject*)m; } -PyObject * -PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_version) +static PyObject * +module_from_def_and_spec( + PyModuleDef* def_like, /* not necessarily a valid Python object */ + PyObject *spec, + int module_api_version, + PyModuleDef* original_def /* NULL if not defined by a def */) { PyModuleDef_Slot* cur_slot; PyObject *(*create)(PyObject *, PyModuleDef*) = NULL; @@ -331,10 +384,10 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio int has_execution_slots = 0; const char *name; int ret; + void *token = NULL; + _Py_modexecfunc m_exec = NULL; PyInterpreterState *interp = _PyInterpreterState_GET(); - PyModuleDef_Init(def); - nameobj = PyObject_GetAttrString(spec, "name"); if (nameobj == NULL) { return NULL; @@ -348,7 +401,7 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio goto error; } - if (def->m_size < 0) { + if (def_like->m_size < 0) { PyErr_Format( PyExc_SystemError, "module %s: m_size may not be negative for multi-phase initialization", @@ -356,7 +409,35 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio goto error; } - for (cur_slot = def->m_slots; cur_slot && cur_slot->slot; cur_slot++) { + for (cur_slot = def_like->m_slots; cur_slot && cur_slot->slot; cur_slot++) { + // Macro to copy a non-NULL, non-repeatable slot that's unusable with + // PyModuleDef. The destination must be initially NULL. +#define COPY_COMMON_SLOT(SLOT, TYPE, DEST) \ + do { \ + if (!(TYPE)(cur_slot->value)) { \ + PyErr_Format( \ + PyExc_SystemError, \ + "module %s: " #SLOT " must not be NULL", \ + name); \ + goto error; \ + } \ + if (original_def) { \ + PyErr_Format( \ + PyExc_SystemError, \ + "module %s: " #SLOT " used with PyModuleDef", \ + name); \ + goto error; \ + } \ + if (DEST) { \ + PyErr_Format( \ + PyExc_SystemError, \ + "module %s has more than one " #SLOT " slot", \ + name); \ + goto error; \ + } \ + DEST = (TYPE)(cur_slot->value); \ + } while (0); \ + ///////////////////////////////////////////////////////////////// switch (cur_slot->slot) { case Py_mod_create: if (create) { @@ -370,6 +451,9 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio break; case Py_mod_exec: has_execution_slots = 1; + if (!original_def) { + COPY_COMMON_SLOT(Py_mod_exec, _Py_modexecfunc, m_exec); + } break; case Py_mod_multiple_interpreters: if (has_multiple_interpreters_slot) { @@ -398,6 +482,35 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio goto error; } break; + case Py_mod_name: + COPY_COMMON_SLOT(Py_mod_name, char*, def_like->m_name); + break; + case Py_mod_doc: + COPY_COMMON_SLOT(Py_mod_doc, char*, def_like->m_doc); + break; + case Py_mod_state_size: + COPY_COMMON_SLOT(Py_mod_state_size, Py_ssize_t, + def_like->m_size); + break; + case Py_mod_methods: + COPY_COMMON_SLOT(Py_mod_methods, PyMethodDef*, + def_like->m_methods); + break; + case Py_mod_state_traverse: + COPY_COMMON_SLOT(Py_mod_state_traverse, traverseproc, + def_like->m_traverse); + break; + case Py_mod_state_clear: + COPY_COMMON_SLOT(Py_mod_state_clear, inquiry, + def_like->m_clear); + break; + case Py_mod_state_free: + COPY_COMMON_SLOT(Py_mod_state_free, freefunc, + def_like->m_free); + break; + case Py_mod_token: + COPY_COMMON_SLOT(Py_mod_token, void*, token); + break; default: assert(cur_slot->slot < 0 || cur_slot->slot > _Py_mod_LAST_SLOT); PyErr_Format( @@ -406,6 +519,7 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio name, cur_slot->slot); goto error; } +#undef COPY_COMMON_SLOT } /* By default, multi-phase init modules are expected @@ -429,7 +543,7 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio } if (create) { - m = create(spec, def); + m = create(spec, original_def); if (m == NULL) { if (!PyErr_Occurred()) { PyErr_Format( @@ -455,15 +569,27 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio } if (PyModule_Check(m)) { - ((PyModuleObject*)m)->md_state = NULL; - ((PyModuleObject*)m)->md_def = def; + PyModuleObject *mod = (PyModuleObject*)m; + mod->md_state = NULL; + module_copy_members_from_deflike(mod, def_like); + if (original_def) { + assert (!token); + mod->md_token = original_def; + mod->md_token_is_def = 1; + } + else { + mod->md_token = token; + } #ifdef Py_GIL_DISABLED - ((PyModuleObject*)m)->md_gil = gil_slot; + mod->md_gil = gil_slot; #else (void)gil_slot; #endif + mod->md_exec = m_exec; } else { - if (def->m_size > 0 || def->m_traverse || def->m_clear || def->m_free) { + if (def_like->m_size > 0 || def_like->m_traverse || def_like->m_clear + || def_like->m_free) + { PyErr_Format( PyExc_SystemError, "module %s is not a module object, but requests module state", @@ -478,17 +604,25 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio name); goto error; } + if (token) { + PyErr_Format( + PyExc_SystemError, + "module %s specifies a token, but did not create " + "a ModuleType instance", + name); + goto error; + } } - if (def->m_methods != NULL) { - ret = _add_methods_to_object(m, nameobj, def->m_methods); + if (def_like->m_methods != NULL) { + ret = _add_methods_to_object(m, nameobj, def_like->m_methods); if (ret != 0) { goto error; } } - if (def->m_doc != NULL) { - ret = PyModule_SetDocString(m, def->m_doc); + if (def_like->m_doc != NULL) { + ret = PyModule_SetDocString(m, def_like->m_doc); if (ret != 0) { goto error; } @@ -503,6 +637,29 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio return NULL; } +PyObject * +PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_version) +{ + PyModuleDef_Init(def); + return module_from_def_and_spec(def, spec, module_api_version, def); +} + +PyObject * +PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *slots, PyObject *spec) +{ + if (!slots) { + PyErr_SetString( + PyExc_SystemError, + "PyModule_FromSlotsAndSpec called with NULL slots"); + return NULL; + } + // Fill in enough of a PyModuleDef to pass to common machinery + PyModuleDef def_like = {.m_slots = (PyModuleDef_Slot *)slots}; + + return module_from_def_and_spec(&def_like, spec, PYTHON_API_VERSION, + NULL); +} + #ifdef Py_GIL_DISABLED int PyUnstable_Module_SetGIL(PyObject *module, void *gil) @@ -516,71 +673,94 @@ PyUnstable_Module_SetGIL(PyObject *module, void *gil) } #endif -int -PyModule_ExecDef(PyObject *module, PyModuleDef *def) +static int +run_exec_func(PyObject *module, int (*exec)(PyObject *)) { - PyModuleDef_Slot *cur_slot; - const char *name; - int ret; - - name = PyModule_GetName(module); - if (name == NULL) { + int ret = exec(module); + if (ret != 0) { + if (!PyErr_Occurred()) { + PyErr_Format( + PyExc_SystemError, + "execution of %R failed without setting an exception", + module); + } return -1; } + if (PyErr_Occurred()) { + _PyErr_FormatFromCause( + PyExc_SystemError, + "execution of module %R raised unreported exception", + module); + return -1; + } + return 0; +} - if (def->m_size >= 0) { - PyModuleObject *md = (PyModuleObject*)module; +static int +alloc_state(PyObject *module) +{ + if (!PyModule_Check(module)) { + PyErr_Format(PyExc_TypeError, "expected module, got %T", module); + return -1; + } + PyModuleObject *md = (PyModuleObject*)module; + + if (md->md_state_size >= 0) { if (md->md_state == NULL) { /* Always set a state pointer; this serves as a marker to skip * multiple initialization (importlib.reload() is no-op) */ - md->md_state = PyMem_Malloc(def->m_size); + md->md_state = PyMem_Malloc(md->md_state_size); if (!md->md_state) { PyErr_NoMemory(); return -1; } - memset(md->md_state, 0, def->m_size); + memset(md->md_state, 0, md->md_state_size); } } + return 0; +} + +int +PyModule_Exec(PyObject *module) +{ + if (alloc_state(module) < 0) { + return -1; + } + PyModuleObject *md = (PyModuleObject*)module; + if (md->md_exec) { + assert(!md->md_token_is_def); + return run_exec_func(module, md->md_exec); + } + + PyModuleDef *def = _PyModule_GetDefOrNull(module); + if (def) { + return PyModule_ExecDef(module, def); + } + return 0; +} + +int +PyModule_ExecDef(PyObject *module, PyModuleDef *def) +{ + PyModuleDef_Slot *cur_slot; + + if (alloc_state(module) < 0) { + return -1; + } + + assert(PyModule_Check(module)); if (def->m_slots == NULL) { return 0; } for (cur_slot = def->m_slots; cur_slot && cur_slot->slot; cur_slot++) { - switch (cur_slot->slot) { - case Py_mod_create: - /* handled in PyModule_FromDefAndSpec2 */ - break; - case Py_mod_exec: - ret = ((int (*)(PyObject *))cur_slot->value)(module); - if (ret != 0) { - if (!PyErr_Occurred()) { - PyErr_Format( - PyExc_SystemError, - "execution of module %s failed without setting an exception", - name); - } - return -1; - } - if (PyErr_Occurred()) { - _PyErr_FormatFromCause( - PyExc_SystemError, - "execution of module %s raised unreported exception", - name); - return -1; - } - break; - case Py_mod_multiple_interpreters: - case Py_mod_gil: - case Py_mod_abi: - /* handled in PyModule_FromDefAndSpec2 */ - break; - default: - PyErr_Format( - PyExc_SystemError, - "module %s initialized with unknown slot %i", - name, cur_slot->slot); + if (cur_slot->slot == Py_mod_exec) { + int (*func)(PyObject *) = cur_slot->value; + if (run_exec_func(module, func) < 0) { return -1; + } + continue; } } return 0; @@ -624,6 +804,31 @@ PyModule_GetDict(PyObject *m) return _PyModule_GetDict(m); // borrowed reference } +int +PyModule_GetStateSize(PyObject *m, Py_ssize_t *size_p) +{ + *size_p = -1; + if (!PyModule_Check(m)) { + PyErr_Format(PyExc_TypeError, "expected module, got %T", m); + return -1; + } + PyModuleObject *mod = (PyModuleObject *)m; + *size_p = mod->md_state_size; + return 0; +} + +int +PyModule_GetToken(PyObject *m, void **token_p) +{ + *token_p = NULL; + if (!PyModule_Check(m)) { + PyErr_Format(PyExc_TypeError, "expected module, got %T", m); + return -1; + } + *token_p = _PyModule_GetToken(m); + return 0; +} + PyObject* PyModule_GetNameObject(PyObject *mod) { @@ -764,7 +969,7 @@ PyModule_GetDef(PyObject* m) PyErr_BadArgument(); return NULL; } - return _PyModule_GetDef(m); + return _PyModule_GetDefOrNull(m); } void* @@ -888,17 +1093,18 @@ module_dealloc(PyObject *self) } FT_CLEAR_WEAKREFS(self, m->md_weaklist); + assert_def_missing_or_redundant(m); /* bpo-39824: Don't call m_free() if m_size > 0 and md_state=NULL */ - if (m->md_def && m->md_def->m_free - && (m->md_def->m_size <= 0 || m->md_state != NULL)) + if (m->md_state_free && (m->md_state_size <= 0 || m->md_state != NULL)) { - m->md_def->m_free(m); + m->md_state_free(m); } Py_XDECREF(m->md_dict); Py_XDECREF(m->md_name); - if (m->md_state != NULL) + if (m->md_state != NULL) { PyMem_Free(m->md_state); + } Py_TYPE(m)->tp_free((PyObject *)m); } @@ -1206,11 +1412,11 @@ module_traverse(PyObject *self, visitproc visit, void *arg) { PyModuleObject *m = _PyModule_CAST(self); + assert_def_missing_or_redundant(m); /* bpo-39824: Don't call m_traverse() if m_size > 0 and md_state=NULL */ - if (m->md_def && m->md_def->m_traverse - && (m->md_def->m_size <= 0 || m->md_state != NULL)) + if (m->md_state_traverse && (m->md_state_size <= 0 || m->md_state != NULL)) { - int res = m->md_def->m_traverse((PyObject*)m, visit, arg); + int res = m->md_state_traverse((PyObject*)m, visit, arg); if (res) return res; } @@ -1224,18 +1430,19 @@ module_clear(PyObject *self) { PyModuleObject *m = _PyModule_CAST(self); + assert_def_missing_or_redundant(m); /* bpo-39824: Don't call m_clear() if m_size > 0 and md_state=NULL */ - if (m->md_def && m->md_def->m_clear - && (m->md_def->m_size <= 0 || m->md_state != NULL)) + if (m->md_state_clear && (m->md_state_size <= 0 || m->md_state != NULL)) { - int res = m->md_def->m_clear((PyObject*)m); + int res = m->md_state_clear((PyObject*)m); if (PyErr_Occurred()) { PyErr_FormatUnraisable("Exception ignored in m_clear of module%s%V", m->md_name ? " " : "", m->md_name, ""); } - if (res) + if (res) { return res; + } } Py_CLEAR(m->md_dict); return 0; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index d5695015807..326f4add896 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5764,11 +5764,11 @@ PyType_GetModuleState(PyTypeObject *type) } -/* Get the module of the first superclass where the module has the - * given PyModuleDef. +/* Return borrowed ref to the module of the first superclass where the module + * has the given token. */ -PyObject * -PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) +static PyObject * +borrow_module_by_token(PyTypeObject *type, const void *token) { assert(PyType_Check(type)); @@ -5780,7 +5780,7 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) else { PyHeapTypeObject *ht = (PyHeapTypeObject*)type; PyObject *module = ht->ht_module; - if (module && _PyModule_GetDef(module) == def) { + if (module && _PyModule_GetToken(module) == token) { return module; } } @@ -5808,7 +5808,7 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) PyHeapTypeObject *ht = (PyHeapTypeObject*)super; PyObject *module = ht->ht_module; - if (module && _PyModule_GetDef(module) == def) { + if (module && _PyModule_GetToken(module) == token) { res = module; break; } @@ -5826,6 +5826,18 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) return NULL; } +PyObject * +PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) +{ + return borrow_module_by_token(type, def); +} + +PyObject * +PyType_GetModuleByToken(PyTypeObject *type, const void *token) +{ + return Py_XNewRef(borrow_module_by_token(type, token)); +} + static PyTypeObject * get_base_by_token_recursive(PyObject *bases, void *token) diff --git a/PC/python3dll.c b/PC/python3dll.c index 05c86e6d592..99e0f05fe03 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -416,8 +416,10 @@ EXPORT_FUNC(PyModule_AddObjectRef) EXPORT_FUNC(PyModule_AddStringConstant) EXPORT_FUNC(PyModule_AddType) EXPORT_FUNC(PyModule_Create2) +EXPORT_FUNC(PyModule_Exec) EXPORT_FUNC(PyModule_ExecDef) EXPORT_FUNC(PyModule_FromDefAndSpec2) +EXPORT_FUNC(PyModule_FromSlotsAndSpec) EXPORT_FUNC(PyModule_GetDef) EXPORT_FUNC(PyModule_GetDict) EXPORT_FUNC(PyModule_GetFilename) @@ -425,6 +427,8 @@ EXPORT_FUNC(PyModule_GetFilenameObject) EXPORT_FUNC(PyModule_GetName) EXPORT_FUNC(PyModule_GetNameObject) EXPORT_FUNC(PyModule_GetState) +EXPORT_FUNC(PyModule_GetStateSize) +EXPORT_FUNC(PyModule_GetToken) EXPORT_FUNC(PyModule_New) EXPORT_FUNC(PyModule_NewObject) EXPORT_FUNC(PyModule_SetDocString) @@ -668,6 +672,7 @@ EXPORT_FUNC(PyType_GetFlags) EXPORT_FUNC(PyType_GetFullyQualifiedName) EXPORT_FUNC(PyType_GetModule) EXPORT_FUNC(PyType_GetModuleByDef) +EXPORT_FUNC(PyType_GetModuleByToken) EXPORT_FUNC(PyType_GetModuleName) EXPORT_FUNC(PyType_GetModuleState) EXPORT_FUNC(PyType_GetName) diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index a355a5fc257..68707a54ff6 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -126,6 +126,7 @@ + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 05128d3ac36..b0e75ce433a 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -111,6 +111,9 @@ Source Files + + Source Files + Source Files diff --git a/Python/import.c b/Python/import.c index d4b574a8828..6cf4a061ca6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -672,8 +672,8 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) (6). first time (not found in _PyRuntime.imports.extensions): A. _imp_create_dynamic_impl() -> import_find_extension() - B. _imp_create_dynamic_impl() -> _PyImport_GetModInitFunc() - C. _PyImport_GetModInitFunc(): load + B. _imp_create_dynamic_impl() -> _PyImport_GetModuleExportHooks() + C. _PyImport_GetModuleExportHooks(): load D. _imp_create_dynamic_impl() -> import_run_extension() E. import_run_extension() -> _PyImport_RunModInitFunc() F. _PyImport_RunModInitFunc(): call @@ -743,16 +743,19 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) A. noop - ...for multi-phase init modules: + ...for multi-phase init modules from PyModInit_* (PyModuleDef): (6). every time: A. _imp_create_dynamic_impl() -> import_find_extension() (not found) - B. _imp_create_dynamic_impl() -> _PyImport_GetModInitFunc() - C. _PyImport_GetModInitFunc(): load + B. _imp_create_dynamic_impl() -> _PyImport_GetModuleExportHooks() + C. _PyImport_GetModuleExportHooks(): load D. _imp_create_dynamic_impl() -> import_run_extension() E. import_run_extension() -> _PyImport_RunModInitFunc() F. _PyImport_RunModInitFunc(): call G. import_run_extension() -> PyModule_FromDefAndSpec() + + PyModule_FromDefAndSpec(): + H. PyModule_FromDefAndSpec(): gather/check moduledef slots I. if there's a Py_mod_create slot: 1. PyModule_FromDefAndSpec(): call its function @@ -765,10 +768,29 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) (10). every time: A. _imp_exec_dynamic_impl() -> exec_builtin_or_dynamic() B. if mod->md_state == NULL (including if m_size == 0): - 1. exec_builtin_or_dynamic() -> PyModule_ExecDef() - 2. PyModule_ExecDef(): allocate mod->md_state + 1. exec_builtin_or_dynamic() -> PyModule_Exec() + 2. PyModule_Exec(): allocate mod->md_state 3. if there's a Py_mod_exec slot: - 1. PyModule_ExecDef(): call its function + 1. PyModule_Exec(): call its function + + + ...for multi-phase init modules from PyModExport_* (slots array): + + (6). every time: + + A. _imp_create_dynamic_impl() -> import_find_extension() (not found) + B. _imp_create_dynamic_impl() -> _PyImport_GetModuleExportHooks() + C. _PyImport_GetModuleExportHooks(): load + D. _imp_create_dynamic_impl() -> import_run_modexport() + E. import_run_modexport(): call + F. import_run_modexport() -> PyModule_FromSlotsAndSpec() + G. PyModule_FromSlotsAndSpec(): create temporary PyModuleDef-like + H. PyModule_FromSlotsAndSpec() -> PyModule_FromDefAndSpec() + + (PyModule_FromDefAndSpec behaves as for PyModInit_*, above) + + (10). every time: as for PyModInit_*, above + */ @@ -825,25 +847,19 @@ _PyImport_SetDLOpenFlags(PyInterpreterState *interp, int new_val) /* Common implementation for _imp.exec_dynamic and _imp.exec_builtin */ static int exec_builtin_or_dynamic(PyObject *mod) { - PyModuleDef *def; void *state; if (!PyModule_Check(mod)) { return 0; } - def = PyModule_GetDef(mod); - if (def == NULL) { - return 0; - } - state = PyModule_GetState(mod); if (state) { /* Already initialized; skip reload */ return 0; } - return PyModule_ExecDef(mod, def); + return PyModule_Exec(mod); } @@ -1787,7 +1803,7 @@ finish_singlephase_extension(PyThreadState *tstate, PyObject *mod, PyObject *name, PyObject *modules) { assert(mod != NULL && PyModule_Check(mod)); - assert(cached->def == _PyModule_GetDef(mod)); + assert(cached->def == _PyModule_GetDefOrNull(mod)); Py_ssize_t index = _get_cached_module_index(cached); if (_modules_by_index_set(tstate->interp, index, mod) < 0) { @@ -1865,8 +1881,8 @@ reload_singlephase_extension(PyThreadState *tstate, * due to violating interpreter isolation. * See the note in set_cached_m_dict(). * Until that is solved, we leave md_def set to NULL. */ - assert(_PyModule_GetDef(mod) == NULL - || _PyModule_GetDef(mod) == def); + assert(_PyModule_GetDefOrNull(mod) == NULL + || _PyModule_GetDefOrNull(mod) == def); } else { assert(cached->m_dict == NULL); @@ -1953,6 +1969,43 @@ import_find_extension(PyThreadState *tstate, return mod; } +static PyObject * +import_run_modexport(PyThreadState *tstate, PyModExportFunction ex0, + struct _Py_ext_module_loader_info *info, + PyObject *spec) +{ + /* This is like import_run_extension, but avoids interpreter switching + * and code for for single-phase modules. + */ + PyModuleDef_Slot *slots = ex0(); + if (!slots) { + if (!PyErr_Occurred()) { + PyErr_Format( + PyExc_SystemError, + "slot export function for module %s failed without setting an exception", + info->name); + } + return NULL; + } + if (PyErr_Occurred()) { + PyErr_Format( + PyExc_SystemError, + "slot export function for module %s raised unreported exception", + info->name); + } + PyObject *result = PyModule_FromSlotsAndSpec(slots, spec); + if (!result) { + return NULL; + } + if (PyModule_Check(result)) { + PyModuleObject *mod = (PyModuleObject *)result; + if (mod && !mod->md_token) { + mod->md_token = slots; + } + } + return result; +} + static PyObject * import_run_extension(PyThreadState *tstate, PyModInitFunction p0, struct _Py_ext_module_loader_info *info, @@ -2125,7 +2178,7 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, assert_multiphase_def(def); assert(mod == NULL); /* Note that we cheat a little by not repeating the calls - * to _PyImport_GetModInitFunc() and _PyImport_RunModInitFunc(). */ + * to _PyImport_GetModuleExportHooks() and _PyImport_RunModInitFunc(). */ mod = PyModule_FromDefAndSpec(def, spec); if (mod == NULL) { goto error; @@ -2239,8 +2292,9 @@ _PyImport_FixupBuiltin(PyThreadState *tstate, PyObject *mod, const char *name, return -1; } - PyModuleDef *def = PyModule_GetDef(mod); + PyModuleDef *def = _PyModule_GetDefOrNull(mod); if (def == NULL) { + assert(!PyErr_Occurred()); PyErr_BadInternalCall(); goto finally; } @@ -2322,8 +2376,8 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) assert(!_PyErr_Occurred(tstate)); assert(cached != NULL); /* The module might not have md_def set in certain reload cases. */ - assert(_PyModule_GetDef(mod) == NULL - || cached->def == _PyModule_GetDef(mod)); + assert(_PyModule_GetDefOrNull(mod) == NULL + || cached->def == _PyModule_GetDefOrNull(mod)); assert_singlephase(cached); goto finally; } @@ -4653,8 +4707,8 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) assert(!_PyErr_Occurred(tstate)); assert(cached != NULL); /* The module might not have md_def set in certain reload cases. */ - assert(_PyModule_GetDef(mod) == NULL - || cached->def == _PyModule_GetDef(mod)); + assert(_PyModule_GetDefOrNull(mod) == NULL + || cached->def == _PyModule_GetDefOrNull(mod)); assert_singlephase(cached); goto finally; } @@ -4679,7 +4733,7 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) } /* We would move this (and the fclose() below) into - * _PyImport_GetModInitFunc(), but it isn't clear if the intervening + * _PyImport_GetModuleExportHooks(), but it isn't clear if the intervening * code relies on fp still being open. */ FILE *fp; if (file != NULL) { @@ -4692,7 +4746,13 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) fp = NULL; } - PyModInitFunction p0 = _PyImport_GetModInitFunc(&info, fp); + PyModInitFunction p0 = NULL; + PyModExportFunction ex0 = NULL; + _PyImport_GetModuleExportHooks(&info, fp, &p0, &ex0); + if (ex0) { + mod = import_run_modexport(tstate, ex0, &info, spec); + goto cleanup; + } if (p0 == NULL) { goto finally; } @@ -4714,6 +4774,7 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) } #endif +cleanup: // XXX Shouldn't this happen in the error cases too (i.e. in "finally")? if (fp) { fclose(fp); diff --git a/Python/importdl.c b/Python/importdl.c index 802843fe7b9..23a55c39677 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -5,7 +5,7 @@ #include "pycore_call.h" // _PyObject_CallMethod() #include "pycore_import.h" // _PyImport_SwapPackageContext() #include "pycore_importdl.h" -#include "pycore_moduleobject.h" // _PyModule_GetDef() +#include "pycore_moduleobject.h" // _PyModule_GetDefOrNull() #include "pycore_pyerrors.h" // _PyErr_FormatFromCause() #include "pycore_runtime.h" // _Py_ID() @@ -35,8 +35,10 @@ extern dl_funcptr _PyImport_FindSharedFuncptr(const char *prefix, /* module info to use when loading */ /***********************************/ -static const char * const ascii_only_prefix = "PyInit"; -static const char * const nonascii_prefix = "PyInitU"; +static const struct hook_prefixes ascii_only_prefixes = { + "PyInit", "PyModExport"}; +static const struct hook_prefixes nonascii_prefixes = { + "PyInitU", "PyModExportU"}; /* Get the variable part of a module's export symbol name. * Returns a bytes instance. For non-ASCII-named modules, the name is @@ -45,7 +47,7 @@ static const char * const nonascii_prefix = "PyInitU"; * nonascii_prefix, as appropriate. */ static PyObject * -get_encoded_name(PyObject *name, const char **hook_prefix) { +get_encoded_name(PyObject *name, const struct hook_prefixes **hook_prefixes) { PyObject *tmp; PyObject *encoded = NULL; PyObject *modname = NULL; @@ -72,7 +74,7 @@ get_encoded_name(PyObject *name, const char **hook_prefix) { /* Encode to ASCII or Punycode, as needed */ encoded = PyUnicode_AsEncodedString(name, "ascii", NULL); if (encoded != NULL) { - *hook_prefix = ascii_only_prefix; + *hook_prefixes = &ascii_only_prefixes; } else { if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) { PyErr_Clear(); @@ -80,7 +82,7 @@ get_encoded_name(PyObject *name, const char **hook_prefix) { if (encoded == NULL) { goto error; } - *hook_prefix = nonascii_prefix; + *hook_prefixes = &nonascii_prefixes; } else { goto error; } @@ -130,7 +132,7 @@ _Py_ext_module_loader_info_init(struct _Py_ext_module_loader_info *p_info, assert(PyUnicode_GetLength(name) > 0); info.name = Py_NewRef(name); - info.name_encoded = get_encoded_name(info.name, &info.hook_prefix); + info.name_encoded = get_encoded_name(info.name, &info.hook_prefixes); if (info.name_encoded == NULL) { _Py_ext_module_loader_info_clear(&info); return -1; @@ -189,7 +191,7 @@ _Py_ext_module_loader_info_init_for_builtin( /* We won't need filename. */ .path=name, .origin=_Py_ext_module_origin_BUILTIN, - .hook_prefix=ascii_only_prefix, + .hook_prefixes=&ascii_only_prefixes, .newcontext=NULL, }; return 0; @@ -377,39 +379,63 @@ _Py_ext_module_loader_result_apply_error( /********************************************/ #ifdef HAVE_DYNAMIC_LOADING -PyModInitFunction -_PyImport_GetModInitFunc(struct _Py_ext_module_loader_info *info, - FILE *fp) +static dl_funcptr +findfuncptr(const char *prefix, const char *name_buf, + struct _Py_ext_module_loader_info *info, + FILE *fp) { +#ifdef MS_WINDOWS + return _PyImport_FindSharedFuncptrWindows( + prefix, name_buf, info->filename, fp); +#else + const char *path_buf = PyBytes_AS_STRING(info->filename_encoded); + return _PyImport_FindSharedFuncptr( + prefix, name_buf, path_buf, fp); +#endif +} + +int +_PyImport_GetModuleExportHooks( + struct _Py_ext_module_loader_info *info, + FILE *fp, + PyModInitFunction *modinit, + PyModExportFunction *modexport) +{ + *modinit = NULL; + *modexport = NULL; + const char *name_buf = PyBytes_AS_STRING(info->name_encoded); dl_funcptr exportfunc; -#ifdef MS_WINDOWS - exportfunc = _PyImport_FindSharedFuncptrWindows( - info->hook_prefix, name_buf, info->filename, fp); -#else - { - const char *path_buf = PyBytes_AS_STRING(info->filename_encoded); - exportfunc = _PyImport_FindSharedFuncptr( - info->hook_prefix, name_buf, path_buf, fp); - } -#endif - if (exportfunc == NULL) { - if (!PyErr_Occurred()) { - PyObject *msg; - msg = PyUnicode_FromFormat( - "dynamic module does not define " - "module export function (%s_%s)", - info->hook_prefix, name_buf); - if (msg != NULL) { - PyErr_SetImportError(msg, info->name, info->filename); - Py_DECREF(msg); - } + exportfunc = findfuncptr( + info->hook_prefixes->export_prefix, + name_buf, info, fp); + if (exportfunc) { + *modexport = (PyModExportFunction)exportfunc; + return 2; + } + + exportfunc = findfuncptr( + info->hook_prefixes->init_prefix, + name_buf, info, fp); + if (exportfunc) { + *modinit = (PyModInitFunction)exportfunc; + return 1; + } + + if (!PyErr_Occurred()) { + PyObject *msg; + msg = PyUnicode_FromFormat( + "dynamic module does not define " + "module export function (%s_%s or %s_%s)", + info->hook_prefixes->export_prefix, name_buf, + info->hook_prefixes->init_prefix, name_buf); + if (msg != NULL) { + PyErr_SetImportError(msg, info->name, info->filename); + Py_DECREF(msg); } - return NULL; } - - return (PyModInitFunction)exportfunc; + return -1; } #endif /* HAVE_DYNAMIC_LOADING */ @@ -477,7 +503,7 @@ _PyImport_RunModInitFunc(PyModInitFunction p0, res.def = (PyModuleDef *)m; /* Run PyModule_FromDefAndSpec() to finish loading the module. */ } - else if (info->hook_prefix == nonascii_prefix) { + else if (info->hook_prefixes == &nonascii_prefixes) { /* Non-ASCII is only supported for multi-phase init. */ res.kind = _Py_ext_module_kind_MULTIPHASE; /* Don't allow legacy init for non-ASCII module names. */ @@ -496,7 +522,7 @@ _PyImport_RunModInitFunc(PyModInitFunction p0, goto error; } - res.def = _PyModule_GetDef(m); + res.def = _PyModule_GetDefOrNull(m); if (res.def == NULL) { PyErr_Clear(); _Py_ext_module_loader_result_set_error( From 35528fccdcaa0890e959eb7884332d1a426819ac Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 5 Nov 2025 09:13:57 -0500 Subject: [PATCH 037/313] gh-141004: Document missing iterator types in the C API (GH-141010) Add documentation for each of the following: - PyByteArrayIter_Type - PyBytesIter_Type - PyListIter_Type - PyListRevIter_Type - PySetIter_Type - PyTupleIter_Type - PyRangeIter_Type - PyLongRangeIter_Type - PyDictIterKey_Type - PyDictRevIterKey_Type - PyDictIterValue_Type - PyDictRevIterValue_Type - PyDictIterItem_Type - PyDictRevIterItem_Type --------- Co-authored-by: Petr Viktorin --- Doc/c-api/iterator.rst | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/Doc/c-api/iterator.rst b/Doc/c-api/iterator.rst index 6b7ba8c9979..4b94970036f 100644 --- a/Doc/c-api/iterator.rst +++ b/Doc/c-api/iterator.rst @@ -50,3 +50,32 @@ sentinel value is returned. callable object that can be called with no parameters; each call to it should return the next item in the iteration. When *callable* returns a value equal to *sentinel*, the iteration will be terminated. + + +Other Iterator Objects +^^^^^^^^^^^^^^^^^^^^^^ + +.. c:var:: PyTypeObject PyByteArrayIter_Type +.. c:var:: PyTypeObject PyBytesIter_Type +.. c:var:: PyTypeObject PyListIter_Type +.. c:var:: PyTypeObject PyListRevIter_Type +.. c:var:: PyTypeObject PySetIter_Type +.. c:var:: PyTypeObject PyTupleIter_Type +.. c:var:: PyTypeObject PyRangeIter_Type +.. c:var:: PyTypeObject PyLongRangeIter_Type +.. c:var:: PyTypeObject PyDictIterKey_Type +.. c:var:: PyTypeObject PyDictRevIterKey_Type +.. c:var:: PyTypeObject PyDictIterValue_Type +.. c:var:: PyTypeObject PyDictRevIterValue_Type +.. c:var:: PyTypeObject PyDictIterItem_Type +.. c:var:: PyTypeObject PyDictRevIterItem_Type + + Type objects for iterators of various built-in objects. + + Do not create instances of these directly; prefer calling + :c:func:`PyObject_GetIter` instead. + + Note that there is no guarantee that a given built-in type uses a given iterator + type. For example, iterating over :class:`range` will use one of two iterator + types depending on the size of the range. Other types may start using a + similar scheme in the future, without warning. From 3f6aca1be49f96c5c5f52040b8e78c73c79c0a86 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 5 Nov 2025 11:45:13 -0500 Subject: [PATCH 038/313] gh-141004: Document `PyMemoryView_Type` (GH-141034) --- Doc/c-api/memoryview.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst index f6038032805..e4ac8b57673 100644 --- a/Doc/c-api/memoryview.rst +++ b/Doc/c-api/memoryview.rst @@ -13,6 +13,12 @@ A :class:`memoryview` object exposes the C level :ref:`buffer interface any other object. +.. c:var:: PyTypeObject PyMemoryView_Type + + This instance of :c:type:`PyTypeObject` represents the Python memoryview + type. This is the same object as :class:`memoryview` in the Python layer. + + .. c:function:: PyObject *PyMemoryView_FromObject(PyObject *obj) Create a memoryview object from an object that provides the buffer interface. From 579b2f8910d6c4b07094d86b01d5421a55b09533 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 5 Nov 2025 17:57:06 +0100 Subject: [PATCH 039/313] gh-140550: Run make regen-limited-abi (#141056) --- Doc/data/stable_abi.dat | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 1359cfa4fbf..5cbf3771950 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -979,6 +979,14 @@ macro,Py_bf_releasebuffer,3.11,, type,Py_buffer,3.11,,full-abi type,Py_intptr_t,3.2,, macro,Py_mod_abi,3.15,, +macro,Py_mod_doc,3.15,, +macro,Py_mod_methods,3.15,, +macro,Py_mod_name,3.15,, +macro,Py_mod_state_clear,3.15,, +macro,Py_mod_state_free,3.15,, +macro,Py_mod_state_size,3.15,, +macro,Py_mod_state_traverse,3.15,, +macro,Py_mod_token,3.15,, macro,Py_mp_ass_subscript,3.2,, macro,Py_mp_length,3.2,, macro,Py_mp_subscript,3.2,, From 30ab627aab050840d17ac14c9d3730d065bda6af Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 5 Nov 2025 18:31:35 +0100 Subject: [PATCH 040/313] gh-83714: Fix a compiler warning in stat_nanosecond_timestamp() (#141043) Disable the fast path on systems with 32-bit long. --- Modules/posixmodule.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 50464b01efb..ecda75ec6ab 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2634,11 +2634,14 @@ _posix_free(void *module) static PyObject * stat_nanosecond_timestamp(_posixstate *state, time_t sec, unsigned long nsec) { +#if SIZEOF_LONG >= 8 /* 1677-09-21 00:12:44 to 2262-04-11 23:47:15 UTC inclusive */ if ((LLONG_MIN/SEC_TO_NS) <= sec && sec <= (LLONG_MAX/SEC_TO_NS - 1)) { return PyLong_FromLongLong(sec * SEC_TO_NS + nsec); } - else { + else +#endif + { PyObject *ns_total = NULL; PyObject *s_in_ns = NULL; PyObject *s = _PyLong_FromTime_t(sec); From 8d55faf2d68bbb6486a3e4509e8912d211748756 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 5 Nov 2025 18:37:06 +0100 Subject: [PATCH 041/313] Remove internal _PyTime_AsLong() function (#141053) * Replace _PyTime_AsLong() with PyLong_FromInt64() * Replace _PyTime_FromLong() with PyLong_AsInt64(). --- Include/internal/pycore_time.h | 9 -------- Modules/_lsprof.c | 4 ++-- Modules/_testinternalcapi/pytime.c | 20 ++++++++--------- Modules/timemodule.c | 14 ++++++------ Python/pytime.c | 35 +----------------------------- 5 files changed, 20 insertions(+), 62 deletions(-) diff --git a/Include/internal/pycore_time.h b/Include/internal/pycore_time.h index 23312471c65..b671225ca6e 100644 --- a/Include/internal/pycore_time.h +++ b/Include/internal/pycore_time.h @@ -147,11 +147,6 @@ extern int _PyTime_FromSecondsDouble( // Clamp to [PyTime_MIN; PyTime_MAX] on overflow. extern PyTime_t _PyTime_FromMicrosecondsClamp(PyTime_t us); -// Create a timestamp from a Python int object (number of nanoseconds). -// Export for '_lsprof' shared extension. -PyAPI_FUNC(int) _PyTime_FromLong(PyTime_t *t, - PyObject *obj); - // Convert a number of seconds (Python float or int) to a timestamp. // Raise an exception and return -1 on error, return 0 on success. // Export for '_socket' shared extension. @@ -182,10 +177,6 @@ extern PyTime_t _PyTime_As100Nanoseconds(PyTime_t t, _PyTime_round_t round); #endif -// Convert a timestamp (number of nanoseconds) as a Python int object. -// Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(PyObject*) _PyTime_AsLong(PyTime_t t); - #ifndef MS_WINDOWS // Create a timestamp from a timeval structure. // Raise an exception and return -1 on overflow, return 0 on success. diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index c20dbc3f4f4..025a3fac46e 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -6,7 +6,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_SetProfile() #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_time.h" // _PyTime_FromLong() +#include "pycore_time.h" // _PyTime_FromSecondsObject() #include "pycore_typeobject.h" // _PyType_GetModuleState() #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() @@ -111,7 +111,7 @@ static PyTime_t CallExternalTimer(ProfilerObject *pObj) if (pObj->externalTimerUnit > 0.0) { /* interpret the result as an integer that will be scaled in profiler_getstats() */ - err = _PyTime_FromLong(&result, o); + err = PyLong_AsInt64(o, &result); } else { /* interpret the result as a double measured in seconds. diff --git a/Modules/_testinternalcapi/pytime.c b/Modules/_testinternalcapi/pytime.c index 2b0a205d158..7fb100c41a1 100644 --- a/Modules/_testinternalcapi/pytime.c +++ b/Modules/_testinternalcapi/pytime.c @@ -17,7 +17,7 @@ test_pytime_fromseconds(PyObject *self, PyObject *args) return NULL; } PyTime_t ts = _PyTime_FromSeconds(seconds); - return _PyTime_AsLong(ts); + return PyLong_FromInt64(ts); } static int @@ -49,7 +49,7 @@ test_pytime_fromsecondsobject(PyObject *self, PyObject *args) if (_PyTime_FromSecondsObject(&ts, obj, round) == -1) { return NULL; } - return _PyTime_AsLong(ts); + return PyLong_FromInt64(ts); } static PyObject * @@ -64,7 +64,7 @@ test_PyTime_AsTimeval(PyObject *self, PyObject *args) return NULL; } PyTime_t t; - if (_PyTime_FromLong(&t, obj) < 0) { + if (PyLong_AsInt64(obj, &t) < 0) { return NULL; } struct timeval tv; @@ -91,7 +91,7 @@ test_PyTime_AsTimeval_clamp(PyObject *self, PyObject *args) return NULL; } PyTime_t t; - if (_PyTime_FromLong(&t, obj) < 0) { + if (PyLong_AsInt64(obj, &t) < 0) { return NULL; } struct timeval tv; @@ -113,7 +113,7 @@ test_PyTime_AsTimespec(PyObject *self, PyObject *args) return NULL; } PyTime_t t; - if (_PyTime_FromLong(&t, obj) < 0) { + if (PyLong_AsInt64(obj, &t) < 0) { return NULL; } struct timespec ts; @@ -131,7 +131,7 @@ test_PyTime_AsTimespec_clamp(PyObject *self, PyObject *args) return NULL; } PyTime_t t; - if (_PyTime_FromLong(&t, obj) < 0) { + if (PyLong_AsInt64(obj, &t) < 0) { return NULL; } struct timespec ts; @@ -149,14 +149,14 @@ test_PyTime_AsMilliseconds(PyObject *self, PyObject *args) return NULL; } PyTime_t t; - if (_PyTime_FromLong(&t, obj) < 0) { + if (PyLong_AsInt64(obj, &t) < 0) { return NULL; } if (check_time_rounding(round) < 0) { return NULL; } PyTime_t ms = _PyTime_AsMilliseconds(t, round); - return _PyTime_AsLong(ms); + return PyLong_FromInt64(ms); } static PyObject * @@ -168,14 +168,14 @@ test_PyTime_AsMicroseconds(PyObject *self, PyObject *args) return NULL; } PyTime_t t; - if (_PyTime_FromLong(&t, obj) < 0) { + if (PyLong_AsInt64(obj, &t) < 0) { return NULL; } if (check_time_rounding(round) < 0) { return NULL; } PyTime_t us = _PyTime_AsMicroseconds(t, round); - return _PyTime_AsLong(us); + return PyLong_FromInt64(us); } static PyObject * diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 3271d87ddc2..3946d18479e 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -128,7 +128,7 @@ time_time_ns(PyObject *self, PyObject *unused) if (PyTime_Time(&t) < 0) { return NULL; } - return _PyTime_AsLong(t); + return PyLong_FromInt64(t); } PyDoc_STRVAR(time_ns_doc, @@ -261,7 +261,7 @@ time_clock_gettime_ns_impl(PyObject *module, clockid_t clk_id) if (_PyTime_FromTimespec(&t, &ts) < 0) { return NULL; } - return _PyTime_AsLong(t); + return PyLong_FromInt64(t); } #endif /* HAVE_CLOCK_GETTIME */ @@ -310,7 +310,7 @@ time_clock_settime_ns(PyObject *self, PyObject *args) return NULL; } - if (_PyTime_FromLong(&t, obj) < 0) { + if (PyLong_AsInt64(obj, &t) < 0) { return NULL; } if (_PyTime_AsTimespec(t, &ts) == -1) { @@ -1216,7 +1216,7 @@ time_monotonic_ns(PyObject *self, PyObject *unused) if (PyTime_Monotonic(&t) < 0) { return NULL; } - return _PyTime_AsLong(t); + return PyLong_FromInt64(t); } PyDoc_STRVAR(monotonic_ns_doc, @@ -1248,7 +1248,7 @@ time_perf_counter_ns(PyObject *self, PyObject *unused) if (PyTime_PerfCounter(&t) < 0) { return NULL; } - return _PyTime_AsLong(t); + return PyLong_FromInt64(t); } PyDoc_STRVAR(perf_counter_ns_doc, @@ -1437,7 +1437,7 @@ time_process_time_ns(PyObject *module, PyObject *unused) if (py_process_time(state, &t, NULL) < 0) { return NULL; } - return _PyTime_AsLong(t); + return PyLong_FromInt64(t); } PyDoc_STRVAR(process_time_ns_doc, @@ -1610,7 +1610,7 @@ time_thread_time_ns(PyObject *self, PyObject *unused) if (_PyTime_GetThreadTimeWithInfo(&t, NULL) < 0) { return NULL; } - return _PyTime_AsLong(t); + return PyLong_FromInt64(t); } PyDoc_STRVAR(thread_time_ns_doc, diff --git a/Python/pytime.c b/Python/pytime.c index 0206467364f..2f3d854428b 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -2,7 +2,7 @@ #include "pycore_initconfig.h" // _PyStatus_ERR #include "pycore_pystate.h" // _Py_AssertHoldsTstate() #include "pycore_runtime.h" // _PyRuntime -#include "pycore_time.h" // PyTime_t +#include "pycore_time.h" // export _PyLong_FromTime_t() #include // gmtime_r() #ifdef HAVE_SYS_TIME_H @@ -472,31 +472,6 @@ _PyTime_FromMicrosecondsClamp(PyTime_t us) } -int -_PyTime_FromLong(PyTime_t *tp, PyObject *obj) -{ - if (!PyLong_Check(obj)) { - PyErr_Format(PyExc_TypeError, "expect int, got %s", - Py_TYPE(obj)->tp_name); - return -1; - } - - static_assert(sizeof(long long) == sizeof(PyTime_t), - "PyTime_t is not long long"); - long long nsec = PyLong_AsLongLong(obj); - if (nsec == -1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) { - pytime_overflow(); - } - return -1; - } - - PyTime_t t = (PyTime_t)nsec; - *tp = t; - return 0; -} - - #ifdef HAVE_CLOCK_GETTIME static int pytime_fromtimespec(PyTime_t *tp, const struct timespec *ts, int raise_exc) @@ -658,14 +633,6 @@ PyTime_AsSecondsDouble(PyTime_t ns) } -PyObject * -_PyTime_AsLong(PyTime_t ns) -{ - static_assert(sizeof(long long) >= sizeof(PyTime_t), - "PyTime_t is larger than long long"); - return PyLong_FromLongLong((long long)ns); -} - int _PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round, PyTime_t *result) { From 4ac16dd10950fad2d3e58e8b0ba5f2e621af3cc1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 5 Nov 2025 19:00:32 +0100 Subject: [PATCH 042/313] Fix a compiler warning in _randommodule.c (#141058) The test just before the cast ensures that the cast cannot overflow. Fix the warning on 32-bit Windows: Modules\_randommodule.c(525,28): warning C4244: '=': conversion from 'uint64_t' to 'Py_ssize_t', possible loss of data --- Modules/_randommodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index aa2fd28c232..544e636d18f 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -522,7 +522,7 @@ _random_Random_getrandbits_impl(RandomObject *self, uint64_t k) PyErr_NoMemory(); return NULL; } - words = (k - 1u) / 32u + 1u; + words = (Py_ssize_t)((k - 1u) / 32u + 1u); wordarray = (uint32_t *)PyMem_Malloc(words * 4); if (wordarray == NULL) { PyErr_NoMemory(); From baa9f338971c6a13433a8232db77cd45e6b87b77 Mon Sep 17 00:00:00 2001 From: Sebastian Pipping Date: Wed, 5 Nov 2025 19:59:59 +0100 Subject: [PATCH 043/313] gh-139313: Improve docs on XML security (GH-139460) Clarify that: - it takes parsing for an attack - that some doors are closed by default - only Expat version 2.7.2 has all the fixes - use of the bundle depends on configuration --- Doc/library/pyexpat.rst | 9 +++++++++ Doc/library/xml.rst | 20 ++++++++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index f533850c0ca..2f5db81955c 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -634,6 +634,15 @@ otherwise stated. .. method:: xmlparser.ExternalEntityRefHandler(context, base, systemId, publicId) + .. warning:: + + Implementing a handler that accesses local files and/or the network + may create a vulnerability to + `external entity attacks `_ + if :class:`xmlparser` is used with user-provided XML content. + Please reflect on your `threat model `_ + before implementing this handler. + Called for references to external entities. *base* is the current base, as set by a previous call to :meth:`SetBase`. The public and system identifiers, *systemId* and *publicId*, are strings if given; if the public identifier is not diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index 3f745573474..acd8d399fe3 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -53,11 +53,22 @@ XML security An attacker can abuse XML features to carry out denial of service attacks, access local files, generate network connections to other machines, or -circumvent firewalls. +circumvent firewalls when attacker-controlled XML is being parsed, +in Python or elsewhere. -Expat versions lower than 2.6.0 may be vulnerable to "billion laughs", -"quadratic blowup" and "large tokens". Python may be vulnerable if it uses such -older versions of Expat as a system-provided library. +The built-in XML parsers of Python rely on the library `libexpat`_, commonly +called Expat, for parsing XML. + +By default, Expat itself does not access local files or create network +connections. + +Expat versions lower than 2.7.2 may be vulnerable to the "billion laughs", +"quadratic blowup" and "large tokens" vulnerabilities, or to disproportional +use of dynamic memory. +Python bundles a copy of Expat, and whether Python uses the bundled or a +system-wide Expat, depends on how the Python interpreter +:option:`has been configured <--with-system-expat>` in your environment. +Python may be vulnerable if it uses such older versions of Expat. Check :const:`!pyexpat.EXPAT_VERSION`. :mod:`xmlrpc` is **vulnerable** to the "decompression bomb" attack. @@ -90,5 +101,6 @@ large tokens be used to cause denial of service in the application parsing XML. The issue is known as :cve:`2023-52425`. +.. _libexpat: https://github.com/libexpat/libexpat .. _Billion Laughs: https://en.wikipedia.org/wiki/Billion_laughs .. _ZIP bomb: https://en.wikipedia.org/wiki/Zip_bomb From 3cb1ab0e5de340861afce50f338b2a9d40b04e68 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Wed, 5 Nov 2025 22:12:56 +0300 Subject: [PATCH 044/313] gh-131527: Stackref debug borrow checker (#140599) Add borrow checking to the stackref debug mode --------- Co-authored-by: mpage --- Include/internal/pycore_stackref.h | 31 ++++++- ...-10-25-21-31-43.gh-issue-131527.V-JVNP.rst | 2 + Python/stackrefs.c | 91 +++++++++++++++++-- 3 files changed, 110 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-21-31-43.gh-issue-131527.V-JVNP.rst diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 15a703a0820..e59611c07fa 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -63,6 +63,8 @@ PyAPI_FUNC(PyObject *) _Py_stackref_get_object(_PyStackRef ref); PyAPI_FUNC(PyObject *) _Py_stackref_close(_PyStackRef ref, const char *filename, int linenumber); PyAPI_FUNC(_PyStackRef) _Py_stackref_create(PyObject *obj, uint16_t flags, const char *filename, int linenumber); PyAPI_FUNC(void) _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber); +PyAPI_FUNC(_PyStackRef) _Py_stackref_get_borrowed_from(_PyStackRef ref, const char *filename, int linenumber); +PyAPI_FUNC(void) _Py_stackref_set_borrowed_from(_PyStackRef ref, _PyStackRef borrowed_from, const char *filename, int linenumber); extern void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref); static const _PyStackRef PyStackRef_NULL = { .index = 0 }; @@ -248,7 +250,12 @@ _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber) } else { flags = Py_TAG_REFCNT; } - return _Py_stackref_create(obj, flags, filename, linenumber); + _PyStackRef new_ref = _Py_stackref_create(obj, flags, filename, linenumber); + if (flags == Py_TAG_REFCNT && !_Py_IsImmortal(obj)) { + _PyStackRef borrowed_from = _Py_stackref_get_borrowed_from(ref, filename, linenumber); + _Py_stackref_set_borrowed_from(new_ref, borrowed_from, filename, linenumber); + } + return new_ref; } #define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__) @@ -259,6 +266,7 @@ _PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct, const char * assert(!PyStackRef_IsNull(ref)); assert(!PyStackRef_IsTaggedInt(ref)); PyObject *obj = _Py_stackref_close(ref, filename, linenumber); + assert(Py_REFCNT(obj) > 0); if (PyStackRef_RefcountOnObject(ref)) { _Py_DECREF_SPECIALIZED(obj, destruct); } @@ -274,7 +282,11 @@ _PyStackRef_Borrow(_PyStackRef ref, const char *filename, int linenumber) return ref; } PyObject *obj = _Py_stackref_get_object(ref); - return _Py_stackref_create(obj, Py_TAG_REFCNT, filename, linenumber); + _PyStackRef new_ref = _Py_stackref_create(obj, Py_TAG_REFCNT, filename, linenumber); + if (!_Py_IsImmortal(obj)) { + _Py_stackref_set_borrowed_from(new_ref, ref, filename, linenumber); + } + return new_ref; } #define PyStackRef_Borrow(REF) _PyStackRef_Borrow((REF), __FILE__, __LINE__) @@ -310,13 +322,22 @@ PyStackRef_IsHeapSafe(_PyStackRef ref) static inline _PyStackRef _PyStackRef_MakeHeapSafe(_PyStackRef ref, const char *filename, int linenumber) { - if (PyStackRef_IsHeapSafe(ref)) { + // Special references that can't be closed. + if (ref.index < INITIAL_STACKREF_INDEX) { return ref; } + bool heap_safe = PyStackRef_IsHeapSafe(ref); PyObject *obj = _Py_stackref_close(ref, filename, linenumber); - Py_INCREF(obj); - return _Py_stackref_create(obj, 0, filename, linenumber); + uint16_t flags = 0; + if (heap_safe) { + // Close old ref and create a new one with the same flags. + // This is necessary for correct borrow checking. + flags = ref.index & Py_TAG_BITS; + } else { + Py_INCREF(obj); + } + return _Py_stackref_create(obj, flags, filename, linenumber); } #define PyStackRef_MakeHeapSafe(REF) _PyStackRef_MakeHeapSafe(REF, __FILE__, __LINE__) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-21-31-43.gh-issue-131527.V-JVNP.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-21-31-43.gh-issue-131527.V-JVNP.rst new file mode 100644 index 00000000000..9969ea058a3 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-21-31-43.gh-issue-131527.V-JVNP.rst @@ -0,0 +1,2 @@ +Dynamic borrow checking for stackrefs is added to ``Py_STACKREF_DEBUG`` +mode. Patch by Mikhail Efimov. diff --git a/Python/stackrefs.c b/Python/stackrefs.c index 720916e0854..0c13cc65510 100644 --- a/Python/stackrefs.c +++ b/Python/stackrefs.c @@ -19,6 +19,8 @@ typedef struct _table_entry { int linenumber; const char *filename_borrow; int linenumber_borrow; + int borrows; + _PyStackRef borrowed_from; } TableEntry; TableEntry * @@ -34,6 +36,8 @@ make_table_entry(PyObject *obj, const char *filename, int linenumber) result->linenumber = linenumber; result->filename_borrow = NULL; result->linenumber_borrow = 0; + result->borrows = 0; + result->borrowed_from = PyStackRef_NULL; return result; } @@ -47,11 +51,13 @@ _Py_stackref_get_object(_PyStackRef ref) PyInterpreterState *interp = PyInterpreterState_Get(); assert(interp != NULL); if (ref.index >= interp->next_stackref) { - _Py_FatalErrorFormat(__func__, "Garbled stack ref with ID %" PRIu64 "\n", ref.index); + _Py_FatalErrorFormat(__func__, + "Garbled stack ref with ID %" PRIu64 "\n", ref.index); } TableEntry *entry = _Py_hashtable_get(interp->open_stackrefs_table, (void *)ref.index); if (entry == NULL) { - _Py_FatalErrorFormat(__func__, "Accessing closed stack ref with ID %" PRIu64 "\n", ref.index); + _Py_FatalErrorFormat(__func__, + "Accessing closed stack ref with ID %" PRIu64 "\n", ref.index); } return entry->obj; } @@ -68,13 +74,16 @@ _Py_stackref_close(_PyStackRef ref, const char *filename, int linenumber) assert(!PyStackRef_IsError(ref)); PyInterpreterState *interp = PyInterpreterState_Get(); if (ref.index >= interp->next_stackref) { - _Py_FatalErrorFormat(__func__, "Invalid StackRef with ID %" PRIu64 " at %s:%d\n", (void *)ref.index, filename, linenumber); - + _Py_FatalErrorFormat(__func__, + "Invalid StackRef with ID %" PRIu64 " at %s:%d\n", + ref.index, filename, linenumber); } PyObject *obj; if (ref.index < INITIAL_STACKREF_INDEX) { if (ref.index == 0) { - _Py_FatalErrorFormat(__func__, "Passing NULL to PyStackRef_CLOSE at %s:%d\n", filename, linenumber); + _Py_FatalErrorFormat(__func__, + "Passing NULL to _Py_stackref_close at %s:%d\n", + filename, linenumber); } // Pre-allocated reference to None, False or True -- Do not clear TableEntry *entry = _Py_hashtable_get(interp->open_stackrefs_table, (void *)ref.index); @@ -88,10 +97,27 @@ _Py_stackref_close(_PyStackRef ref, const char *filename, int linenumber) if (entry != NULL) { _Py_FatalErrorFormat(__func__, "Double close of ref ID %" PRIu64 " at %s:%d. Referred to instance of %s at %p. Closed at %s:%d\n", - (void *)ref.index, filename, linenumber, entry->classname, entry->obj, entry->filename, entry->linenumber); + ref.index, filename, linenumber, entry->classname, entry->obj, entry->filename, entry->linenumber); } #endif - _Py_FatalErrorFormat(__func__, "Invalid StackRef with ID %" PRIu64 "\n", (void *)ref.index); + _Py_FatalErrorFormat(__func__, + "Invalid StackRef with ID %" PRIu64 " at %s:%d\n", + ref.index, filename, linenumber); + } + if (!PyStackRef_IsNull(entry->borrowed_from)) { + _PyStackRef borrowed_from = entry->borrowed_from; + TableEntry *entry_borrowed = _Py_hashtable_get(interp->open_stackrefs_table, (void *)borrowed_from.index); + if (entry_borrowed == NULL) { + _Py_FatalErrorFormat(__func__, + "Invalid borrowed StackRef with ID %" PRIu64 " at %s:%d\n", + borrowed_from.index, filename, linenumber); + } + entry_borrowed->borrows--; + } + if (entry->borrows > 0) { + _Py_FatalErrorFormat(__func__, + "StackRef with ID %" PRIu64 " closed with %d borrowed refs at %s:%d. Opened at %s:%d\n", + ref.index, entry->borrows, filename, linenumber, entry->filename, entry->linenumber); } obj = entry->obj; free(entry); @@ -143,15 +169,62 @@ _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber if (entry != NULL) { _Py_FatalErrorFormat(__func__, "Borrow of closed ref ID %" PRIu64 " at %s:%d. Referred to instance of %s at %p. Closed at %s:%d\n", - (void *)ref.index, filename, linenumber, entry->classname, entry->obj, entry->filename, entry->linenumber); + ref.index, filename, linenumber, entry->classname, entry->obj, entry->filename, entry->linenumber); } #endif - _Py_FatalErrorFormat(__func__, "Invalid StackRef with ID %" PRIu64 " at %s:%d\n", (void *)ref.index, filename, linenumber); + _Py_FatalErrorFormat(__func__, + "Invalid StackRef with ID %" PRIu64 " at %s:%d\n", + ref.index, filename, linenumber); } entry->filename_borrow = filename; entry->linenumber_borrow = linenumber; } +_PyStackRef +_Py_stackref_get_borrowed_from(_PyStackRef ref, const char *filename, int linenumber) +{ + assert(!PyStackRef_IsError(ref)); + PyInterpreterState *interp = PyInterpreterState_Get(); + + TableEntry *entry = _Py_hashtable_get(interp->open_stackrefs_table, (void *)ref.index); + if (entry == NULL) { + _Py_FatalErrorFormat(__func__, + "Invalid StackRef with ID %" PRIu64 " at %s:%d\n", + ref.index, filename, linenumber); + } + + return entry->borrowed_from; +} + +// This function should be used no more than once per ref. +void +_Py_stackref_set_borrowed_from(_PyStackRef ref, _PyStackRef borrowed_from, const char *filename, int linenumber) +{ + assert(!PyStackRef_IsError(ref)); + PyInterpreterState *interp = PyInterpreterState_Get(); + + TableEntry *entry = _Py_hashtable_get(interp->open_stackrefs_table, (void *)ref.index); + if (entry == NULL) { + _Py_FatalErrorFormat(__func__, + "Invalid StackRef (ref) with ID %" PRIu64 " at %s:%d\n", + ref.index, filename, linenumber); + } + + assert(PyStackRef_IsNull(entry->borrowed_from)); + if (PyStackRef_IsNull(borrowed_from)) { + return; + } + + TableEntry *entry_borrowed = _Py_hashtable_get(interp->open_stackrefs_table, (void *)borrowed_from.index); + if (entry_borrowed == NULL) { + _Py_FatalErrorFormat(__func__, + "Invalid StackRef (borrowed_from) with ID %" PRIu64 " at %s:%d\n", + borrowed_from.index, filename, linenumber); + } + + entry->borrowed_from = borrowed_from; + entry_borrowed->borrows++; +} void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref) From 1d25b751c5382aa808dbdfd7eacd77cd793418fc Mon Sep 17 00:00:00 2001 From: Sachin Shah <39803835+inventshah@users.noreply.github.com> Date: Wed, 5 Nov 2025 14:15:27 -0500 Subject: [PATCH 045/313] gh-140650: Fix write(), flush() and close() methods of io.BufferedWriter (GH-140653) They could raise SystemError or crash when getting the "closed" attribute or converting it to boolean raises an exception. --- Lib/test/test_io/test_bufferedio.py | 21 ++++++++++++++++ ...-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst | 3 +++ Modules/_io/bufferedio.c | 25 ++++++++++++++----- 3 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst diff --git a/Lib/test/test_io/test_bufferedio.py b/Lib/test/test_io/test_bufferedio.py index 6e9e96b0e55..30c34e818b1 100644 --- a/Lib/test/test_io/test_bufferedio.py +++ b/Lib/test/test_io/test_bufferedio.py @@ -962,6 +962,27 @@ def test_args_error(self): with self.assertRaisesRegex(TypeError, "BufferedWriter"): self.tp(self.BytesIO(), 1024, 1024, 1024) + def test_non_boolean_closed_attr(self): + # gh-140650: check TypeError is raised + class MockRawIOWithoutClosed(self.MockRawIO): + closed = NotImplemented + + bufio = self.tp(MockRawIOWithoutClosed()) + self.assertRaises(TypeError, bufio.write, b"") + self.assertRaises(TypeError, bufio.flush) + self.assertRaises(TypeError, bufio.close) + + def test_closed_attr_raises(self): + class MockRawIOClosedRaises(self.MockRawIO): + @property + def closed(self): + raise ValueError("test") + + bufio = self.tp(MockRawIOClosedRaises()) + self.assertRaisesRegex(ValueError, "test", bufio.write, b"") + self.assertRaisesRegex(ValueError, "test", bufio.flush) + self.assertRaisesRegex(ValueError, "test", bufio.close) + class PyBufferedWriterTest(BufferedWriterTest, PyTestCase): tp = pyio.BufferedWriter diff --git a/Misc/NEWS.d/next/Library/2025-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst b/Misc/NEWS.d/next/Library/2025-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst new file mode 100644 index 00000000000..2ae153a6480 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst @@ -0,0 +1,3 @@ +Fix an issue where closing :class:`io.BufferedWriter` could crash if +the closed attribute raised an exception on access or could not be +converted to a boolean. diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 0a2b3502532..0b4bc4c6b8a 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -362,16 +362,24 @@ _enter_buffered_busy(buffered *self) } #define IS_CLOSED(self) \ - (!self->buffer || \ + (!self->buffer ? 1 : \ (self->fast_closed_checks \ ? _PyFileIO_closed(self->raw) \ : buffered_closed(self))) #define CHECK_CLOSED(self, error_msg) \ - if (IS_CLOSED(self) && (Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t) == 0)) { \ - PyErr_SetString(PyExc_ValueError, error_msg); \ - return NULL; \ - } \ + do { \ + int _closed = IS_CLOSED(self); \ + if (_closed < 0) { \ + return NULL; \ + } \ + if (_closed && \ + (Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t) == 0)) \ + { \ + PyErr_SetString(PyExc_ValueError, error_msg); \ + return NULL; \ + } \ + } while (0); #define VALID_READ_BUFFER(self) \ (self->readable && self->read_end != -1) @@ -2079,6 +2087,7 @@ _io_BufferedWriter_write_impl(buffered *self, Py_buffer *buffer) PyObject *res = NULL; Py_ssize_t written, avail, remaining; Py_off_t offset; + int r; CHECK_INITIALIZED(self) @@ -2087,7 +2096,11 @@ _io_BufferedWriter_write_impl(buffered *self, Py_buffer *buffer) /* Issue #31976: Check for closed file after acquiring the lock. Another thread could be holding the lock while closing the file. */ - if (IS_CLOSED(self)) { + r = IS_CLOSED(self); + if (r < 0) { + goto error; + } + if (r > 0) { PyErr_SetString(PyExc_ValueError, "write to closed file"); goto error; } From f458ac01ba522cc7f94c0c0ee9a00c82f1be6d69 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 5 Nov 2025 20:18:45 +0100 Subject: [PATCH 046/313] Fix compiler warnings in remote debugging (#141060) Example of fixed warnings on 32-bit Windows: Python\remote_debugging.c(24,53): warning C4244: 'function': conversion from 'uint64_t' to 'uintptr_t', possible loss of data Modules\_remote_debugging_module.c(789,44): warning C4244: 'function': conversion from 'uint64_t' to 'size_t', possible loss of data --- Modules/_remote_debugging_module.c | 58 +++++++++++++++--------------- Python/remote_debugging.c | 14 ++++---- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index 5937d4892f5..c6ced39c70c 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -503,7 +503,7 @@ iterate_threads( if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, - unwinder->interpreter_addr + unwinder->debug_offsets.interpreter_state.threads_main, + unwinder->interpreter_addr + (uintptr_t)unwinder->debug_offsets.interpreter_state.threads_main, sizeof(void*), &thread_state_addr)) { @@ -514,7 +514,7 @@ iterate_threads( while (thread_state_addr != 0) { if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, - thread_state_addr + unwinder->debug_offsets.thread_state.native_thread_id, + thread_state_addr + (uintptr_t)unwinder->debug_offsets.thread_state.native_thread_id, sizeof(tid), &tid)) { @@ -530,7 +530,7 @@ iterate_threads( // Move to next thread if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, - thread_state_addr + unwinder->debug_offsets.thread_state.next, + thread_state_addr + (uintptr_t)unwinder->debug_offsets.thread_state.next, sizeof(void*), &thread_state_addr)) { @@ -686,7 +686,7 @@ read_py_str( return NULL; } - size_t offset = unwinder->debug_offsets.unicode_object.asciiobject_size; + size_t offset = (size_t)unwinder->debug_offsets.unicode_object.asciiobject_size; res = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, address + offset, len, buf); if (res < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read string data from remote memory"); @@ -748,7 +748,7 @@ read_py_bytes( return NULL; } - size_t offset = unwinder->debug_offsets.bytes_object.ob_sval; + size_t offset = (size_t)unwinder->debug_offsets.bytes_object.ob_sval; res = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, address + offset, len, buf); if (res < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read bytes data from remote memory"); @@ -786,7 +786,7 @@ read_py_long( int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, address, - unwinder->debug_offsets.long_object.size, + (size_t)unwinder->debug_offsets.long_object.size, long_obj); if (bytes_read < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read PyLongObject"); @@ -823,7 +823,7 @@ read_py_long( bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, - address + unwinder->debug_offsets.long_object.ob_digit, + address + (uintptr_t)unwinder->debug_offsets.long_object.ob_digit, sizeof(digit) * size, digits ); @@ -933,7 +933,7 @@ parse_task_name( int err = _Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, task_address, - unwinder->async_debug_offsets.asyncio_task_object.size, + (size_t)unwinder->async_debug_offsets.asyncio_task_object.size, task_obj); if (err < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task object"); @@ -1040,7 +1040,7 @@ handle_yield_from_frame( uintptr_t gi_await_addr_type_addr; err = read_ptr( unwinder, - gi_await_addr + unwinder->debug_offsets.pyobject.ob_type, + gi_await_addr + (uintptr_t)unwinder->debug_offsets.pyobject.ob_type, &gi_await_addr_type_addr); if (err) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read gi_await type address"); @@ -1101,7 +1101,7 @@ parse_coro_chain( // Parse the previous frame using the gi_iframe from local copy uintptr_t prev_frame; - uintptr_t gi_iframe_addr = coro_address + unwinder->debug_offsets.gen_object.gi_iframe; + uintptr_t gi_iframe_addr = coro_address + (uintptr_t)unwinder->debug_offsets.gen_object.gi_iframe; uintptr_t address_of_code_object = 0; if (parse_frame_object(unwinder, &name, gi_iframe_addr, &address_of_code_object, &prev_frame) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse frame object in coro chain"); @@ -1153,7 +1153,7 @@ create_task_result( // Parse coroutine chain if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, task_address, - unwinder->async_debug_offsets.asyncio_task_object.size, + (size_t)unwinder->async_debug_offsets.asyncio_task_object.size, task_obj) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task object for coro chain"); goto error; @@ -1206,7 +1206,7 @@ parse_task( err = read_char( unwinder, - task_address + unwinder->async_debug_offsets.asyncio_task_object.task_is_task, + task_address + (uintptr_t)unwinder->async_debug_offsets.asyncio_task_object.task_is_task, &is_task); if (err) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read is_task flag"); @@ -1354,7 +1354,7 @@ process_thread_for_awaited_by( void *context ) { PyObject *result = (PyObject *)context; - uintptr_t head_addr = thread_state_addr + unwinder->async_debug_offsets.asyncio_thread_state.asyncio_tasks_head; + uintptr_t head_addr = thread_state_addr + (uintptr_t)unwinder->async_debug_offsets.asyncio_thread_state.asyncio_tasks_head; return append_awaited_by(unwinder, tid, head_addr, result); } @@ -1369,7 +1369,7 @@ process_task_awaited_by( // Read the entire TaskObj at once char task_obj[SIZEOF_TASK_OBJ]; if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, task_address, - unwinder->async_debug_offsets.asyncio_task_object.size, + (size_t)unwinder->async_debug_offsets.asyncio_task_object.size, task_obj) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task object"); return -1; @@ -1526,7 +1526,7 @@ find_running_task_in_thread( uintptr_t address_of_running_loop; int bytes_read = read_py_ptr( unwinder, - thread_state_addr + unwinder->async_debug_offsets.asyncio_thread_state.asyncio_running_loop, + thread_state_addr + (uintptr_t)unwinder->async_debug_offsets.asyncio_thread_state.asyncio_running_loop, &address_of_running_loop); if (bytes_read == -1) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running loop address"); @@ -1540,7 +1540,7 @@ find_running_task_in_thread( int err = read_ptr( unwinder, - thread_state_addr + unwinder->async_debug_offsets.asyncio_thread_state.asyncio_running_task, + thread_state_addr + (uintptr_t)unwinder->async_debug_offsets.asyncio_thread_state.asyncio_running_task, running_task_addr); if (err) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running task address"); @@ -1556,7 +1556,7 @@ get_task_code_object(RemoteUnwinderObject *unwinder, uintptr_t task_addr, uintpt if(read_py_ptr( unwinder, - task_addr + unwinder->async_debug_offsets.asyncio_task_object.task_coro, + task_addr + (uintptr_t)unwinder->async_debug_offsets.asyncio_task_object.task_coro, &running_coro_addr) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Running task coro read failed"); return -1; @@ -1572,7 +1572,7 @@ get_task_code_object(RemoteUnwinderObject *unwinder, uintptr_t task_addr, uintpt // the offset leads directly to its first field: f_executable if (read_py_ptr( unwinder, - running_coro_addr + unwinder->debug_offsets.gen_object.gi_iframe, code_obj_addr) < 0) { + running_coro_addr + (uintptr_t)unwinder->debug_offsets.gen_object.gi_iframe, code_obj_addr) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running task code object"); return -1; } @@ -1741,7 +1741,7 @@ static bool parse_linetable(const uintptr_t addrq, const char* linetable, int firstlineno, LocationInfo* info) { const uint8_t* ptr = (const uint8_t*)(linetable); - uint64_t addr = 0; + uintptr_t addr = 0; info->lineno = firstlineno; while (*ptr != '\0') { @@ -1870,7 +1870,7 @@ parse_code_object(RemoteUnwinderObject *unwinder, meta->file_name = file; meta->linetable = linetable; meta->first_lineno = GET_MEMBER(int, code_object, unwinder->debug_offsets.code_object.firstlineno); - meta->addr_code_adaptive = real_address + unwinder->debug_offsets.code_object.co_code_adaptive; + meta->addr_code_adaptive = real_address + (uintptr_t)unwinder->debug_offsets.code_object.co_code_adaptive; if (unwinder && unwinder->code_object_cache && _Py_hashtable_set(unwinder->code_object_cache, key, meta) < 0) { cached_code_metadata_destroy(meta); @@ -2037,7 +2037,7 @@ copy_stack_chunks(RemoteUnwinderObject *unwinder, size_t count = 0; size_t max_chunks = 16; - if (read_ptr(unwinder, tstate_addr + unwinder->debug_offsets.thread_state.datastack_chunk, &chunk_addr)) { + if (read_ptr(unwinder, tstate_addr + (uintptr_t)unwinder->debug_offsets.thread_state.datastack_chunk, &chunk_addr)) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read initial stack chunk address"); return -1; } @@ -2146,8 +2146,8 @@ populate_initial_state_data( uintptr_t *interpreter_state, uintptr_t *tstate ) { - uint64_t interpreter_state_list_head = - unwinder->debug_offsets.runtime_state.interpreters_head; + uintptr_t interpreter_state_list_head = + (uintptr_t)unwinder->debug_offsets.runtime_state.interpreters_head; uintptr_t address_of_interpreter_state; int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( @@ -2174,7 +2174,7 @@ populate_initial_state_data( } uintptr_t address_of_thread = address_of_interpreter_state + - unwinder->debug_offsets.interpreter_state.threads_main; + (uintptr_t)unwinder->debug_offsets.interpreter_state.threads_main; if (_Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, @@ -2198,7 +2198,7 @@ find_running_frame( if ((void*)address_of_thread != NULL) { int err = read_ptr( unwinder, - address_of_thread + unwinder->debug_offsets.thread_state.current_frame, + address_of_thread + (uintptr_t)unwinder->debug_offsets.thread_state.current_frame, frame); if (err) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read current frame pointer"); @@ -2370,7 +2370,7 @@ append_awaited_by_for_thread( } uintptr_t task_addr = (uintptr_t)GET_MEMBER(uintptr_t, task_node, unwinder->debug_offsets.llist_node.next) - - unwinder->async_debug_offsets.asyncio_task_object.task_node; + - (uintptr_t)unwinder->async_debug_offsets.asyncio_task_object.task_node; if (process_single_task_node(unwinder, task_addr, NULL, result) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to process task node in awaited_by"); @@ -2605,7 +2605,7 @@ get_thread_status(RemoteUnwinderObject *unwinder, uint64_t tid, uint64_t pthread } SYSTEM_THREAD_INFORMATION *ti = (SYSTEM_THREAD_INFORMATION *)((char *)pi + sizeof(SYSTEM_PROCESS_INFORMATION)); - for (Py_ssize_t i = 0; i < pi->NumberOfThreads; i++, ti++) { + for (size_t i = 0; i < pi->NumberOfThreads; i++, ti++) { if (ti->ClientId.UniqueThread == (HANDLE)tid) { return ti->ThreadState != WIN32_THREADSTATE_RUNNING ? THREAD_STATE_IDLE : THREAD_STATE_RUNNING; } @@ -2642,7 +2642,7 @@ unwind_stack_for_thread( char ts[SIZEOF_THREAD_STATE]; int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, *current_tstate, unwinder->debug_offsets.thread_state.size, ts); + &unwinder->handle, *current_tstate, (size_t)unwinder->debug_offsets.thread_state.size, ts); if (bytes_read < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread state"); goto error; @@ -3174,7 +3174,7 @@ _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *s } uintptr_t head_addr = self->interpreter_addr - + self->async_debug_offsets.asyncio_interpreter_state.asyncio_tasks_head; + + (uintptr_t)self->async_debug_offsets.asyncio_interpreter_state.asyncio_tasks_head; // On top of a per-thread task lists used by default by asyncio to avoid // contention, there is also a fallback per-interpreter list of tasks; diff --git a/Python/remote_debugging.c b/Python/remote_debugging.c index 7aee87ef05a..71ffb17ed68 100644 --- a/Python/remote_debugging.c +++ b/Python/remote_debugging.c @@ -19,7 +19,7 @@ cleanup_proc_handle(proc_handle_t *handle) { } static int -read_memory(proc_handle_t *handle, uint64_t remote_address, size_t len, void* dst) +read_memory(proc_handle_t *handle, uintptr_t remote_address, size_t len, void* dst) { return _Py_RemoteDebug_ReadRemoteMemory(handle, remote_address, len, dst); } @@ -235,7 +235,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc int is_remote_debugging_enabled = 0; if (0 != read_memory( handle, - interpreter_state_addr + debug_offsets.debugger_support.remote_debugging_enabled, + interpreter_state_addr + (uintptr_t)debug_offsets.debugger_support.remote_debugging_enabled, sizeof(int), &is_remote_debugging_enabled)) { @@ -255,7 +255,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc if (tid != 0) { if (0 != read_memory( handle, - interpreter_state_addr + debug_offsets.interpreter_state.threads_head, + interpreter_state_addr + (uintptr_t)debug_offsets.interpreter_state.threads_head, sizeof(void*), &thread_state_addr)) { @@ -264,7 +264,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc while (thread_state_addr != 0) { if (0 != read_memory( handle, - thread_state_addr + debug_offsets.thread_state.native_thread_id, + thread_state_addr + (uintptr_t)debug_offsets.thread_state.native_thread_id, sizeof(this_tid), &this_tid)) { @@ -277,7 +277,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc if (0 != read_memory( handle, - thread_state_addr + debug_offsets.thread_state.next, + thread_state_addr + (uintptr_t)debug_offsets.thread_state.next, sizeof(void*), &thread_state_addr)) { @@ -294,7 +294,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc } else { if (0 != read_memory( handle, - interpreter_state_addr + debug_offsets.interpreter_state.threads_main, + interpreter_state_addr + (uintptr_t)debug_offsets.interpreter_state.threads_main, sizeof(void*), &thread_state_addr)) { @@ -346,7 +346,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc uintptr_t eval_breaker; if (0 != read_memory( handle, - thread_state_addr + debug_offsets.debugger_support.eval_breaker, + thread_state_addr + (uintptr_t)debug_offsets.debugger_support.eval_breaker, sizeof(uintptr_t), &eval_breaker)) { From 986bb0a1a2bd290f5da347e455b23468aa3f62f0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 5 Nov 2025 21:16:37 +0100 Subject: [PATCH 047/313] gh-83714: Fix stat_nanosecond_timestamp() for 32-bit time_t (#141069) --- Modules/posixmodule.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index ecda75ec6ab..6390f1fc5fe 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2634,13 +2634,14 @@ _posix_free(void *module) static PyObject * stat_nanosecond_timestamp(_posixstate *state, time_t sec, unsigned long nsec) { -#if SIZEOF_LONG >= 8 +#if SIZEOF_TIME_T == 4 + return PyLong_FromLongLong(sec * SEC_TO_NS + nsec); +#else /* 1677-09-21 00:12:44 to 2262-04-11 23:47:15 UTC inclusive */ if ((LLONG_MIN/SEC_TO_NS) <= sec && sec <= (LLONG_MAX/SEC_TO_NS - 1)) { return PyLong_FromLongLong(sec * SEC_TO_NS + nsec); } else -#endif { PyObject *ns_total = NULL; PyObject *s_in_ns = NULL; @@ -2663,6 +2664,7 @@ stat_nanosecond_timestamp(_posixstate *state, time_t sec, unsigned long nsec) Py_XDECREF(s_in_ns); return ns_total; } +#endif } static int From b83f379a972c001864d3593cd64fc07e7c7f375f Mon Sep 17 00:00:00 2001 From: Edward Xu Date: Thu, 6 Nov 2025 05:20:40 +0800 Subject: [PATCH 048/313] gh-133467: Fix typeobject `tp_base` race in free threading (gh-140549) --- Lib/test/test_free_threading/test_type.py | 19 +++++++++++++++++++ ...-10-24-14-29-12.gh-issue-133467.A5d6TM.rst | 1 + Objects/typeobject.c | 10 ++++++++++ Tools/tsan/suppressions_free_threading.txt | 4 ---- 4 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-14-29-12.gh-issue-133467.A5d6TM.rst diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py index 2d995751005..1255d842dbf 100644 --- a/Lib/test/test_free_threading/test_type.py +++ b/Lib/test/test_free_threading/test_type.py @@ -141,6 +141,25 @@ def reader(): self.run_one(writer, reader) + def test_bases_change(self): + class BaseA: + pass + + class Derived(BaseA): + pass + + def writer(): + for _ in range(1000): + class BaseB: + pass + Derived.__bases__ = (BaseB,) + + def reader(): + for _ in range(1000): + Derived.__base__ + + self.run_one(writer, reader) + def run_one(self, writer_func, reader_func): barrier = threading.Barrier(NTHREADS) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-14-29-12.gh-issue-133467.A5d6TM.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-14-29-12.gh-issue-133467.A5d6TM.rst new file mode 100644 index 00000000000..f69786866e9 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-14-29-12.gh-issue-133467.A5d6TM.rst @@ -0,0 +1 @@ +Fix race when updating :attr:`!type.__bases__` that could allow a read of :attr:`!type.__base__` to observe an inconsistent value on the free threaded build. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 326f4add896..58228d62485 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -189,6 +189,8 @@ type_lock_allow_release(void) #define types_world_is_stopped() 1 #define types_stop_world() #define types_start_world() +#define type_lock_prevent_release() +#define type_lock_allow_release() #endif @@ -1920,8 +1922,12 @@ type_set_bases_unlocked(PyTypeObject *type, PyObject *new_bases, PyTypeObject *b assert(old_bases != NULL); PyTypeObject *old_base = type->tp_base; + type_lock_prevent_release(); + types_stop_world(); set_tp_bases(type, Py_NewRef(new_bases), 0); type->tp_base = (PyTypeObject *)Py_NewRef(best_base); + types_start_world(); + type_lock_allow_release(); PyObject *temp = PyList_New(0); if (temp == NULL) { @@ -1982,8 +1988,12 @@ type_set_bases_unlocked(PyTypeObject *type, PyObject *new_bases, PyTypeObject *b if (lookup_tp_bases(type) == new_bases) { assert(type->tp_base == best_base); + type_lock_prevent_release(); + types_stop_world(); set_tp_bases(type, old_bases, 0); type->tp_base = old_base; + types_start_world(); + type_lock_allow_release(); Py_DECREF(new_bases); Py_DECREF(best_base); diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 6bd31e8e6ec..404c3015736 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -41,7 +41,3 @@ race:list_inplace_repeat_lock_held # PyObject_Realloc internally does memcpy which isn't atomic so can race # with non-locking reads. See #132070 race:PyObject_Realloc - -# gh-133467. Some of these could be hard to trigger. -race_top:set_tp_bases -race_top:type_set_bases_unlocked From 11fc411f98a04947a2a21329c29fe0f35ff52dba Mon Sep 17 00:00:00 2001 From: AN Long Date: Thu, 6 Nov 2025 06:49:45 +0900 Subject: [PATCH 049/313] gh-140916: Remove unused codes in winreg.c (#140934) --- PC/winreg.c | 55 ----------------------------------------------------- 1 file changed, 55 deletions(-) diff --git a/PC/winreg.c b/PC/winreg.c index c7bc74728f1..3cc6123fc3a 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -425,19 +425,6 @@ static PyType_Spec pyhkey_type_spec = { /************************************************************************ The public PyHKEY API (well, not public yet :-) ************************************************************************/ -PyObject * -PyHKEY_New(PyObject *m, HKEY hInit) -{ - winreg_state *st = _PyModule_GetState(m); - PyHKEYObject *key = PyObject_GC_New(PyHKEYObject, st->PyHKEY_Type); - if (key == NULL) { - return NULL; - } - key->hkey = hInit; - PyObject_GC_Track(key); - return (PyObject *)key; -} - BOOL PyHKEY_Close(winreg_state *st, PyObject *ob_handle) { @@ -513,48 +500,6 @@ PyHKEY_FromHKEY(winreg_state *st, HKEY h) } -/************************************************************************ - The module methods -************************************************************************/ -BOOL -PyWinObject_CloseHKEY(winreg_state *st, PyObject *obHandle) -{ - BOOL ok; - if (PyHKEY_Check(st, obHandle)) { - ok = PyHKEY_Close(st, obHandle); - } -#if SIZEOF_LONG >= SIZEOF_HKEY - else if (PyLong_Check(obHandle)) { - long rc; - Py_BEGIN_ALLOW_THREADS - rc = RegCloseKey((HKEY)PyLong_AsLong(obHandle)); - Py_END_ALLOW_THREADS - ok = (rc == ERROR_SUCCESS); - if (!ok) - PyErr_SetFromWindowsErrWithFunction(rc, "RegCloseKey"); - } -#else - else if (PyLong_Check(obHandle)) { - long rc; - HKEY hkey = (HKEY)PyLong_AsVoidPtr(obHandle); - Py_BEGIN_ALLOW_THREADS - rc = RegCloseKey(hkey); - Py_END_ALLOW_THREADS - ok = (rc == ERROR_SUCCESS); - if (!ok) - PyErr_SetFromWindowsErrWithFunction(rc, "RegCloseKey"); - } -#endif - else { - PyErr_SetString( - PyExc_TypeError, - "A handle must be a HKEY object or an integer"); - return FALSE; - } - return ok; -} - - /* Private Helper functions for the registry interfaces From 5b02c6e920aaef4b202fc19186f742d008460fd3 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 5 Nov 2025 17:00:26 -0500 Subject: [PATCH 050/313] gh-141004: Document `Py_RETURN_NAN` and `Py_RETURN_INF` (GH-141029) Co-authored-by: Sergey B Kirpichev --- Doc/c-api/float.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 489676caa3a..1085c32a537 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -78,6 +78,23 @@ Floating-Point Objects Return the minimum normalized positive float *DBL_MIN* as C :c:expr:`double`. +.. c:macro:: Py_RETURN_NAN + + Return :data:`math.nan` from a function. + + On most platforms, this is equivalent to ``return PyFloat_FromDouble(NAN)``. + + +.. c:macro:: Py_RETURN_INF(sign) + + Return :data:`math.inf` or :data:`-math.inf ` from a function, + depending on the sign of *sign*. + + On most platforms, this is equivalent to the following:: + + return PyFloat_FromDouble(copysign(INFINITY, sign)); + + Pack and Unpack functions ------------------------- From 227f4abacdd89bb3816c172a7f6fdaa2024dbada Mon Sep 17 00:00:00 2001 From: Zenith Date: Wed, 5 Nov 2025 17:00:36 -0500 Subject: [PATCH 051/313] gh-76007: remove curses.__version__ doc (#141052) --- Doc/library/curses.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index fb84cf32246..e60197ddd89 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -1349,7 +1349,6 @@ The :mod:`curses` module defines the following data members: .. data:: version -.. data:: __version__ A bytes object representing the current version of the module. From f0ab07f22c5fd18058a3ece7a1e745b3922af908 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 5 Nov 2025 17:32:12 -0500 Subject: [PATCH 052/313] gh-141004: Document `PyDict_GET_SIZE` (GH-141078) --- Doc/c-api/dict.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 0abbd662dad..246ce5391e1 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -245,6 +245,11 @@ Dictionary Objects ``len(p)`` on a dictionary. +.. c:function:: Py_ssize_t PyDict_GET_SIZE(PyObject *p) + + Similar to :c:func:`PyDict_Size`, but without error checking. + + .. c:function:: int PyDict_Next(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) Iterate over all key-value pairs in the dictionary *p*. The From 95f6e1275b1c9de550d978cb2b4351cc4ed24fe4 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Wed, 5 Nov 2025 14:46:30 -0800 Subject: [PATCH 053/313] GH-108009: Add clarification of parser and argument defaults in argparse docs (#124154) Co-authored-by: C.A.M. Gerlach --- Doc/library/argparse.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 9655db4f301..5a8f0bde2e3 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -2070,7 +2070,9 @@ Parser defaults >>> parser.parse_args(['736']) Namespace(bar=42, baz='badger', foo=736) - Note that parser-level defaults always override argument-level defaults:: + Note that defaults can be set at both the parser level using :meth:`set_defaults` + and at the argument level using :meth:`add_argument`. If both are called for the + same argument, the last default set for an argument is used:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('--foo', default='bar') From 101c9c0a2187940900f684086cb9ba0d456fda49 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Thu, 6 Nov 2025 10:21:13 +0530 Subject: [PATCH 054/313] gh-118516: clarify that subprocess are automatically killed if transport gets garbage collected (#140997) --- Doc/library/asyncio-eventloop.rst | 6 ++++++ Doc/library/asyncio-subprocess.rst | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 0ccc7a2b448..72f484fd1cb 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -1631,6 +1631,9 @@ async/await code consider using the high-level conforms to the :class:`asyncio.SubprocessTransport` base class and *protocol* is an object instantiated by the *protocol_factory*. + If the transport is closed or is garbage collected, the child process + is killed if it is still running. + .. method:: loop.subprocess_shell(protocol_factory, cmd, *, \ stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ stderr=subprocess.PIPE, **kwargs) @@ -1654,6 +1657,9 @@ async/await code consider using the high-level conforms to the :class:`SubprocessTransport` base class and *protocol* is an object instantiated by the *protocol_factory*. + If the transport is closed or is garbage collected, the child process + is killed if it is still running. + .. note:: It is the application's responsibility to ensure that all whitespace and special characters are quoted appropriately to avoid `shell injection diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index 03e76bc8689..9416c758e51 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -76,6 +76,9 @@ Creating Subprocesses See the documentation of :meth:`loop.subprocess_exec` for other parameters. + If the process object is garbage collected while the process is still + running, the child process will be killed. + .. versionchanged:: 3.10 Removed the *loop* parameter. @@ -95,6 +98,9 @@ Creating Subprocesses See the documentation of :meth:`loop.subprocess_shell` for other parameters. + If the process object is garbage collected while the process is still + running, the child process will be killed. + .. important:: It is the application's responsibility to ensure that all whitespace and From 8822166200ddb4a7635337b97b626e658a443cef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20S=C5=82awecki?= Date: Thu, 6 Nov 2025 06:39:07 +0100 Subject: [PATCH 055/313] gh-140569: recommend the new REPL in the asyncio REPL docs (#140570) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/library/asyncio.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 444db01390d..0f72e31dee5 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -79,6 +79,10 @@ You can experiment with an ``asyncio`` concurrent context in the :term:`REPL`: >>> await asyncio.sleep(10, result='hello') 'hello' +This REPL provides limited compatibility with :envvar:`PYTHON_BASIC_REPL`. +It is recommended that the default REPL is used +for full functionality and the latest features. + .. audit-event:: cpython.run_stdin "" "" .. versionchanged:: 3.12.5 (also 3.11.10, 3.10.15, 3.9.20, and 3.8.20) From 9037a386c6ed0c71cf8525ef91c55694ebeedc36 Mon Sep 17 00:00:00 2001 From: RayXu <140802139+F18-Maverick@users.noreply.github.com> Date: Thu, 6 Nov 2025 13:48:30 +0800 Subject: [PATCH 056/313] docs: fix a grammatical error in function.rst (#140990) --- Doc/c-api/function.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 764b2ac610b..0bac6389571 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -200,7 +200,7 @@ There are a few functions specific to Python functions. runtime behavior depending on optimization decisions, it does not change the semantics of the Python code being executed. - If *event* is ``PyFunction_EVENT_DESTROY``, Taking a reference in the + If *event* is ``PyFunction_EVENT_DESTROY``, taking a reference in the callback to the about-to-be-destroyed function will resurrect it, preventing it from being freed at this time. When the resurrected object is destroyed later, any watcher callbacks active at that time will be called again. From d6c89a2df2c8b7603125883494e9058a88348f66 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Thu, 6 Nov 2025 05:50:57 +0000 Subject: [PATCH 057/313] gh-140939: Fix memory leak in `_PyBytes_FormatEx` error path (#140957) --- Lib/test/test_bytes.py | 7 +++++++ .../2025-11-03-17-21-38.gh-issue-140939.FVboAw.rst | 2 ++ Objects/bytesobject.c | 1 + 3 files changed, 10 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-17-21-38.gh-issue-140939.FVboAw.rst diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index f10e4041937..e012042159d 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -802,6 +802,13 @@ def __int__(self): with self.assertRaisesRegex(TypeError, msg): operator.mod(format_bytes, value) + def test_memory_leak_gh_140939(self): + # gh-140939: MemoryError is raised without leaking + _testcapi = import_helper.import_module('_testcapi') + with self.assertRaises(MemoryError): + b = self.type2test(b'%*b') + b % (_testcapi.PY_SSIZE_T_MAX, b'abc') + def test_imod(self): b = self.type2test(b'hello, %b!') orig = b diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-17-21-38.gh-issue-140939.FVboAw.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-17-21-38.gh-issue-140939.FVboAw.rst new file mode 100644 index 00000000000..a2921761f75 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-17-21-38.gh-issue-140939.FVboAw.rst @@ -0,0 +1,2 @@ +Fix memory leak when :class:`bytearray` or :class:`bytes` is formated with the +``%*b`` format with a large width that results in a :exc:`MemoryError`. diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 9c807b3dd16..2b9513abe91 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -985,6 +985,7 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, if (alloc > 2) { res = PyBytesWriter_GrowAndUpdatePointer(writer, alloc - 2, res); if (res == NULL) { + Py_XDECREF(temp); goto error; } } From 6a7c969d003d3ba932d5c7f14a58e2a6408f4a3d Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Thu, 6 Nov 2025 03:08:24 -0500 Subject: [PATCH 058/313] gh-129876: Move misplaced IDLE news item (#141118) --- Lib/idlelib/News3.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/idlelib/News3.txt b/Lib/idlelib/News3.txt index 30784578cc6..53d83762f99 100644 --- a/Lib/idlelib/News3.txt +++ b/Lib/idlelib/News3.txt @@ -4,6 +4,9 @@ Released on 2025-10-07 ========================= +gh-129873: Simplify displaying the IDLE doc by only copying the text +section of idle.html to idlelib/help.html. Patch by Stan Ulbrych. + gh-112936: IDLE - Include Shell menu in single-process mode, though with Restart Shell and View Last Restart disabled. Patch by Zhikang Yan. @@ -26,9 +29,6 @@ Released on 2024-10-07 gh-120104: Fix padding in config and search dialog windows in IDLE. -gh-129873: Simplify displaying the IDLE doc by only copying the text -section of idle.html to idlelib/help.html. Patch by Stan Ulbrych. - gh-120083: Add explicit black IDLE Hovertip foreground color needed for recent macOS. Fixes Sonoma showing unreadable white on pale yellow. Patch by John Riggles. From 4e6e208be9d1c52d1b55a8bb3a83682cb078e55e Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Thu, 6 Nov 2025 08:21:02 +0000 Subject: [PATCH 059/313] Minor fixes to `idle.rst` and regenerate `help.html` (#140037) --- .gitattributes | 1 + Doc/library/idle.rst | 6 +-- Lib/idlelib/help.html | 110 ++++++++++++++++++++++++++++++------------ 3 files changed, 82 insertions(+), 35 deletions(-) diff --git a/.gitattributes b/.gitattributes index 823e3e975a2..d6547212393 100644 --- a/.gitattributes +++ b/.gitattributes @@ -83,6 +83,7 @@ Include/opcode_ids.h generated Include/token.h generated Lib/_opcode_metadata.py generated Lib/keyword.py generated +Lib/idlelib/help.html generated Lib/test/certdata/*.pem generated Lib/test/certdata/*.0 generated Lib/test/levenshtein_examples.json generated diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index e547c96b580..10ec7f0a6f1 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -13,7 +13,7 @@ IDLE --- Python editor and shell single: Integrated Development Environment .. - Remember to update Lib/idlelib/help.html with idlelib.help.copy_source() when modifying this file. + Remember to update Lib/idlelib/help.html with idlelib.help.copy_strip() when modifying this file. -------------- @@ -88,7 +88,7 @@ Save Save As... Save the current window with a Save As dialog. The file saved becomes the - new associated file for the window. (If your file namager is set to hide + new associated file for the window. (If your file manager is set to hide extensions, the current extension will be omitted in the file name box. If the new filename has no '.', '.py' and '.txt' will be added for Python and text files, except that on macOS Aqua,'.py' is added for all files.) @@ -206,7 +206,7 @@ New Indent Width Strip Trailing Whitespace Remove trailing space and other whitespace characters after the last - non-whitespace character of a line by applying str.rstrip to each line, + non-whitespace character of a line by applying :meth:`str.rstrip` to each line, including lines within multiline strings. Except for Shell windows, remove extra newlines at the end of the file. diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index ebff9a309d9..fc618ab727d 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -53,7 +53,7 @@ and after the window title. If there is no associated file, do Save As instead.

Save As…

Save the current window with a Save As dialog. The file saved becomes the -new associated file for the window. (If your file namager is set to hide +new associated file for the window. (If your file manager is set to hide extensions, the current extension will be omitted in the file name box. If the new filename has no ‘.’, ‘.py’ and ‘.txt’ will be added for Python and text files, except that on macOS Aqua,’.py’ is added for all files.)

@@ -143,8 +143,8 @@ paragraph will be formatted to less than N columns, where N defaults to 72.

New Indent Width

Open a dialog to change indent width. The accepted default by the Python community is 4 spaces.

-
Strip Trailing Chitespace

Remove trailing space and other whitespace characters after the last -non-whitespace character of a line by applying str.rstrip to each line, +

Strip Trailing Whitespace

Remove trailing space and other whitespace characters after the last +non-whitespace character of a line by applying str.rstrip() to each line, including lines within multiline strings. Except for Shell windows, remove extra newlines at the end of the file.

@@ -337,16 +337,16 @@ Unix and the Command key on assume that the keys have not been re-bound to something else.)

  • Arrow keys move the cursor one character or line.

    • -

  • C-LeftArrow and C-RightArrow moves left or right one word.

    • +

  • C-LeftArrow and C-RightArrow moves left or right one word.

  • Home and End go to the beginning or end of the line.

  • Page Up and Page Down go up or down one screen.

    • -

  • C-Home and C-End go to beginning or end of the file.

    • -

  • Backspace and Del (or C-d) delete the previous +

  • C-Home and C-End go to beginning or end of the file.

    • +

  • Backspace and Del (or C-d) delete the previous or next character.

    • -

  • C-Backspace and C-Del delete one word left or right.

    • -

  • C-k deletes (‘kills’) everything to the right.

    • +

  • C-Backspace and C-Del delete one word left or right.

    • +

  • C-k deletes (‘kills’) everything to the right.

-

Standard keybindings (like C-c to copy and C-v to paste) +

Standard keybindings (like C-c to copy and C-v to paste) may work. Keybindings are selected in the Configure IDLE dialog.

@@ -390,7 +390,7 @@ one can specify a drive first.) Move into subdirectories by typing a directory name and a separator.

Instead of waiting, or after a box is closed, open a completion box immediately with Show Completions on the Edit menu. The default hot -key is C-space. If one types a prefix for the desired name +key is C-space. If one types a prefix for the desired name before opening the box, the first match or near miss is made visible. The result is the same as if one enters a prefix after the box is displayed. Show Completions after a quote completes @@ -473,9 +473,9 @@ in an editor window.

The editing features described in previous subsections work when entering code interactively. IDLE’s Shell window also responds to the following:

    -

  • C-c attempts to interrupt statement execution (but may fail).

    • -

  • C-d closes Shell if typed at a >>> prompt.

    • -

  • Alt-p and Alt-n (C-p and C-n on macOS) +

  • C-c attempts to interrupt statement execution (but may fail).

    • +

  • C-d closes Shell if typed at a >>> prompt.

    • +

  • Alt-p and Alt-n (C-p and C-n on macOS) retrieve to the current prompt the previous or next previously entered statement that matches anything already typed.

  • Return while the cursor is on any previous statement @@ -517,27 +517,73 @@ executed in the Tk namespace, so this file is not useful for importing functions to be used from IDLE’s Python shell.

    Command line usage

    -
    idle.py [-c command] [-d] [-e] [-h] [-i] [-r file] [-s] [-t title] [-] [arg] ...
-
--c command  run command in the shell window
--d          enable debugger and open shell window
--e          open editor window
--h          print help message with legal combinations and exit
--i          open shell window
--r file     run file in shell window
--s          run $IDLESTARTUP or $PYTHONSTARTUP first, in shell window
--t title    set title of shell window
--           run stdin in shell (- must be last option before args)
+
    IDLE can be invoked from the command line with various options. The general syntax is:
    
+
    python -m idlelib [options] [file ...]
 
    
 
    
-
    If there are arguments:
    
+
    The following options are available:
    
+
    
+
    
+-c <command>
    
+
    Run the specified Python command in the shell window.
+For example, pass -c "print('Hello, World!')".
+On Windows, the outer quotes must be double quotes as shown.
    
+
    
+
+
    
+
    
+-d
    
+
    Enable the debugger and open the shell window.
    
+
    
+
+
    
+
    
+-e
    
+
    Open an editor window.
    
+
    
+
+
    
+
    
+-h
    
+
    Print a help message with legal combinations of options and exit.
    
+
    
+
+
    
+
    
+-i
    
+
    Open a shell window.
    
+
    
+
+
    
+
    
+-r <file>
    
+
    Run the specified file in the shell window.
    
+
    
+
+
    
+
    
+-s
    
+
    Run the startup file (as defined by the environment variables IDLESTARTUP or PYTHONSTARTUP) before opening the shell window.
    
+
    
+
+
    
+
    
+-t <title>
    
+
    Set the title of the shell window.
    
+
    
+
+
    
+
    
+-
    
+
    Read and execute standard input in the shell window. This option must be the last one before any arguments.
    
+
    
+
+
    If arguments are provided:
    
 
      
-
    • If -, -c, or r is used, all arguments are placed in
-sys.argv[1:...] and sys.argv[0] is set to '', '-c',
-or '-r'.  No editor window is opened, even if that is the default
-set in the Options dialog.
      • 
-
    • Otherwise, arguments are files opened for editing and
-sys.argv reflects the arguments passed to IDLE itself.
      • 
+
    • If -, -c, or -r is used, all arguments are placed in sys.argv[1:],
+and sys.argv[0] is set to '', '-c', or '-r' respectively.
+No editor window is opened, even if that is the default set in the Options dialog.
      • 
+
    • Otherwise, arguments are treated as files to be opened for editing, and sys.argv reflects the arguments passed to IDLE itself.
      •
    @@ -798,7 +844,7 @@ of this page for how to use IDLE.

    either in idlelib or click Help => About IDLE on the IDLE menu. This file also maps IDLE menu items to the code that implements the item. Except for files listed under ‘Startup’, the idlelib code is ‘private’ in -sense that feature changes can be backported (see PEP 434).

    +sense that feature changes can be backported (see PEP 434).

From 13360efd385d1a7d0659beba03787ea3d063ef9b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 6 Nov 2025 11:34:32 +0200 Subject: [PATCH 060/313] gh-125346: Add more base64 tests (GH-141061) Add more tests for the altchars argument of b64decode() and for the map01 argument of b32decode(). --- Lib/test/test_base64.py | 64 ++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index 6b5c65a56d8..65977ca8c9f 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -231,18 +231,6 @@ def test_b64decode(self): self.check_other_types(base64.b64decode, b"YWJj", b"abc") self.check_decode_type_errors(base64.b64decode) - # Test with arbitrary alternative characters - tests_altchars = {(b'01a*b$cd', b'*$'): b'\xd3V\xbeo\xf7\x1d', - } - for (data, altchars), res in tests_altchars.items(): - data_str = data.decode('ascii') - altchars_str = altchars.decode('ascii') - - eq(base64.b64decode(data, altchars=altchars), res) - eq(base64.b64decode(data_str, altchars=altchars), res) - eq(base64.b64decode(data, altchars=altchars_str), res) - eq(base64.b64decode(data_str, altchars=altchars_str), res) - # Test standard alphabet for data, res in tests.items(): eq(base64.standard_b64decode(data), res) @@ -263,6 +251,20 @@ def test_b64decode(self): b'\xd3V\xbeo\xf7\x1d') self.check_decode_type_errors(base64.urlsafe_b64decode) + def test_b64decode_altchars(self): + # Test with arbitrary alternative characters + eq = self.assertEqual + res = b'\xd3V\xbeo\xf7\x1d' + for altchars in b'*$', b'+/', b'/+', b'+_', b'-+', b'-/', b'/_': + data = b'01a%cb%ccd' % tuple(altchars) + data_str = data.decode('ascii') + altchars_str = altchars.decode('ascii') + + eq(base64.b64decode(data, altchars=altchars), res) + eq(base64.b64decode(data_str, altchars=altchars), res) + eq(base64.b64decode(data, altchars=altchars_str), res) + eq(base64.b64decode(data_str, altchars=altchars_str), res) + def test_b64decode_padding_error(self): self.assertRaises(binascii.Error, base64.b64decode, b'abc') self.assertRaises(binascii.Error, base64.b64decode, 'abc') @@ -295,10 +297,12 @@ def test_b64decode_invalid_chars(self): base64.b64decode(bstr.decode('ascii'), validate=True) # Normal alphabet characters not discarded when alternative given - res = b'\xFB\xEF\xBE\xFF\xFF\xFF' - self.assertEqual(base64.b64decode(b'++[[//]]', b'[]'), res) - self.assertEqual(base64.urlsafe_b64decode(b'++--//__'), res) - + res = b'\xfb\xef\xff' + self.assertEqual(base64.b64decode(b'++//', validate=True), res) + self.assertEqual(base64.b64decode(b'++//', '-_', validate=True), res) + self.assertEqual(base64.b64decode(b'--__', '-_', validate=True), res) + self.assertEqual(base64.urlsafe_b64decode(b'++//'), res) + self.assertEqual(base64.urlsafe_b64decode(b'--__'), res) def _altchars_strategy(): """Generate 'altchars' for base64 encoding.""" @@ -394,19 +398,33 @@ def test_b32decode_casefold(self): eq(base64.b32decode(b'MLO23456'), b'b\xdd\xad\xf3\xbe') eq(base64.b32decode('MLO23456'), b'b\xdd\xad\xf3\xbe') - map_tests = {(b'M1023456', b'L'): b'b\xdd\xad\xf3\xbe', - (b'M1023456', b'I'): b'b\x1d\xad\xf3\xbe', - } - for (data, map01), res in map_tests.items(): - data_str = data.decode('ascii') + def test_b32decode_map01(self): + # Mapping zero and one + eq = self.assertEqual + res_L = b'b\xdd\xad\xf3\xbe' + res_I = b'b\x1d\xad\xf3\xbe' + eq(base64.b32decode(b'MLO23456'), res_L) + eq(base64.b32decode('MLO23456'), res_L) + eq(base64.b32decode(b'MIO23456'), res_I) + eq(base64.b32decode('MIO23456'), res_I) + self.assertRaises(binascii.Error, base64.b32decode, b'M1023456') + self.assertRaises(binascii.Error, base64.b32decode, b'M1O23456') + self.assertRaises(binascii.Error, base64.b32decode, b'ML023456') + self.assertRaises(binascii.Error, base64.b32decode, b'MI023456') + + data = b'M1023456' + data_str = data.decode('ascii') + for map01, res in [(b'L', res_L), (b'I', res_I)]: map01_str = map01.decode('ascii') eq(base64.b32decode(data, map01=map01), res) eq(base64.b32decode(data_str, map01=map01), res) eq(base64.b32decode(data, map01=map01_str), res) eq(base64.b32decode(data_str, map01=map01_str), res) - self.assertRaises(binascii.Error, base64.b32decode, data) - self.assertRaises(binascii.Error, base64.b32decode, data_str) + + eq(base64.b32decode(b'M1O23456', map01=map01), res) + eq(base64.b32decode(b'M%c023456' % map01, map01=map01), res) + eq(base64.b32decode(b'M%cO23456' % map01, map01=map01), res) def test_b32decode_error(self): tests = [b'abc', b'ABCDEF==', b'==ABCDEF'] From 86ab7bb87a3b8c7d617763bffc1992791c0e9bde Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 6 Nov 2025 11:20:02 +0100 Subject: [PATCH 061/313] gh-137232: Update free-threading HOWTOs with up-to-date info for 3.14 (#140817) --- Doc/howto/free-threading-extensions.rst | 11 ++++++----- Doc/howto/free-threading-python.rst | 21 ++++++++++++--------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index 3776132c685..5647ab2d87c 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -203,7 +203,7 @@ Memory Allocation APIs Python's memory management C API provides functions in three different :ref:`allocation domains `: "raw", "mem", and "object". For thread-safety, the free-threaded build requires that only Python objects -are allocated using the object domain, and that all Python object are +are allocated using the object domain, and that all Python objects are allocated using that domain. This differs from the prior Python versions, where this was only a best practice and not a hard requirement. @@ -344,12 +344,12 @@ This means you cannot rely on nested critical sections to lock multiple objects at once, as the inner critical section may suspend the outer ones. Instead, use :c:macro:`Py_BEGIN_CRITICAL_SECTION2` to lock two objects simultaneously. -Note that the locks described above are only :c:type:`!PyMutex` based locks. +Note that the locks described above are only :c:type:`PyMutex` based locks. The critical section implementation does not know about or affect other locking mechanisms that might be in use, like POSIX mutexes. Also note that while -blocking on any :c:type:`!PyMutex` causes the critical sections to be +blocking on any :c:type:`PyMutex` causes the critical sections to be suspended, only the mutexes that are part of the critical sections are -released. If :c:type:`!PyMutex` is used without a critical section, it will +released. If :c:type:`PyMutex` is used without a critical section, it will not be released and therefore does not get the same deadlock avoidance. Important Considerations @@ -397,7 +397,8 @@ The wheels, shared libraries, and binaries are indicated by a ``t`` suffix. * `pypa/manylinux `_ supports the free-threaded build, with the ``t`` suffix, such as ``python3.13t``. * `pypa/cibuildwheel `_ supports the - free-threaded build if you set + free-threaded build on Python 3.13 and 3.14. On Python 3.14, free-threaded + wheels will be built by default. On Python 3.13, you will need to set `CIBW_ENABLE to cpython-freethreading `_. Limited C API and Stable ABI diff --git a/Doc/howto/free-threading-python.rst b/Doc/howto/free-threading-python.rst index 24069617c47..e4df7a787a2 100644 --- a/Doc/howto/free-threading-python.rst +++ b/Doc/howto/free-threading-python.rst @@ -116,12 +116,14 @@ after the main thread is running. The following objects are immortalized: * :ref:`classes ` (type objects) Because immortal objects are never deallocated, applications that create many -objects of these types may see increased memory usage. This is expected to be -addressed in the 3.14 release. +objects of these types may see increased memory usage under Python 3.13. This +has been addressed in the 3.14 release, where the aforementioned objects use +deferred reference counting to avoid reference count contention. Additionally, numeric and string literals in the code as well as strings -returned by :func:`sys.intern` are also immortalized. This behavior is -expected to remain in the 3.14 free-threaded build. +returned by :func:`sys.intern` are also immortalized in the 3.13 release. This +behavior is part of the 3.14 release as well and it is expected to remain in +future free-threaded builds. Frame objects @@ -150,11 +152,12 @@ compared to the default GIL-enabled build. In 3.13, this overhead is about 40% on the `pyperformance `_ suite. Programs that spend most of their time in C extensions or I/O will see less of an impact. The largest impact is because the specializing adaptive -interpreter (:pep:`659`) is disabled in the free-threaded build. We expect -to re-enable it in a thread-safe way in the 3.14 release. This overhead is -expected to be reduced in upcoming Python release. We are aiming for an -overhead of 10% or less on the pyperformance suite compared to the default -GIL-enabled build. +interpreter (:pep:`659`) is disabled in the free-threaded build. + +The specializing adaptive interpreter has been re-enabled in a thread-safe way +in the 3.14 release. The performance penalty on single-threaded code in +free-threaded mode is now roughly 5-10%, depending on the platform and C +compiler used. Behavioral changes From d2ce6d708a9eaac4e546744ca4da359ee6901ebc Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 6 Nov 2025 11:49:44 +0100 Subject: [PATCH 062/313] gh-139707: Add docs for optional modules (GH-140171) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: Emma Smith Co-authored-by: Author: Terry Jan Reedy Co-authored-by: Victor Stinner --- Doc/glossary.rst | 9 ++ Doc/includes/optional-module.rst | 9 ++ Doc/library/bz2.rst | 2 + Doc/library/compression.zstd.rst | 2 + Doc/library/ctypes.rst | 2 + Doc/library/curses.rst | 2 + Doc/library/ensurepip.rst | 2 + Doc/library/gzip.rst | 2 + Doc/library/idle.rst | 4 + Doc/library/lzma.rst | 2 + Doc/library/readline.rst | 2 + Doc/library/sqlite3.rst | 4 +- Doc/library/ssl.rst | 5 +- Doc/library/tarfile.rst | 8 ++ Doc/library/tkinter.rst | 2 + Doc/library/turtle.rst | 2 + Doc/library/zipfile.rst | 10 ++ Doc/library/zlib.rst | 6 +- Doc/using/configure.rst | 216 ++++++++++++++++++++----------- Doc/using/unix.rst | 12 +- 20 files changed, 221 insertions(+), 82 deletions(-) create mode 100644 Doc/includes/optional-module.rst diff --git a/Doc/glossary.rst b/Doc/glossary.rst index c0ca0be304e..a4066d42927 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -1025,6 +1025,15 @@ Glossary applied to all scopes, only those relying on a known set of local and nonlocal variable names are restricted to optimized scopes. + optional module + An :term:`extension module` that is part of the :term:`standard library`, + but may be absent in some builds of :term:`CPython`, + usually due to missing third-party libraries or because the module + is not available for a given platform. + + See :ref:`optional-module-requirements` for a list of optional modules + that require third-party libraries. + package A Python :term:`module` which can contain submodules or recursively, subpackages. Technically, a package is a Python module with a diff --git a/Doc/includes/optional-module.rst b/Doc/includes/optional-module.rst new file mode 100644 index 00000000000..262e73f2eaa --- /dev/null +++ b/Doc/includes/optional-module.rst @@ -0,0 +1,9 @@ +This is an :term:`optional module`. +If it is missing from your copy of CPython, +look for documentation from your distributor (that is, +whoever provided Python to you). +If you are the distributor, see :ref:`optional-module-requirements`. + +.. Similar notes appear in the docs of the modules: + - zipfile + - tarfile diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index ebe2e43feba..12650861c0f 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -25,6 +25,8 @@ The :mod:`bz2` module contains: * The :func:`compress` and :func:`decompress` functions for one-shot (de)compression. +.. include:: ../includes/optional-module.rst + (De)compression of files ------------------------ diff --git a/Doc/library/compression.zstd.rst b/Doc/library/compression.zstd.rst index a901403621b..89b6fe540f5 100644 --- a/Doc/library/compression.zstd.rst +++ b/Doc/library/compression.zstd.rst @@ -33,6 +33,8 @@ The :mod:`!compression.zstd` module contains: * The :class:`CompressionParameter`, :class:`DecompressionParameter`, and :class:`Strategy` classes for setting advanced (de)compression parameters. +.. include:: ../includes/optional-module.rst + Exceptions ---------- diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index d8dac24c8ab..9c0b246c095 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -14,6 +14,8 @@ data types, and allows calling functions in DLLs or shared libraries. It can be used to wrap these libraries in pure Python. +.. include:: ../includes/optional-module.rst + .. _ctypes-ctypes-tutorial: diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index e60197ddd89..057d338edda 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -23,6 +23,8 @@ Linux and the BSD variants of Unix. .. include:: ../includes/wasm-mobile-notavail.rst +.. include:: ../includes/optional-module.rst + .. note:: Whenever the documentation mentions a *character* it can be specified diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index fa102c4a080..165b9a9f823 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -30,6 +30,8 @@ when creating a virtual environment) or after explicitly uninstalling needed to bootstrap ``pip`` are included as internal parts of the package. +.. include:: ../includes/optional-module.rst + .. seealso:: :ref:`installing-index` diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index 4bdcec66088..cb36be42a83 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -11,6 +11,8 @@ This module provides a simple interface to compress and decompress files just like the GNU programs :program:`gzip` and :program:`gunzip` would. +.. include:: ../includes/optional-module.rst + The data compression is provided by the :mod:`zlib` module. The :mod:`gzip` module provides the :class:`GzipFile` class, as well as the diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 10ec7f0a6f1..52e3726a0f5 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -37,6 +37,10 @@ IDLE has the following features: * configuration, browsers, and other dialogs +The IDLE application is implemented in the :mod:`idlelib` package. + +.. include:: ../includes/optional-module.rst + Menus ----- diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 69f7cb8d48d..8a4f68f3502 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -23,6 +23,8 @@ module. Note that :class:`LZMAFile` and :class:`bz2.BZ2File` are *not* thread-safe, so if you need to use a single :class:`LZMAFile` instance from multiple threads, it is necessary to protect it with a lock. +.. include:: ../includes/optional-module.rst + .. exception:: LZMAError diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index f649fce5efc..75db832c546 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -26,6 +26,8 @@ Readline library in general. .. include:: ../includes/wasm-mobile-notavail.rst +.. include:: ../includes/optional-module.rst + .. note:: The underlying Readline library API may be implemented by diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 9d56e81dee1..3b1a9c2f6ee 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -31,7 +31,9 @@ PostgreSQL or Oracle. The :mod:`!sqlite3` module was written by Gerhard Häring. It provides an SQL interface compliant with the DB-API 2.0 specification described by :pep:`249`, and -requires SQLite 3.15.2 or newer. +requires the third-party `SQLite `_ library. + +.. include:: ../includes/optional-module.rst This document includes four main sections: diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index e0d85c852fa..fa0a5234720 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -18,8 +18,9 @@ This module provides access to Transport Layer Security (often known as "Secure Sockets Layer") encryption and peer authentication facilities for network sockets, both client-side and server-side. This module uses the OpenSSL -library. It is available on all modern Unix systems, Windows, macOS, and -probably additional platforms, as long as OpenSSL is installed on that platform. +library. + +.. include:: ../includes/optional-module.rst .. note:: diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index c4614bf28a4..5ff8502bbe2 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -21,6 +21,14 @@ Some facts and figures: * reads and writes :mod:`gzip`, :mod:`bz2`, :mod:`compression.zstd`, and :mod:`lzma` compressed archives if the respective modules are available. + .. + The following paragraph should be similar to ../includes/optional-module.rst + + If any of these :term:`optional modules ` are missing from + your copy of CPython, look for documentation from your distributor (that is, + whoever provided Python to you). + If you are the distributor, see :ref:`optional-module-requirements`. + * read/write support for the POSIX.1-1988 (ustar) format. * read/write support for the GNU tar format including *longname* and *longlink* diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index 22e08c45d01..81177533be8 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -36,6 +36,8 @@ details that are unchanged. Most documentation you will find online still uses the old API and can be woefully outdated. +.. include:: ../includes/optional-module.rst + .. seealso:: * `TkDocs `_ diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index b687231bd48..58b99e0d441 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -29,6 +29,8 @@ introduced in Logo `_, developed by Wally Feurzeig, Seymour Papert and Cynthia Solomon in 1967. +.. include:: ../includes/optional-module.rst + Get started =========== diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index f6ec33640b6..5a8bbc8c1ae 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -23,6 +23,16 @@ decryption of encrypted files in ZIP archives, but it currently cannot create an encrypted file. Decryption is extremely slow as it is implemented in native Python rather than C. +.. + The following paragraph should be similar to ../includes/optional-module.rst + +Handling compressed archives requires :term:`optional modules ` +such as :mod:`zlib`, :mod:`bz2`, :mod:`lzma`, and :mod:`compression.zstd`. +If any of them are missing from your copy of CPython, +look for documentation from your distributor (that is, +whoever provided Python to you). +If you are the distributor, see :ref:`optional-module-requirements`. + The module defines the following items: .. exception:: BadZipFile diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index b961f7113d3..ce0a22b9456 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -8,9 +8,9 @@ -------------- For applications that require data compression, the functions in this module -allow compression and decompression, using the zlib library. The zlib library -has its own home page at https://www.zlib.net. zlib 1.2.2.1 is the minium -supported version. +allow compression and decompression, using the `zlib library `_. + +.. include:: ../includes/optional-module.rst zlib's functions have many options and often need to be used in a particular order. This documentation doesn't attempt to cover all of the permutations; diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 1f773a3a547..cdadbe51417 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -4,10 +4,13 @@ Configure Python .. highlight:: sh + +.. _build-requirements: + Build Requirements ================== -Features and minimum versions required to build CPython: +To build CPython, you will need: * A `C11 `_ compiler. `Optional C11 features @@ -22,85 +25,138 @@ Features and minimum versions required to build CPython: * Support for threads. -To build optional modules: - -* `libbz2 `_ for the :mod:`bz2` module. - -* `libb2 `_ (:ref:`BLAKE2 `) - for the :mod:`hashlib` module. - -* `libffi `_ 3.3.0 is the recommended - minimum version for the :mod:`ctypes` module. - -* ``liblzma`` for the :mod:`lzma` module. - -* `libmpdec `_ 2.5.0 - for the :mod:`decimal` module. - -* ``libncurses`` or ``libncursesw`` for the :mod:`curses` module. - -* ``libpanel`` or ``libpanelw`` for the :mod:`curses.panel` module. - -* `libreadline `_ or - `libedit `_ - for the :mod:`readline` module. - -* `libuuid `_ for the :mod:`uuid` module. - -* `OpenSSL `_ 1.1.1 is the minimum version and - OpenSSL 3.0.18 is the recommended minimum version for the - :mod:`ssl` and :mod:`hashlib` extension modules. - -* `SQLite `_ 3.15.2 for the :mod:`sqlite3` extension module. - -* `Tcl/Tk `_ 8.5.12 for the :mod:`tkinter` module. - -* `zlib `_ 1.2.2.1 is the minimum version for the - :mod:`zlib` module. - -* `zstd `_ 1.4.5 is the minimum version for - the :mod:`compression.zstd` module. - -For a full list of dependencies required to build all modules and how to install -them, see the -`devguide `_. - -* Autoconf 2.72 and aclocal 1.16.5 are required to regenerate the - :file:`configure` script. - -.. versionchanged:: 3.1 - Tcl/Tk version 8.3.1 is now required. - .. versionchanged:: 3.5 On Windows, Visual Studio 2015 or later is now required. - Tcl/Tk version 8.4 is now required. .. versionchanged:: 3.6 - Selected C99 features are now required, like ```` and ``static - inline`` functions. + Selected C99 features, like ```` and ``static inline`` functions, + are now required. .. versionchanged:: 3.7 - Thread support and OpenSSL 1.0.2 are now required. - -.. versionchanged:: 3.10 - OpenSSL 1.1.1 is now required. - Require SQLite 3.7.15. + Thread support is now required. .. versionchanged:: 3.11 C11 compiler, IEEE 754 and NaN support are now required. On Windows, Visual Studio 2017 or later is required. - Tcl/Tk version 8.5.12 is now required for the :mod:`tkinter` module. - -.. versionchanged:: 3.13 - Autoconf 2.71, aclocal 1.16.5 and SQLite 3.15.2 are now required. - -.. versionchanged:: 3.14 - Autoconf 2.72 is now required. See also :pep:`7` "Style Guide for C Code" and :pep:`11` "CPython platform support". +.. _optional-module-requirements: + +Requirements for optional modules +--------------------------------- + +Some :term:`optional modules ` of the standard library +require third-party libraries installed for development +(for example, header files must be available). + +Missing requirements are reported in the ``configure`` output. +Modules that are missing due to missing dependencies are listed near the end +of the ``make`` output, +sometimes using an internal name, for example, ``_ctypes`` for :mod:`ctypes` +module. + +If you distribute a CPython interpreter without optional modules, +it's best practice to advise users, who generally expect that +standard library modules are available. + +Dependencies to build optional modules are: + +.. list-table:: + :header-rows: 1 + :align: left + + * - Dependency + - Minimum version + - Python module + * - `libbz2 `_ + - + - :mod:`bz2` + * - `libffi `_ + - 3.3.0 recommended + - :mod:`ctypes` + * - `liblzma `_ + - + - :mod:`lzma` + * - `libmpdec `_ + - 2.5.0 + - :mod:`decimal` [1]_ + * - `libreadline `_ or + `libedit `_ [2]_ + - + - :mod:`readline` + * - `libuuid `_ + - + - ``_uuid`` [3]_ + * - `ncurses `_ [4]_ + - + - :mod:`curses` + * - `OpenSSL `_ + - | 3.0.18 recommended + | (1.1.1 minimum) + - :mod:`ssl`, :mod:`hashlib` [5]_ + * - `SQLite `_ + - 3.15.2 + - :mod:`sqlite3` + * - `Tcl/Tk `_ + - 8.5.12 + - :mod:`tkinter`, :ref:`IDLE `, :mod:`turtle` + * - `zlib `_ + - 1.2.2.1 + - :mod:`zlib`, :mod:`gzip`, :mod:`ensurepip` + * - `zstd `_ + - 1.4.5 + - :mod:`compression.zstd` + +.. [1] If *libmpdec* is not available, the :mod:`decimal` module will use + a pure-Python implementation. + See :option:`--with-system-libmpdec` for details. +.. [2] See :option:`--with-readline` for choosing the backend for the + :mod:`readline` module. +.. [3] The :mod:`uuid` module uses ``_uuid`` to generate "safe" UUIDs. + See the module documentation for details. +.. [4] The :mod:`curses` module requires the ``libncurses`` or ``libncursesw`` + library. + The :mod:`curses.panel` module additionally requires the ``libpanel`` or + ``libpanelw`` library. +.. [5] If OpenSSL is not available, the :mod:`hashlib` module will use + bundled implementations of several hash functions. + See :option:`--with-builtin-hashlib-hashes` for *forcing* usage of OpenSSL. + +Note that the table does not include all optional modules; in particular, +platform-specific modules like :mod:`winreg` are not listed here. + +.. seealso:: + + * The `devguide `_ + includes a full list of dependencies required to build all modules and + instructions on how to install them on common platforms. + * :option:`--with-system-expat` allows building with an external + `libexpat `_ library. + * :ref:`configure-options-for-dependencies` + +.. versionchanged:: 3.1 + Tcl/Tk version 8.3.1 is now required for :mod:`tkinter`. + +.. versionchanged:: 3.5 + Tcl/Tk version 8.4 is now required for :mod:`tkinter`. + +.. versionchanged:: 3.7 + OpenSSL 1.0.2 is now required for :mod:`hashlib` and :mod:`ssl`. + +.. versionchanged:: 3.10 + OpenSSL 1.1.1 is now required for :mod:`hashlib` and :mod:`ssl`. + SQLite 3.7.15 is now required for :mod:`sqlite3`. + +.. versionchanged:: 3.11 + Tcl/Tk version 8.5.12 is now required for :mod:`tkinter`. + +.. versionchanged:: 3.13 + SQLite 3.15.2 is now required for :mod:`sqlite3`. + + Generated files =============== @@ -127,8 +183,19 @@ The container is optional, the following command can be run locally:: autoreconf -ivf -Werror -The generated files can change depending on the exact ``autoconf-archive``, -``aclocal`` and ``pkg-config`` versions. +The generated files can change depending on the exact versions of the +tools used. +The container that CPython uses has +`Autoconf `_ 2.72, +``aclocal`` from `Automake `_ 1.16.5, +and `pkg-config `_ 1.8.1. + +.. versionchanged:: 3.13 + Autoconf 2.71 and aclocal 1.16.5 and are now used to regenerate + :file:`configure`. + +.. versionchanged:: 3.14 + Autoconf 2.72 is now used to regenerate :file:`configure`. .. _configure-options: @@ -409,6 +476,8 @@ Linker options Name for machine-dependent library files. +.. _configure-options-for-dependencies: + Options for third-party dependencies ------------------------------------ @@ -431,12 +500,6 @@ Options for third-party dependencies C compiler and linker flags for ``gdbm``. -.. option:: LIBB2_CFLAGS -.. option:: LIBB2_LIBS - - C compiler and linker flags for ``libb2`` (:ref:`BLAKE2 `), - used by :mod:`hashlib` module, overriding ``pkg-config``. - .. option:: LIBEDIT_CFLAGS .. option:: LIBEDIT_LIBS @@ -902,6 +965,13 @@ Libraries options .. versionchanged:: 3.13 Default to using the installed ``mpdecimal`` library. + .. versionchanged:: 3.15 + + A bundled copy of the library will no longer be selected + implicitly if an installed ``mpdecimal`` library is not found. + In Python 3.15 only, it can still be selected explicitly using + ``--with-system-libmpdec=no`` or ``--without-system-libmpdec``. + .. deprecated-removed:: 3.13 3.16 A copy of the ``mpdecimal`` library sources will no longer be distributed with Python 3.16. diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index 9ec4e341932..a9950ef7525 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -84,11 +84,17 @@ On FreeBSD and OpenBSD Building Python =============== +.. seealso:: + + If you want to contribute to CPython, refer to the + `devguide `_, + which includes build instructions and other tips on setting up environment. + If you want to compile CPython yourself, first thing you should do is get the `source `_. You can download either the -latest release's source or just grab a fresh `clone -`_. (If you want -to contribute patches, you will need a clone.) +latest release's source or grab a fresh `clone +`_. +You will also need to install the :ref:`build requirements `. The build process consists of the usual commands:: From 1697cb5710f526d38816bb00ca3dcd4434e5e773 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 6 Nov 2025 07:29:04 -0500 Subject: [PATCH 063/313] gh-141004: Document built-in iterator types in the C API (GH-141006) Adds documentation for each of the following: - PyEnum_Type - PyFilter_Type - PyMap_Type - PyReversed_Type - PyZip_Type In addition, PyRange_Type and PyRange_Check are also documented. --- Doc/c-api/iterator.rst | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/Doc/c-api/iterator.rst b/Doc/c-api/iterator.rst index 4b94970036f..7eaf72ec55f 100644 --- a/Doc/c-api/iterator.rst +++ b/Doc/c-api/iterator.rst @@ -52,6 +52,45 @@ sentinel value is returned. *sentinel*, the iteration will be terminated. +Range Objects +^^^^^^^^^^^^^ + +.. c:var:: PyTypeObject PyRange_Type + + The type object for :class:`range` objects. + + +.. c:function:: int PyRange_Check(PyObject *o) + + Return true if the object *o* is an instance of a :class:`range` object. + This function always succeeds. + + +Builtin Iterator Types +^^^^^^^^^^^^^^^^^^^^^^ + +These are built-in iteration types that are included in Python's C API, but +provide no additional functions. They are here for completeness. + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * C type + * Python type + * * .. c:var:: PyTypeObject PyEnum_Type + * :py:class:`enumerate` + * * .. c:var:: PyTypeObject PyFilter_Type + * :py:class:`filter` + * * .. c:var:: PyTypeObject PyMap_Type + * :py:class:`map` + * * .. c:var:: PyTypeObject PyReversed_Type + * :py:class:`reversed` + * * .. c:var:: PyTypeObject PyZip_Type + * :py:class:`zip` + + Other Iterator Objects ^^^^^^^^^^^^^^^^^^^^^^ From 54110e20e0ed0584e159c42d9f57516c1a3b997a Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Thu, 6 Nov 2025 13:48:42 +0000 Subject: [PATCH 064/313] gh-141004: Document `Py_hexdigits` (GH-141059) Co-authored-by: Victor Stinner --- Doc/c-api/codec.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Doc/c-api/codec.rst b/Doc/c-api/codec.rst index 08a99245ad6..35ee048bd5f 100644 --- a/Doc/c-api/codec.rst +++ b/Doc/c-api/codec.rst @@ -129,3 +129,13 @@ Registry API for Unicode encoding error handlers Replace the unicode encode error with ``\N{...}`` escapes. .. versionadded:: 3.5 + + +Codec utility variables +----------------------- + +.. c:var:: const char *Py_hexdigits + + A string constant containing the lowercase hexadecimal digits: ``"0123456789abcdef"``. + + .. versionadded:: 3.3 From bcc524f82d8548707046ce90f5bc56f60018767a Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 6 Nov 2025 09:01:48 -0500 Subject: [PATCH 065/313] gh-141004: Document `PyLong_FromPid` and `PyLong_AsPid` (GH-141028) Co-authored-by: Victor Stinner --- Doc/c-api/long.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index fcb20f7c93c..ed34efe716d 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -161,6 +161,17 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.13 +.. c:macro:: PyLong_FromPid(pid) + + Macro for creating a Python integer from a process identifier. + + This can be defined as an alias to :c:func:`PyLong_FromLong` or + :c:func:`PyLong_FromLongLong`, depending on the size of the system's + PID type. + + .. versionadded:: 3.2 + + .. c:function:: long PyLong_AsLong(PyObject *obj) .. index:: @@ -575,6 +586,17 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.13 +.. c:macro:: PyLong_AsPid(pid) + + Macro for converting a Python integer into a process identifier. + + This can be defined as an alias to :c:func:`PyLong_AsLong`, + :c:func:`PyLong_FromLongLong`, or :c:func:`PyLong_AsInt`, depending on the + size of the system's PID type. + + .. versionadded:: 3.2 + + .. c:function:: int PyLong_GetSign(PyObject *obj, int *sign) Get the sign of the integer object *obj*. From 2e5e6fd380eb747bffeac151ca6f609779ef36f3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 6 Nov 2025 16:10:39 +0100 Subject: [PATCH 066/313] gh-134745: Use "pymutex" for sys.thread_info on Windows (#141140) --- Python/thread.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Python/thread.c b/Python/thread.c index 18c4af7f634..0365f977d82 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -334,14 +334,12 @@ PyThread_GetInfo(void) #ifdef HAVE_PTHREAD_STUBS value = Py_NewRef(Py_None); -#elif defined(_POSIX_THREADS) +#else value = PyUnicode_FromString("pymutex"); if (value == NULL) { Py_DECREF(threadinfo); return NULL; } -#else - value = Py_NewRef(Py_None); #endif PyStructSequence_SET_ITEM(threadinfo, pos++, value); From 13f09a60f4d6fca019bff9dec05dabe5ad390d21 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Thu, 6 Nov 2025 10:16:56 -0500 Subject: [PATCH 067/313] =?UTF-8?q?gh-141150:=20Don't=20rely=20on=20implic?= =?UTF-8?q?it=20conversion=20from=20void=20*=20to=20pointer=20in=20=5FPyMo?= =?UTF-8?q?dule=E2=80=A6=20(#141147)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don't rely on implicit conversion from void * to pointer in _PyModule_GetToken --- Include/internal/pycore_moduleobject.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index c34e42e826e..6eef6eaa5df 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -53,7 +53,7 @@ static inline PyModuleDef *_PyModule_GetDefOrNull(PyObject *arg) { static inline PyModuleDef *_PyModule_GetToken(PyObject *arg) { PyModuleObject *mod = _PyModule_CAST(arg); - return mod->md_token; + return (PyModuleDef *)mod->md_token; } static inline void* _PyModule_GetState(PyObject* mod) { From bea0d3d12bcd122d8498b92cdd6c724822fd6505 Mon Sep 17 00:00:00 2001 From: AN Long Date: Fri, 7 Nov 2025 00:33:30 +0900 Subject: [PATCH 068/313] gh-140826: Update winreg's docstring (GH-141050) --- PC/winreg.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/PC/winreg.c b/PC/winreg.c index 3cc6123fc3a..c1be920fc1d 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -53,6 +53,7 @@ PyDoc_STRVAR(module_doc, "DeleteKey() - Deletes the specified key.\n" "DeleteKeyEx() - Deletes the specified key.\n" "DeleteValue() - Removes a named value from the specified registry key.\n" +"DeleteTree() - Deletes the specified key and all its subkeys and values recursively.\n" "EnumKey() - Enumerates subkeys of the specified open registry key.\n" "EnumValue() - Enumerates values of the specified open registry key.\n" "ExpandEnvironmentStrings() - Expand the env strings in a REG_EXPAND_SZ\n" @@ -107,7 +108,9 @@ PyDoc_STRVAR(PyHKEY_doc, "Operations:\n" "__bool__ - Handles with an open object return true, otherwise false.\n" "__int__ - Converting a handle to an integer returns the Win32 handle.\n" -"rich comparison - Handle objects are compared using the handle value."); +"__enter__, __exit__ - Context manager support for 'with' statement,\n" +"automatically closes handle.\n" +"__eq__, __ne__ - Equality comparison based on Windows handle value."); From 0b260305d302eace7d59931ca582a1953d894018 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 6 Nov 2025 11:37:52 -0500 Subject: [PATCH 069/313] gh-141004: Document `Py_GetRecursionLimit` and `Py_SetRecursionLimit` (GH-141151) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/exceptions.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 3ff4631a8e5..9c75f66f683 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -979,6 +979,27 @@ these are the C equivalent to :func:`reprlib.recursive_repr`. Ends a :c:func:`Py_ReprEnter`. Must be called once for each invocation of :c:func:`Py_ReprEnter` that returns zero. +.. c:function:: int Py_GetRecursionLimit(void) + + Get the recursion limit for the current interpreter. It can be set with + :c:func:`Py_SetRecursionLimit`. The recursion limit prevents the + Python interpreter stack from growing infinitely. + + This function cannot fail, and the caller must hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`sys.getrecursionlimit` + +.. c:function:: void Py_SetRecursionLimit(int new_limit) + + Set the recursion limit for the current interpreter. + + This function cannot fail, and the caller must hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`sys.setrecursionlimit` .. _standardexceptions: From e95e783dff443b68e8179fdb57737025bf02ba76 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 6 Nov 2025 18:48:58 +0200 Subject: [PATCH 070/313] Remove duplicated tests in test_base64 (gh-125346) (GH-141153) --- Lib/test/test_base64.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index 65977ca8c9f..ac3f0940545 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -394,10 +394,6 @@ def test_b32decode_casefold(self): self.assertRaises(binascii.Error, base64.b32decode, b'me======') self.assertRaises(binascii.Error, base64.b32decode, 'me======') - # Mapping zero and one - eq(base64.b32decode(b'MLO23456'), b'b\xdd\xad\xf3\xbe') - eq(base64.b32decode('MLO23456'), b'b\xdd\xad\xf3\xbe') - def test_b32decode_map01(self): # Mapping zero and one eq = self.assertEqual From 9bf5100037f661f3a369d3ee539bec06f063b650 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rodrigo=20Gir=C3=A3o=20Serr=C3=A3o?= <5621605+rodrigogiraoserrao@users.noreply.github.com> Date: Thu, 6 Nov 2025 17:11:50 +0000 Subject: [PATCH 071/313] Minor documentation improvements (#140626) --- Doc/library/concurrent.interpreters.rst | 6 +++--- Doc/library/heapq.rst | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Doc/library/concurrent.interpreters.rst b/Doc/library/concurrent.interpreters.rst index 41ea6af3b22..55036090e8d 100644 --- a/Doc/library/concurrent.interpreters.rst +++ b/Doc/library/concurrent.interpreters.rst @@ -29,12 +29,12 @@ Actual concurrency is available separately through .. seealso:: :class:`~concurrent.futures.InterpreterPoolExecutor` - combines threads with interpreters in a familiar interface. + Combines threads with interpreters in a familiar interface. - .. XXX Add references to the upcoming HOWTO docs in the seealso block. + .. XXX Add references to the upcoming HOWTO docs in the seealso block. :ref:`isolating-extensions-howto` - how to update an extension module to support multiple interpreters + How to update an extension module to support multiple interpreters. :pep:`554` diff --git a/Doc/library/heapq.rst b/Doc/library/heapq.rst index 95ef72469b1..5049262306a 100644 --- a/Doc/library/heapq.rst +++ b/Doc/library/heapq.rst @@ -58,6 +58,11 @@ functions, respectively. The following functions are provided for min-heaps: +.. function:: heapify(x) + + Transform list *x* into a min-heap, in-place, in linear time. + + .. function:: heappush(heap, item) Push the value *item* onto the *heap*, maintaining the min-heap invariant. @@ -77,11 +82,6 @@ The following functions are provided for min-heaps: followed by a separate call to :func:`heappop`. -.. function:: heapify(x) - - Transform list *x* into a min-heap, in-place, in linear time. - - .. function:: heapreplace(heap, item) Pop and return the smallest item from the *heap*, and also push the new *item*. From 42d014086098d3d70cacb4d8993f04cace120c12 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Thu, 6 Nov 2025 11:58:01 -0800 Subject: [PATCH 072/313] GH-136895: Fixes for pulling LLVM as a release artifact (#141002) --- PCbuild/get_external.py | 37 ++++++++++++++++++++----------------- Tools/jit/_llvm.py | 5 +++++ 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index 07970624e86..edf14ce578b 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -3,8 +3,8 @@ import argparse import os import pathlib -import shutil import sys +import tarfile import time import urllib.error import urllib.request @@ -56,7 +56,8 @@ def fetch_release(tag, tarball_dir, *, org='python', verbose=False): def extract_tarball(externals_dir, tarball_path, tag): output_path = externals_dir / tag - shutil.unpack_archive(os.fspath(tarball_path), os.fspath(output_path)) + with tarfile.open(tarball_path) as tf: + tf.extractall(os.fspath(externals_dir)) return output_path @@ -115,21 +116,23 @@ def main(): verbose=args.verbose, ) extracted = extract_zip(args.externals_dir, zip_path) - for wait in [1, 2, 3, 5, 8, 0]: - try: - extracted.replace(final_name) - break - except PermissionError as ex: - retry = f" Retrying in {wait}s..." if wait else "" - print(f"Encountered permission error '{ex}'.{retry}", file=sys.stderr) - time.sleep(wait) - else: - print( - f"ERROR: Failed to extract {final_name}.", - "You may need to restart your build", - file=sys.stderr, - ) - sys.exit(1) + + if extracted != final_name: + for wait in [1, 2, 3, 5, 8, 0]: + try: + extracted.replace(final_name) + break + except PermissionError as ex: + retry = f" Retrying in {wait}s..." if wait else "" + print(f"Encountered permission error '{ex}'.{retry}", file=sys.stderr) + time.sleep(wait) + else: + print( + f"ERROR: Failed to rename {extracted} to {final_name}.", + "You may need to restart your build", + file=sys.stderr, + ) + sys.exit(1) if __name__ == '__main__': diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 54c2bf86a36..f1b0ad3f5db 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -83,6 +83,11 @@ async def _find_tool(tool: str, llvm_version: str, *, echo: bool = False) -> str # PCbuild externals: externals = os.environ.get("EXTERNALS_DIR", _targets.EXTERNALS) path = os.path.join(externals, _EXTERNALS_LLVM_TAG, "bin", tool) + # On Windows, executables need .exe extension + if os.name == "nt" and not path.endswith(".exe"): + path_with_exe = path + ".exe" + if os.path.exists(path_with_exe): + path = path_with_exe if await _check_tool_version(path, llvm_version, echo=echo): return path # Homebrew-installed executables: From c77441ef1d1f3182280bd14d11516d54f38fe90b Mon Sep 17 00:00:00 2001 From: yihong Date: Fri, 7 Nov 2025 08:02:47 +0800 Subject: [PATCH 073/313] gh-141125: delete unused import textwrap in interpreter.py (#141126) --- Lib/concurrent/futures/interpreter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/concurrent/futures/interpreter.py b/Lib/concurrent/futures/interpreter.py index 53c6e757ded..85c1da2c722 100644 --- a/Lib/concurrent/futures/interpreter.py +++ b/Lib/concurrent/futures/interpreter.py @@ -2,7 +2,6 @@ from concurrent import interpreters import sys -import textwrap from . import thread as _thread import traceback From 9a199006733dae999f96c0f596c2035f4b9847b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 7 Nov 2025 10:54:02 +0100 Subject: [PATCH 074/313] gh-140734: fix off-by-one error when comparing to `_SUN_PATH_MAX` (#140903) The limit includes a NULL terminator. --- Lib/multiprocessing/util.py | 20 +++++++++++-------- ...-11-02-09-37-22.gh-issue-140734.f8gST9.rst | 2 ++ 2 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-02-09-37-22.gh-issue-140734.f8gST9.rst diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index a1a537dd48d..549fb07c275 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -126,12 +126,14 @@ def is_abstract_socket_namespace(address): # Function returning a temp directory which will be removed on exit # -# Maximum length of a socket file path is usually between 92 and 108 [1], -# but Linux is known to use a size of 108 [2]. BSD-based systems usually -# use a size of 104 or 108 and Windows does not create AF_UNIX sockets. +# Maximum length of a NULL-terminated [1] socket file path is usually +# between 92 and 108 [2], but Linux is known to use a size of 108 [3]. +# BSD-based systems usually use a size of 104 or 108 and Windows does +# not create AF_UNIX sockets. # -# [1]: https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/sys_un.h.html -# [2]: https://man7.org/linux/man-pages/man7/unix.7.html. +# [1]: https://github.com/python/cpython/issues/140734 +# [2]: https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/sys_un.h.html +# [3]: https://man7.org/linux/man-pages/man7/unix.7.html if sys.platform == 'linux': _SUN_PATH_MAX = 108 @@ -171,11 +173,13 @@ def _get_base_temp_dir(tempfile): # generated by tempfile._RandomNameSequence, which, by design, # is 8 characters long. # - # Thus, the length of socket filename will be: + # Thus, the socket file path length (without NULL terminator) will be: # # len(base_tempdir + '/pymp-XXXXXXXX' + '/sock-XXXXXXXX') sun_path_len = len(base_tempdir) + 14 + 14 - if sun_path_len <= _SUN_PATH_MAX: + # Strict inequality to account for the NULL terminator. + # See https://github.com/python/cpython/issues/140734. + if sun_path_len < _SUN_PATH_MAX: return base_tempdir # Fallback to the default system-wide temporary directory. # This ignores user-defined environment variables. @@ -201,7 +205,7 @@ def _get_base_temp_dir(tempfile): return base_tempdir warn("Ignoring user-defined temporary directory: %s", base_tempdir) # at most max(map(len, dirlist)) + 14 + 14 = 36 characters - assert len(base_system_tempdir) + 14 + 14 <= _SUN_PATH_MAX + assert len(base_system_tempdir) + 14 + 14 < _SUN_PATH_MAX return base_system_tempdir def get_temp_dir(): diff --git a/Misc/NEWS.d/next/Library/2025-11-02-09-37-22.gh-issue-140734.f8gST9.rst b/Misc/NEWS.d/next/Library/2025-11-02-09-37-22.gh-issue-140734.f8gST9.rst new file mode 100644 index 00000000000..46582f7fcf4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-02-09-37-22.gh-issue-140734.f8gST9.rst @@ -0,0 +1,2 @@ +:mod:`multiprocessing`: fix off-by-one error when checking the length +of a temporary socket file path. Patch by Bénédikt Tran. From a7bf27f7f521384a8964718bdb58a5cb113bb3ec Mon Sep 17 00:00:00 2001 From: Benel Tayar <86257734+beneltayar@users.noreply.github.com> Date: Fri, 7 Nov 2025 12:47:25 +0200 Subject: [PATCH 075/313] gh-141141: Make base64.b85decode() thread safe (GH-141149) --- Lib/base64.py | 7 +++++-- .../Library/2025-11-06-15-11-50.gh-issue-141141.tgIfgH.rst | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-06-15-11-50.gh-issue-141141.tgIfgH.rst diff --git a/Lib/base64.py b/Lib/base64.py index 5d78cc09f40..cfc57626c40 100644 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -462,9 +462,12 @@ def b85decode(b): # Delay the initialization of tables to not waste memory # if the function is never called if _b85dec is None: - _b85dec = [None] * 256 + # we don't assign to _b85dec directly to avoid issues when + # multiple threads call this function simultaneously + b85dec_tmp = [None] * 256 for i, c in enumerate(_b85alphabet): - _b85dec[c] = i + b85dec_tmp[c] = i + _b85dec = b85dec_tmp b = _bytes_from_decode_data(b) padding = (-len(b)) % 5 diff --git a/Misc/NEWS.d/next/Library/2025-11-06-15-11-50.gh-issue-141141.tgIfgH.rst b/Misc/NEWS.d/next/Library/2025-11-06-15-11-50.gh-issue-141141.tgIfgH.rst new file mode 100644 index 00000000000..f59ccfb33e7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-06-15-11-50.gh-issue-141141.tgIfgH.rst @@ -0,0 +1 @@ +Fix a thread safety issue with :func:`base64.b85decode`. Contributed by Benel Tayar. From ffd64737d00277eea1c4721d278a0951168d07ca Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 7 Nov 2025 14:17:47 +0100 Subject: [PATCH 076/313] Clarify argument/result ownership/validity for PyModule_* functions (GH-141159) Co-authored-by: Peter Bierma --- Doc/c-api/module.rst | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index ed2a7663375..1994a3c7d01 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -13,7 +13,7 @@ Module Objects .. index:: single: ModuleType (in module types) This instance of :c:type:`PyTypeObject` represents the Python module type. This - is exposed to Python programs as ``types.ModuleType``. + is exposed to Python programs as :py:class:`types.ModuleType`. .. c:function:: int PyModule_Check(PyObject *p) @@ -71,6 +71,9 @@ Module Objects ``PyObject_*`` functions rather than directly manipulate a module's :attr:`~object.__dict__`. + The returned reference is borrowed from the module; it is valid until + the module is destroyed. + .. c:function:: PyObject* PyModule_GetNameObject(PyObject *module) @@ -90,6 +93,10 @@ Module Objects Similar to :c:func:`PyModule_GetNameObject` but return the name encoded to ``'utf-8'``. + The returned buffer is only valid until the module is renamed or destroyed. + Note that Python code may rename a module by setting its :py:attr:`~module.__name__` + attribute. + .. c:function:: void* PyModule_GetState(PyObject *module) Return the "state" of the module, that is, a pointer to the block of memory @@ -126,6 +133,9 @@ Module Objects Similar to :c:func:`PyModule_GetFilenameObject` but return the filename encoded to 'utf-8'. + The returned buffer is only valid until the module's :py:attr:`~module.__file__` attribute + is reassigned or the module is destroyed. + .. deprecated:: 3.2 :c:func:`PyModule_GetFilename` raises :exc:`UnicodeEncodeError` on unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead. @@ -671,6 +681,9 @@ or code that creates modules dynamically. :c:type:`PyMethodDef` arrays; in that case they should call this function directly. + The *functions* array must be statically allocated (or otherwise guaranteed + to outlive the module object). + .. versionadded:: 3.5 .. c:function:: int PyModule_SetDocString(PyObject *module, const char *docstring) From 920286d6b296f9971fc79e14ec22966f8f7a7b90 Mon Sep 17 00:00:00 2001 From: "W. H. Wang" Date: Fri, 7 Nov 2025 21:34:49 +0800 Subject: [PATCH 077/313] Update NaNs handling description in `c-api/float.rst` (#141179) Clarified the behavior of NaNs on IEEE platforms regarding signaling and quiet NaNs. --- Doc/c-api/float.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 1085c32a537..edee498a0b8 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -113,8 +113,8 @@ NaNs (if such things exist on the platform) isn't handled correctly, and attempting to unpack a bytes string containing an IEEE INF or NaN will raise an exception. -Note that NaNs type may not be preserved on IEEE platforms (silent NaN become -quiet), for example on x86 systems in 32-bit mode. +Note that NaNs type may not be preserved on IEEE platforms (signaling NaN become +quiet NaN), for example on x86 systems in 32-bit mode. On non-IEEE platforms with more precision, or larger dynamic range, than IEEE 754 supports, not all values can be packed; on non-IEEE platforms with less From 7af9b5354dd7633df422b9f720633989b3090199 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Fri, 7 Nov 2025 09:09:38 -0500 Subject: [PATCH 078/313] gh-141004: Document `PyCapsule_Type` (GH-141079) --- Doc/c-api/capsule.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/c-api/capsule.rst b/Doc/c-api/capsule.rst index 6da020efc7f..03a848d68ed 100644 --- a/Doc/c-api/capsule.rst +++ b/Doc/c-api/capsule.rst @@ -22,6 +22,12 @@ Refer to :ref:`using-capsules` for more information on using these objects. loaded modules. +.. c:var:: PyTypeObject PyCapsule_Type + + The type object corresponding to capsule objects. This is the same object + as :class:`types.CapsuleType` in the Python layer. + + .. c:type:: PyCapsule_Destructor The type of a destructor callback for a capsule. Defined as:: From 9420795b47ac88f31315a8d1041e2c66c2cd9a8b Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Fri, 7 Nov 2025 11:19:14 -0500 Subject: [PATCH 079/313] gh-141004: Document `PyErr_WarnExplicitFormat` (GH-141187) --- Doc/c-api/exceptions.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 9c75f66f683..b6f9399337e 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -394,6 +394,15 @@ an error value). .. versionadded:: 3.2 +.. c:function:: int PyErr_WarnExplicitFormat(PyObject *category, const char *filename, int lineno, const char *module, PyObject *registry, const char *format, ...) + + Similar to :c:func:`PyErr_WarnExplicit`, but uses + :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is + an ASCII-encoded string. + + .. versionadded:: 3.2 + + .. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) Function similar to :c:func:`PyErr_WarnFormat`, but *category* is From 3989e12d39bfe2587e5ba80873c37e0c2d449088 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 7 Nov 2025 19:25:32 +0100 Subject: [PATCH 080/313] gh-141004: Document `Py_HASH_*` macros (#141205) --- Doc/c-api/hash.rst | 56 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/hash.rst b/Doc/c-api/hash.rst index b5fe93573a1..e74f5b4d18e 100644 --- a/Doc/c-api/hash.rst +++ b/Doc/c-api/hash.rst @@ -11,42 +11,94 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. .. versionadded:: 3.2 + .. c:type:: Py_uhash_t Hash value type: unsigned integer. .. versionadded:: 3.2 -.. c:macro:: PyHASH_MODULUS - The `Mersenne prime `_ ``P = 2**n -1``, used for numeric hash scheme. +.. c:macro:: Py_HASH_ALGORITHM + + A numerical value indicating the algorithm for hashing of :class:`str`, + :class:`bytes`, and :class:`memoryview`. + + The algorithm name is exposed by :data:`sys.hash_info.algorithm`. + + .. versionadded:: 3.4 + + +.. c:macro:: Py_HASH_FNV + Py_HASH_SIPHASH24 + Py_HASH_SIPHASH13 + + Numerical values to compare to :c:macro:`Py_HASH_ALGORITHM` to determine + which algorithm is used for hashing. The hash algorithm can be configured + via the configure :option:`--with-hash-algorithm` option. + + .. versionadded:: 3.4 + Add :c:macro:`!Py_HASH_FNV` and :c:macro:`!Py_HASH_SIPHASH24`. .. versionadded:: 3.13 + Add :c:macro:`!Py_HASH_SIPHASH13`. + + +.. c:macro:: Py_HASH_CUTOFF + + Buffers of length in range ``[1, Py_HASH_CUTOFF)`` are hashed using DJBX33A + instead of the algorithm described by :c:macro:`Py_HASH_ALGORITHM`. + + - A :c:macro:`!Py_HASH_CUTOFF` of 0 disables the optimization. + - :c:macro:`!Py_HASH_CUTOFF` must be non-negative and less or equal than 7. + + 32-bit platforms should use a cutoff smaller than 64-bit platforms because + it is easier to create colliding strings. A cutoff of 7 on 64-bit platforms + and 5 on 32-bit platforms should provide a decent safety margin. + + .. versionadded:: 3.4 + + +.. c:macro:: PyHASH_MODULUS + + The `Mersenne prime `_ ``P = 2**n -1``, + used for numeric hash scheme. + This corresponds to the :data:`sys.hash_info.modulus` constant. + + .. versionadded:: 3.13 + .. c:macro:: PyHASH_BITS The exponent ``n`` of ``P`` in :c:macro:`PyHASH_MODULUS`. + This corresponds to the :data:`sys.hash_info.hash_bits` constant. .. versionadded:: 3.13 + .. c:macro:: PyHASH_MULTIPLIER Prime multiplier used in string and various other hashes. .. versionadded:: 3.13 + .. c:macro:: PyHASH_INF The hash value returned for a positive infinity. + This corresponds to the :data:`sys.hash_info.inf` constant. .. versionadded:: 3.13 + .. c:macro:: PyHASH_IMAG The multiplier used for the imaginary part of a complex number. + This corresponds to the :data:`sys.hash_info.imag` constant. .. versionadded:: 3.13 + .. c:type:: PyHash_FuncDef Hash function definition used by :c:func:`PyHash_GetFuncDef`. From d13ee0ae186f4704f3b6016dd52f7727b81f9194 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Fri, 7 Nov 2025 13:46:47 -0500 Subject: [PATCH 081/313] gh-141004: Document `PyTraceBack*` APIs (GH-141192) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/exceptions.rst | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index b6f9399337e..f525ee7a046 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -1237,3 +1237,37 @@ Warning types .. versionadded:: 3.10 :c:data:`PyExc_EncodingWarning`. + + +Tracebacks +========== + +.. c:var:: PyTypeObject PyTraceBack_Type + + Type object for traceback objects. This is available as + :class:`types.TracebackType` in the Python layer. + + +.. c:function:: int PyTraceBack_Check(PyObject *op) + + Return true if *op* is a traceback object, false otherwise. This function + does not account for subtypes. + + +.. c:function:: int PyTraceBack_Here(PyFrameObject *f) + + Replace the :attr:`~BaseException.__traceback__` attribute on the current + exception with a new traceback prepending *f* to the existing chain. + + Calling this function without an exception set is undefined behavior. + + This function returns ``0`` on success, and returns ``-1`` with an + exception set on failure. + + +.. c:function:: int PyTraceBack_Print(PyObject *tb, PyObject *f) + + Write the traceback *tb* into the file *f*. + + This function returns ``0`` on success, and returns ``-1`` with an + exception set on failure. From cd0b3e5d12ed57d41878a89aac581375d021e25a Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Sat, 8 Nov 2025 06:28:01 +0000 Subject: [PATCH 082/313] gh-140849: Update bundled liblzma to 5.8.1 on Windows (#141022) --- .../2025-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst | 1 + Misc/externals.spdx.json | 8 ++++---- PCbuild/get_externals.bat | 2 +- PCbuild/liblzma.vcxproj | 7 +++---- PCbuild/liblzma.vcxproj.filters | 11 ++++------- PCbuild/python.props | 2 +- 6 files changed, 14 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2025-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst diff --git a/Misc/NEWS.d/next/Windows/2025-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst b/Misc/NEWS.d/next/Windows/2025-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst new file mode 100644 index 00000000000..6f25b867566 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2025-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst @@ -0,0 +1 @@ +Update bundled liblzma to version 5.8.1. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 59aceedb94d..dba01de8352 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -154,21 +154,21 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "a15c168e39e87d750c3dc766edc7f19bdda57dacf01e509678467eace91ad282" + "checksumValue": "1bfaba0ccacc6681d3ba85335cc7f49c24cf6f9d16f848cbd153b896d8a7d631" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/xz-5.2.5.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/xz-5.8.1.1.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:tukaani:xz:5.2.5:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:tukaani:xz:5.8.1.1:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "xz", "primaryPackagePurpose": "SOURCE", - "versionInfo": "5.2.5" + "versionInfo": "5.8.1.1" }, { "SPDXID": "SPDXRef-PACKAGE-zlib-ng", diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 319024e0f50..115203cecc8 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -59,7 +59,7 @@ set libraries=%libraries% mpdecimal-4.0.0 set libraries=%libraries% sqlite-3.50.4.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.15.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.15.0 -set libraries=%libraries% xz-5.2.5 +set libraries=%libraries% xz-5.8.1.1 set libraries=%libraries% zlib-ng-2.2.4 set libraries=%libraries% zstd-1.5.7 diff --git a/PCbuild/liblzma.vcxproj b/PCbuild/liblzma.vcxproj index 97938692328..75d4e162346 100644 --- a/PCbuild/liblzma.vcxproj +++ b/PCbuild/liblzma.vcxproj @@ -92,7 +92,7 @@ WIN32;HAVE_CONFIG_H;_LIB;%(PreprocessorDefinitions) - $(lzmaDir)windows/vs2019;$(lzmaDir)src/liblzma/common;$(lzmaDir)src/common;$(lzmaDir)src/liblzma/api;$(lzmaDir)src/liblzma/check;$(lzmaDir)src/liblzma/delta;$(lzmaDir)src/liblzma/lz;$(lzmaDir)src/liblzma/lzma;$(lzmaDir)src/liblzma/rangecoder;$(lzmaDir)src/liblzma/simple;%(AdditionalIncludeDirectories) + $(lzmaDir)windows;$(lzmaDir)src/liblzma/common;$(lzmaDir)src/common;$(lzmaDir)src/liblzma/api;$(lzmaDir)src/liblzma/check;$(lzmaDir)src/liblzma/delta;$(lzmaDir)src/liblzma/lz;$(lzmaDir)src/liblzma/lzma;$(lzmaDir)src/liblzma/rangecoder;$(lzmaDir)src/liblzma/simple;%(AdditionalIncludeDirectories) 4244;4267;4996;%(DisableSpecificWarnings) %(AdditionalOptions) -Wno-deprecated-declarations @@ -102,9 +102,7 @@ - - @@ -163,6 +161,7 @@ + @@ -239,7 +238,7 @@ - + diff --git a/PCbuild/liblzma.vcxproj.filters b/PCbuild/liblzma.vcxproj.filters index ebe2a7d5fa9..42feca5a341 100644 --- a/PCbuild/liblzma.vcxproj.filters +++ b/PCbuild/liblzma.vcxproj.filters @@ -18,6 +18,9 @@ Source Files + + Source Files + Source Files @@ -54,15 +57,9 @@ Source Files - - Source Files - Source Files - - Source Files - Source Files @@ -428,7 +425,7 @@ Header Files - + Header Files diff --git a/PCbuild/python.props b/PCbuild/python.props index cc157252655..7840e2a1cfc 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -77,7 +77,7 @@ $(ExternalsDir)sqlite-3.50.4.0\ $(ExternalsDir)bzip2-1.0.8\ - $(ExternalsDir)xz-5.2.5\ + $(ExternalsDir)xz-5.8.1.1\ $(ExternalsDir)libffi-3.4.4\ $(libffiDir)$(ArchName)\ $(libffiOutDir)include From c1785129c3b1b020e1523ff4f0d4314b9854d0e3 Mon Sep 17 00:00:00 2001 From: Brandon Hubacher Date: Sat, 8 Nov 2025 03:08:19 -0600 Subject: [PATCH 083/313] fix typos in contextvars asyncio support example docs (#141219) Co-authored-by: Kumar Aditya --- Doc/library/contextvars.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 57580ce026e..b218468a084 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -313,7 +313,7 @@ client:: addr = writer.transport.get_extra_info('socket').getpeername() client_addr_var.set(addr) - # In any code that we call is now possible to get + # In any code that we call, it is now possible to get the # client's address by calling 'client_addr_var.get()'. while True: From 8cec3d3a9d827aadc7008ab4312121fcf28329c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 8 Nov 2025 10:10:27 +0100 Subject: [PATCH 084/313] gh-141004: fix `versionadded` typo for `Py_HASH_SIPHASH13` (#141223) --- Doc/c-api/hash.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/hash.rst b/Doc/c-api/hash.rst index e74f5b4d18e..ecd604c81bc 100644 --- a/Doc/c-api/hash.rst +++ b/Doc/c-api/hash.rst @@ -40,7 +40,7 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. .. versionadded:: 3.4 Add :c:macro:`!Py_HASH_FNV` and :c:macro:`!Py_HASH_SIPHASH24`. - .. versionadded:: 3.13 + .. versionadded:: 3.11 Add :c:macro:`!Py_HASH_SIPHASH13`. From 7e90bac3cc6fd68fe6696ab4bce1262751de7531 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 8 Nov 2025 12:07:27 +0200 Subject: [PATCH 085/313] gh-140793: Improve documentatation and tests for the ensure_ascii option in the json module (GH-140906) * Document that ensure_ascii=True forces escaping not only non-ASCII, but also non-printable characters (the only affected ASCII character is U+007F). * Ensure that the help output for the json module does not exceed 80 columns (except one long line in an example and generated lines). * Add more tests. --- Doc/library/json.rst | 14 +++-- Lib/json/__init__.py | 52 +++++++++++-------- Lib/json/decoder.py | 8 +-- Lib/json/encoder.py | 20 +++---- .../test_json/test_encode_basestring_ascii.py | 3 +- Lib/test/test_json/test_unicode.py | 35 +++++++++++++ 6 files changed, 89 insertions(+), 43 deletions(-) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 12a5a96a3c5..8b4217c210d 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -183,8 +183,10 @@ Basic Usage :param bool ensure_ascii: If ``True`` (the default), the output is guaranteed to - have all incoming non-ASCII characters escaped. - If ``False``, these characters will be outputted as-is. + have all incoming non-ASCII and non-printable characters escaped. + If ``False``, all characters will be outputted as-is, except for + the characters that must be escaped: quotation mark, reverse solidus, + and the control characters U+0000 through U+001F. :param bool check_circular: If ``False``, the circular reference check for container types is skipped @@ -495,8 +497,10 @@ Encoders and Decoders :class:`bool` or ``None``. If *skipkeys* is true, such items are simply skipped. If *ensure_ascii* is true (the default), the output is guaranteed to - have all incoming non-ASCII characters escaped. If *ensure_ascii* is - false, these characters will be output as-is. + have all incoming non-ASCII and non-printable characters escaped. + If *ensure_ascii* is false, all characters will be output as-is, except for + the characters that must be escaped: quotation mark, reverse solidus, + and the control characters U+0000 through U+001F. If *check_circular* is true (the default), then lists, dicts, and custom encoded objects will be checked for circular references during encoding to @@ -636,7 +640,7 @@ UTF-32, with UTF-8 being the recommended default for maximum interoperability. As permitted, though not required, by the RFC, this module's serializer sets *ensure_ascii=True* by default, thus escaping the output so that the resulting -strings only contain ASCII characters. +strings only contain printable ASCII characters. Other than the *ensure_ascii* parameter, this module is defined strictly in terms of conversion between Python objects and diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index c8fdd0d99a0..89396b25a2c 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -127,8 +127,9 @@ def dump(obj, fp, *, skipkeys=False, ensure_ascii=True, check_circular=True, instead of raising a ``TypeError``. If ``ensure_ascii`` is false, then the strings written to ``fp`` can - contain non-ASCII characters if they appear in strings contained in - ``obj``. Otherwise, all such characters are escaped in JSON strings. + contain non-ASCII and non-printable characters if they appear in strings + contained in ``obj``. Otherwise, all such characters are escaped in JSON + strings. If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will @@ -144,10 +145,11 @@ def dump(obj, fp, *, skipkeys=False, ensure_ascii=True, check_circular=True, level of 0 will only insert newlines. ``None`` is the most compact representation. - If specified, ``separators`` should be an ``(item_separator, key_separator)`` - tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and - ``(',', ': ')`` otherwise. To get the most compact JSON representation, - you should specify ``(',', ':')`` to eliminate whitespace. + If specified, ``separators`` should be an ``(item_separator, + key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is + ``None`` and ``(',', ': ')`` otherwise. To get the most compact JSON + representation, you should specify ``(',', ':')`` to eliminate + whitespace. ``default(obj)`` is a function that should return a serializable version of obj or raise TypeError. The default simply raises TypeError. @@ -188,9 +190,10 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, (``str``, ``int``, ``float``, ``bool``, ``None``) will be skipped instead of raising a ``TypeError``. - If ``ensure_ascii`` is false, then the return value can contain non-ASCII - characters if they appear in strings contained in ``obj``. Otherwise, all - such characters are escaped in JSON strings. + If ``ensure_ascii`` is false, then the return value can contain + non-ASCII and non-printable characters if they appear in strings + contained in ``obj``. Otherwise, all such characters are escaped in + JSON strings. If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will @@ -206,10 +209,11 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, level of 0 will only insert newlines. ``None`` is the most compact representation. - If specified, ``separators`` should be an ``(item_separator, key_separator)`` - tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and - ``(',', ': ')`` otherwise. To get the most compact JSON representation, - you should specify ``(',', ':')`` to eliminate whitespace. + If specified, ``separators`` should be an ``(item_separator, + key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is + ``None`` and ``(',', ': ')`` otherwise. To get the most compact JSON + representation, you should specify ``(',', ':')`` to eliminate + whitespace. ``default(obj)`` is a function that should return a serializable version of obj or raise TypeError. The default simply raises TypeError. @@ -280,11 +284,12 @@ def load(fp, *, cls=None, object_hook=None, parse_float=None, ``object_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders (e.g. JSON-RPC class hinting). - ``object_pairs_hook`` is an optional function that will be called with the - result of any object literal decoded with an ordered list of pairs. The - return value of ``object_pairs_hook`` will be used instead of the ``dict``. - This feature can be used to implement custom decoders. If ``object_hook`` - is also defined, the ``object_pairs_hook`` takes priority. + ``object_pairs_hook`` is an optional function that will be called with + the result of any object literal decoded with an ordered list of pairs. + The return value of ``object_pairs_hook`` will be used instead of the + ``dict``. This feature can be used to implement custom decoders. If + ``object_hook`` is also defined, the ``object_pairs_hook`` takes + priority. To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` kwarg; otherwise ``JSONDecoder`` is used. @@ -305,11 +310,12 @@ def loads(s, *, cls=None, object_hook=None, parse_float=None, ``object_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders (e.g. JSON-RPC class hinting). - ``object_pairs_hook`` is an optional function that will be called with the - result of any object literal decoded with an ordered list of pairs. The - return value of ``object_pairs_hook`` will be used instead of the ``dict``. - This feature can be used to implement custom decoders. If ``object_hook`` - is also defined, the ``object_pairs_hook`` takes priority. + ``object_pairs_hook`` is an optional function that will be called with + the result of any object literal decoded with an ordered list of pairs. + The return value of ``object_pairs_hook`` will be used instead of the + ``dict``. This feature can be used to implement custom decoders. If + ``object_hook`` is also defined, the ``object_pairs_hook`` takes + priority. ``parse_float``, if specified, will be called with the string of every JSON float to be decoded. By default this is equivalent to diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py index ff4bfcdcc40..92ad6352557 100644 --- a/Lib/json/decoder.py +++ b/Lib/json/decoder.py @@ -297,10 +297,10 @@ def __init__(self, *, object_hook=None, parse_float=None, place of the given ``dict``. This can be used to provide custom deserializations (e.g. to support JSON-RPC class hinting). - ``object_pairs_hook``, if specified will be called with the result of - every JSON object decoded with an ordered list of pairs. The return - value of ``object_pairs_hook`` will be used instead of the ``dict``. - This feature can be used to implement custom decoders. + ``object_pairs_hook``, if specified will be called with the result + of every JSON object decoded with an ordered list of pairs. The + return value of ``object_pairs_hook`` will be used instead of the + ``dict``. This feature can be used to implement custom decoders. If ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index bc446e0f377..5cf6d64f3ea 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -111,9 +111,10 @@ def __init__(self, *, skipkeys=False, ensure_ascii=True, encoding of keys that are not str, int, float, bool or None. If skipkeys is True, such items are simply skipped. - If ensure_ascii is true, the output is guaranteed to be str - objects with all incoming non-ASCII characters escaped. If - ensure_ascii is false, the output can contain non-ASCII characters. + If ensure_ascii is true, the output is guaranteed to be str objects + with all incoming non-ASCII and non-printable characters escaped. + If ensure_ascii is false, the output can contain non-ASCII and + non-printable characters. If check_circular is true, then lists, dicts, and custom encoded objects will be checked for circular references during encoding to @@ -134,14 +135,15 @@ def __init__(self, *, skipkeys=False, ensure_ascii=True, indent level. An indent level of 0 will only insert newlines. None is the most compact representation. - If specified, separators should be an (item_separator, key_separator) - tuple. The default is (', ', ': ') if *indent* is ``None`` and - (',', ': ') otherwise. To get the most compact JSON representation, - you should specify (',', ':') to eliminate whitespace. + If specified, separators should be an (item_separator, + key_separator) tuple. The default is (', ', ': ') if *indent* is + ``None`` and (',', ': ') otherwise. To get the most compact JSON + representation, you should specify (',', ':') to eliminate + whitespace. If specified, default is a function that gets called for objects - that can't otherwise be serialized. It should return a JSON encodable - version of the object or raise a ``TypeError``. + that can't otherwise be serialized. It should return a JSON + encodable version of the object or raise a ``TypeError``. """ diff --git a/Lib/test/test_json/test_encode_basestring_ascii.py b/Lib/test/test_json/test_encode_basestring_ascii.py index 6a39b72a09d..c90d3e968e5 100644 --- a/Lib/test/test_json/test_encode_basestring_ascii.py +++ b/Lib/test/test_json/test_encode_basestring_ascii.py @@ -8,13 +8,12 @@ ('\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), ('controls', '"controls"'), ('\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), + ('\x00\x1f\x7f', '"\\u0000\\u001f\\u007f"'), ('{"object with 1 member":["array with 1 element"]}', '"{\\"object with 1 member\\":[\\"array with 1 element\\"]}"'), (' s p a c e d ', '" s p a c e d "'), ('\U0001d120', '"\\ud834\\udd20"'), ('\u03b1\u03a9', '"\\u03b1\\u03a9"'), ("`1~!@#$%^&*()_+-={':[,]}|;.?", '"`1~!@#$%^&*()_+-={\':[,]}|;.?"'), - ('\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), - ('\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), ] class TestEncodeBasestringAscii: diff --git a/Lib/test/test_json/test_unicode.py b/Lib/test/test_json/test_unicode.py index 68629cceeb9..1aa9546dc46 100644 --- a/Lib/test/test_json/test_unicode.py +++ b/Lib/test/test_json/test_unicode.py @@ -32,6 +32,29 @@ def test_encoding7(self): j = self.dumps(u + "\n", ensure_ascii=False) self.assertEqual(j, f'"{u}\\n"') + def test_ascii_non_printable_encode(self): + u = '\b\t\n\f\r\x00\x1f\x7f' + self.assertEqual(self.dumps(u), + '"\\b\\t\\n\\f\\r\\u0000\\u001f\\u007f"') + self.assertEqual(self.dumps(u, ensure_ascii=False), + '"\\b\\t\\n\\f\\r\\u0000\\u001f\x7f"') + + def test_ascii_non_printable_decode(self): + self.assertEqual(self.loads('"\\b\\t\\n\\f\\r"'), + '\b\t\n\f\r') + s = ''.join(map(chr, range(32))) + for c in s: + self.assertRaises(self.JSONDecodeError, self.loads, f'"{c}"') + self.assertEqual(self.loads(f'"{s}"', strict=False), s) + self.assertEqual(self.loads('"\x7f"'), '\x7f') + + def test_escaped_decode(self): + self.assertEqual(self.loads('"\\b\\t\\n\\f\\r"'), '\b\t\n\f\r') + self.assertEqual(self.loads('"\\"\\\\\\/"'), '"\\/') + for c in set(map(chr, range(0x100))) - set('"\\/bfnrt'): + self.assertRaises(self.JSONDecodeError, self.loads, f'"\\{c}"') + self.assertRaises(self.JSONDecodeError, self.loads, f'"\\{c}"', strict=False) + def test_big_unicode_encode(self): u = '\U0001d120' self.assertEqual(self.dumps(u), '"\\ud834\\udd20"') @@ -48,6 +71,18 @@ def test_unicode_decode(self): s = f'"\\u{i:04x}"' self.assertEqual(self.loads(s), u) + def test_single_surrogate_encode(self): + self.assertEqual(self.dumps('\uD83D'), '"\\ud83d"') + self.assertEqual(self.dumps('\uD83D', ensure_ascii=False), '"\ud83d"') + self.assertEqual(self.dumps('\uDC0D'), '"\\udc0d"') + self.assertEqual(self.dumps('\uDC0D', ensure_ascii=False), '"\udc0d"') + + def test_single_surrogate_decode(self): + self.assertEqual(self.loads('"\uD83D"'), '\ud83d') + self.assertEqual(self.loads('"\\uD83D"'), '\ud83d') + self.assertEqual(self.loads('"\udc0d"'), '\udc0d') + self.assertEqual(self.loads('"\\udc0d"'), '\udc0d') + def test_unicode_preservation(self): self.assertEqual(type(self.loads('""')), str) self.assertEqual(type(self.loads('"a"')), str) From 610377056bad696915d70590429e68002bee9006 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 8 Nov 2025 12:17:59 +0200 Subject: [PATCH 086/313] gh-140615: Update docstrings in the fcntl module (GH-140619) * Refer to bytes objects or bytes-like objects instead of strings. * Remove backticks -- they do not have effect on formatting. * Re-wrap lines to ensure the pydoc output fits in 80 coluimns. * Remove references to the 1024 bytes limit. --- Modules/clinic/fcntlmodule.c.h | 90 ++++++++++++++++--------------- Modules/fcntlmodule.c | 98 +++++++++++++++++----------------- 2 files changed, 97 insertions(+), 91 deletions(-) diff --git a/Modules/clinic/fcntlmodule.c.h b/Modules/clinic/fcntlmodule.c.h index 2b61d9f8708..718f80bfe73 100644 --- a/Modules/clinic/fcntlmodule.c.h +++ b/Modules/clinic/fcntlmodule.c.h @@ -8,17 +8,22 @@ PyDoc_STRVAR(fcntl_fcntl__doc__, "fcntl($module, fd, cmd, arg=0, /)\n" "--\n" "\n" -"Perform the operation `cmd` on file descriptor fd.\n" +"Perform the operation cmd on file descriptor fd.\n" "\n" -"The values used for `cmd` are operating system dependent, and are available\n" -"as constants in the fcntl module, using the same names as used in\n" -"the relevant C header files. The argument arg is optional, and\n" -"defaults to 0; it may be an int or a string. If arg is given as a string,\n" -"the return value of fcntl is a string of that length, containing the\n" -"resulting value put in the arg buffer by the operating system. The length\n" -"of the arg string is not allowed to exceed 1024 bytes. If the arg given\n" -"is an integer or if none is specified, the result value is an integer\n" -"corresponding to the return value of the fcntl call in the C code."); +"The values used for cmd are operating system dependent, and are\n" +"available as constants in the fcntl module, using the same names as used\n" +"in the relevant C header files. The argument arg is optional, and\n" +"defaults to 0; it may be an integer, a bytes-like object or a string.\n" +"If arg is given as a string, it will be encoded to binary using the\n" +"UTF-8 encoding.\n" +"\n" +"If the arg given is an integer or if none is specified, the result value\n" +"is an integer corresponding to the return value of the fcntl() call in\n" +"the C code.\n" +"\n" +"If arg is given as a bytes-like object, the return value of fcntl() is a\n" +"bytes object of that length, containing the resulting value put in the\n" +"arg buffer by the operating system."); #define FCNTL_FCNTL_METHODDEF \ {"fcntl", _PyCFunction_CAST(fcntl_fcntl), METH_FASTCALL, fcntl_fcntl__doc__}, @@ -60,34 +65,33 @@ PyDoc_STRVAR(fcntl_ioctl__doc__, "ioctl($module, fd, request, arg=0, mutate_flag=True, /)\n" "--\n" "\n" -"Perform the operation `request` on file descriptor `fd`.\n" +"Perform the operation request on file descriptor fd.\n" "\n" -"The values used for `request` are operating system dependent, and are available\n" -"as constants in the fcntl or termios library modules, using the same names as\n" -"used in the relevant C header files.\n" +"The values used for request are operating system dependent, and are\n" +"available as constants in the fcntl or termios library modules, using\n" +"the same names as used in the relevant C header files.\n" "\n" -"The argument `arg` is optional, and defaults to 0; it may be an int or a\n" -"buffer containing character data (most likely a string or an array).\n" +"The argument arg is optional, and defaults to 0; it may be an integer, a\n" +"bytes-like object or a string. If arg is given as a string, it will be\n" +"encoded to binary using the UTF-8 encoding.\n" "\n" -"If the argument is a mutable buffer (such as an array) and if the\n" -"mutate_flag argument (which is only allowed in this case) is true then the\n" -"buffer is (in effect) passed to the operating system and changes made by\n" -"the OS will be reflected in the contents of the buffer after the call has\n" -"returned. The return value is the integer returned by the ioctl system\n" -"call.\n" +"If the arg given is an integer or if none is specified, the result value\n" +"is an integer corresponding to the return value of the ioctl() call in\n" +"the C code.\n" "\n" -"If the argument is a mutable buffer and the mutable_flag argument is false,\n" -"the behavior is as if a string had been passed.\n" +"If the argument is a mutable buffer (such as a bytearray) and the\n" +"mutate_flag argument is true (default) then the buffer is (in effect)\n" +"passed to the operating system and changes made by the OS will be\n" +"reflected in the contents of the buffer after the call has returned.\n" +"The return value is the integer returned by the ioctl() system call.\n" "\n" -"If the argument is an immutable buffer (most likely a string) then a copy\n" -"of the buffer is passed to the operating system and the return value is a\n" -"string of the same length containing whatever the operating system put in\n" -"the buffer. The length of the arg buffer in this case is not allowed to\n" -"exceed 1024 bytes.\n" +"If the argument is a mutable buffer and the mutable_flag argument is\n" +"false, the behavior is as if an immutable buffer had been passed.\n" "\n" -"If the arg given is an integer or if none is specified, the result value is\n" -"an integer corresponding to the return value of the ioctl call in the C\n" -"code."); +"If the argument is an immutable buffer then a copy of the buffer is\n" +"passed to the operating system and the return value is a bytes object of\n" +"the same length containing whatever the operating system put in the\n" +"buffer."); #define FCNTL_IOCTL_METHODDEF \ {"ioctl", _PyCFunction_CAST(fcntl_ioctl), METH_FASTCALL, fcntl_ioctl__doc__}, @@ -154,7 +158,7 @@ PyDoc_STRVAR(fcntl_flock__doc__, "flock($module, fd, operation, /)\n" "--\n" "\n" -"Perform the lock operation `operation` on file descriptor `fd`.\n" +"Perform the lock operation on file descriptor fd.\n" "\n" "See the Unix manual page for flock(2) for details (On some systems, this\n" "function is emulated using fcntl())."); @@ -195,22 +199,22 @@ PyDoc_STRVAR(fcntl_lockf__doc__, "\n" "A wrapper around the fcntl() locking calls.\n" "\n" -"`fd` is the file descriptor of the file to lock or unlock, and operation is one\n" -"of the following values:\n" +"fd is the file descriptor of the file to lock or unlock, and operation\n" +"is one of the following values:\n" "\n" " LOCK_UN - unlock\n" " LOCK_SH - acquire a shared lock\n" " LOCK_EX - acquire an exclusive lock\n" "\n" "When operation is LOCK_SH or LOCK_EX, it can also be bitwise ORed with\n" -"LOCK_NB to avoid blocking on lock acquisition. If LOCK_NB is used and the\n" -"lock cannot be acquired, an OSError will be raised and the exception will\n" -"have an errno attribute set to EACCES or EAGAIN (depending on the operating\n" -"system -- for portability, check for either value).\n" +"LOCK_NB to avoid blocking on lock acquisition. If LOCK_NB is used and\n" +"the lock cannot be acquired, an OSError will be raised and the exception\n" +"will have an errno attribute set to EACCES or EAGAIN (depending on the\n" +"operating system -- for portability, check for either value).\n" "\n" -"`len` is the number of bytes to lock, with the default meaning to lock to\n" -"EOF. `start` is the byte offset, relative to `whence`, to that the lock\n" -"starts. `whence` is as with fileobj.seek(), specifically:\n" +"len is the number of bytes to lock, with the default meaning to lock to\n" +"EOF. start is the byte offset, relative to whence, to that the lock\n" +"starts. whence is as with fileobj.seek(), specifically:\n" "\n" " 0 - relative to the start of the file (SEEK_SET)\n" " 1 - relative to the current buffer position (SEEK_CUR)\n" @@ -265,4 +269,4 @@ skip_optional: exit: return return_value; } -/*[clinic end generated code: output=9773e44da302dc7c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c782fcf9dd6690e0 input=a9049054013a1b77]*/ diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index df2c9994127..e373bf36881 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -40,22 +40,27 @@ fcntl.fcntl arg: object(c_default='NULL') = 0 / -Perform the operation `cmd` on file descriptor fd. +Perform the operation cmd on file descriptor fd. -The values used for `cmd` are operating system dependent, and are available -as constants in the fcntl module, using the same names as used in -the relevant C header files. The argument arg is optional, and -defaults to 0; it may be an int or a string. If arg is given as a string, -the return value of fcntl is a string of that length, containing the -resulting value put in the arg buffer by the operating system. The length -of the arg string is not allowed to exceed 1024 bytes. If the arg given -is an integer or if none is specified, the result value is an integer -corresponding to the return value of the fcntl call in the C code. +The values used for cmd are operating system dependent, and are +available as constants in the fcntl module, using the same names as used +in the relevant C header files. The argument arg is optional, and +defaults to 0; it may be an integer, a bytes-like object or a string. +If arg is given as a string, it will be encoded to binary using the +UTF-8 encoding. + +If the arg given is an integer or if none is specified, the result value +is an integer corresponding to the return value of the fcntl() call in +the C code. + +If arg is given as a bytes-like object, the return value of fcntl() is a +bytes object of that length, containing the resulting value put in the +arg buffer by the operating system. [clinic start generated code]*/ static PyObject * fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg) -/*[clinic end generated code: output=888fc93b51c295bd input=7955340198e5f334]*/ +/*[clinic end generated code: output=888fc93b51c295bd input=77340720f11665da]*/ { int ret; int async_err = 0; @@ -151,7 +156,6 @@ fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg) /*[clinic input] -@permit_long_docstring_body fcntl.ioctl fd: fildes @@ -160,40 +164,39 @@ fcntl.ioctl mutate_flag as mutate_arg: bool = True / -Perform the operation `request` on file descriptor `fd`. +Perform the operation request on file descriptor fd. -The values used for `request` are operating system dependent, and are available -as constants in the fcntl or termios library modules, using the same names as -used in the relevant C header files. +The values used for request are operating system dependent, and are +available as constants in the fcntl or termios library modules, using +the same names as used in the relevant C header files. -The argument `arg` is optional, and defaults to 0; it may be an int or a -buffer containing character data (most likely a string or an array). +The argument arg is optional, and defaults to 0; it may be an integer, a +bytes-like object or a string. If arg is given as a string, it will be +encoded to binary using the UTF-8 encoding. -If the argument is a mutable buffer (such as an array) and if the -mutate_flag argument (which is only allowed in this case) is true then the -buffer is (in effect) passed to the operating system and changes made by -the OS will be reflected in the contents of the buffer after the call has -returned. The return value is the integer returned by the ioctl system -call. +If the arg given is an integer or if none is specified, the result value +is an integer corresponding to the return value of the ioctl() call in +the C code. -If the argument is a mutable buffer and the mutable_flag argument is false, -the behavior is as if a string had been passed. +If the argument is a mutable buffer (such as a bytearray) and the +mutate_flag argument is true (default) then the buffer is (in effect) +passed to the operating system and changes made by the OS will be +reflected in the contents of the buffer after the call has returned. +The return value is the integer returned by the ioctl() system call. -If the argument is an immutable buffer (most likely a string) then a copy -of the buffer is passed to the operating system and the return value is a -string of the same length containing whatever the operating system put in -the buffer. The length of the arg buffer in this case is not allowed to -exceed 1024 bytes. +If the argument is a mutable buffer and the mutable_flag argument is +false, the behavior is as if an immutable buffer had been passed. -If the arg given is an integer or if none is specified, the result value is -an integer corresponding to the return value of the ioctl call in the C -code. +If the argument is an immutable buffer then a copy of the buffer is +passed to the operating system and the return value is a bytes object of +the same length containing whatever the operating system put in the +buffer. [clinic start generated code]*/ static PyObject * fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *arg, int mutate_arg) -/*[clinic end generated code: output=f72baba2454d7a62 input=d7fe504d335449e2]*/ +/*[clinic end generated code: output=f72baba2454d7a62 input=954fe75c208cc492]*/ { /* We use the unsigned non-checked 'I' format for the 'code' parameter because the system expects it to be a 32bit bit field value @@ -340,7 +343,7 @@ fcntl.flock operation as code: int / -Perform the lock operation `operation` on file descriptor `fd`. +Perform the lock operation on file descriptor fd. See the Unix manual page for flock(2) for details (On some systems, this function is emulated using fcntl()). @@ -348,7 +351,7 @@ function is emulated using fcntl()). static PyObject * fcntl_flock_impl(PyObject *module, int fd, int code) -/*[clinic end generated code: output=84059e2b37d2fc64 input=0bfc00f795953452]*/ +/*[clinic end generated code: output=84059e2b37d2fc64 input=ade68943e8599f0a]*/ { int ret; int async_err = 0; @@ -400,7 +403,6 @@ fcntl_flock_impl(PyObject *module, int fd, int code) /*[clinic input] -@permit_long_docstring_body fcntl.lockf fd: fildes @@ -412,22 +414,22 @@ fcntl.lockf A wrapper around the fcntl() locking calls. -`fd` is the file descriptor of the file to lock or unlock, and operation is one -of the following values: +fd is the file descriptor of the file to lock or unlock, and operation +is one of the following values: LOCK_UN - unlock LOCK_SH - acquire a shared lock LOCK_EX - acquire an exclusive lock When operation is LOCK_SH or LOCK_EX, it can also be bitwise ORed with -LOCK_NB to avoid blocking on lock acquisition. If LOCK_NB is used and the -lock cannot be acquired, an OSError will be raised and the exception will -have an errno attribute set to EACCES or EAGAIN (depending on the operating -system -- for portability, check for either value). +LOCK_NB to avoid blocking on lock acquisition. If LOCK_NB is used and +the lock cannot be acquired, an OSError will be raised and the exception +will have an errno attribute set to EACCES or EAGAIN (depending on the +operating system -- for portability, check for either value). -`len` is the number of bytes to lock, with the default meaning to lock to -EOF. `start` is the byte offset, relative to `whence`, to that the lock -starts. `whence` is as with fileobj.seek(), specifically: +len is the number of bytes to lock, with the default meaning to lock to +EOF. start is the byte offset, relative to whence, to that the lock +starts. whence is as with fileobj.seek(), specifically: 0 - relative to the start of the file (SEEK_SET) 1 - relative to the current buffer position (SEEK_CUR) @@ -437,7 +439,7 @@ starts. `whence` is as with fileobj.seek(), specifically: static PyObject * fcntl_lockf_impl(PyObject *module, int fd, int code, PyObject *lenobj, PyObject *startobj, int whence) -/*[clinic end generated code: output=4985e7a172e7461a input=f666662ec2edd775]*/ +/*[clinic end generated code: output=4985e7a172e7461a input=369bef4d7a1c5ff4]*/ { int ret; int async_err = 0; From 9c3994663b1d8f128f18f53fd4e7871fd0301c4f Mon Sep 17 00:00:00 2001 From: Naitree Zhu Date: Sat, 8 Nov 2025 19:19:42 +0900 Subject: [PATCH 087/313] gh-139741: Make `dist-pdf` docs archive build work for macOS (#140837) --- Doc/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/Makefile b/Doc/Makefile index f6f4c721080..f16d9cacb1b 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -241,7 +241,8 @@ dist-pdf: # as otherwise the full latexmk process is run twice. # ($$ is needed to escape the $; https://www.gnu.org/software/make/manual/make.html#Basics-of-Variable-References) -sed -i 's/: all-$$(FMT)/:/' build/latex/Makefile - (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`nproc`+1)) --output-sync LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) + if [ -n "$(filter output-sync,$(value .FEATURES))" ]; then OUTPUTSYNC=--output-sync; else OUTPUTSYNC=; fi && \ + (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`getconf _NPROCESSORS_ONLN`+1)) $$OUTPUTSYNC LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-a4.zip cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-a4.tar.bz2 @echo "Build finished and archived!" From 87942d911b8bc9e83caee3c0b699f0b0ba15daa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 8 Nov 2025 13:22:02 +0100 Subject: [PATCH 088/313] gh-141004: correctly document `Py_HASH_*` and `PyHASH_*` as `hash_info` attributes (#141233) --- Doc/c-api/hash.rst | 12 +++++++++++- Doc/library/sys.rst | 6 +++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/hash.rst b/Doc/c-api/hash.rst index ecd604c81bc..1ad712b0ce4 100644 --- a/Doc/c-api/hash.rst +++ b/Doc/c-api/hash.rst @@ -56,6 +56,8 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. it is easier to create colliding strings. A cutoff of 7 on 64-bit platforms and 5 on 32-bit platforms should provide a decent safety margin. + This corresponds to the :data:`sys.hash_info.cutoff` constant. + .. versionadded:: 3.4 @@ -63,6 +65,7 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. The `Mersenne prime `_ ``P = 2**n -1``, used for numeric hash scheme. + This corresponds to the :data:`sys.hash_info.modulus` constant. .. versionadded:: 3.13 @@ -71,7 +74,6 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. .. c:macro:: PyHASH_BITS The exponent ``n`` of ``P`` in :c:macro:`PyHASH_MODULUS`. - This corresponds to the :data:`sys.hash_info.hash_bits` constant. .. versionadded:: 3.13 @@ -86,6 +88,7 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. .. c:macro:: PyHASH_INF The hash value returned for a positive infinity. + This corresponds to the :data:`sys.hash_info.inf` constant. .. versionadded:: 3.13 @@ -94,6 +97,7 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. .. c:macro:: PyHASH_IMAG The multiplier used for the imaginary part of a complex number. + This corresponds to the :data:`sys.hash_info.imag` constant. .. versionadded:: 3.13 @@ -111,14 +115,20 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. Hash function name (UTF-8 encoded string). + This corresponds to the :data:`sys.hash_info.algorithm` constant. + .. c:member:: const int hash_bits Internal size of the hash value in bits. + This corresponds to the :data:`sys.hash_info.hash_bits` constant. + .. c:member:: const int seed_bits Size of seed input in bits. + This corresponds to the :data:`sys.hash_info.seed_bits` constant. + .. versionadded:: 3.4 diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 698a9d0689d..a0621d4b0db 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1176,10 +1176,14 @@ always available. Unless explicitly noted otherwise, all variables are read-only The size of the seed key of the hash algorithm + .. attribute:: hash_info.cutoff + + Cutoff for small string DJBX33A optimization in range ``[1, cutoff)``. + .. versionadded:: 3.2 .. versionchanged:: 3.4 - Added *algorithm*, *hash_bits* and *seed_bits* + Added *algorithm*, *hash_bits*, *seed_bits*, and *cutoff*. .. data:: hexversion From 6545a4e8f83c27996fc771ed7c8c96ae0ce8d2e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=AF=E5=87=9B?= <1348292515@qq.com> Date: Sat, 8 Nov 2025 22:56:48 +0800 Subject: [PATCH 089/313] gh-141246: Link to correct Windows docs in `time.sleep()` doc (#141248) --- Doc/library/time.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 69e6433e898..a931134331f 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -407,9 +407,9 @@ Functions On Windows, if *secs* is zero, the thread relinquishes the remainder of its time slice to any other thread that is ready to run. If there are no other threads ready to run, the function returns immediately, and the thread - continues execution. On Windows 8.1 and newer the implementation uses + continues execution. On Windows 10 and newer the implementation uses a `high-resolution timer - `_ + `_ which provides resolution of 100 nanoseconds. If *secs* is zero, ``Sleep(0)`` is used. .. rubric:: Unix implementation From be1c72a45d54cdd35e0a830e18224c4c74be808c Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sat, 8 Nov 2025 10:47:09 -0500 Subject: [PATCH 090/313] gh-141004: Document `PyErr_ProgramTextObject` and `PyErr_ProgramText` (GH-141250) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/exceptions.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index f525ee7a046..c58aa659e1b 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -331,6 +331,23 @@ For convenience, some of these functions will always return a use. +.. c:function:: PyObject *PyErr_ProgramTextObject(PyObject *filename, int lineno) + + Get the source line in *filename* at line *lineno*. *filename* should be a + Python :class:`str` object. + + On success, this function returns a Python string object with the found line. + On failure, this function returns ``NULL`` without an exception set. + + +.. c:function:: PyObject *PyErr_ProgramText(const char *filename, int lineno) + + Similar to :c:func:`PyErr_ProgramTextObject`, but *filename* is a + :c:expr:`const char *`, which is decoded with the + :term:`filesystem encoding and error handler`, instead of a + Python object reference. + + Issuing warnings ================ From 5e5fc0404ed983bb37a19793a5c802d0d9852e5d Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sat, 8 Nov 2025 12:29:31 -0500 Subject: [PATCH 091/313] gh-141004: Document `PyBUF_WRITEABLE` (GH-141255) --- Doc/c-api/buffer.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index d3081894ead..6bb72a2312b 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -261,6 +261,10 @@ readonly, format MUST be consistent for all consumers. For example, :c:expr:`PyBUF_SIMPLE | PyBUF_WRITABLE` can be used to request a simple writable buffer. + .. c:macro:: PyBUF_WRITEABLE + + This is a :term:`soft deprecated` alias to :c:macro:`PyBUF_WRITABLE`. + .. c:macro:: PyBUF_FORMAT Controls the :c:member:`~Py_buffer.format` field. If set, this field MUST From 545299773b40fb589cbd5e54d1d597207d9a2a76 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sat, 8 Nov 2025 18:12:03 +0000 Subject: [PATCH 092/313] gh-141004: Document the `PyDoc_VAR` macro (GH-141263) --- Doc/c-api/intro.rst | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index acce3dc215d..4e7d1630ab3 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -235,7 +235,7 @@ complete listing. .. c:macro:: PyDoc_STRVAR(name, str) - Creates a variable with name ``name`` that can be used in docstrings. + Creates a variable with name *name* that can be used in docstrings. If Python is built without docstrings, the value will be empty. Use :c:macro:`PyDoc_STRVAR` for docstrings to support building @@ -267,6 +267,15 @@ complete listing. {NULL, NULL} }; +.. c:macro:: PyDoc_VAR(name) + + Declares a static character array variable with the given name *name*. + + For example:: + + PyDoc_VAR(python_doc) = PyDoc_STR("A genus of constricting snakes in the Pythonidae family native " + "to the tropics and subtropics of the Eastern Hemisphere."); + .. _api-objects: From 0ac890bea79d3e0162c8909b0999f626f1141d89 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sat, 8 Nov 2025 19:22:05 +0000 Subject: [PATCH 093/313] gh-141004: Document `Py_BUILD_ASSERT*` macros (GH-141266) --- Doc/c-api/intro.rst | 23 +++++++++++++++++++++++ Doc/conf.py | 3 --- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 4e7d1630ab3..6e1a9dcb355 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -233,6 +233,29 @@ complete listing. .. versionadded:: 3.4 +.. c:macro:: Py_BUILD_ASSERT(cond) + + Asserts a compile-time condition *cond*, as a statement. + The build will fail if the condition is false or cannot be evaluated at compile time. + + For example:: + + Py_BUILD_ASSERT(sizeof(PyTime_t) == sizeof(int64_t)); + + .. versionadded:: 3.3 + +.. c:macro:: Py_BUILD_ASSERT_EXPR(cond) + + Asserts a compile-time condition *cond*, as an expression that evaluates to ``0``. + The build will fail if the condition is false or cannot be evaluated at compile time. + + For example:: + + #define foo_to_char(foo) \ + ((char *)(foo) + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) + + .. versionadded:: 3.3 + .. c:macro:: PyDoc_STRVAR(name, str) Creates a variable with name *name* that can be used in docstrings. diff --git a/Doc/conf.py b/Doc/conf.py index f1dda10052e..0f1412d1007 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -226,9 +226,6 @@ # Temporary undocumented names. # In future this list must be empty. nitpick_ignore += [ - # Undocumented public C macros - ('c:macro', 'Py_BUILD_ASSERT'), - ('c:macro', 'Py_BUILD_ASSERT_EXPR'), # Do not error nit-picky mode builds when _SubParsersAction.add_parser cannot # be resolved, as the method is currently undocumented. For context, see # https://github.com/python/cpython/pull/103289. From b36f01d03f8f89cadd8436a7261ca29e4030d30c Mon Sep 17 00:00:00 2001 From: Mohsin Mehmood <55545648+mohsinm-dev@users.noreply.github.com> Date: Sun, 9 Nov 2025 09:49:29 +0500 Subject: [PATCH 094/313] gh-141186: document `asyncio.Task` cancellation propagation behavior (#141249) --- Doc/library/asyncio-task.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index f825ae92ec7..863b3e33657 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -1221,8 +1221,8 @@ Task Object To cancel a running Task use the :meth:`cancel` method. Calling it will cause the Task to throw a :exc:`CancelledError` exception into - the wrapped coroutine. If a coroutine is awaiting on a Future - object during cancellation, the Future object will be cancelled. + the wrapped coroutine. If a coroutine is awaiting on a future-like + object during cancellation, the awaited object will be cancelled. :meth:`cancelled` can be used to check if the Task was cancelled. The method returns ``True`` if the wrapped coroutine did not @@ -1411,6 +1411,10 @@ Task Object the cancellation, it needs to call :meth:`Task.uncancel` in addition to catching the exception. + If the Task being cancelled is currently awaiting on a future-like + object, that awaited object will also be cancelled. This cancellation + propagates down the entire chain of awaited objects. + .. versionchanged:: 3.9 Added the *msg* parameter. From 7ae440f262c99ba9a3327237f83c9290dc963028 Mon Sep 17 00:00:00 2001 From: Chilla Kalyan <127284726+chillakalyan@users.noreply.github.com> Date: Sun, 9 Nov 2025 14:27:34 +0530 Subject: [PATCH 095/313] gh-141127: Clarify o?s.symlink() documentation for argument order (#141144) --- Doc/library/os.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index d31d0ce9c85..dbc3c92c879 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -3873,6 +3873,9 @@ features: Create a symbolic link pointing to *src* named *dst*. + The *src* parameter refers to the target of the link (the file or directory being linked to), + and *dst* is the name of the link being created. + On Windows, a symlink represents either a file or a directory, and does not morph to the target dynamically. If the target is present, the type of the symlink will be created to match. Otherwise, the symlink will be created From 5ba0a1aa1fe386fbc863d3fe8f32dfbfe2b1bded Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sun, 9 Nov 2025 12:37:34 +0000 Subject: [PATCH 096/313] gh-136702: Deprecate passing non-ascii *encoding* (str) to `encodings.normalize_encoding` (#140030) Closes #136702 --- Doc/deprecations/pending-removal-in-3.17.rst | 6 ++++++ Lib/email/_header_value_parser.py | 4 ++++ Lib/email/utils.py | 4 ++++ Lib/encodings/__init__.py | 8 +++++++- Lib/test/test_codecs.py | 10 +++++++--- Lib/test/test_email/test_email.py | 3 ++- Lib/test/test_email/test_headerregistry.py | 10 +++++++++- .../2025-10-13-11-25-41.gh-issue-136702.uvLGK1.rst | 3 +++ 8 files changed, 42 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-13-11-25-41.gh-issue-136702.uvLGK1.rst diff --git a/Doc/deprecations/pending-removal-in-3.17.rst b/Doc/deprecations/pending-removal-in-3.17.rst index 0a1c2f08cab..e769c9d371e 100644 --- a/Doc/deprecations/pending-removal-in-3.17.rst +++ b/Doc/deprecations/pending-removal-in-3.17.rst @@ -23,6 +23,12 @@ Pending removal in Python 3.17 (Contributed by Shantanu Jain in :gh:`91896`.) +* :mod:`encodings`: + + - Passing non-ascii *encoding* names to :func:`encodings.normalize_encoding` + is deprecated and scheduled for removal in Python 3.17. + (Contributed by Stan Ulbrych in :gh:`136702`) + * :mod:`typing`: - Before Python 3.14, old-style unions were implemented using the private class diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 91243378dc0..c7f665b3990 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -796,6 +796,10 @@ def params(self): value = urllib.parse.unquote(value, encoding='latin-1') else: try: + # Explicitly look up the codec for warning generation, see gh-140030 + # Can be removed in 3.17 + import codecs + codecs.lookup(charset) value = value.decode(charset, 'surrogateescape') except (LookupError, UnicodeEncodeError): # XXX: there should really be a custom defect for diff --git a/Lib/email/utils.py b/Lib/email/utils.py index 3de1f0d24a1..d4824dc3601 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -460,6 +460,10 @@ def collapse_rfc2231_value(value, errors='replace', charset = fallback_charset rawbytes = bytes(text, 'raw-unicode-escape') try: + # Explicitly look up the codec for warning generation, see gh-140030 + # Can be removed in 3.17 + import codecs + codecs.lookup(charset) return str(rawbytes, charset, errors) except LookupError: # charset is not a known codec. diff --git a/Lib/encodings/__init__.py b/Lib/encodings/__init__.py index e7e4ca3358e..e205ec32637 100644 --- a/Lib/encodings/__init__.py +++ b/Lib/encodings/__init__.py @@ -26,7 +26,7 @@ (c) Copyright CNRI, All Rights Reserved. NO WARRANTY. -"""#" +""" import codecs import sys @@ -56,6 +56,12 @@ def normalize_encoding(encoding): if isinstance(encoding, bytes): encoding = str(encoding, "ascii") + if not encoding.isascii(): + import warnings + warnings.warn( + "Support for non-ascii encoding names will be removed in 3.17", + DeprecationWarning, stacklevel=2) + return _normalize_encoding(encoding) def search_function(encoding): diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index c35a4508943..f1f0ac5ad36 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3886,15 +3886,14 @@ def search_function(encoding): self.assertEqual(codecs.lookup('TEST.AAA 8'), ('test.aaa-8', 2, 3, 4)) self.assertEqual(codecs.lookup('TEST.AAA---8'), ('test.aaa---8', 2, 3, 4)) self.assertEqual(codecs.lookup('TEST.AAA 8'), ('test.aaa---8', 2, 3, 4)) - self.assertEqual(codecs.lookup('TEST.AAA\xe9\u20ac-8'), ('test.aaa\xe9\u20ac-8', 2, 3, 4)) self.assertEqual(codecs.lookup('TEST.AAA.8'), ('test.aaa.8', 2, 3, 4)) self.assertEqual(codecs.lookup('TEST.AAA...8'), ('test.aaa...8', 2, 3, 4)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(codecs.lookup('TEST.AAA\xe9\u20ac-8'), ('test.aaa\xe9\u20ac-8', 2, 3, 4)) def test_encodings_normalize_encoding(self): - # encodings.normalize_encoding() ignores non-ASCII characters. normalize = encodings.normalize_encoding self.assertEqual(normalize('utf_8'), 'utf_8') - self.assertEqual(normalize('utf\xE9\u20AC\U0010ffff-8'), 'utf_8') self.assertEqual(normalize('utf 8'), 'utf_8') # encodings.normalize_encoding() doesn't convert # characters to lower case. @@ -3902,6 +3901,11 @@ def test_encodings_normalize_encoding(self): self.assertEqual(normalize('utf.8'), 'utf.8') self.assertEqual(normalize('utf...8'), 'utf...8') + # Non-ASCII *encoding* is deprecated. + with self.assertWarnsRegex(DeprecationWarning, + "Support for non-ascii encoding names will be removed in 3.17"): + self.assertEqual(normalize('utf\xE9\u20AC\U0010ffff-8'), 'utf_8') + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 4cd587bcd76..1900adf463b 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -5738,7 +5738,8 @@ def test_rfc2231_bad_character_in_encoding(self): """ msg = email.message_from_string(m) - self.assertEqual(msg.get_filename(), 'myfile.txt') + with self.assertWarns(DeprecationWarning): + self.assertEqual(msg.get_filename(), 'myfile.txt') def test_rfc2231_single_tick_in_filename_extended(self): eq = self.assertEqual diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index ff7a6da644d..1d0d0a49a82 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -247,7 +247,15 @@ def content_type_as_value(self, decoded = args[2] if l>2 and args[2] is not DITTO else source header = 'Content-Type:' + ' ' if source else '' folded = args[3] if l>3 else header + decoded + '\n' - h = self.make_header('Content-Type', source) + # Both rfc2231 test cases with utf-8%E2%80%9D raise warnings, + # clear encoding cache to ensure test isolation. + if 'utf-8%E2%80%9D' in source and 'ascii' not in source: + import encodings + encodings._cache.clear() + with self.assertWarns(DeprecationWarning): + h = self.make_header('Content-Type', source) + else: + h = self.make_header('Content-Type', source) self.assertEqual(h.content_type, content_type) self.assertEqual(h.maintype, maintype) self.assertEqual(h.subtype, subtype) diff --git a/Misc/NEWS.d/next/Library/2025-10-13-11-25-41.gh-issue-136702.uvLGK1.rst b/Misc/NEWS.d/next/Library/2025-10-13-11-25-41.gh-issue-136702.uvLGK1.rst new file mode 100644 index 00000000000..88303f017f5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-13-11-25-41.gh-issue-136702.uvLGK1.rst @@ -0,0 +1,3 @@ +:mod:`encodings`: Deprecate passing a non-ascii *encoding* name to +:func:`encodings.normalize_encoding` and schedule removal of support for +Python 3.17. From 0c77e7c23b5c270a3142105542c56c59b59c52a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 9 Nov 2025 13:41:08 +0100 Subject: [PATCH 097/313] gh-140530: fix a reference leak in an error path for `raise exc from cause` (#140908) Fix a reference leak in `raise E from T` when `T` is an exception subtype for which `T.__new__` does not return an exception instance. --- Lib/test/test_raise.py | 18 +++++++----------- ...5-11-02-12-47-38.gh-issue-140530.S934bp.rst | 2 ++ Python/ceval.c | 1 + 3 files changed, 10 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-12-47-38.gh-issue-140530.S934bp.rst diff --git a/Lib/test/test_raise.py b/Lib/test/test_raise.py index dcf0753bc82..645ef291a58 100644 --- a/Lib/test/test_raise.py +++ b/Lib/test/test_raise.py @@ -186,18 +186,14 @@ def test_class_cause(self): self.fail("No exception raised") def test_class_cause_nonexception_result(self): - class ConstructsNone(BaseException): - @classmethod + # See https://github.com/python/cpython/issues/140530. + class ConstructMortal(BaseException): def __new__(*args, **kwargs): - return None - try: - raise IndexError from ConstructsNone - except TypeError as e: - self.assertIn("should have returned an instance of BaseException", str(e)) - except IndexError: - self.fail("Wrong kind of exception raised") - else: - self.fail("No exception raised") + return ["mortal value"] + + msg = ".*should have returned an instance of BaseException.*" + with self.assertRaisesRegex(TypeError, msg): + raise IndexError from ConstructMortal def test_instance_cause(self): cause = KeyError() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-12-47-38.gh-issue-140530.S934bp.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-12-47-38.gh-issue-140530.S934bp.rst new file mode 100644 index 00000000000..e3af493893a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-12-47-38.gh-issue-140530.S934bp.rst @@ -0,0 +1,2 @@ +Fix a reference leak when ``raise exc from cause`` fails. Patch by Bénédikt +Tran. diff --git a/Python/ceval.c b/Python/ceval.c index 7ca7ae48b5c..43e8ee71206 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2148,6 +2148,7 @@ do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause) "calling %R should have returned an instance of " "BaseException, not %R", cause, Py_TYPE(fixed_cause)); + Py_DECREF(fixed_cause); goto raise_error; } Py_DECREF(cause); From 3ce2d57b2f02030353af314d89c5f6215d2f5c96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 9 Nov 2025 13:45:38 +0100 Subject: [PATCH 098/313] gh-100218: correctly set `errno` when `socket.if_{nametoindex,indextoname}` raise `OSError` (#140905) Previously, socket.if_nametoindex() and socket.if_indextoname() could raise an `OSError` with a `None` errno. Now, the errno from libc is propagated. --- Lib/test/test_socket.py | 10 ++++++++-- .../2025-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst | 3 +++ Modules/socketmodule.c | 5 +++-- 3 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 24ee0f2c280..934b7137096 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1176,7 +1176,10 @@ def testInterfaceNameIndex(self): 'socket.if_indextoname() not available.') @support.skip_android_selinux('if_indextoname') def testInvalidInterfaceIndexToName(self): - self.assertRaises(OSError, socket.if_indextoname, 0) + with self.assertRaises(OSError) as cm: + socket.if_indextoname(0) + self.assertIsNotNone(cm.exception.errno) + self.assertRaises(ValueError, socket.if_indextoname, -1) self.assertRaises(OverflowError, socket.if_indextoname, 2**1000) self.assertRaises(TypeError, socket.if_indextoname, '_DEADBEEF') @@ -1196,8 +1199,11 @@ def testInvalidInterfaceIndexToName(self): 'socket.if_nametoindex() not available.') @support.skip_android_selinux('if_nametoindex') def testInvalidInterfaceNameToIndex(self): + with self.assertRaises(OSError) as cm: + socket.if_nametoindex("_DEADBEEF") + self.assertIsNotNone(cm.exception.errno) + self.assertRaises(TypeError, socket.if_nametoindex, 0) - self.assertRaises(OSError, socket.if_nametoindex, '_DEADBEEF') @unittest.skipUnless(hasattr(sys, 'getrefcount'), 'test needs sys.getrefcount()') diff --git a/Misc/NEWS.d/next/Library/2025-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst b/Misc/NEWS.d/next/Library/2025-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst new file mode 100644 index 00000000000..2f7500d2955 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst @@ -0,0 +1,3 @@ +Correctly set :attr:`~OSError.errno` when :func:`socket.if_nametoindex` or +:func:`socket.if_indextoname` raise an :exc:`OSError`. Patch by Bénédikt +Tran. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index dd4b6892977..6a844d44bf0 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -7294,10 +7294,10 @@ _socket_if_nametoindex_impl(PyObject *module, PyObject *oname) unsigned long index; #endif + errno = ENODEV; // in case 'if_nametoindex' does not set errno index = if_nametoindex(PyBytes_AS_STRING(oname)); if (index == 0) { - /* if_nametoindex() doesn't set errno */ - PyErr_SetString(PyExc_OSError, "no interface with this name"); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } @@ -7317,6 +7317,7 @@ static PyObject * _socket_if_indextoname_impl(PyObject *module, NET_IFINDEX index) /*[clinic end generated code: output=e48bc324993052e0 input=c93f753d0cf6d7d1]*/ { + errno = ENXIO; // in case 'if_indextoname' does not set errno char name[IF_NAMESIZE + 1]; if (if_indextoname(index, name) == NULL) { PyErr_SetFromErrno(PyExc_OSError); From ae1f435071148571422e19be598336726eb25b7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 9 Nov 2025 15:14:08 +0100 Subject: [PATCH 099/313] gh-111389: replace deprecated occurrences of `_PyHASH_*` macros (#141236) --- Include/cpython/pyhash.h | 4 ++-- Modules/_decimal/_decimal.c | 6 +++--- Objects/complexobject.c | 2 +- Objects/longobject.c | 38 ++++++++++++++++++------------------- Python/pyhash.c | 26 ++++++++++++------------- Python/sysmodule.c | 6 +++--- 6 files changed, 41 insertions(+), 41 deletions(-) diff --git a/Include/cpython/pyhash.h b/Include/cpython/pyhash.h index a33ba10b8d3..dac223368db 100644 --- a/Include/cpython/pyhash.h +++ b/Include/cpython/pyhash.h @@ -7,7 +7,7 @@ /* Parameters used for the numeric hash implementation. See notes for _Py_HashDouble in Python/pyhash.c. Numeric hashes are based on - reduction modulo the prime 2**_PyHASH_BITS - 1. */ + reduction modulo the prime 2**PyHASH_BITS - 1. */ #if SIZEOF_VOID_P >= 8 # define PyHASH_BITS 61 @@ -15,7 +15,7 @@ # define PyHASH_BITS 31 #endif -#define PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) +#define PyHASH_MODULUS (((size_t)1 << PyHASH_BITS) - 1) #define PyHASH_INF 314159 #define PyHASH_IMAG PyHASH_MULTIPLIER diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 44917ed7357..0484d9896a1 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -5799,7 +5799,7 @@ _decimal_Decimal___floor___impl(PyObject *self, PyTypeObject *cls) static Py_hash_t _dec_hash(PyDecObject *v) { -#if defined(CONFIG_64) && _PyHASH_BITS == 61 +#if defined(CONFIG_64) && PyHASH_BITS == 61 /* 2**61 - 1 */ mpd_uint_t p_data[1] = {2305843009213693951ULL}; mpd_t p = {MPD_POS|MPD_STATIC|MPD_CONST_DATA, 0, 19, 1, 1, p_data}; @@ -5807,7 +5807,7 @@ _dec_hash(PyDecObject *v) mpd_uint_t inv10_p_data[1] = {2075258708292324556ULL}; mpd_t inv10_p = {MPD_POS|MPD_STATIC|MPD_CONST_DATA, 0, 19, 1, 1, inv10_p_data}; -#elif defined(CONFIG_32) && _PyHASH_BITS == 31 +#elif defined(CONFIG_32) && PyHASH_BITS == 31 /* 2**31 - 1 */ mpd_uint_t p_data[2] = {147483647UL, 2}; mpd_t p = {MPD_POS|MPD_STATIC|MPD_CONST_DATA, 0, 10, 2, 2, p_data}; @@ -5816,7 +5816,7 @@ _dec_hash(PyDecObject *v) mpd_t inv10_p = {MPD_POS|MPD_STATIC|MPD_CONST_DATA, 0, 10, 2, 2, inv10_p_data}; #else - #error "No valid combination of CONFIG_64, CONFIG_32 and _PyHASH_BITS" + #error "No valid combination of CONFIG_64, CONFIG_32 and PyHASH_BITS" #endif const Py_hash_t py_hash_inf = 314159; mpd_uint_t ten_data[1] = {10}; diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 03fc137c345..6247376a0e6 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -644,7 +644,7 @@ complex_hash(PyObject *op) * compare equal must have the same hash value, so that * hash(x + 0*j) must equal hash(x). */ - combined = hashreal + _PyHASH_IMAG * hashimag; + combined = hashreal + PyHASH_IMAG * hashimag; if (combined == (Py_uhash_t)-1) combined = (Py_uhash_t)-2; return (Py_hash_t)combined; diff --git a/Objects/longobject.c b/Objects/longobject.c index 298399210af..43c0db753a0 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3703,36 +3703,36 @@ long_hash(PyObject *obj) #endif while (--i >= 0) { - /* Here x is a quantity in the range [0, _PyHASH_MODULUS); we + /* Here x is a quantity in the range [0, PyHASH_MODULUS); we want to compute x * 2**PyLong_SHIFT + v->long_value.ob_digit[i] modulo - _PyHASH_MODULUS. + PyHASH_MODULUS. - The computation of x * 2**PyLong_SHIFT % _PyHASH_MODULUS + The computation of x * 2**PyLong_SHIFT % PyHASH_MODULUS amounts to a rotation of the bits of x. To see this, write - x * 2**PyLong_SHIFT = y * 2**_PyHASH_BITS + z + x * 2**PyLong_SHIFT = y * 2**PyHASH_BITS + z - where y = x >> (_PyHASH_BITS - PyLong_SHIFT) gives the top + where y = x >> (PyHASH_BITS - PyLong_SHIFT) gives the top PyLong_SHIFT bits of x (those that are shifted out of the - original _PyHASH_BITS bits, and z = (x << PyLong_SHIFT) & - _PyHASH_MODULUS gives the bottom _PyHASH_BITS - PyLong_SHIFT - bits of x, shifted up. Then since 2**_PyHASH_BITS is - congruent to 1 modulo _PyHASH_MODULUS, y*2**_PyHASH_BITS is - congruent to y modulo _PyHASH_MODULUS. So + original PyHASH_BITS bits, and z = (x << PyLong_SHIFT) & + PyHASH_MODULUS gives the bottom PyHASH_BITS - PyLong_SHIFT + bits of x, shifted up. Then since 2**PyHASH_BITS is + congruent to 1 modulo PyHASH_MODULUS, y*2**PyHASH_BITS is + congruent to y modulo PyHASH_MODULUS. So - x * 2**PyLong_SHIFT = y + z (mod _PyHASH_MODULUS). + x * 2**PyLong_SHIFT = y + z (mod PyHASH_MODULUS). The right-hand side is just the result of rotating the - _PyHASH_BITS bits of x left by PyLong_SHIFT places; since - not all _PyHASH_BITS bits of x are 1s, the same is true - after rotation, so 0 <= y+z < _PyHASH_MODULUS and y + z is + PyHASH_BITS bits of x left by PyLong_SHIFT places; since + not all PyHASH_BITS bits of x are 1s, the same is true + after rotation, so 0 <= y+z < PyHASH_MODULUS and y + z is the reduction of x*2**PyLong_SHIFT modulo - _PyHASH_MODULUS. */ - x = ((x << PyLong_SHIFT) & _PyHASH_MODULUS) | - (x >> (_PyHASH_BITS - PyLong_SHIFT)); + PyHASH_MODULUS. */ + x = ((x << PyLong_SHIFT) & PyHASH_MODULUS) | + (x >> (PyHASH_BITS - PyLong_SHIFT)); x += v->long_value.ob_digit[i]; - if (x >= _PyHASH_MODULUS) - x -= _PyHASH_MODULUS; + if (x >= PyHASH_MODULUS) + x -= PyHASH_MODULUS; } x = x * sign; if (x == (Py_uhash_t)-1) diff --git a/Python/pyhash.c b/Python/pyhash.c index 216f437dd9a..157312a936b 100644 --- a/Python/pyhash.c +++ b/Python/pyhash.c @@ -29,7 +29,7 @@ static Py_ssize_t hashstats[Py_HASH_STATS_MAX + 1] = {0}; #endif /* For numeric types, the hash of a number x is based on the reduction - of x modulo the prime P = 2**_PyHASH_BITS - 1. It's designed so that + of x modulo the prime P = 2**PyHASH_BITS - 1. It's designed so that hash(x) == hash(y) whenever x and y are numerically equal, even if x and y have different types. @@ -52,8 +52,8 @@ static Py_ssize_t hashstats[Py_HASH_STATS_MAX + 1] = {0}; If the result of the reduction is infinity (this is impossible for integers, floats and Decimals) then use the predefined hash value - _PyHASH_INF for x >= 0, or -_PyHASH_INF for x < 0, instead. - _PyHASH_INF and -_PyHASH_INF are also used for the + PyHASH_INF for x >= 0, or -PyHASH_INF for x < 0, instead. + PyHASH_INF and -PyHASH_INF are also used for the hashes of float and Decimal infinities. NaNs hash with a pointer hash. Having distinct hash values prevents @@ -65,16 +65,16 @@ static Py_ssize_t hashstats[Py_HASH_STATS_MAX + 1] = {0}; efficiently, even if the exponent of the binary or decimal number is large. The key point is that - reduce(x * y) == reduce(x) * reduce(y) (modulo _PyHASH_MODULUS) + reduce(x * y) == reduce(x) * reduce(y) (modulo PyHASH_MODULUS) provided that {reduce(x), reduce(y)} != {0, infinity}. The reduction of a binary or decimal float is never infinity, since the denominator is a power of 2 (for binary) or a divisor of a power of 10 (for decimal). So we have, for nonnegative x, - reduce(x * 2**e) == reduce(x) * reduce(2**e) % _PyHASH_MODULUS + reduce(x * 2**e) == reduce(x) * reduce(2**e) % PyHASH_MODULUS - reduce(x * 10**e) == reduce(x) * reduce(10**e) % _PyHASH_MODULUS + reduce(x * 10**e) == reduce(x) * reduce(10**e) % PyHASH_MODULUS and reduce(10**e) can be computed efficiently by the usual modular exponentiation algorithm. For reduce(2**e) it's even better: since @@ -92,7 +92,7 @@ _Py_HashDouble(PyObject *inst, double v) if (!isfinite(v)) { if (isinf(v)) - return v > 0 ? _PyHASH_INF : -_PyHASH_INF; + return v > 0 ? PyHASH_INF : -PyHASH_INF; else return PyObject_GenericHash(inst); } @@ -109,19 +109,19 @@ _Py_HashDouble(PyObject *inst, double v) and hexadecimal floating point. */ x = 0; while (m) { - x = ((x << 28) & _PyHASH_MODULUS) | x >> (_PyHASH_BITS - 28); + x = ((x << 28) & PyHASH_MODULUS) | x >> (PyHASH_BITS - 28); m *= 268435456.0; /* 2**28 */ e -= 28; y = (Py_uhash_t)m; /* pull out integer part */ m -= y; x += y; - if (x >= _PyHASH_MODULUS) - x -= _PyHASH_MODULUS; + if (x >= PyHASH_MODULUS) + x -= PyHASH_MODULUS; } - /* adjust for the exponent; first reduce it modulo _PyHASH_BITS */ - e = e >= 0 ? e % _PyHASH_BITS : _PyHASH_BITS-1-((-1-e) % _PyHASH_BITS); - x = ((x << e) & _PyHASH_MODULUS) | x >> (_PyHASH_BITS - e); + /* adjust for the exponent; first reduce it modulo PyHASH_BITS */ + e = e >= 0 ? e % PyHASH_BITS : PyHASH_BITS-1-((-1-e) % PyHASH_BITS); + x = ((x << e) & PyHASH_MODULUS) | x >> (PyHASH_BITS - e); x = x * sign; if (x == (Py_uhash_t)-1) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 86dd1395cae..a611844f76e 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1587,10 +1587,10 @@ get_hash_info(PyThreadState *tstate) } while(0) SET_HASH_INFO_ITEM(PyLong_FromLong(8 * sizeof(Py_hash_t))); - SET_HASH_INFO_ITEM(PyLong_FromSsize_t(_PyHASH_MODULUS)); - SET_HASH_INFO_ITEM(PyLong_FromLong(_PyHASH_INF)); + SET_HASH_INFO_ITEM(PyLong_FromSsize_t(PyHASH_MODULUS)); + SET_HASH_INFO_ITEM(PyLong_FromLong(PyHASH_INF)); SET_HASH_INFO_ITEM(PyLong_FromLong(0)); // This is no longer used - SET_HASH_INFO_ITEM(PyLong_FromLong(_PyHASH_IMAG)); + SET_HASH_INFO_ITEM(PyLong_FromLong(PyHASH_IMAG)); SET_HASH_INFO_ITEM(PyUnicode_FromString(hashfunc->name)); SET_HASH_INFO_ITEM(PyLong_FromLong(hashfunc->hash_bits)); SET_HASH_INFO_ITEM(PyLong_FromLong(hashfunc->seed_bits)); From 1d738dea6364de004f8cec7c6309d6bbd3b996c7 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 9 Nov 2025 10:06:38 -0500 Subject: [PATCH 100/313] gh-141004: Document deprecated aliases for memory allocation (GH-141146) --- Doc/c-api/allocation.rst | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 59d913a0462..59044d2d88c 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -140,10 +140,6 @@ Allocating Objects on the Heap * :c:member:`~PyTypeObject.tp_alloc` -.. c:function:: void PyObject_Del(void *op) - - Same as :c:func:`PyObject_Free`. - .. c:var:: PyObject _Py_NoneStruct Object which is visible in Python as ``None``. This should only be accessed @@ -156,3 +152,35 @@ Allocating Objects on the Heap :ref:`moduleobjects` To allocate and create extension modules. + +Deprecated aliases +^^^^^^^^^^^^^^^^^^ + +These are :term:`soft deprecated` aliases to existing functions and macros. +They exist solely for backwards compatibility. + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * Deprecated alias + * Function + * * .. c:macro:: PyObject_NEW(type, typeobj) + * :c:macro:`PyObject_New` + * * .. c:macro:: PyObject_NEW_VAR(type, typeobj, n) + * :c:macro:`PyObject_NewVar` + * * .. c:macro:: PyObject_INIT(op, typeobj) + * :c:func:`PyObject_Init` + * * .. c:macro:: PyObject_INIT_VAR(op, typeobj, n) + * :c:func:`PyObject_InitVar` + * * .. c:macro:: PyObject_MALLOC(n) + * :c:func:`PyObject_Malloc` + * * .. c:macro:: PyObject_REALLOC(p, n) + * :c:func:`PyObject_Realloc` + * * .. c:macro:: PyObject_FREE(p) + * :c:func:`PyObject_Free` + * * .. c:macro:: PyObject_DEL(p) + * :c:func:`PyObject_Free` + * * .. c:macro:: PyObject_Del(p) + * :c:func:`PyObject_Free` From 60155329a0a83a2b9e740f0c0de41c9d44f5a053 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sun, 9 Nov 2025 15:32:39 +0000 Subject: [PATCH 101/313] gh-141004: Document `PyWeakref_CheckRefExact` (GH-141279) --- Doc/c-api/weakref.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index 14ec9d951c4..39e4febd3ef 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -19,7 +19,14 @@ as much as it can. .. c:function:: int PyWeakref_CheckRef(PyObject *ob) - Return non-zero if *ob* is a reference object. This function always succeeds. + Return non-zero if *ob* is a reference object or a subclass of the reference + type. This function always succeeds. + + +.. c:function:: int PyWeakref_CheckRefExact(PyObject *ob) + + Return non-zero if *ob* is a reference object, but not a subclass of the + reference type. This function always succeeds. .. c:function:: int PyWeakref_CheckProxy(PyObject *ob) From dbe40904a78a0c8ffa25fb64e6ff1e14e6e7ba5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 9 Nov 2025 16:44:26 +0100 Subject: [PATCH 102/313] gh-141004: document `curses` C API (#141254) --- Doc/c-api/concrete.rst | 10 ++- Doc/c-api/curses.rst | 138 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 Doc/c-api/curses.rst diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst index 880f7b15ce6..a5c5a53236c 100644 --- a/Doc/c-api/concrete.rst +++ b/Doc/c-api/concrete.rst @@ -115,5 +115,13 @@ Other Objects gen.rst coro.rst contextvars.rst - datetime.rst typehints.rst + + +C API for extension modules +=========================== + +.. toctree:: + + curses.rst + datetime.rst diff --git a/Doc/c-api/curses.rst b/Doc/c-api/curses.rst new file mode 100644 index 00000000000..5a1697c43cc --- /dev/null +++ b/Doc/c-api/curses.rst @@ -0,0 +1,138 @@ +.. highlight:: c + +Curses C API +------------ + +:mod:`curses` exposes a small C interface for extension modules. +Consumers must include the header file :file:`py_curses.h` (which is not +included by default by :file:`Python.h`) and :c:func:`import_curses` must +be invoked, usually as part of the module initialisation function, to populate +:c:var:`PyCurses_API`. + +.. warning:: + + Neither the C API nor the pure Python :mod:`curses` module are compatible + with subinterpreters. + +.. c:macro:: import_curses() + + Import the curses C API. The macro does not need a semi-colon to be called. + + On success, populate the :c:var:`PyCurses_API` pointer. + + On failure, set :c:var:`PyCurses_API` to NULL and set an exception. + The caller must check if an error occurred via :c:func:`PyErr_Occurred`: + + .. code-block:: + + import_curses(); // semi-colon is optional but recommended + if (PyErr_Occurred()) { /* cleanup */ } + + +.. c:var:: void **PyCurses_API + + Dynamically allocated object containing the curses C API. + This variable is only available once :c:macro:`import_curses` succeeds. + + ``PyCurses_API[0]`` corresponds to :c:data:`PyCursesWindow_Type`. + + ``PyCurses_API[1]``, ``PyCurses_API[2]``, and ``PyCurses_API[3]`` + are pointers to predicate functions of type ``int (*)(void)``. + + When called, these predicates return whether :func:`curses.setupterm`, + :func:`curses.initscr`, and :func:`curses.start_color` have been called + respectively. + + See also the convenience macros :c:macro:`PyCursesSetupTermCalled`, + :c:macro:`PyCursesInitialised`, and :c:macro:`PyCursesInitialisedColor`. + + .. note:: + + The number of entries in this structure is subject to changes. + Consider using :c:macro:`PyCurses_API_pointers` to check if + new fields are available or not. + + +.. c:macro:: PyCurses_API_pointers + + The number of accessible fields (``4``) in :c:var:`PyCurses_API`. + This number is incremented whenever new fields are added. + + +.. c:var:: PyTypeObject PyCursesWindow_Type + + The :ref:`heap type ` corresponding to :class:`curses.window`. + + +.. c:function:: int PyCursesWindow_Check(PyObject *op) + + Return true if *op* is a :class:`curses.window` instance, false otherwise. + + +The following macros are convenience macros expanding into C statements. +In particular, they can only be used as ``macro;`` or ``macro``, but not +``macro()`` or ``macro();``. + +.. c:macro:: PyCursesSetupTermCalled + + Macro checking if :func:`curses.setupterm` has been called. + + The macro expansion is roughly equivalent to: + + .. code-block:: + + { + typedef int (*predicate_t)(void); + predicate_t was_setupterm_called = (predicate_t)PyCurses_API[1]; + if (!was_setupterm_called()) { + return NULL; + } + } + + +.. c:macro:: PyCursesInitialised + + Macro checking if :func:`curses.initscr` has been called. + + The macro expansion is roughly equivalent to: + + .. code-block:: + + { + typedef int (*predicate_t)(void); + predicate_t was_initscr_called = (predicate_t)PyCurses_API[2]; + if (!was_initscr_called()) { + return NULL; + } + } + + +.. c:macro:: PyCursesInitialisedColor + + Macro checking if :func:`curses.start_color` has been called. + + The macro expansion is roughly equivalent to: + + .. code-block:: + + { + typedef int (*predicate_t)(void); + predicate_t was_start_color_called = (predicate_t)PyCurses_API[3]; + if (!was_start_color_called()) { + return NULL; + } + } + + +Internal data +------------- + +The following objects are exposed by the C API but should be considered +internal-only. + +.. c:macro:: PyCurses_CAPSULE_NAME + + Name of the curses capsule to pass to :c:func:`PyCapsule_Import`. + + Internal usage only. Use :c:macro:`import_curses` instead. + From 8dd849828636bb3989c6d5d20f8790a3fb770fc4 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 9 Nov 2025 11:21:44 -0500 Subject: [PATCH 103/313] gh-141004: Document `Py_func_type_input` (GH-141273) --- Doc/c-api/veryhigh.rst | 87 ++++++++++++++++++++++++++---------------- 1 file changed, 55 insertions(+), 32 deletions(-) diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 0b2b55b6387..916c616dfee 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -13,8 +13,9 @@ the interpreter. Several of these functions accept a start symbol from the grammar as a parameter. The available start symbols are :c:data:`Py_eval_input`, -:c:data:`Py_file_input`, and :c:data:`Py_single_input`. These are described -following the functions which accept them as parameters. +:c:data:`Py_file_input`, :c:data:`Py_single_input`, and +:c:data:`Py_func_type_input`. These are described following the functions +which accept them as parameters. Note also that several of these functions take :c:expr:`FILE*` parameters. One particular issue which needs to be handled carefully is that the :c:type:`FILE` @@ -183,8 +184,7 @@ the same library that the Python runtime is using. objects *globals* and *locals* with the compiler flags specified by *flags*. *globals* must be a dictionary; *locals* can be any object that implements the mapping protocol. The parameter *start* specifies - the start symbol and must one of the following: - :c:data:`Py_eval_input`, :c:data:`Py_file_input`, or :c:data:`Py_single_input`. + the start symbol and must one of the :ref:`available start symbols `. Returns the result of executing the code as a Python object, or ``NULL`` if an exception was raised. @@ -233,8 +233,8 @@ the same library that the Python runtime is using. Parse and compile the Python source code in *str*, returning the resulting code object. The start symbol is given by *start*; this can be used to constrain the - code which can be compiled and should be :c:data:`Py_eval_input`, - :c:data:`Py_file_input`, or :c:data:`Py_single_input`. The filename specified by + code which can be compiled and should be :ref:`available start symbols + `. The filename specified by *filename* is used to construct the code object and may appear in tracebacks or :exc:`SyntaxError` exception messages. This returns ``NULL`` if the code cannot be parsed or compiled. @@ -297,32 +297,6 @@ the same library that the Python runtime is using. true on success, false on failure. -.. c:var:: int Py_eval_input - - .. index:: single: Py_CompileString (C function) - - The start symbol from the Python grammar for isolated expressions; for use with - :c:func:`Py_CompileString`. - - -.. c:var:: int Py_file_input - - .. index:: single: Py_CompileString (C function) - - The start symbol from the Python grammar for sequences of statements as read - from a file or other source; for use with :c:func:`Py_CompileString`. This is - the symbol to use when compiling arbitrarily long Python source code. - - -.. c:var:: int Py_single_input - - .. index:: single: Py_CompileString (C function) - - The start symbol from the Python grammar for a single statement; for use with - :c:func:`Py_CompileString`. This is the symbol used for the interactive - interpreter loop. - - .. c:struct:: PyCompilerFlags This is the structure used to hold compiler flags. In cases where code is only @@ -366,3 +340,52 @@ the same library that the Python runtime is using. as :c:macro:`CO_FUTURE_ANNOTATIONS` to enable features normally selectable using :ref:`future statements `. See :ref:`c_codeobject_flags` for a complete list. + + +.. _start-symbols: + +Available start symbols +^^^^^^^^^^^^^^^^^^^^^^^ + + +.. c:var:: int Py_eval_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for isolated expressions; for use with + :c:func:`Py_CompileString`. + + +.. c:var:: int Py_file_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for sequences of statements as read + from a file or other source; for use with :c:func:`Py_CompileString`. This is + the symbol to use when compiling arbitrarily long Python source code. + + +.. c:var:: int Py_single_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for a single statement; for use with + :c:func:`Py_CompileString`. This is the symbol used for the interactive + interpreter loop. + + +.. c:var:: int Py_func_type_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for a function type; for use with + :c:func:`Py_CompileString`. This is used to parse "signature type comments" + from :pep:`484`. + + This requires the :c:macro:`PyCF_ONLY_AST` flag to be set. + + .. seealso:: + * :py:class:`ast.FunctionType` + * :pep:`484` + + .. versionadded:: 3.8 From b5a0c72492800c7e999b87adfcfeabaacb4ecb97 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 9 Nov 2025 12:09:09 -0500 Subject: [PATCH 104/313] gh-141004: Document `PyExceptionInstance*` APIs (GH-141301) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/exceptions.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index c58aa659e1b..5241533e112 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -788,6 +788,17 @@ Exception Classes Exception Objects ================= +.. c:function:: int PyExceptionInstance_Check(PyObject *op) + + Return true if *op* is an instance of :class:`BaseException`, false + otherwise. This function always succeeds. + + +.. c:macro:: PyExceptionInstance_Class(op) + + Equivalent to :c:func:`Py_TYPE(op) `. + + .. c:function:: PyObject* PyException_GetTraceback(PyObject *ex) Return the traceback associated with the exception as a new reference, as From 18529b580b59b8d075641da6c685bef377eb0a7b Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 9 Nov 2025 12:49:17 -0500 Subject: [PATCH 105/313] gh-141004: Document `PyFunction_SetKwDefaults` (GH-141294) --- Doc/c-api/function.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 0bac6389571..609b5e885b6 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -102,6 +102,15 @@ There are a few functions specific to Python functions. dictionary of arguments or ``NULL``. +.. c:function:: int PyFunction_SetKwDefaults(PyObject *op, PyObject *defaults) + + Set the keyword-only argument default values of the function object *op*. + *defaults* must be a dictionary of keyword-only arguments or ``Py_None``. + + This function returns ``0`` on success, and returns ``-1`` with an exception + set on failure. + + .. c:function:: PyObject* PyFunction_GetClosure(PyObject *op) Return the closure associated with the function object *op*. This can be ``NULL`` From 807db68ddd8572cfa825373bc13461b02691f4d9 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 9 Nov 2025 13:03:38 -0500 Subject: [PATCH 106/313] gh-141004: Document `PyClassMethod*` and `PyStaticMethod*` APIs (GH-141296) --- Doc/c-api/descriptor.rst | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index ff0df575279..9a4093a7708 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -38,3 +38,39 @@ found in the dictionary of type objects. .. c:function:: PyObject* PyWrapper_New(PyObject *, PyObject *) + + +Built-in descriptors +^^^^^^^^^^^^^^^^^^^^ + +.. c:var:: PyTypeObject PyClassMethod_Type + + The type of class method objects. This is the same object as + :class:`classmethod` in the Python layer. + + +.. c:function:: PyObject *PyClassMethod_New(PyObject *callable) + + Create a new :class:`classmethod` object wrapping *callable*. + *callable* must be a callable object and must not be ``NULL``. + + On success, this function returns a :term:`strong reference` to a new class + method descriptor. On failure, this function returns ``NULL`` with an + exception set. + + +.. c:var:: PyTypeObject PyStaticMethod_Type + + The type of static method objects. This is the same object as + :class:`staticmethod` in the Python layer. + + +.. c:function:: PyObject *PyStaticMethod_New(PyObject *callable) + + Create a new :class:`staticmethod` object wrapping *callable*. + *callable* must be a callable object and must not be ``NULL``. + + On success, this function returns a :term:`strong reference` to a new static + method descriptor. On failure, this function returns ``NULL`` with an + exception set. + From 6f20ea1e2d302b7b88d64b6786abbad1747ff950 Mon Sep 17 00:00:00 2001 From: Lakshya Upadhyaya Date: Mon, 10 Nov 2025 00:29:06 +0530 Subject: [PATCH 107/313] gh-140980: document `SET_FUNCTION_ATTRIBUTE` flag for `annotate` function (#141306) --- Doc/library/dis.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 284eeff5e4d..a24589fd0a5 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1673,9 +1673,13 @@ iterations of the loop. * ``0x02`` a dictionary of keyword-only parameters' default values * ``0x04`` a tuple of strings containing parameters' annotations * ``0x08`` a tuple containing cells for free variables, making a closure + * ``0x10`` the :term:`annotate function` for the function object .. versionadded:: 3.13 + .. versionchanged:: 3.14 + Added ``0x10`` to indicate the annotate function for the function object. + .. opcode:: BUILD_SLICE (argc) From 14c62227f9fa11fb743f9e03dcc5aab553de1098 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sun, 9 Nov 2025 19:53:56 +0000 Subject: [PATCH 108/313] gh-141004: Document `PySuper_Type` (GH-141315) --- Doc/c-api/descriptor.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index 9a4093a7708..22c3b790cc3 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -43,6 +43,12 @@ found in the dictionary of type objects. Built-in descriptors ^^^^^^^^^^^^^^^^^^^^ +.. c:var:: PyTypeObject PySuper_Type + + The type object for super objects. This is the same object as + :class:`super` in the Python layer. + + .. c:var:: PyTypeObject PyClassMethod_Type The type of class method objects. This is the same object as From ec85d3cbfe315086805c33bb64c28a8509098829 Mon Sep 17 00:00:00 2001 From: Elena O <31424287+oklena@users.noreply.github.com> Date: Sun, 9 Nov 2025 15:42:22 -0800 Subject: [PATCH 109/313] gh-62480: De-personalize "Mocking Unbound Methods" section in `unittest.mock` examples (#141322) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Rewrite Mocking Unbound Methods paragraph to second person Co-authored-by: C.A.M. Gerlach Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/unittest.mock-examples.rst | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index 00cc9bfc0a5..e2b0322dae0 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -743,16 +743,15 @@ exception is raised in the setUp then tearDown is not called. Mocking Unbound Methods ~~~~~~~~~~~~~~~~~~~~~~~ -Whilst writing tests today I needed to patch an *unbound method* (patching the -method on the class rather than on the instance). I needed self to be passed -in as the first argument because I want to make asserts about which objects -were calling this particular method. The issue is that you can't patch with a -mock for this, because if you replace an unbound method with a mock it doesn't -become a bound method when fetched from the instance, and so it doesn't get -self passed in. The workaround is to patch the unbound method with a real -function instead. The :func:`patch` decorator makes it so simple to -patch out methods with a mock that having to create a real function becomes a -nuisance. +Sometimes a test needs to patch an *unbound method*, which means patching the +method on the class rather than on the instance. In order to make assertions +about which objects were calling this particular method, you need to pass +``self`` as the first argument. The issue is that you can't patch with a mock for +this, because if you replace an unbound method with a mock it doesn't become +a bound method when fetched from the instance, and so it doesn't get ``self`` +passed in. The workaround is to patch the unbound method with a real function +instead. The :func:`patch` decorator makes it so simple to patch out methods +with a mock that having to create a real function becomes a nuisance. If you pass ``autospec=True`` to patch then it does the patching with a *real* function object. This function object has the same signature as the one @@ -760,8 +759,8 @@ it is replacing, but delegates to a mock under the hood. You still get your mock auto-created in exactly the same way as before. What it means though, is that if you use it to patch out an unbound method on a class the mocked function will be turned into a bound method if it is fetched from an instance. -It will have ``self`` passed in as the first argument, which is exactly what I -wanted: +It will have ``self`` passed in as the first argument, which is exactly what +was needed: >>> class Foo: ... def foo(self): From b618731781c31d4b5b75d199dfc14993ffd66e37 Mon Sep 17 00:00:00 2001 From: KarnbirKhera <166065758+KarnbirKhera@users.noreply.github.com> Date: Sun, 9 Nov 2025 15:45:38 -0800 Subject: [PATCH 110/313] gh-62480: De-personalize "Partial mocking" section in `unittest.mock` examples (#141321) * Refine some wording in unittest partial mock doc Some of the descriptions were addressed in first person, but have now been changed to address the user reading the documentation instead. Co-authored-by: C.A.M. Gerlach --- Doc/library/unittest.mock-examples.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index e2b0322dae0..6af4298d44f 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -600,13 +600,13 @@ this list of calls for us:: Partial mocking ~~~~~~~~~~~~~~~ -In some tests I wanted to mock out a call to :meth:`datetime.date.today` -to return a known date, but I didn't want to prevent the code under test from -creating new date objects. Unfortunately :class:`datetime.date` is written in C, and -so I couldn't just monkey-patch out the static :meth:`datetime.date.today` method. +For some tests, you may want to mock out a call to :meth:`datetime.date.today` +to return a known date, but don't want to prevent the code under test from +creating new date objects. Unfortunately :class:`datetime.date` is written in C, +so you cannot just monkey-patch out the static :meth:`datetime.date.today` method. -I found a simple way of doing this that involved effectively wrapping the date -class with a mock, but passing through calls to the constructor to the real +Instead, you can effectively wrap the date +class with a mock, while passing through calls to the constructor to the real class (and returning real instances). The :func:`patch decorator ` is used here to From 9b0179fa87fee39df0f75bd84fc2dd75f1d00553 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Mon, 10 Nov 2025 00:43:03 +0000 Subject: [PATCH 111/313] gh-141004: Document `Py_DTSF_*` macros (GH-141310) --- Doc/c-api/conversion.rst | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index cc7a3d9d956..e9d866c647d 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -128,18 +128,28 @@ The following functions provide locale-independent string to number conversions. must be 0 and is ignored. The ``'r'`` format code specifies the standard :func:`repr` format. - *flags* can be zero or more of the values ``Py_DTSF_SIGN``, - ``Py_DTSF_ADD_DOT_0``, or ``Py_DTSF_ALT``, or-ed together: + *flags* can be zero or more of the following values or-ed together: - * ``Py_DTSF_SIGN`` means to always precede the returned string with a sign - character, even if *val* is non-negative. + .. c:macro:: Py_DTSF_SIGN - * ``Py_DTSF_ADD_DOT_0`` means to ensure that the returned string will not look - like an integer. + Always precede the returned string with a sign + character, even if *val* is non-negative. - * ``Py_DTSF_ALT`` means to apply "alternate" formatting rules. See the - documentation for the :c:func:`PyOS_snprintf` ``'#'`` specifier for - details. + .. c:macro:: Py_DTSF_ADD_DOT_0 + + Ensure that the returned string will not look like an integer. + + .. c:macro:: Py_DTSF_ALT + + Apply "alternate" formatting rules. + See the documentation for the :c:func:`PyOS_snprintf` ``'#'`` specifier for + details. + + .. c:macro:: Py_DTSF_NO_NEG_0 + + Negative zero is converted to positive zero. + + .. versionadded:: 3.11 If *ptype* is non-``NULL``, then the value it points to will be set to one of ``Py_DTST_FINITE``, ``Py_DTST_INFINITE``, or ``Py_DTST_NAN``, signifying that From df192616212f80aaa2f672b722b925943dbd3b78 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Mon, 10 Nov 2025 10:05:06 +0000 Subject: [PATCH 112/313] gh-141004: Document `pyctype.h` macros (GH-141272) --- Doc/c-api/conversion.rst | 55 ++++++++++++++++++++++++++++++++++++++++ Doc/library/locale.rst | 4 +-- Doc/whatsnew/2.7.rst | 18 ++++++------- 3 files changed, 65 insertions(+), 12 deletions(-) diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index e9d866c647d..533e5460da8 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -172,3 +172,58 @@ The following functions provide locale-independent string to number conversions. Case insensitive comparison of strings. The function works almost identically to :c:func:`!strncmp` except that it ignores the case. + + +Character classification and conversion +======================================= + +The following macros provide locale-independent (unlike the C standard library +``ctype.h``) character classification and conversion. +The argument must be a signed or unsigned :c:expr:`char`. + + +.. c:macro:: Py_ISALNUM(c) + + Return true if the character *c* is an alphanumeric character. + + +.. c:macro:: Py_ISALPHA(c) + + Return true if the character *c* is an alphabetic character (``a-z`` and ``A-Z``). + + +.. c:macro:: Py_ISDIGIT(c) + + Return true if the character *c* is a decimal digit (``0-9``). + + +.. c:macro:: Py_ISLOWER(c) + + Return true if the character *c* is a lowercase ASCII letter (``a-z``). + + +.. c:macro:: Py_ISUPPER(c) + + Return true if the character *c* is an uppercase ASCII letter (``A-Z``). + + +.. c:macro:: Py_ISSPACE(c) + + Return true if the character *c* is a whitespace character (space, tab, + carriage return, newline, vertical tab, or form feed). + + +.. c:macro:: Py_ISXDIGIT(c) + + Return true if the character *c* is a hexadecimal digit (``0-9``, ``a-f``, and + ``A-F``). + + +.. c:macro:: Py_TOLOWER(c) + + Return the lowercase equivalent of the character *c*. + + +.. c:macro:: Py_TOUPPER(c) + + Return the uppercase equivalent of the character *c*. diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index 4824391e597..94fc046d3f3 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -524,8 +524,8 @@ The :mod:`locale` module defines the following exception and functions: SSH connections. Python doesn't internally use locale-dependent character transformation functions - from ``ctype.h``. Instead, an internal ``pyctype.h`` provides locale-independent - equivalents like :c:macro:`!Py_TOLOWER`. + from ``ctype.h``. Instead, ``pyctype.h`` provides locale-independent + equivalents like :c:macro:`Py_TOLOWER`. .. data:: LC_COLLATE diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 09feb185b82..7296296d144 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -2181,14 +2181,14 @@ Changes to Python's build process and to the C API include: discussed in :issue:`5753`, and fixed by Antoine Pitrou. * New macros: the Python header files now define the following macros: - :c:macro:`!Py_ISALNUM`, - :c:macro:`!Py_ISALPHA`, - :c:macro:`!Py_ISDIGIT`, - :c:macro:`!Py_ISLOWER`, - :c:macro:`!Py_ISSPACE`, - :c:macro:`!Py_ISUPPER`, - :c:macro:`!Py_ISXDIGIT`, - :c:macro:`!Py_TOLOWER`, and :c:macro:`!Py_TOUPPER`. + :c:macro:`Py_ISALNUM`, + :c:macro:`Py_ISALPHA`, + :c:macro:`Py_ISDIGIT`, + :c:macro:`Py_ISLOWER`, + :c:macro:`Py_ISSPACE`, + :c:macro:`Py_ISUPPER`, + :c:macro:`Py_ISXDIGIT`, + :c:macro:`Py_TOLOWER`, and :c:macro:`Py_TOUPPER`. All of these functions are analogous to the C standard macros for classifying characters, but ignore the current locale setting, because in @@ -2196,8 +2196,6 @@ Changes to Python's build process and to the C API include: locale-independent way. (Added by Eric Smith; :issue:`5793`.) - .. XXX these macros don't seem to be described in the c-api docs. - * Removed function: :c:func:`!PyEval_CallObject` is now only available as a macro. A function version was being kept around to preserve ABI linking compatibility, but that was in 1997; it can certainly be From 6d710a79eaf085bf826c6f147e813495519a0897 Mon Sep 17 00:00:00 2001 From: Karina Souza <97332018+KarinaS0uza@users.noreply.github.com> Date: Mon, 10 Nov 2025 04:39:49 -0800 Subject: [PATCH 113/313] gh-140500: Update download.html instructions (#141320) Co-authored-by: Joseph Anthony Pasquale Holsten --- Doc/tools/templates/download.html | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index f914ad86211..c78c650b1cb 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -31,8 +31,7 @@

{% trans %}Download Python {{ dl_version }} documentation{% endtrans %}

{% if last_updated %}

{% trans %}Last updated on: {{ last_updated }}.{% endtrans %}

{% endif %} -

{% trans %}To download an archive containing all the documents for this version of -Python in one of various formats, follow one of links in this table.{% endtrans %}

+

{% trans %}Download an archive containing all the documentation for this version of Python:{% endtrans %}

@@ -62,8 +61,6 @@

{% trans %}Download Python {{ dl_version }} documentation{% endtrans %}

-

{% trans %}These archives contain all the content in the documentation.{% endtrans %}

-

{% trans %} We no longer provide pre-built PDFs of the documentation. To build a PDF archive, follow the instructions in the @@ -75,7 +72,6 @@

{% trans %}Download Python {{ dl_version }} documentation{% endtrans %}

See the directory listing for file sizes.{% endtrans %}

-

{% trans %}Problems{% endtrans %}

{% set bugs = pathto('bugs') %}

{% trans bugs = bugs %}Open an issue From 13fa313bebed71d8bc64f1cfdaf4b2f1ddd3ce5f Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Mon, 10 Nov 2025 13:37:34 +0000 Subject: [PATCH 114/313] gh-139707: Specify `winreg`, `msvcrt` and `winsound` module availability in docs (GH-140429) --- Doc/library/msvcrt.rst | 2 ++ Doc/library/winreg.rst | 2 ++ Doc/library/winsound.rst | 2 ++ 3 files changed, 6 insertions(+) diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index 327cc3602b1..a2c5e375d2c 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -22,6 +22,8 @@ api. The normal API deals only with ASCII characters and is of limited use for internationalized applications. The wide char API should be used where ever possible. +.. availability:: Windows. + .. versionchanged:: 3.3 Operations in this module now raise :exc:`OSError` where :exc:`IOError` was raised. diff --git a/Doc/library/winreg.rst b/Doc/library/winreg.rst index df8fb83a018..b150c53735d 100644 --- a/Doc/library/winreg.rst +++ b/Doc/library/winreg.rst @@ -14,6 +14,8 @@ integer as the registry handle, a :ref:`handle object ` is used to ensure that the handles are closed correctly, even if the programmer neglects to explicitly close them. +.. availability:: Windows. + .. _exception-changed: .. versionchanged:: 3.3 diff --git a/Doc/library/winsound.rst b/Doc/library/winsound.rst index 925984c3cdb..93c0c025982 100644 --- a/Doc/library/winsound.rst +++ b/Doc/library/winsound.rst @@ -13,6 +13,8 @@ The :mod:`winsound` module provides access to the basic sound-playing machinery provided by Windows platforms. It includes functions and several constants. +.. availability:: Windows. + .. function:: Beep(frequency, duration) From 9f5152441d32166134c3c64f56f974b9476f9478 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 10 Nov 2025 14:42:18 +0100 Subject: [PATCH 115/313] gh-136702: Clear codec caches for refleak tests; use test.support helpers (GH-141345) This should fix refleak buildbots. --- Lib/test/libregrtest/utils.py | 19 +++++++++++++++++++ Lib/test/test_codecs.py | 5 +++-- Lib/test/test_email/test_email.py | 3 ++- Lib/test/test_email/test_headerregistry.py | 3 ++- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index d94fb84a743..cfb009c203e 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -294,6 +294,25 @@ def clear_caches(): else: importlib_metadata.FastPath.__new__.cache_clear() + try: + encodings = sys.modules['encodings'] + except KeyError: + pass + else: + encodings._cache.clear() + + try: + codecs = sys.modules['codecs'] + except KeyError: + pass + else: + # There's no direct API to clear the codecs search cache, but + # `unregister` clears it implicitly. + def noop_search_function(name): + return None + codecs.register(noop_search_function) + codecs.unregister(noop_search_function) + def get_build_info(): # Get most important configure and build options as a list of strings. diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index f1f0ac5ad36..c31faec9ee5 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -13,6 +13,7 @@ from test import support from test.support import os_helper +from test.support import warnings_helper try: import _testlimitedcapi @@ -3902,8 +3903,8 @@ def test_encodings_normalize_encoding(self): self.assertEqual(normalize('utf...8'), 'utf...8') # Non-ASCII *encoding* is deprecated. - with self.assertWarnsRegex(DeprecationWarning, - "Support for non-ascii encoding names will be removed in 3.17"): + msg = "Support for non-ascii encoding names will be removed in 3.17" + with warnings_helper.check_warnings((msg, DeprecationWarning)): self.assertEqual(normalize('utf\xE9\u20AC\U0010ffff-8'), 'utf_8') diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 1900adf463b..4020f1041c4 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -41,6 +41,7 @@ from test import support from test.support import threading_helper +from test.support import warnings_helper from test.support.os_helper import unlink from test.test_email import openfile, TestEmailBase @@ -5738,7 +5739,7 @@ def test_rfc2231_bad_character_in_encoding(self): """ msg = email.message_from_string(m) - with self.assertWarns(DeprecationWarning): + with warnings_helper.check_warnings(('', DeprecationWarning)): self.assertEqual(msg.get_filename(), 'myfile.txt') def test_rfc2231_single_tick_in_filename_extended(self): diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index 1d0d0a49a82..7138aa4c556 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -8,6 +8,7 @@ from email import headerregistry from email.headerregistry import Address, Group from test.support import ALWAYS_EQ +from test.support import warnings_helper DITTO = object() @@ -252,7 +253,7 @@ def content_type_as_value(self, if 'utf-8%E2%80%9D' in source and 'ascii' not in source: import encodings encodings._cache.clear() - with self.assertWarns(DeprecationWarning): + with warnings_helper.check_warnings(('', DeprecationWarning)): h = self.make_header('Content-Type', source) else: h = self.make_header('Content-Type', source) From 12837c63635559873a5abddf511d38456d69617b Mon Sep 17 00:00:00 2001 From: David Ellis Date: Mon, 10 Nov 2025 13:57:11 +0000 Subject: [PATCH 116/313] gh-137530: generate an __annotate__ function for dataclasses __init__ (GH-137711) --- Doc/whatsnew/3.15.rst | 8 + Lib/dataclasses.py | 94 ++++++++++-- Lib/test/test_dataclasses/__init__.py | 139 +++++++++++++++++- ...-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst | 1 + 4 files changed, 227 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 5379ac3abba..e0b0471567c 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -368,6 +368,14 @@ collections.abc previously emitted if it was merely imported or accessed from the :mod:`!collections.abc` module. + +dataclasses +----------- + +* Annotations for generated ``__init__`` methods no longer include internal + type names. + + dbm --- diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index b98f21dcbe9..3ccb7246928 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -441,9 +441,11 @@ def __init__(self, globals): self.locals = {} self.overwrite_errors = {} self.unconditional_adds = {} + self.method_annotations = {} def add_fn(self, name, args, body, *, locals=None, return_type=MISSING, - overwrite_error=False, unconditional_add=False, decorator=None): + overwrite_error=False, unconditional_add=False, decorator=None, + annotation_fields=None): if locals is not None: self.locals.update(locals) @@ -464,16 +466,14 @@ def add_fn(self, name, args, body, *, locals=None, return_type=MISSING, self.names.append(name) - if return_type is not MISSING: - self.locals[f'__dataclass_{name}_return_type__'] = return_type - return_annotation = f'->__dataclass_{name}_return_type__' - else: - return_annotation = '' + if annotation_fields is not None: + self.method_annotations[name] = (annotation_fields, return_type) + args = ','.join(args) body = '\n'.join(body) # Compute the text of the entire function, add it to the text we're generating. - self.src.append(f'{f' {decorator}\n' if decorator else ''} def {name}({args}){return_annotation}:\n{body}') + self.src.append(f'{f' {decorator}\n' if decorator else ''} def {name}({args}):\n{body}') def add_fns_to_class(self, cls): # The source to all of the functions we're generating. @@ -509,6 +509,15 @@ def add_fns_to_class(self, cls): # Now that we've generated the functions, assign them into cls. for name, fn in zip(self.names, fns): fn.__qualname__ = f"{cls.__qualname__}.{fn.__name__}" + + try: + annotation_fields, return_type = self.method_annotations[name] + except KeyError: + pass + else: + annotate_fn = _make_annotate_function(cls, name, annotation_fields, return_type) + fn.__annotate__ = annotate_fn + if self.unconditional_adds.get(name, False): setattr(cls, name, fn) else: @@ -524,6 +533,44 @@ def add_fns_to_class(self, cls): raise TypeError(error_msg) +def _make_annotate_function(__class__, method_name, annotation_fields, return_type): + # Create an __annotate__ function for a dataclass + # Try to return annotations in the same format as they would be + # from a regular __init__ function + + def __annotate__(format, /): + Format = annotationlib.Format + match format: + case Format.VALUE | Format.FORWARDREF | Format.STRING: + cls_annotations = {} + for base in reversed(__class__.__mro__): + cls_annotations.update( + annotationlib.get_annotations(base, format=format) + ) + + new_annotations = {} + for k in annotation_fields: + new_annotations[k] = cls_annotations[k] + + if return_type is not MISSING: + if format == Format.STRING: + new_annotations["return"] = annotationlib.type_repr(return_type) + else: + new_annotations["return"] = return_type + + return new_annotations + + case _: + raise NotImplementedError(format) + + # This is a flag for _add_slots to know it needs to regenerate this method + # In order to remove references to the original class when it is replaced + __annotate__.__generated_by_dataclasses__ = True + __annotate__.__qualname__ = f"{__class__.__qualname__}.{method_name}.__annotate__" + + return __annotate__ + + def _field_assign(frozen, name, value, self_name): # If we're a frozen class, then assign to our fields in __init__ # via object.__setattr__. Otherwise, just use a simple @@ -612,7 +659,7 @@ def _init_param(f): elif f.default_factory is not MISSING: # There's a factory function. Set a marker. default = '=__dataclass_HAS_DEFAULT_FACTORY__' - return f'{f.name}:__dataclass_type_{f.name}__{default}' + return f'{f.name}{default}' def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, @@ -635,11 +682,10 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, raise TypeError(f'non-default argument {f.name!r} ' f'follows default argument {seen_default.name!r}') - locals = {**{f'__dataclass_type_{f.name}__': f.type for f in fields}, - **{'__dataclass_HAS_DEFAULT_FACTORY__': _HAS_DEFAULT_FACTORY, - '__dataclass_builtins_object__': object, - } - } + annotation_fields = [f.name for f in fields if f.init] + + locals = {'__dataclass_HAS_DEFAULT_FACTORY__': _HAS_DEFAULT_FACTORY, + '__dataclass_builtins_object__': object} body_lines = [] for f in fields: @@ -670,7 +716,8 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, [self_name] + _init_params, body_lines, locals=locals, - return_type=None) + return_type=None, + annotation_fields=annotation_fields) def _frozen_get_del_attr(cls, fields, func_builder): @@ -1337,6 +1384,25 @@ def _add_slots(cls, is_frozen, weakref_slot, defined_fields): or _update_func_cell_for__class__(member.fdel, cls, newcls)): break + # Get new annotations to remove references to the original class + # in forward references + newcls_ann = annotationlib.get_annotations( + newcls, format=annotationlib.Format.FORWARDREF) + + # Fix references in dataclass Fields + for f in getattr(newcls, _FIELDS).values(): + try: + ann = newcls_ann[f.name] + except KeyError: + pass + else: + f.type = ann + + # Fix the class reference in the __annotate__ method + init_annotate = newcls.__init__.__annotate__ + if getattr(init_annotate, "__generated_by_dataclasses__", False): + _update_func_cell_for__class__(init_annotate, cls, newcls) + return newcls diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index 6bf5e5b3e55..513dd78c438 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -2471,6 +2471,135 @@ def __init__(self, a): self.assertEqual(D(5).a, 10) +class TestInitAnnotate(unittest.TestCase): + # Tests for the generated __annotate__ function for __init__ + # See: https://github.com/python/cpython/issues/137530 + + def test_annotate_function(self): + # No forward references + @dataclass + class A: + a: int + + value_annos = annotationlib.get_annotations(A.__init__, format=annotationlib.Format.VALUE) + forwardref_annos = annotationlib.get_annotations(A.__init__, format=annotationlib.Format.FORWARDREF) + string_annos = annotationlib.get_annotations(A.__init__, format=annotationlib.Format.STRING) + + self.assertEqual(value_annos, {'a': int, 'return': None}) + self.assertEqual(forwardref_annos, {'a': int, 'return': None}) + self.assertEqual(string_annos, {'a': 'int', 'return': 'None'}) + + self.assertTrue(getattr(A.__init__.__annotate__, "__generated_by_dataclasses__")) + + def test_annotate_function_forwardref(self): + # With forward references + @dataclass + class B: + b: undefined + + # VALUE annotations should raise while unresolvable + with self.assertRaises(NameError): + _ = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.VALUE) + + forwardref_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.FORWARDREF) + string_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.STRING) + + self.assertEqual(forwardref_annos, {'b': support.EqualToForwardRef('undefined', owner=B, is_class=True), 'return': None}) + self.assertEqual(string_annos, {'b': 'undefined', 'return': 'None'}) + + # Now VALUE and FORWARDREF should resolve, STRING should be unchanged + undefined = int + + value_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.VALUE) + forwardref_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.FORWARDREF) + string_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.STRING) + + self.assertEqual(value_annos, {'b': int, 'return': None}) + self.assertEqual(forwardref_annos, {'b': int, 'return': None}) + self.assertEqual(string_annos, {'b': 'undefined', 'return': 'None'}) + + def test_annotate_function_init_false(self): + # Check `init=False` attributes don't get into the annotations of the __init__ function + @dataclass + class C: + c: str = field(init=False) + + self.assertEqual(annotationlib.get_annotations(C.__init__), {'return': None}) + + def test_annotate_function_contains_forwardref(self): + # Check string annotations on objects containing a ForwardRef + @dataclass + class D: + d: list[undefined] + + with self.assertRaises(NameError): + annotationlib.get_annotations(D.__init__) + + self.assertEqual( + annotationlib.get_annotations(D.__init__, format=annotationlib.Format.FORWARDREF), + {"d": list[support.EqualToForwardRef("undefined", is_class=True, owner=D)], "return": None} + ) + + self.assertEqual( + annotationlib.get_annotations(D.__init__, format=annotationlib.Format.STRING), + {"d": "list[undefined]", "return": "None"} + ) + + # Now test when it is defined + undefined = str + + # VALUE should now resolve + self.assertEqual( + annotationlib.get_annotations(D.__init__), + {"d": list[str], "return": None} + ) + + self.assertEqual( + annotationlib.get_annotations(D.__init__, format=annotationlib.Format.FORWARDREF), + {"d": list[str], "return": None} + ) + + self.assertEqual( + annotationlib.get_annotations(D.__init__, format=annotationlib.Format.STRING), + {"d": "list[undefined]", "return": "None"} + ) + + def test_annotate_function_not_replaced(self): + # Check that __annotate__ is not replaced on non-generated __init__ functions + @dataclass(slots=True) + class E: + x: str + def __init__(self, x: int) -> None: + self.x = x + + self.assertEqual( + annotationlib.get_annotations(E.__init__), {"x": int, "return": None} + ) + + self.assertFalse(hasattr(E.__init__.__annotate__, "__generated_by_dataclasses__")) + + def test_init_false_forwardref(self): + # Test forward references in fields not required for __init__ annotations. + + # At the moment this raises a NameError for VALUE annotations even though the + # undefined annotation is not required for the __init__ annotations. + # Ideally this will be fixed but currently there is no good way to resolve this + + @dataclass + class F: + not_in_init: list[undefined] = field(init=False, default=None) + in_init: int + + annos = annotationlib.get_annotations(F.__init__, format=annotationlib.Format.FORWARDREF) + self.assertEqual( + annos, + {"in_init": int, "return": None}, + ) + + with self.assertRaises(NameError): + annos = annotationlib.get_annotations(F.__init__) # NameError on not_in_init + + class TestRepr(unittest.TestCase): def test_repr(self): @dataclass @@ -3831,7 +3960,15 @@ def method(self) -> int: return SlotsTest - for make in (make_simple, make_with_annotations, make_with_annotations_and_method): + def make_with_forwardref(): + @dataclass(slots=True) + class SlotsTest: + x: undefined + y: list[undefined] + + return SlotsTest + + for make in (make_simple, make_with_annotations, make_with_annotations_and_method, make_with_forwardref): with self.subTest(make=make): C = make() support.gc_collect() diff --git a/Misc/NEWS.d/next/Library/2025-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst b/Misc/NEWS.d/next/Library/2025-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst new file mode 100644 index 00000000000..4ff55b41dea --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst @@ -0,0 +1 @@ +:mod:`dataclasses` Fix annotations for generated ``__init__`` methods by replacing the annotations that were in-line in the generated source code with ``__annotate__`` functions attached to the methods. From 06b62282c79dd69293a3eefb4c55f5acc6312cb2 Mon Sep 17 00:00:00 2001 From: dr-carlos <77367421+dr-carlos@users.noreply.github.com> Date: Tue, 11 Nov 2025 01:15:22 +1030 Subject: [PATCH 117/313] gh-141174: Improve `annotationlib.get_annotations()` test coverage (#141286) * Test `get_annotations(format=Format.VALUE)` for stringized annotations on custom objects * Test `get_annotations(format=Format.VALUE)` for stringized annotations on wrapped partial functions * Update test_stringized_annotations_with_star_unpack() to actually test stringized annotations * Test __annotate__ returning a non-dict * Test passing globals and locals to stringized `get_annotations()` --- Lib/test/test_annotationlib.py | 70 +++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index fd5d43b09b9..f1d32ab50cf 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -9,6 +9,7 @@ import pickle from string.templatelib import Template, Interpolation import typing +import sys import unittest from annotationlib import ( Format, @@ -755,6 +756,8 @@ def test_stringized_annotations_in_module(self): for kwargs in [ {"eval_str": True}, + {"eval_str": True, "globals": isa.__dict__, "locals": {}}, + {"eval_str": True, "globals": {}, "locals": isa.__dict__}, {"format": Format.VALUE, "eval_str": True}, ]: with self.subTest(**kwargs): @@ -788,7 +791,7 @@ def test_stringized_annotations_in_empty_module(self): self.assertEqual(get_annotations(isa2, eval_str=False), {}) def test_stringized_annotations_with_star_unpack(self): - def f(*args: *tuple[int, ...]): ... + def f(*args: "*tuple[int, ...]"): ... self.assertEqual(get_annotations(f, eval_str=True), {'args': (*tuple[int, ...],)[0]}) @@ -811,6 +814,44 @@ def test_stringized_annotations_on_wrapper(self): {"a": "int", "b": "str", "return": "MyClass"}, ) + def test_stringized_annotations_on_partial_wrapper(self): + isa = inspect_stringized_annotations + + def times_three_str(fn: typing.Callable[[str], isa.MyClass]): + @functools.wraps(fn) + def wrapper(b: "str") -> "MyClass": + return fn(b * 3) + + return wrapper + + wrapped = times_three_str(functools.partial(isa.function, 1)) + self.assertEqual(wrapped("x"), isa.MyClass(1, "xxx")) + self.assertIsNot(wrapped.__globals__, isa.function.__globals__) + self.assertEqual( + get_annotations(wrapped, eval_str=True), + {"b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=False), + {"b": "str", "return": "MyClass"}, + ) + + # If functools is not loaded, names will be evaluated in the current + # module instead of being unwrapped to the original. + functools_mod = sys.modules["functools"] + del sys.modules["functools"] + + self.assertEqual( + get_annotations(wrapped, eval_str=True), + {"b": str, "return": MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=False), + {"b": "str", "return": "MyClass"}, + ) + + sys.modules["functools"] = functools_mod + def test_stringized_annotations_on_class(self): isa = inspect_stringized_annotations # test that local namespace lookups work @@ -823,6 +864,16 @@ def test_stringized_annotations_on_class(self): {"x": int}, ) + def test_stringized_annotations_on_custom_object(self): + class HasAnnotations: + @property + def __annotations__(self): + return {"x": "int"} + + ha = HasAnnotations() + self.assertEqual(get_annotations(ha), {"x": "int"}) + self.assertEqual(get_annotations(ha, eval_str=True), {"x": int}) + def test_stringized_annotation_permutations(self): def define_class(name, has_future, has_annos, base_text, extra_names=None): lines = [] @@ -990,6 +1041,23 @@ def __annotate__(self): {"x": "int"}, ) + def test_non_dict_annotate(self): + class WeirdAnnotate: + def __annotate__(self, *args, **kwargs): + return "not a dict" + + wa = WeirdAnnotate() + for format in Format: + if format == Format.VALUE_WITH_FAKE_GLOBALS: + continue + with ( + self.subTest(format=format), + self.assertRaisesRegex( + ValueError, r".*__annotate__ returned a non-dict" + ), + ): + get_annotations(wa, format=format) + def test_no_annotations(self): class CustomClass: pass From 68266c1f01e5791558cb088dfb0e26ecd577295e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 10 Nov 2025 15:50:51 +0100 Subject: [PATCH 118/313] gh-141341: Rename COMPILER macro to _Py_COMPILER on Windows (#141342) --- ...-11-10-11-26-26.gh-issue-141341.OsO6-y.rst | 2 ++ PC/pyconfig.h | 28 +++++++++---------- Python/getcompiler.c | 4 +++ 3 files changed, 20 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-11-10-11-26-26.gh-issue-141341.OsO6-y.rst diff --git a/Misc/NEWS.d/next/C_API/2025-11-10-11-26-26.gh-issue-141341.OsO6-y.rst b/Misc/NEWS.d/next/C_API/2025-11-10-11-26-26.gh-issue-141341.OsO6-y.rst new file mode 100644 index 00000000000..460923b4d62 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-10-11-26-26.gh-issue-141341.OsO6-y.rst @@ -0,0 +1,2 @@ +On Windows, rename the ``COMPILER`` macro to ``_Py_COMPILER`` to avoid name +conflicts. Patch by Victor Stinner. diff --git a/PC/pyconfig.h b/PC/pyconfig.h index 0e8379387cd..a126fca6f5a 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -118,7 +118,7 @@ WIN32 is still required for the locale module. /* Microsoft C defines _MSC_VER, as does clang-cl.exe */ #ifdef _MSC_VER -/* We want COMPILER to expand to a string containing _MSC_VER's *value*. +/* We want _Py_COMPILER to expand to a string containing _MSC_VER's *value*. * This is horridly tricky, because the stringization operator only works * on macro arguments, and doesn't evaluate macros passed *as* arguments. */ @@ -148,7 +148,7 @@ WIN32 is still required for the locale module. #define MS_WIN64 #endif -/* set the COMPILER and support tier +/* set the _Py_COMPILER and support tier * * win_amd64 MSVC (x86_64-pc-windows-msvc): 1 * win32 MSVC (i686-pc-windows-msvc): 1 @@ -158,22 +158,22 @@ WIN32 is still required for the locale module. #ifdef MS_WIN64 #if defined(_M_X64) || defined(_M_AMD64) #if defined(__clang__) -#define COMPILER ("[Clang " __clang_version__ "] 64 bit (AMD64) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") +#define _Py_COMPILER ("[Clang " __clang_version__ "] 64 bit (AMD64) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") #define PY_SUPPORT_TIER 0 #elif defined(__INTEL_COMPILER) -#define COMPILER ("[ICC v." _Py_STRINGIZE(__INTEL_COMPILER) " 64 bit (amd64) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") +#define _Py_COMPILER ("[ICC v." _Py_STRINGIZE(__INTEL_COMPILER) " 64 bit (amd64) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") #define PY_SUPPORT_TIER 0 #else -#define COMPILER _Py_PASTE_VERSION("64 bit (AMD64)") +#define _Py_COMPILER _Py_PASTE_VERSION("64 bit (AMD64)") #define PY_SUPPORT_TIER 1 #endif /* __clang__ */ #define PYD_PLATFORM_TAG "win_amd64" #elif defined(_M_ARM64) -#define COMPILER _Py_PASTE_VERSION("64 bit (ARM64)") +#define _Py_COMPILER _Py_PASTE_VERSION("64 bit (ARM64)") #define PY_SUPPORT_TIER 3 #define PYD_PLATFORM_TAG "win_arm64" #else -#define COMPILER _Py_PASTE_VERSION("64 bit (Unknown)") +#define _Py_COMPILER _Py_PASTE_VERSION("64 bit (Unknown)") #define PY_SUPPORT_TIER 0 #endif #endif /* MS_WIN64 */ @@ -220,22 +220,22 @@ typedef _W64 int Py_ssize_t; #if defined(MS_WIN32) && !defined(MS_WIN64) #if defined(_M_IX86) #if defined(__clang__) -#define COMPILER ("[Clang " __clang_version__ "] 32 bit (Intel) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") +#define _Py_COMPILER ("[Clang " __clang_version__ "] 32 bit (Intel) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") #define PY_SUPPORT_TIER 0 #elif defined(__INTEL_COMPILER) -#define COMPILER ("[ICC v." _Py_STRINGIZE(__INTEL_COMPILER) " 32 bit (Intel) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") +#define _Py_COMPILER ("[ICC v." _Py_STRINGIZE(__INTEL_COMPILER) " 32 bit (Intel) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") #define PY_SUPPORT_TIER 0 #else -#define COMPILER _Py_PASTE_VERSION("32 bit (Intel)") +#define _Py_COMPILER _Py_PASTE_VERSION("32 bit (Intel)") #define PY_SUPPORT_TIER 1 #endif /* __clang__ */ #define PYD_PLATFORM_TAG "win32" #elif defined(_M_ARM) -#define COMPILER _Py_PASTE_VERSION("32 bit (ARM)") +#define _Py_COMPILER _Py_PASTE_VERSION("32 bit (ARM)") #define PYD_PLATFORM_TAG "win_arm32" #define PY_SUPPORT_TIER 0 #else -#define COMPILER _Py_PASTE_VERSION("32 bit (Unknown)") +#define _Py_COMPILER _Py_PASTE_VERSION("32 bit (Unknown)") #define PY_SUPPORT_TIER 0 #endif #endif /* MS_WIN32 && !MS_WIN64 */ @@ -273,7 +273,7 @@ typedef int pid_t; #warning "Please use an up-to-date version of gcc! (>2.91 recommended)" #endif -#define COMPILER "[gcc]" +#define _Py_COMPILER "[gcc]" #define PY_LONG_LONG long long #define PY_LLONG_MIN LLONG_MIN #define PY_LLONG_MAX LLONG_MAX @@ -286,7 +286,7 @@ typedef int pid_t; /* XXX These defines are likely incomplete, but should be easy to fix. They should be complete enough to build extension modules. */ -#define COMPILER "[lcc-win32]" +#define _Py_COMPILER "[lcc-win32]" typedef int pid_t; /* __declspec() is supported here too - do nothing to get the defaults */ diff --git a/Python/getcompiler.c b/Python/getcompiler.c index a5d26239e87..cc56ad8c895 100644 --- a/Python/getcompiler.c +++ b/Python/getcompiler.c @@ -3,6 +3,10 @@ #include "Python.h" +#ifdef _Py_COMPILER +# define COMPILER _Py_COMPILER +#endif + #ifndef COMPILER // Note the __clang__ conditional has to come before the __GNUC__ one because From 19b573025e0aa569e7a34081116280133e33979a Mon Sep 17 00:00:00 2001 From: dr-carlos <77367421+dr-carlos@users.noreply.github.com> Date: Tue, 11 Nov 2025 01:23:40 +1030 Subject: [PATCH 119/313] gh-141174: Improve `ForwardRef` test coverage (#141175) * Test unsupported format in ForwardRef.evaluate() * Test dict cell closure with multiple variables * Test all options in ForwardRef repr * Test ForwardRef being a final class --- Lib/test/test_annotationlib.py | 50 ++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index f1d32ab50cf..d196801eede 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -74,6 +74,30 @@ def inner(arg: x): anno = get_annotations(inner, format=Format.FORWARDREF) self.assertEqual(anno["arg"], x) + def test_multiple_closure(self): + def inner(arg: x[y]): + pass + + fwdref = get_annotations(inner, format=Format.FORWARDREF)["arg"] + self.assertIsInstance(fwdref, ForwardRef) + self.assertEqual(fwdref.__forward_arg__, "x[y]") + with self.assertRaises(NameError): + fwdref.evaluate() + + y = str + fwdref = get_annotations(inner, format=Format.FORWARDREF)["arg"] + self.assertIsInstance(fwdref, ForwardRef) + extra_name, extra_val = next(iter(fwdref.__extra_names__.items())) + self.assertEqual(fwdref.__forward_arg__.replace(extra_name, extra_val.__name__), "x[str]") + with self.assertRaises(NameError): + fwdref.evaluate() + + x = list + self.assertEqual(fwdref.evaluate(), x[y]) + + fwdref = get_annotations(inner, format=Format.FORWARDREF)["arg"] + self.assertEqual(fwdref, x[y]) + def test_function(self): def f(x: int, y: doesntexist): pass @@ -1756,6 +1780,14 @@ def test_forward_repr(self): repr(List[ForwardRef("int", module="mod")]), "typing.List[ForwardRef('int', module='mod')]", ) + self.assertEqual( + repr(List[ForwardRef("int", module="mod", is_class=True)]), + "typing.List[ForwardRef('int', module='mod', is_class=True)]", + ) + self.assertEqual( + repr(List[ForwardRef("int", owner="class")]), + "typing.List[ForwardRef('int', owner='class')]", + ) def test_forward_recursion_actually(self): def namespace1(): @@ -1861,6 +1893,19 @@ def test_evaluate_forwardref_format(self): support.EqualToForwardRef('"a" + 1'), ) + def test_evaluate_notimplemented_format(self): + class C: + x: alias + + fwdref = get_annotations(C, format=Format.FORWARDREF)["x"] + + with self.assertRaises(NotImplementedError): + fwdref.evaluate(format=Format.VALUE_WITH_FAKE_GLOBALS) + + with self.assertRaises(NotImplementedError): + # Some other unsupported value + fwdref.evaluate(format=7) + def test_evaluate_with_type_params(self): class Gen[T]: alias = int @@ -1994,6 +2039,11 @@ def test_fwdref_invalid_syntax(self): with self.assertRaises(SyntaxError): fr.evaluate() + def test_fwdref_final_class(self): + with self.assertRaises(TypeError): + class C(ForwardRef): + pass + class TestAnnotationLib(unittest.TestCase): def test__all__(self): From 1110e8f6a4a767f6d09b121017442528733b380b Mon Sep 17 00:00:00 2001 From: dr-carlos <77367421+dr-carlos@users.noreply.github.com> Date: Tue, 11 Nov 2025 01:24:50 +1030 Subject: [PATCH 120/313] gh-141174: Improve `annotationlib.call_annotate_function()` test coverage (#141176) * Test passing unsupported Format values to call_annotate_function() * Test call_evaluate_function with fake globals that raise errors * Fix typo and comparison in test_fake_global_evaluation --- Lib/test/test_annotationlib.py | 43 ++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index d196801eede..0ae598b6839 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1339,6 +1339,32 @@ def evaluate(format, exc=NotImplementedError): "undefined", ) + def test_fake_global_evaluation(self): + # This will raise an AttributeError + def evaluate_union(format, exc=NotImplementedError): + if format == Format.VALUE_WITH_FAKE_GLOBALS: + # Return a ForwardRef + return builtins.undefined | list[int] + raise exc + + self.assertEqual( + annotationlib.call_evaluate_function(evaluate_union, Format.FORWARDREF), + support.EqualToForwardRef("builtins.undefined | list[int]"), + ) + + # This will raise an AttributeError + def evaluate_intermediate(format, exc=NotImplementedError): + if format == Format.VALUE_WITH_FAKE_GLOBALS: + intermediate = builtins.undefined + # Return a literal + return intermediate is None + raise exc + + self.assertIs( + annotationlib.call_evaluate_function(evaluate_intermediate, Format.FORWARDREF), + False, + ) + class TestCallAnnotateFunction(unittest.TestCase): # Tests for user defined annotate functions. @@ -1480,6 +1506,23 @@ def annotate(format, /): with self.assertRaises(NotImplementedError): annotationlib.call_annotate_function(annotate, Format.STRING) + def test_unsupported_formats(self): + def annotate(format, /): + if format == Format.FORWARDREF: + return {"x": str} + else: + raise NotImplementedError(format) + + with self.assertRaises(ValueError): + annotationlib.call_annotate_function(annotate, Format.VALUE_WITH_FAKE_GLOBALS) + + with self.assertRaises(RuntimeError): + annotationlib.call_annotate_function(annotate, Format.VALUE) + + with self.assertRaises(ValueError): + # Some non-Format value + annotationlib.call_annotate_function(annotate, 7) + def test_error_from_value_raised(self): # Test that the error from format.VALUE is raised # if all formats fail From 59b793b0dd76d37229fe6d379cd5fe76023d15f1 Mon Sep 17 00:00:00 2001 From: Yongzi Li <204532581+Yzi-Li@users.noreply.github.com> Date: Mon, 10 Nov 2025 22:55:15 +0800 Subject: [PATCH 121/313] gh-141343: Fix swapped words in `sorted` doc (GH-141348) --- Doc/library/functions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 61799e303a1..e9879397555 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1859,7 +1859,7 @@ are always available. They are listed here in alphabetical order. the same data with other ordering tools such as :func:`max` that rely on a different underlying method. Implementing all six comparisons also helps avoid confusion for mixed type comparisons which can call - reflected the :meth:`~object.__gt__` method. + the reflected :meth:`~object.__gt__` method. For sorting examples and a brief sorting tutorial, see :ref:`sortinghowto`. From 55ea13231313a2133e6f5a6112409d349081f273 Mon Sep 17 00:00:00 2001 From: dr-carlos <77367421+dr-carlos@users.noreply.github.com> Date: Tue, 11 Nov 2025 01:26:45 +1030 Subject: [PATCH 122/313] gh-141174: Improve `annotationlib._Stringifier` test coverage (#141220) * Test `_Stringifier.__convert_to_ast()` for containers * Test partial evaluation of `ForwardRef`s in `_Stringifier` --- Lib/test/test_annotationlib.py | 67 ++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 0ae598b6839..9f3275d5071 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -119,6 +119,10 @@ def f( alpha: some | obj, beta: +some, gamma: some < obj, + delta: some | {obj: module}, + epsilon: some | {obj, module}, + zeta: some | [obj], + eta: some | (), ): pass @@ -147,6 +151,69 @@ def f( self.assertIsInstance(gamma_anno, ForwardRef) self.assertEqual(gamma_anno, support.EqualToForwardRef("some < obj", owner=f)) + delta_anno = anno["delta"] + self.assertIsInstance(delta_anno, ForwardRef) + self.assertEqual(delta_anno, support.EqualToForwardRef("some | {obj: module}", owner=f)) + + epsilon_anno = anno["epsilon"] + self.assertIsInstance(epsilon_anno, ForwardRef) + self.assertEqual(epsilon_anno, support.EqualToForwardRef("some | {obj, module}", owner=f)) + + zeta_anno = anno["zeta"] + self.assertIsInstance(zeta_anno, ForwardRef) + self.assertEqual(zeta_anno, support.EqualToForwardRef("some | [obj]", owner=f)) + + eta_anno = anno["eta"] + self.assertIsInstance(eta_anno, ForwardRef) + self.assertEqual(eta_anno, support.EqualToForwardRef("some | ()", owner=f)) + + def test_partially_nonexistent(self): + # These annotations start with a non-existent variable and then use + # global types with defined values. This partially evaluates by putting + # those globals into `fwdref.__extra_names__`. + def f( + x: obj | int, + y: container[int:obj, int], + z: dict_val | {str: int}, + alpha: set_val | {str, int}, + beta: obj | bool | int, + gamma: obj | call_func(int, kwd=bool), + ): + pass + + def func(*args, **kwargs): + return Union[*args, *(kwargs.values())] + + anno = get_annotations(f, format=Format.FORWARDREF) + globals_ = { + "obj": str, "container": list, "dict_val": {1: 2}, "set_val": {1, 2}, + "call_func": func + } + + x_anno = anno["x"] + self.assertIsInstance(x_anno, ForwardRef) + self.assertEqual(x_anno.evaluate(globals=globals_), str | int) + + y_anno = anno["y"] + self.assertIsInstance(y_anno, ForwardRef) + self.assertEqual(y_anno.evaluate(globals=globals_), list[int:str, int]) + + z_anno = anno["z"] + self.assertIsInstance(z_anno, ForwardRef) + self.assertEqual(z_anno.evaluate(globals=globals_), {1: 2} | {str: int}) + + alpha_anno = anno["alpha"] + self.assertIsInstance(alpha_anno, ForwardRef) + self.assertEqual(alpha_anno.evaluate(globals=globals_), {1, 2} | {str, int}) + + beta_anno = anno["beta"] + self.assertIsInstance(beta_anno, ForwardRef) + self.assertEqual(beta_anno.evaluate(globals=globals_), str | bool | int) + + gamma_anno = anno["gamma"] + self.assertIsInstance(gamma_anno, ForwardRef) + self.assertEqual(gamma_anno.evaluate(globals=globals_), str | func(int, kwd=bool)) + def test_partially_nonexistent_union(self): # Test unions with '|' syntax equal unions with typing.Union[] with some forwardrefs class UnionForwardrefs: From 88953d5debf08dfaa1cdb314d62262f770addf5b Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Mon, 10 Nov 2025 18:36:01 +0300 Subject: [PATCH 123/313] gh-141004: Deprecate Py_MATH_El and Py_MATH_PIl macros (#141035) Co-authored-by: Victor Stinner --- Doc/c-api/float.rst | 14 ++++++++++++++ Doc/deprecations/c-api-pending-removal-in-3.20.rst | 2 ++ Doc/whatsnew/3.15.rst | 4 ++++ Include/pymath.h | 2 ++ .../2025-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst | 1 + 5 files changed, 23 insertions(+) create mode 100644 Misc/NEWS.d/next/C_API/2025-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index edee498a0b8..9e703a46445 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -78,6 +78,20 @@ Floating-Point Objects Return the minimum normalized positive float *DBL_MIN* as C :c:expr:`double`. +.. c:macro:: Py_MATH_El + + High precision (long double) definition of :data:`~math.e` constant. + + .. deprecated-removed:: 3.15 3.20 + + +.. c:macro:: Py_MATH_PIl + + High precision (long double) definition of :data:`~math.pi` constant. + + .. deprecated-removed:: 3.15 3.20 + + .. c:macro:: Py_RETURN_NAN Return :data:`math.nan` from a function. diff --git a/Doc/deprecations/c-api-pending-removal-in-3.20.rst b/Doc/deprecations/c-api-pending-removal-in-3.20.rst index 82f975d6ed4..18623b19a2a 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.20.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.20.rst @@ -5,3 +5,5 @@ Pending removal in Python 3.20 Use :c:func:`PyComplex_AsCComplex` and :c:func:`PyComplex_FromCComplex` to convert a Python complex number to/from the C :c:type:`Py_complex` representation. + +* Macros :c:macro:`!Py_MATH_PIl` and :c:macro:`!Py_MATH_El`. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index e0b0471567c..1ba394a1967 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1076,6 +1076,10 @@ Deprecated C APIs since 3.15 and will be removed in 3.17. (Contributed by Nikita Sobolev in :gh:`136355`.) +* :c:macro:`!Py_MATH_El` and :c:macro:`!Py_MATH_PIl` are deprecated + since 3.15 and will be removed in 3.20. + (Contributed by Sergey B Kirpichev in :gh:`141004`.) + .. Add C API deprecations above alphabetically, not here at the end. diff --git a/Include/pymath.h b/Include/pymath.h index e2919c7b527..0f9f0f3b299 100644 --- a/Include/pymath.h +++ b/Include/pymath.h @@ -7,6 +7,7 @@ /* High precision definition of pi and e (Euler) * The values are taken from libc6's math.h. */ +// Deprecated since Python 3.15. #ifndef Py_MATH_PIl #define Py_MATH_PIl 3.1415926535897932384626433832795029L #endif @@ -14,6 +15,7 @@ #define Py_MATH_PI 3.14159265358979323846 #endif +// Deprecated since Python 3.15. #ifndef Py_MATH_El #define Py_MATH_El 2.7182818284590452353602874713526625L #endif diff --git a/Misc/NEWS.d/next/C_API/2025-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst b/Misc/NEWS.d/next/C_API/2025-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst new file mode 100644 index 00000000000..5f3ccd62016 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst @@ -0,0 +1 @@ +:c:macro:`!Py_MATH_El` and :c:macro:`!Py_MATH_PIl` are deprecated. From f835552946e29ec20144c359b8822f9e421d4d64 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Mon, 10 Nov 2025 21:19:13 +0500 Subject: [PATCH 124/313] GH-141212: Fix possible memory leak in gc_mark_span_push (gh-141213) --- Python/gc_free_threading.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index f39793c3eeb..b183062eff7 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -675,10 +675,11 @@ gc_mark_span_push(gc_span_stack_t *ss, PyObject **start, PyObject **end) else { ss->capacity *= 2; } - ss->stack = (gc_span_t *)PyMem_Realloc(ss->stack, ss->capacity * sizeof(gc_span_t)); - if (ss->stack == NULL) { + gc_span_t *new_stack = (gc_span_t *)PyMem_Realloc(ss->stack, ss->capacity * sizeof(gc_span_t)); + if (new_stack == NULL) { return -1; } + ss->stack = new_stack; } assert(end > start); ss->stack[ss->size].start = start; From ed0a5fd8cacb1964111d03ff37627f6bea5e6026 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Mon, 10 Nov 2025 17:46:41 +0000 Subject: [PATCH 125/313] gh-141004: Document `PyType_FastSubclass` (GH-141313) Co-authored-by: Peter Bierma --- Doc/c-api/type.rst | 12 ++++++++++++ Doc/c-api/typeobj.rst | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 5bdbff4e0ad..479ede70b01 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -133,6 +133,18 @@ Type Objects Type features are denoted by single bit flags. +.. c:function:: int PyType_FastSubclass(PyTypeObject *type, int flag) + + Return non-zero if the type object *type* sets the subclass flag *flag*. + Subclass flags are denoted by + :c:macro:`Py_TPFLAGS_*_SUBCLASS `. + This function is used by many ``_Check`` functions for common types. + + .. seealso:: + :c:func:`PyObject_TypeCheck`, which is used as a slower alternative in + ``_Check`` functions for types that don't come with subclass flags. + + .. c:function:: int PyType_IS_GC(PyTypeObject *o) Return true if the type object includes support for the cycle detector; this diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 9d23aea5734..34d19acdf17 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1351,8 +1351,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_BASE_EXC_SUBCLASS .. c:macro:: Py_TPFLAGS_TYPE_SUBCLASS - These flags are used by functions such as - :c:func:`PyLong_Check` to quickly determine if a type is a subclass + Functions such as :c:func:`PyLong_Check` will call :c:func:`PyType_FastSubclass` + with one of these flags to quickly determine if a type is a subclass of a built-in type; such specific checks are faster than a generic check, like :c:func:`PyObject_IsInstance`. Custom types that inherit from built-ins should have their :c:member:`~PyTypeObject.tp_flags` From 86513f6c2ebdd1fb692c39b84786ea41d88c84fd Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Mon, 10 Nov 2025 16:35:47 -0500 Subject: [PATCH 126/313] gh-141004: Document missing frame APIs (GH-141189) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/frame.rst | 57 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 1a52e146a69..fb17cf7f1da 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -29,6 +29,12 @@ See also :ref:`Reflection `. Previously, this type was only available after including ````. +.. c:function:: PyFrameObject *PyFrame_New(PyThreadState *tstate, PyCodeObject *code, PyObject *globals, PyObject *locals) + + Create a new frame object. This function returns a :term:`strong reference` + to the new frame object on success, and returns ``NULL`` with an exception + set on failure. + .. c:function:: int PyFrame_Check(PyObject *obj) Return non-zero if *obj* is a frame object. @@ -161,6 +167,57 @@ See :pep:`667` for more information. Return non-zero if *obj* is a frame :func:`locals` proxy. + +Legacy Local Variable APIs +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These APIs are :term:`soft deprecated`. As of Python 3.13, they do nothing. +They exist solely for backwards compatibility. + + +.. c:function:: void PyFrame_LocalsToFast(PyFrameObject *f, int clear) + + This function is :term:`soft deprecated` and does nothing. + + Prior to Python 3.13, this function would copy the :attr:`~frame.f_locals` + attribute of *f* to the internal "fast" array of local variables, allowing + changes in frame objects to be visible to the interpreter. If *clear* was + true, this function would process variables that were unset in the locals + dictionary. + + .. versionchanged:: 3.13 + This function now does nothing. + + +.. c:function:: void PyFrame_FastToLocals(PyFrameObject *f) + + This function is :term:`soft deprecated` and does nothing. + + Prior to Python 3.13, this function would copy the internal "fast" array + of local variables (which is used by the interpreter) to the + :attr:`~frame.f_locals` attribute of *f*, allowing changes in local + variables to be visible to frame objects. + + .. versionchanged:: 3.13 + This function now does nothing. + + +.. c:function:: int PyFrame_FastToLocalsWithError(PyFrameObject *f) + + This function is :term:`soft deprecated` and does nothing. + + Prior to Python 3.13, this function was similar to + :c:func:`PyFrame_FastToLocals`, but would return ``0`` on success, and + ``-1`` with an exception set on failure. + + .. versionchanged:: 3.13 + This function now does nothing. + + +.. seealso:: + :pep:`667` + + Internal Frames ^^^^^^^^^^^^^^^ From 46b58e1bb9e1e17d855588935f5a259be960a3a1 Mon Sep 17 00:00:00 2001 From: Louis Date: Tue, 11 Nov 2025 05:50:30 +0100 Subject: [PATCH 127/313] gh-140578: Doc: Remove sencence implying that concurrent.futures.ThreadPoolExecutor does not exist (#140689) * Doc: Remove sencence implying that concurrent.futures.ThreadPoolExecutor does not exist Closes #140578 * Add NEWS.d entry for gh-140578 --------- Co-authored-by: Louis Paternault --- Doc/library/multiprocessing.rst | 7 +++++-- .../2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index d18ada3511d..714207cb0ae 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -22,8 +22,7 @@ to this, the :mod:`multiprocessing` module allows the programmer to fully leverage multiple processors on a given machine. It runs on both POSIX and Windows. -The :mod:`multiprocessing` module also introduces APIs which do not have -analogs in the :mod:`threading` module. A prime example of this is the +The :mod:`multiprocessing` module also introduces the :class:`~multiprocessing.pool.Pool` object which offers a convenient means of parallelizing the execution of a function across multiple input values, distributing the input data across processes (data parallelism). The following @@ -44,6 +43,10 @@ will print to standard output :: [1, 4, 9] +The :mod:`multiprocessing` module also introduces APIs which do not have +analogs in the :mod:`threading` module, like the ability to :meth:`terminate +`, :meth:`interrupt ` or :meth:`kill +` a running process. .. seealso:: diff --git a/Misc/NEWS.d/next/Documentation/2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst b/Misc/NEWS.d/next/Documentation/2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst new file mode 100644 index 00000000000..702d38d4d24 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst @@ -0,0 +1,3 @@ +Remove outdated sencence in the documentation for :mod:`multiprocessing`, +that implied that :class:`concurrent.futures.ThreadPoolExecutor` did not +exist. From 9cb8c52d5e9a83efe4fa3878db06befd9df52f54 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 11 Nov 2025 05:59:16 +0100 Subject: [PATCH 128/313] gh-140485: Catch ChildProcessError in multiprocessing resource tracker (GH-141132) --- Lib/multiprocessing/resource_tracker.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index c53092f6e34..38fcaed48fa 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -111,7 +111,12 @@ def _stop_locked( close(self._fd) self._fd = None - _, status = waitpid(self._pid, 0) + try: + _, status = waitpid(self._pid, 0) + except ChildProcessError: + self._pid = None + self._exitcode = None + return self._pid = None From 92741c59f89e114474bdb2cb539107ef6bae0b9c Mon Sep 17 00:00:00 2001 From: Krishna Chaitanya <141550576+XChaitanyaX@users.noreply.github.com> Date: Tue, 11 Nov 2025 11:32:46 +0530 Subject: [PATCH 129/313] gh-140379: add hyperlinks to list and set (GH-140399) add hyperlinks to list and set --- Doc/tutorial/datastructures.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 1332c53f396..7e02e74177c 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -12,9 +12,8 @@ and adds some new things as well. More on Lists ============= -The list data type has some more methods. Here are all of the methods of list -objects: - +The :ref:`list ` data type has some more methods. Here are all +of the methods of list objects: .. method:: list.append(x) :noindex: @@ -445,10 +444,11 @@ packing and sequence unpacking. Sets ==== -Python also includes a data type for *sets*. A set is an unordered collection -with no duplicate elements. Basic uses include membership testing and -eliminating duplicate entries. Set objects also support mathematical operations -like union, intersection, difference, and symmetric difference. +Python also includes a data type for :ref:`sets `. A set is +an unordered collection with no duplicate elements. Basic uses include +membership testing and eliminating duplicate entries. Set objects also +support mathematical operations like union, intersection, difference, and +symmetric difference. Curly braces or the :func:`set` function can be used to create sets. Note: to create an empty set you have to use ``set()``, not ``{}``; the latter creates an From 8435a2278f964f48d36edbc5092be5ebecfcb120 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 11 Nov 2025 09:21:24 +0100 Subject: [PATCH 130/313] gh-141376: Fix exported symbols (GH-141377) * gh-141376: Fix exported symbols * _io module: add "_Py_" prefix to "spec" variables. For example, rename bufferedrandom_spec to _Py_bufferedrandom_spec. * typevarobject.c: add "static" to "spec" and "slots" variables. * import.c: add "static" to "pkgcontext" variable. * No longer export textiowrapper_slots --- Modules/_io/_iomodule.c | 30 +++++++++++++++--------------- Modules/_io/_iomodule.h | 30 +++++++++++++++--------------- Modules/_io/bufferedio.c | 10 +++++----- Modules/_io/bytesio.c | 4 ++-- Modules/_io/fileio.c | 2 +- Modules/_io/iobase.c | 4 ++-- Modules/_io/stringio.c | 2 +- Modules/_io/textio.c | 8 ++++---- Modules/_io/winconsoleio.c | 2 +- Objects/typevarobject.c | 16 ++++++++-------- Python/import.c | 2 +- 11 files changed, 55 insertions(+), 55 deletions(-) diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 27483494559..433d68d515c 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -681,40 +681,40 @@ iomodule_exec(PyObject *m) } // Base classes - ADD_TYPE(m, state->PyIncrementalNewlineDecoder_Type, &nldecoder_spec, NULL); - ADD_TYPE(m, state->PyBytesIOBuffer_Type, &bytesiobuf_spec, NULL); - ADD_TYPE(m, state->PyIOBase_Type, &iobase_spec, NULL); + ADD_TYPE(m, state->PyIncrementalNewlineDecoder_Type, &_Py_nldecoder_spec, NULL); + ADD_TYPE(m, state->PyBytesIOBuffer_Type, &_Py_bytesiobuf_spec, NULL); + ADD_TYPE(m, state->PyIOBase_Type, &_Py_iobase_spec, NULL); // PyIOBase_Type subclasses - ADD_TYPE(m, state->PyTextIOBase_Type, &textiobase_spec, + ADD_TYPE(m, state->PyTextIOBase_Type, &_Py_textiobase_spec, state->PyIOBase_Type); - ADD_TYPE(m, state->PyBufferedIOBase_Type, &bufferediobase_spec, + ADD_TYPE(m, state->PyBufferedIOBase_Type, &_Py_bufferediobase_spec, state->PyIOBase_Type); - ADD_TYPE(m, state->PyRawIOBase_Type, &rawiobase_spec, + ADD_TYPE(m, state->PyRawIOBase_Type, &_Py_rawiobase_spec, state->PyIOBase_Type); // PyBufferedIOBase_Type(PyIOBase_Type) subclasses - ADD_TYPE(m, state->PyBytesIO_Type, &bytesio_spec, state->PyBufferedIOBase_Type); - ADD_TYPE(m, state->PyBufferedWriter_Type, &bufferedwriter_spec, + ADD_TYPE(m, state->PyBytesIO_Type, &_Py_bytesio_spec, state->PyBufferedIOBase_Type); + ADD_TYPE(m, state->PyBufferedWriter_Type, &_Py_bufferedwriter_spec, state->PyBufferedIOBase_Type); - ADD_TYPE(m, state->PyBufferedReader_Type, &bufferedreader_spec, + ADD_TYPE(m, state->PyBufferedReader_Type, &_Py_bufferedreader_spec, state->PyBufferedIOBase_Type); - ADD_TYPE(m, state->PyBufferedRWPair_Type, &bufferedrwpair_spec, + ADD_TYPE(m, state->PyBufferedRWPair_Type, &_Py_bufferedrwpair_spec, state->PyBufferedIOBase_Type); - ADD_TYPE(m, state->PyBufferedRandom_Type, &bufferedrandom_spec, + ADD_TYPE(m, state->PyBufferedRandom_Type, &_Py_bufferedrandom_spec, state->PyBufferedIOBase_Type); // PyRawIOBase_Type(PyIOBase_Type) subclasses - ADD_TYPE(m, state->PyFileIO_Type, &fileio_spec, state->PyRawIOBase_Type); + ADD_TYPE(m, state->PyFileIO_Type, &_Py_fileio_spec, state->PyRawIOBase_Type); #ifdef HAVE_WINDOWS_CONSOLE_IO - ADD_TYPE(m, state->PyWindowsConsoleIO_Type, &winconsoleio_spec, + ADD_TYPE(m, state->PyWindowsConsoleIO_Type, &_Py_winconsoleio_spec, state->PyRawIOBase_Type); #endif // PyTextIOBase_Type(PyIOBase_Type) subclasses - ADD_TYPE(m, state->PyStringIO_Type, &stringio_spec, state->PyTextIOBase_Type); - ADD_TYPE(m, state->PyTextIOWrapper_Type, &textiowrapper_spec, + ADD_TYPE(m, state->PyStringIO_Type, &_Py_stringio_spec, state->PyTextIOBase_Type); + ADD_TYPE(m, state->PyTextIOWrapper_Type, &_Py_textiowrapper_spec, state->PyTextIOBase_Type); #undef ADD_TYPE diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index 18cf20edf26..4ae487c8e2a 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -9,23 +9,23 @@ #include "structmember.h" /* Type specs */ -extern PyType_Spec bufferediobase_spec; -extern PyType_Spec bufferedrandom_spec; -extern PyType_Spec bufferedreader_spec; -extern PyType_Spec bufferedrwpair_spec; -extern PyType_Spec bufferedwriter_spec; -extern PyType_Spec bytesio_spec; -extern PyType_Spec bytesiobuf_spec; -extern PyType_Spec fileio_spec; -extern PyType_Spec iobase_spec; -extern PyType_Spec nldecoder_spec; -extern PyType_Spec rawiobase_spec; -extern PyType_Spec stringio_spec; -extern PyType_Spec textiobase_spec; -extern PyType_Spec textiowrapper_spec; +extern PyType_Spec _Py_bufferediobase_spec; +extern PyType_Spec _Py_bufferedrandom_spec; +extern PyType_Spec _Py_bufferedreader_spec; +extern PyType_Spec _Py_bufferedrwpair_spec; +extern PyType_Spec _Py_bufferedwriter_spec; +extern PyType_Spec _Py_bytesio_spec; +extern PyType_Spec _Py_bytesiobuf_spec; +extern PyType_Spec _Py_fileio_spec; +extern PyType_Spec _Py_iobase_spec; +extern PyType_Spec _Py_nldecoder_spec; +extern PyType_Spec _Py_rawiobase_spec; +extern PyType_Spec _Py_stringio_spec; +extern PyType_Spec _Py_textiobase_spec; +extern PyType_Spec _Py_textiowrapper_spec; #ifdef HAVE_WINDOWS_CONSOLE_IO -extern PyType_Spec winconsoleio_spec; +extern PyType_Spec _Py_winconsoleio_spec; #endif /* These functions are used as METH_NOARGS methods, are normally called diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 0b4bc4c6b8a..4602f2b42a6 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -2537,7 +2537,7 @@ static PyType_Slot bufferediobase_slots[] = { }; /* Do not set Py_TPFLAGS_HAVE_GC so that tp_traverse and tp_clear are inherited */ -PyType_Spec bufferediobase_spec = { +PyType_Spec _Py_bufferediobase_spec = { .name = "_io._BufferedIOBase", .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), @@ -2600,7 +2600,7 @@ static PyType_Slot bufferedreader_slots[] = { {0, NULL}, }; -PyType_Spec bufferedreader_spec = { +PyType_Spec _Py_bufferedreader_spec = { .name = "_io.BufferedReader", .basicsize = sizeof(buffered), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | @@ -2658,7 +2658,7 @@ static PyType_Slot bufferedwriter_slots[] = { {0, NULL}, }; -PyType_Spec bufferedwriter_spec = { +PyType_Spec _Py_bufferedwriter_spec = { .name = "_io.BufferedWriter", .basicsize = sizeof(buffered), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | @@ -2708,7 +2708,7 @@ static PyType_Slot bufferedrwpair_slots[] = { {0, NULL}, }; -PyType_Spec bufferedrwpair_spec = { +PyType_Spec _Py_bufferedrwpair_spec = { .name = "_io.BufferedRWPair", .basicsize = sizeof(rwpair), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | @@ -2776,7 +2776,7 @@ static PyType_Slot bufferedrandom_slots[] = { {0, NULL}, }; -PyType_Spec bufferedrandom_spec = { +PyType_Spec _Py_bufferedrandom_spec = { .name = "_io.BufferedRandom", .basicsize = sizeof(buffered), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 30d61f9d68e..d6bfb93177c 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -1156,7 +1156,7 @@ static PyType_Slot bytesio_slots[] = { {0, NULL}, }; -PyType_Spec bytesio_spec = { +PyType_Spec _Py_bytesio_spec = { .name = "_io.BytesIO", .basicsize = sizeof(bytesio), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | @@ -1246,7 +1246,7 @@ static PyType_Slot bytesiobuf_slots[] = { {0, NULL}, }; -PyType_Spec bytesiobuf_spec = { +PyType_Spec _Py_bytesiobuf_spec = { .name = "_io._BytesIOBuffer", .basicsize = sizeof(bytesiobuf), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index b84c1bd3e22..2544ff4ea91 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -1329,7 +1329,7 @@ static PyType_Slot fileio_slots[] = { {0, NULL}, }; -PyType_Spec fileio_spec = { +PyType_Spec _Py_fileio_spec = { .name = "_io.FileIO", .basicsize = sizeof(fileio), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index e304fc8bee2..f1c2fe17801 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -885,7 +885,7 @@ static PyType_Slot iobase_slots[] = { {0, NULL}, }; -PyType_Spec iobase_spec = { +PyType_Spec _Py_iobase_spec = { .name = "_io._IOBase", .basicsize = sizeof(iobase), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | @@ -1046,7 +1046,7 @@ static PyType_Slot rawiobase_slots[] = { }; /* Do not set Py_TPFLAGS_HAVE_GC so that tp_traverse and tp_clear are inherited */ -PyType_Spec rawiobase_spec = { +PyType_Spec _Py_rawiobase_spec = { .name = "_io._RawIOBase", .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 20b7cfc0088..781ca4327f9 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -1094,7 +1094,7 @@ static PyType_Slot stringio_slots[] = { {0, NULL}, }; -PyType_Spec stringio_spec = { +PyType_Spec _Py_stringio_spec = { .name = "_io.StringIO", .basicsize = sizeof(stringio), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index c462bd2ac57..84b7d9df400 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -208,7 +208,7 @@ static PyType_Slot textiobase_slots[] = { }; /* Do not set Py_TPFLAGS_HAVE_GC so that tp_traverse and tp_clear are inherited */ -PyType_Spec textiobase_spec = { +PyType_Spec _Py_textiobase_spec = { .name = "_io._TextIOBase", .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), @@ -3352,7 +3352,7 @@ static PyType_Slot nldecoder_slots[] = { {0, NULL}, }; -PyType_Spec nldecoder_spec = { +PyType_Spec _Py_nldecoder_spec = { .name = "_io.IncrementalNewlineDecoder", .basicsize = sizeof(nldecoder_object), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | @@ -3404,7 +3404,7 @@ static PyGetSetDef textiowrapper_getset[] = { {NULL} }; -PyType_Slot textiowrapper_slots[] = { +static PyType_Slot textiowrapper_slots[] = { {Py_tp_dealloc, textiowrapper_dealloc}, {Py_tp_repr, textiowrapper_repr}, {Py_tp_doc, (void *)_io_TextIOWrapper___init____doc__}, @@ -3418,7 +3418,7 @@ PyType_Slot textiowrapper_slots[] = { {0, NULL}, }; -PyType_Spec textiowrapper_spec = { +PyType_Spec _Py_textiowrapper_spec = { .name = "_io.TextIOWrapper", .basicsize = sizeof(textio), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 950b7fe241c..677d7e85d4e 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -1253,7 +1253,7 @@ static PyType_Slot winconsoleio_slots[] = { {0, NULL}, }; -PyType_Spec winconsoleio_spec = { +PyType_Spec _Py_winconsoleio_spec = { .name = "_io._WindowsConsoleIO", .basicsize = sizeof(winconsoleio), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 75a69d4bc3e..8e43962c7e3 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -251,7 +251,7 @@ static PyType_Slot constevaluator_slots[] = { {0, NULL}, }; -PyType_Spec constevaluator_spec = { +static PyType_Spec constevaluator_spec = { .name = "_typing._ConstEvaluator", .basicsize = sizeof(constevaluatorobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE @@ -930,7 +930,7 @@ static PyType_Slot typevar_slots[] = { {0, NULL}, }; -PyType_Spec typevar_spec = { +static PyType_Spec typevar_spec = { .name = "typing.TypeVar", .basicsize = sizeof(typevarobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE @@ -1078,7 +1078,7 @@ static PyType_Slot paramspecargs_slots[] = { {0, NULL}, }; -PyType_Spec paramspecargs_spec = { +static PyType_Spec paramspecargs_spec = { .name = "typing.ParamSpecArgs", .basicsize = sizeof(paramspecattrobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE @@ -1158,7 +1158,7 @@ static PyType_Slot paramspeckwargs_slots[] = { {0, NULL}, }; -PyType_Spec paramspeckwargs_spec = { +static PyType_Spec paramspeckwargs_spec = { .name = "typing.ParamSpecKwargs", .basicsize = sizeof(paramspecattrobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE @@ -1509,7 +1509,7 @@ static PyType_Slot paramspec_slots[] = { {0, 0}, }; -PyType_Spec paramspec_spec = { +static PyType_Spec paramspec_spec = { .name = "typing.ParamSpec", .basicsize = sizeof(paramspecobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE @@ -1789,7 +1789,7 @@ Note that only TypeVarTuples defined in the global scope can be\n\ pickled.\n\ "); -PyType_Slot typevartuple_slots[] = { +static PyType_Slot typevartuple_slots[] = { {Py_tp_doc, (void *)typevartuple_doc}, {Py_tp_members, typevartuple_members}, {Py_tp_methods, typevartuple_methods}, @@ -1805,7 +1805,7 @@ PyType_Slot typevartuple_slots[] = { {0, 0}, }; -PyType_Spec typevartuple_spec = { +static PyType_Spec typevartuple_spec = { .name = "typing.TypeVarTuple", .basicsize = sizeof(typevartupleobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_DICT @@ -2347,7 +2347,7 @@ static PyType_Slot generic_slots[] = { {0, NULL}, }; -PyType_Spec generic_spec = { +static PyType_Spec generic_spec = { .name = "typing.Generic", .basicsize = sizeof(PyObject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, diff --git a/Python/import.c b/Python/import.c index 6cf4a061ca6..2afa7c15e6a 100644 --- a/Python/import.c +++ b/Python/import.c @@ -804,7 +804,7 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) substitute this (if the name actually matches). */ -_Py_thread_local const char *pkgcontext = NULL; +static _Py_thread_local const char *pkgcontext = NULL; # undef PKGCONTEXT # define PKGCONTEXT pkgcontext From d69447445cbacf7537bf59c5c683a3b17060312d Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Tue, 11 Nov 2025 13:13:59 +0300 Subject: [PATCH 131/313] gh-141004: document Py_INFINITY and Py_NAN macros (#141145) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/float.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 9e703a46445..eae4792af7d 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -78,6 +78,24 @@ Floating-Point Objects Return the minimum normalized positive float *DBL_MIN* as C :c:expr:`double`. +.. c:macro:: Py_INFINITY + + This macro expands a to constant expression of type :c:expr:`double`, that + represents the positive infinity. + + On most platforms, this is equivalent to the :c:macro:`!INFINITY` macro from + the C11 standard ```` header. + + +.. c:macro:: Py_NAN + + This macro expands a to constant expression of type :c:expr:`double`, that + represents a quiet not-a-number (qNaN) value. + + On most platforms, this is equivalent to the :c:macro:`!NAN` macro from + the C11 standard ```` header. + + .. c:macro:: Py_MATH_El High precision (long double) definition of :data:`~math.e` constant. From 799326b0a93ae6375f153d5a6607e7dc5e0690b2 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 11 Nov 2025 13:52:13 +0100 Subject: [PATCH 132/313] gh-141169: Re-raise exception from findfuncptr (GH-141349) --- Include/internal/pycore_importdl.h | 32 ++++++++++++++++++++++++++---- Python/importdl.c | 24 +++------------------- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index 12a32a5f70e..f60c5510d20 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -14,6 +14,34 @@ extern "C" { extern const char *_PyImport_DynLoadFiletab[]; +#ifdef HAVE_DYNAMIC_LOADING +/* ./configure sets HAVE_DYNAMIC_LOADING if dynamic loading of modules is + supported on this platform. configure will then compile and link in one + of the dynload_*.c files, as appropriate. We will call a function in + those modules to get a function pointer to the module's init function. + + The function should return: + - The function pointer on success + - NULL with exception set if the library cannot be loaded + - NULL *without* an extension set if the library could be loaded but the + function cannot be found in it. +*/ +#ifdef MS_WINDOWS +#include +typedef FARPROC dl_funcptr; +extern dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, + const char *shortname, + PyObject *pathname, + FILE *fp); +#else +typedef void (*dl_funcptr)(void); +extern dl_funcptr _PyImport_FindSharedFuncptr(const char *prefix, + const char *shortname, + const char *pathname, FILE *fp); +#endif + +#endif /* HAVE_DYNAMIC_LOADING */ + typedef enum ext_module_kind { _Py_ext_module_kind_UNKNOWN = 0, @@ -112,8 +140,6 @@ extern int _PyImport_RunModInitFunc( #define MAXSUFFIXSIZE 12 #ifdef MS_WINDOWS -#include -typedef FARPROC dl_funcptr; #ifdef Py_DEBUG # define PYD_DEBUG_SUFFIX "_d" @@ -136,8 +162,6 @@ typedef FARPROC dl_funcptr; #define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX "." PYD_SOABI ".pyd" #define PYD_UNTAGGED_SUFFIX PYD_DEBUG_SUFFIX ".pyd" -#else -typedef void (*dl_funcptr)(void); #endif diff --git a/Python/importdl.c b/Python/importdl.c index 23a55c39677..61a9cdaf375 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -10,27 +10,6 @@ #include "pycore_runtime.h" // _Py_ID() -/* ./configure sets HAVE_DYNAMIC_LOADING if dynamic loading of modules is - supported on this platform. configure will then compile and link in one - of the dynload_*.c files, as appropriate. We will call a function in - those modules to get a function pointer to the module's init function. -*/ -#ifdef HAVE_DYNAMIC_LOADING - -#ifdef MS_WINDOWS -extern dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, - const char *shortname, - PyObject *pathname, - FILE *fp); -#else -extern dl_funcptr _PyImport_FindSharedFuncptr(const char *prefix, - const char *shortname, - const char *pathname, FILE *fp); -#endif - -#endif /* HAVE_DYNAMIC_LOADING */ - - /***********************************/ /* module info to use when loading */ /***********************************/ @@ -414,6 +393,9 @@ _PyImport_GetModuleExportHooks( *modexport = (PyModExportFunction)exportfunc; return 2; } + if (PyErr_Occurred()) { + return -1; + } exportfunc = findfuncptr( info->hook_prefixes->init_prefix, From 7211a34fe1d9704935342af8c9b46725629f2d97 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Tue, 11 Nov 2025 20:02:32 +0530 Subject: [PATCH 133/313] gh-132657: optimize `PySet_Contains` for `frozenset` (#141183) --- Objects/setobject.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 213bd821d8a..2401176576e 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2747,7 +2747,9 @@ PySet_Contains(PyObject *anyset, PyObject *key) PyErr_BadInternalCall(); return -1; } - + if (PyFrozenSet_CheckExact(anyset)) { + return set_contains_key((PySetObject *)anyset, key); + } int rv; Py_BEGIN_CRITICAL_SECTION(anyset); rv = set_contains_key((PySetObject *)anyset, key); From d890aba748e5213585f9f906888999227dc3fa9c Mon Sep 17 00:00:00 2001 From: John Franey <1728528+johnfraney@users.noreply.github.com> Date: Tue, 11 Nov 2025 10:33:56 -0400 Subject: [PATCH 134/313] gh-140942: Add MIME type for .cjs extension (#140937) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/whatsnew/3.15.rst | 1 + Lib/mimetypes.py | 1 + .../2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst | 2 ++ 3 files changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 1ba394a1967..ef18d36e4d4 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -451,6 +451,7 @@ math mimetypes --------- +* Add ``application/node`` MIME type for ``.cjs`` extension. (Contributed by John Franey in :gh:`140937`.) * Add ``application/toml``. (Contributed by Gil Forcada in :gh:`139959`.) * Rename ``application/x-texinfo`` to ``application/texinfo``. (Contributed by Charlie Lin in :gh:`140165`) diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 48a9f430d45..d6896fc4042 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -486,6 +486,7 @@ def _default_mime_types(): '.wiz' : 'application/msword', '.nq' : 'application/n-quads', '.nt' : 'application/n-triples', + '.cjs' : 'application/node', '.bin' : 'application/octet-stream', '.a' : 'application/octet-stream', '.dll' : 'application/octet-stream', diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst new file mode 100644 index 00000000000..20cfeca1e71 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst @@ -0,0 +1,2 @@ +Add ``.cjs`` to :mod:`mimetypes` to give CommonJS modules a MIME type of +``application/node``. From 759a048d4bea522fda2fe929be0fba1650c62b0e Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Tue, 11 Nov 2025 12:22:16 -0500 Subject: [PATCH 135/313] gh-141004: Document `PyType_Unwatch` (GH-141414) --- Doc/c-api/type.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 479ede70b01..29ffeb7c483 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -116,6 +116,20 @@ Type Objects .. versionadded:: 3.12 +.. c:function:: int PyType_Unwatch(int watcher_id, PyObject *type) + + Mark *type* as not watched. This undoes a previous call to + :c:func:`PyType_Watch`. *type* must not be ``NULL``. + + An extension should never call this function with a *watcher_id* that was + not returned to it by a previous call to :c:func:`PyType_AddWatcher`. + + On success, this function returns ``0``. On failure, this function returns + ``-1`` with an exception set. + + .. versionadded:: 3.12 + + .. c:type:: int (*PyType_WatchCallback)(PyObject *type) Type of a type-watcher callback function. From 713edbcebfdb5aa83e9bf376ebc40255ccacd235 Mon Sep 17 00:00:00 2001 From: Tan Long Date: Wed, 12 Nov 2025 03:27:21 +0800 Subject: [PATCH 136/313] gh-141415: Remove unused variables and comment in `_pyrepl.windows_console.py` (#141416) --- Lib/_pyrepl/windows_console.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py index c56dcd6d7dd..f9f5988af0b 100644 --- a/Lib/_pyrepl/windows_console.py +++ b/Lib/_pyrepl/windows_console.py @@ -249,22 +249,10 @@ def input_hook(self): def __write_changed_line( self, y: int, oldline: str, newline: str, px_coord: int ) -> None: - # this is frustrating; there's no reason to test (say) - # self.dch1 inside the loop -- but alternative ways of - # structuring this function are equally painful (I'm trying to - # avoid writing code generators these days...) minlen = min(wlen(oldline), wlen(newline)) x_pos = 0 x_coord = 0 - px_pos = 0 - j = 0 - for c in oldline: - if j >= px_coord: - break - j += wlen(c) - px_pos += 1 - # reuse the oldline as much as possible, but stop as soon as we # encounter an ESCAPE, because it might be the start of an escape # sequence @@ -358,7 +346,6 @@ def prepare(self) -> None: self.height, self.width = self.getheightwidth() self.posxy = 0, 0 - self.__gone_tall = 0 self.__offset = 0 if self.__vt_support: From 0f09bda643d778fb20fb79fecdfd09f20f9d9717 Mon Sep 17 00:00:00 2001 From: yihong Date: Wed, 12 Nov 2025 03:27:56 +0800 Subject: [PATCH 137/313] gh-140193: Forward port test_exec_set_nomemory_hang from 3.13 (GH-140187) * chore: test_exec_set_nomemory_hang from 3.13 Signed-off-by: yihong0618 * fix: apply comments Signed-off-by: yihong0618 * Update Lib/test/test_exceptions.py Co-authored-by: Peter Bierma * Update Lib/test/test_exceptions.py Co-authored-by: Peter Bierma * fix: windows too long name 60 times is enough Signed-off-by: yihong0618 --------- Signed-off-by: yihong0618 Co-authored-by: Peter Bierma --- Lib/test/test_exceptions.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 5262b58908a..6f212d2f91e 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1923,6 +1923,39 @@ def test_keyerror_context(self): exc2 = None + @cpython_only + # Python built with Py_TRACE_REFS fail with a fatal error in + # _PyRefchain_Trace() on memory allocation error. + @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') + def test_exec_set_nomemory_hang(self): + import_module("_testcapi") + # gh-134163: A MemoryError inside code that was wrapped by a try/except + # block would lead to an infinite loop. + + # The frame_lasti needs to be greater than 257 to prevent + # PyLong_FromLong() from returning cached integers, which + # don't require a memory allocation. Prepend some dummy code + # to artificially increase the instruction index. + warmup_code = "a = list(range(0, 1))\n" * 60 + user_input = warmup_code + dedent(""" + try: + import _testcapi + _testcapi.set_nomemory(0) + b = list(range(1000, 2000)) + except Exception as e: + import traceback + traceback.print_exc() + """) + with SuppressCrashReport(): + with script_helper.spawn_python('-c', user_input) as p: + p.wait() + output = p.stdout.read() + + self.assertIn(p.returncode, (0, 1)) + self.assertGreater(len(output), 0) # At minimum, should not hang + self.assertIn(b"MemoryError", output) + + class NameErrorTests(unittest.TestCase): def test_name_error_has_name(self): try: From c903d768322989e9f8ba79e38ee87e14c85c5430 Mon Sep 17 00:00:00 2001 From: Marco Barbosa Date: Tue, 11 Nov 2025 16:35:55 -0300 Subject: [PATCH 138/313] gh-139533: fix refs to code without proper markups on turtledemo doc (GH-139534) gh-139533: fix refs to code without proper markups on turtledemo documentation --- Doc/library/turtle.rst | 124 ++++++++++++++++++++--------------------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 58b99e0d441..95a57c57e71 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -2801,68 +2801,68 @@ The demo scripts are: .. tabularcolumns:: |l|L|L| -+----------------+------------------------------+-----------------------+ -| Name | Description | Features | -+================+==============================+=======================+ -| bytedesign | complex classical | :func:`tracer`, delay,| -| | turtle graphics pattern | :func:`update` | -+----------------+------------------------------+-----------------------+ -| chaos | graphs Verhulst dynamics, | world coordinates | -| | shows that computer's | | -| | computations can generate | | -| | results sometimes against the| | -| | common sense expectations | | -+----------------+------------------------------+-----------------------+ -| clock | analog clock showing time | turtles as clock's | -| | of your computer | hands, ontimer | -+----------------+------------------------------+-----------------------+ -| colormixer | experiment with r, g, b | :func:`ondrag` | -+----------------+------------------------------+-----------------------+ -| forest | 3 breadth-first trees | randomization | -+----------------+------------------------------+-----------------------+ -| fractalcurves | Hilbert & Koch curves | recursion | -+----------------+------------------------------+-----------------------+ -| lindenmayer | ethnomathematics | L-System | -| | (indian kolams) | | -+----------------+------------------------------+-----------------------+ -| minimal_hanoi | Towers of Hanoi | Rectangular Turtles | -| | | as Hanoi discs | -| | | (shape, shapesize) | -+----------------+------------------------------+-----------------------+ -| nim | play the classical nim game | turtles as nimsticks, | -| | with three heaps of sticks | event driven (mouse, | -| | against the computer. | keyboard) | -+----------------+------------------------------+-----------------------+ -| paint | super minimalistic | :func:`onclick` | -| | drawing program | | -+----------------+------------------------------+-----------------------+ -| peace | elementary | turtle: appearance | -| | | and animation | -+----------------+------------------------------+-----------------------+ -| penrose | aperiodic tiling with | :func:`stamp` | -| | kites and darts | | -+----------------+------------------------------+-----------------------+ -| planet_and_moon| simulation of | compound shapes, | -| | gravitational system | :class:`Vec2D` | -+----------------+------------------------------+-----------------------+ -| rosette | a pattern from the wikipedia | :func:`clone`, | -| | article on turtle graphics | :func:`undo` | -+----------------+------------------------------+-----------------------+ -| round_dance | dancing turtles rotating | compound shapes, clone| -| | pairwise in opposite | shapesize, tilt, | -| | direction | get_shapepoly, update | -+----------------+------------------------------+-----------------------+ -| sorting_animate| visual demonstration of | simple alignment, | -| | different sorting methods | randomization | -+----------------+------------------------------+-----------------------+ -| tree | a (graphical) breadth | :func:`clone` | -| | first tree (using generators)| | -+----------------+------------------------------+-----------------------+ -| two_canvases | simple design | turtles on two | -| | | canvases | -+----------------+------------------------------+-----------------------+ -| yinyang | another elementary example | :func:`circle` | -+----------------+------------------------------+-----------------------+ ++------------------------+------------------------------+--------------------------------------+ +| Name | Description | Features | ++========================+==============================+======================================+ +| ``bytedesign`` | complex classical | :func:`tracer`, :func:`delay`, | +| | turtle graphics pattern | :func:`update` | ++------------------------+------------------------------+--------------------------------------+ +| ``chaos`` | graphs Verhulst dynamics, | world coordinates | +| | shows that computer's | | +| | computations can generate | | +| | results sometimes against the| | +| | common sense expectations | | ++------------------------+------------------------------+--------------------------------------+ +| ``clock`` | analog clock showing time | turtles as clock's | +| | of your computer | hands, :func:`ontimer` | ++------------------------+------------------------------+--------------------------------------+ +| ``colormixer`` | experiment with r, g, b | :func:`ondrag` | ++------------------------+------------------------------+--------------------------------------+ +| ``forest`` | 3 breadth-first trees | randomization | ++------------------------+------------------------------+--------------------------------------+ +| ``fractalcurves`` | Hilbert & Koch curves | recursion | ++------------------------+------------------------------+--------------------------------------+ +| ``lindenmayer`` | ethnomathematics | L-System | +| | (indian kolams) | | ++------------------------+------------------------------+--------------------------------------+ +| ``minimal_hanoi`` | Towers of Hanoi | Rectangular Turtles | +| | | as Hanoi discs | +| | | (:func:`shape`, :func:`shapesize`) | ++------------------------+------------------------------+--------------------------------------+ +| ``nim`` | play the classical nim game | turtles as nimsticks, | +| | with three heaps of sticks | event driven (mouse, | +| | against the computer. | keyboard) | ++------------------------+------------------------------+--------------------------------------+ +| ``paint`` | super minimalistic | :func:`onclick` | +| | drawing program | | ++------------------------+------------------------------+--------------------------------------+ +| ``peace`` | elementary | turtle: appearance | +| | | and animation | ++------------------------+------------------------------+--------------------------------------+ +| ``penrose`` | aperiodic tiling with | :func:`stamp` | +| | kites and darts | | ++------------------------+------------------------------+--------------------------------------+ +| ``planet_and_moon`` | simulation of | compound shapes, | +| | gravitational system | :class:`Vec2D` | ++------------------------+------------------------------+--------------------------------------+ +| ``rosette`` | a pattern from the wikipedia | :func:`clone`, | +| | article on turtle graphics | :func:`undo` | ++------------------------+------------------------------+--------------------------------------+ +| ``round_dance`` | dancing turtles rotating | compound shapes, :func:`clone` | +| | pairwise in opposite | :func:`shapesize`, :func:`tilt`, | +| | direction | :func:`get_shapepoly`, :func:`update`| ++------------------------+------------------------------+--------------------------------------+ +| ``sorting_animate`` | visual demonstration of | simple alignment, | +| | different sorting methods | randomization | ++------------------------+------------------------------+--------------------------------------+ +| ``tree`` | a (graphical) breadth | :func:`clone` | +| | first tree (using generators)| | ++------------------------+------------------------------+--------------------------------------+ +| ``two_canvases`` | simple design | turtles on two | +| | | canvases | ++------------------------+------------------------------+--------------------------------------+ +| ``yinyang`` | another elementary example | :func:`circle` | ++------------------------+------------------------------+--------------------------------------+ Have fun! From 336154f4b0dbcf1d9dbb461ae962d558ba60f452 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 11 Nov 2025 20:02:49 +0000 Subject: [PATCH 139/313] Add documentation for Python install manager's install_dir, global_dir and download_dir (GH-140223) --- Doc/using/windows.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 0b98cfb8d27..e6619b73bd2 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -457,6 +457,25 @@ customization. - Specify the default format used by the ``py list`` command. By default, ``table``. + * - ``install_dir`` + - (none) + - Specify the root directory that runtimes will be installed into. + If you change this setting, previously installed runtimes will not be + usable unless you move them to the new location. + + * - ``global_dir`` + - (none) + - Specify the directory where global commands (such as ``python3.14.exe``) + are stored. + This directory should be added to your :envvar:`PATH` to make the + commands available from your terminal. + + * - ``download_dir`` + - (none) + - Specify the directory where downloaded files are stored. + This directory is a temporary cache, and can be cleaned up from time to + time. + Dotted names should be nested inside JSON objects, for example, ``list.format`` would be specified as ``{"list": {"format": "table"}}``. From 2fb2b82161c6df57c4a247cb743816b79134e932 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Tue, 11 Nov 2025 23:16:46 +0300 Subject: [PATCH 140/313] gh-141367: Use actual SPECIALIZATION_THRESHOLD value in specialization related test (GH-141417) --- Lib/test/test_list.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 223f34fb696..642b54d3484 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -349,10 +349,12 @@ def test_deopt_from_append_list(self): # gh-132011: it used to crash, because # of `CALL_LIST_APPEND` specialization failure. code = textwrap.dedent(""" + import _testinternalcapi + l = [] def lappend(l, x, y): l.append((x, y)) - for x in range(3): + for x in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): lappend(l, None, None) try: lappend(list, None, None) From b5196fa15a6c5aaa90eafff06206f8e44a9da216 Mon Sep 17 00:00:00 2001 From: Aniket <148300120+Aniketsy@users.noreply.github.com> Date: Wed, 12 Nov 2025 01:55:26 +0530 Subject: [PATCH 141/313] gh-137339: Clarify host and port parameter behavior in smtplib.SMTP{_SSL} initialization (#137340) This also documents the previously undocumented default_port parameter. Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/library/smtplib.rst | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index c5a3de52090..3ee8b82a188 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -24,10 +24,13 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). .. class:: SMTP(host='', port=0, local_hostname=None[, timeout], source_address=None) An :class:`SMTP` instance encapsulates an SMTP connection. It has methods - that support a full repertoire of SMTP and ESMTP operations. If the optional - *host* and *port* parameters are given, the SMTP :meth:`connect` method is - called with those parameters during initialization. If specified, - *local_hostname* is used as the FQDN of the local host in the HELO/EHLO + that support a full repertoire of SMTP and ESMTP operations. + + If the host parameter is set to a truthy value, :meth:`SMTP.connect` is called with + host and port automatically when the object is created; otherwise, :meth:`!connect` must + be called manually. + + If specified, *local_hostname* is used as the FQDN of the local host in the HELO/EHLO command. Otherwise, the local hostname is found using :func:`socket.getfqdn`. If the :meth:`connect` call returns anything other than a success code, an :exc:`SMTPConnectError` is raised. The optional @@ -62,6 +65,10 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). ``smtplib.SMTP.send`` with arguments ``self`` and ``data``, where ``data`` is the bytes about to be sent to the remote host. + .. attribute:: SMTP.default_port + + The default port used for SMTP connections (25). + .. versionchanged:: 3.3 Support for the :keyword:`with` statement was added. @@ -80,15 +87,23 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). An :class:`SMTP_SSL` instance behaves exactly the same as instances of :class:`SMTP`. :class:`SMTP_SSL` should be used for situations where SSL is - required from the beginning of the connection and using :meth:`~SMTP.starttls` - is not appropriate. If *host* is not specified, the local host is used. If - *port* is zero, the standard SMTP-over-SSL port (465) is used. The optional - arguments *local_hostname*, *timeout* and *source_address* have the same + required from the beginning of the connection and using :meth:`SMTP.starttls` is + not appropriate. + + If the host parameter is set to a truthy value, :meth:`SMTP.connect` is called with host + and port automatically when the object is created; otherwise, :meth:`!SMTP.connect` must + be called manually. + + The optional arguments *local_hostname*, *timeout* and *source_address* have the same meaning as they do in the :class:`SMTP` class. *context*, also optional, can contain a :class:`~ssl.SSLContext` and allows configuring various aspects of the secure connection. Please read :ref:`ssl-security` for best practices. + .. attribute:: SMTP_SSL.default_port + + The default port used for SMTP-over-SSL connections (465). + .. versionchanged:: 3.3 *context* was added. @@ -259,6 +274,9 @@ An :class:`SMTP` instance has the following methods: 2-tuple of the response code and message sent by the server in its connection response. + If port is not changed from its default value of 0, the value of the :attr:`default_port` + attribute is used. + .. audit-event:: smtplib.connect self,host,port smtplib.SMTP.connect From 298e9074cdffb09d518e6aceea556e8f4a8a745d Mon Sep 17 00:00:00 2001 From: Alper Date: Tue, 11 Nov 2025 12:27:21 -0800 Subject: [PATCH 142/313] gh-140476: optimize `PySet_Add` for `frozenset` in free-threading (#140440) Avoids critical section in `PySet_Add` when adding items to newly created frozensets. Co-authored-by: Kumar Aditya --- ...-10-22-12-48-05.gh-issue-140476.F3-d1P.rst | 2 ++ Objects/setobject.c | 25 ++++++++++++------- 2 files changed, 18 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst new file mode 100644 index 00000000000..a24033208c5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst @@ -0,0 +1,2 @@ +Optimize :c:func:`PySet_Add` for :class:`frozenset` in :term:`free threaded +` build. diff --git a/Objects/setobject.c b/Objects/setobject.c index 2401176576e..85f4d7d4031 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2775,17 +2775,24 @@ PySet_Discard(PyObject *set, PyObject *key) int PySet_Add(PyObject *anyset, PyObject *key) { - if (!PySet_Check(anyset) && - (!PyFrozenSet_Check(anyset) || !_PyObject_IsUniquelyReferenced(anyset))) { - PyErr_BadInternalCall(); - return -1; + if (PySet_Check(anyset)) { + int rv; + Py_BEGIN_CRITICAL_SECTION(anyset); + rv = set_add_key((PySetObject *)anyset, key); + Py_END_CRITICAL_SECTION(); + return rv; } - int rv; - Py_BEGIN_CRITICAL_SECTION(anyset); - rv = set_add_key((PySetObject *)anyset, key); - Py_END_CRITICAL_SECTION(); - return rv; + if (PyFrozenSet_Check(anyset) && _PyObject_IsUniquelyReferenced(anyset)) { + // We can only change frozensets if they are uniquely referenced. The + // API limits the usage of `PySet_Add` to "fill in the values of brand + // new frozensets before they are exposed to other code". In this case, + // this can be done without a lock. + return set_add_key((PySetObject *)anyset, key); + } + + PyErr_BadInternalCall(); + return -1; } int From 2befce86e699fdbb6610949b029bad56a0d0780f Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Tue, 11 Nov 2025 15:31:29 -0500 Subject: [PATCH 143/313] gh-141004: Document `PyFile_OpenCode` and `PyFile_OpenCodeObject` (GH-141413) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/file.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index e9019a0d500..9d01254ddb2 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -93,6 +93,29 @@ the :mod:`io` APIs instead. .. versionadded:: 3.8 +.. c:function:: PyObject *PyFile_OpenCodeObject(PyObject *path) + + Open *path* with the mode ``'rb'``. *path* must be a Python :class:`str` + object. The behavior of this function may be overridden by + :c:func:`PyFile_SetOpenCodeHook` to allow for some preprocessing of the + text. + + This is analogous to :func:`io.open_code` in Python. + + On success, this function returns a :term:`strong reference` to a Python + file object. On failure, this function returns ``NULL`` with an exception + set. + + .. versionadded:: 3.8 + + +.. c:function:: PyObject *PyFile_OpenCode(const char *path) + + Similar to :c:func:`PyFile_OpenCodeObject`, but *path* is a + UTF-8 encoded :c:expr:`const char*`. + + .. versionadded:: 3.8 + .. c:function:: int PyFile_WriteObject(PyObject *obj, PyObject *p, int flags) From c13b59204af562bfb022eb8f6a5c03eb82659531 Mon Sep 17 00:00:00 2001 From: Alper Date: Tue, 11 Nov 2025 12:31:55 -0800 Subject: [PATCH 144/313] gh-116738: use `PyMutex` in `lzma` module (#140711) Co-authored-by: Kumar Aditya --- Lib/test/test_free_threading/test_lzma.py | 56 +++++++++++++++++++++++ Modules/_lzmamodule.c | 46 ++++++------------- 2 files changed, 69 insertions(+), 33 deletions(-) create mode 100644 Lib/test/test_free_threading/test_lzma.py diff --git a/Lib/test/test_free_threading/test_lzma.py b/Lib/test/test_free_threading/test_lzma.py new file mode 100644 index 00000000000..38d7e5db489 --- /dev/null +++ b/Lib/test/test_free_threading/test_lzma.py @@ -0,0 +1,56 @@ +import unittest + +from test.support import import_helper, threading_helper +from test.support.threading_helper import run_concurrently + +lzma = import_helper.import_module("lzma") +from lzma import LZMACompressor, LZMADecompressor + +from test.test_lzma import INPUT + + +NTHREADS = 10 + + +@threading_helper.requires_working_threading() +class TestLZMA(unittest.TestCase): + def test_compressor(self): + lzc = LZMACompressor() + + # First compress() outputs LZMA header + header = lzc.compress(INPUT) + self.assertGreater(len(header), 0) + + def worker(): + # it should return empty bytes as it buffers data internally + data = lzc.compress(INPUT) + self.assertEqual(data, b"") + + run_concurrently(worker_func=worker, nthreads=NTHREADS - 1) + full_compressed = header + lzc.flush() + decompressed = lzma.decompress(full_compressed) + # The decompressed data should be INPUT repeated NTHREADS times + self.assertEqual(decompressed, INPUT * NTHREADS) + + def test_decompressor(self): + chunk_size = 128 + chunks = [bytes([ord("a") + i]) * chunk_size for i in range(NTHREADS)] + input_data = b"".join(chunks) + compressed = lzma.compress(input_data) + + lzd = LZMADecompressor() + output = [] + + def worker(): + data = lzd.decompress(compressed, chunk_size) + self.assertEqual(len(data), chunk_size) + output.append(data) + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + self.assertEqual(len(output), NTHREADS) + # Verify the expected chunks (order doesn't matter due to append race) + self.assertSetEqual(set(output), set(chunks)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index 6fc072f6d0a..58766233998 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -72,13 +72,6 @@ OutputBuffer_OnError(_BlocksOutputBuffer *buffer) } -#define ACQUIRE_LOCK(obj) do { \ - if (!PyThread_acquire_lock((obj)->lock, 0)) { \ - Py_BEGIN_ALLOW_THREADS \ - PyThread_acquire_lock((obj)->lock, 1); \ - Py_END_ALLOW_THREADS \ - } } while (0) -#define RELEASE_LOCK(obj) PyThread_release_lock((obj)->lock) typedef struct { PyTypeObject *lzma_compressor_type; @@ -111,7 +104,7 @@ typedef struct { lzma_allocator alloc; lzma_stream lzs; int flushed; - PyThread_type_lock lock; + PyMutex mutex; } Compressor; typedef struct { @@ -124,7 +117,7 @@ typedef struct { char needs_input; uint8_t *input_buffer; size_t input_buffer_size; - PyThread_type_lock lock; + PyMutex mutex; } Decompressor; #define Compressor_CAST(op) ((Compressor *)(op)) @@ -617,14 +610,14 @@ _lzma_LZMACompressor_compress_impl(Compressor *self, Py_buffer *data) { PyObject *result = NULL; - ACQUIRE_LOCK(self); + PyMutex_Lock(&self->mutex); if (self->flushed) { PyErr_SetString(PyExc_ValueError, "Compressor has been flushed"); } else { result = compress(self, data->buf, data->len, LZMA_RUN); } - RELEASE_LOCK(self); + PyMutex_Unlock(&self->mutex); return result; } @@ -644,14 +637,14 @@ _lzma_LZMACompressor_flush_impl(Compressor *self) { PyObject *result = NULL; - ACQUIRE_LOCK(self); + PyMutex_Lock(&self->mutex); if (self->flushed) { PyErr_SetString(PyExc_ValueError, "Repeated call to flush()"); } else { self->flushed = 1; result = compress(self, NULL, 0, LZMA_FINISH); } - RELEASE_LOCK(self); + PyMutex_Unlock(&self->mutex); return result; } @@ -820,12 +813,7 @@ Compressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) self->alloc.free = PyLzma_Free; self->lzs.allocator = &self->alloc; - self->lock = PyThread_allocate_lock(); - if (self->lock == NULL) { - Py_DECREF(self); - PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); - return NULL; - } + self->mutex = (PyMutex){0}; self->flushed = 0; switch (format) { @@ -867,10 +855,8 @@ static void Compressor_dealloc(PyObject *op) { Compressor *self = Compressor_CAST(op); + assert(!PyMutex_IsLocked(&self->mutex)); lzma_end(&self->lzs); - if (self->lock != NULL) { - PyThread_free_lock(self->lock); - } PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); Py_DECREF(tp); @@ -1146,12 +1132,12 @@ _lzma_LZMADecompressor_decompress_impl(Decompressor *self, Py_buffer *data, { PyObject *result = NULL; - ACQUIRE_LOCK(self); + PyMutex_Lock(&self->mutex); if (self->eof) PyErr_SetString(PyExc_EOFError, "Already at end of stream"); else result = decompress(self, data->buf, data->len, max_length); - RELEASE_LOCK(self); + PyMutex_Unlock(&self->mutex); return result; } @@ -1244,12 +1230,7 @@ _lzma_LZMADecompressor_impl(PyTypeObject *type, int format, self->lzs.allocator = &self->alloc; self->lzs.next_in = NULL; - self->lock = PyThread_allocate_lock(); - if (self->lock == NULL) { - Py_DECREF(self); - PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); - return NULL; - } + self->mutex = (PyMutex){0}; self->check = LZMA_CHECK_UNKNOWN; self->needs_input = 1; @@ -1304,14 +1285,13 @@ static void Decompressor_dealloc(PyObject *op) { Decompressor *self = Decompressor_CAST(op); + assert(!PyMutex_IsLocked(&self->mutex)); + if(self->input_buffer != NULL) PyMem_Free(self->input_buffer); lzma_end(&self->lzs); Py_CLEAR(self->unused_data); - if (self->lock != NULL) { - PyThread_free_lock(self->lock); - } PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); Py_DECREF(tp); From 37e2762ee12c2d7fc465938d7161a9a0640bd71f Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Tue, 11 Nov 2025 15:32:54 -0500 Subject: [PATCH 145/313] gh-141004: Document `PyBytes_Repr` and `PyBytes_DecodeEscape` (GH-141407) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/bytes.rst | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index 865a9e5d2bf..82c25573683 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -228,6 +228,42 @@ called with a non-bytes parameter. The function is :term:`soft deprecated`, use the :c:type:`PyBytesWriter` API instead. + +.. c:function:: PyObject *PyBytes_Repr(PyObject *bytes, int smartquotes) + + Get the string representation of *bytes*. This function is currently used to + implement :meth:`!bytes.__repr__` in Python. + + This function does not do type checking; it is undefined behavior to pass + *bytes* as a non-bytes object or ``NULL``. + + If *smartquotes* is true, the representation will use a double-quoted string + instead of single-quoted string when single-quotes are present in *bytes*. + For example, the byte string ``'Python'`` would be represented as + ``b"'Python'"`` when *smartquotes* is true, or ``b'\'Python\''`` when it is + false. + + On success, this function returns a :term:`strong reference` to a + :class:`str` object containing the representation. On failure, this + returns ``NULL`` with an exception set. + + +.. c:function:: PyObject *PyBytes_DecodeEscape(const char *s, Py_ssize_t len, const char *errors, Py_ssize_t unicode, const char *recode_encoding) + + Unescape a backslash-escaped string *s*. *s* must not be ``NULL``. + *len* must be the size of *s*. + + *errors* must be one of ``"strict"``, ``"replace"``, or ``"ignore"``. If + *errors* is ``NULL``, then ``"strict"`` is used by default. + + On success, this function returns a :term:`strong reference` to a Python + :class:`bytes` object containing the unescaped string. On failure, this + function returns ``NULL`` with an exception set. + + .. versionchanged:: 3.9 + *unicode* and *recode_encoding* are now unused. + + .. _pybyteswriter: PyBytesWriter From af80fac42548719ede7241bfbab3c2c0775b4760 Mon Sep 17 00:00:00 2001 From: Mohsin Mehmood <55545648+mohsinm-dev@users.noreply.github.com> Date: Wed, 12 Nov 2025 02:49:54 +0500 Subject: [PATCH 146/313] gh-141314: Fix TextIOWrapper.tell() assertion failure with standalone carriage return (GH-141331) The assertion was checking wrong variable (skip_back vs skip_bytes). --- Lib/test/test_io/test_textio.py | 19 +++++++++++++++++++ ...-11-10-01-47-18.gh-issue-141314.baaa28.rst | 1 + Modules/_io/textio.c | 2 +- 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst diff --git a/Lib/test/test_io/test_textio.py b/Lib/test/test_io/test_textio.py index d8d0928b4ba..6331ed2b958 100644 --- a/Lib/test/test_io/test_textio.py +++ b/Lib/test/test_io/test_textio.py @@ -686,6 +686,25 @@ def test_multibyte_seek_and_tell(self): self.assertEqual(f.tell(), p1) f.close() + def test_tell_after_readline_with_cr(self): + # Test for gh-141314: TextIOWrapper.tell() assertion failure + # when dealing with standalone carriage returns + data = b'line1\r' + with self.open(os_helper.TESTFN, "wb") as f: + f.write(data) + + with self.open(os_helper.TESTFN, "r") as f: + # Read line that ends with \r + line = f.readline() + self.assertEqual(line, "line1\n") + # This should not cause an assertion failure + pos = f.tell() + # Verify we can seek back to this position + f.seek(pos) + remaining = f.read() + self.assertEqual(remaining, "") + + def test_seek_with_encoder_state(self): f = self.open(os_helper.TESTFN, "w", encoding="euc_jis_2004") f.write("\u00e6\u0300") diff --git a/Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst b/Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst new file mode 100644 index 00000000000..37acaabfa3e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst @@ -0,0 +1 @@ +Fix assertion failure in :meth:`io.TextIOWrapper.tell` when reading files with standalone carriage return (``\r``) line endings. diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 84b7d9df400..65da300abcf 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -2845,7 +2845,7 @@ _io_TextIOWrapper_tell_impl(textio *self) current pos */ skip_bytes = (Py_ssize_t) (self->b2cratio * chars_to_skip); skip_back = 1; - assert(skip_back <= PyBytes_GET_SIZE(next_input)); + assert(skip_bytes <= PyBytes_GET_SIZE(next_input)); input = PyBytes_AS_STRING(next_input); while (skip_bytes > 0) { /* Decode up to temptative start point */ From c744ccb2c92746bc7be6316ab478dbc13e176e97 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 11 Nov 2025 21:51:22 +0000 Subject: [PATCH 147/313] GH-139596: Cease caching config.cache & ccache in GH Actions (GH-139623) * Cease caching config.cache in GH Actions\ * Remove ccache action --- .github/workflows/build.yml | 52 --------------------------- .github/workflows/reusable-macos.yml | 5 --- .github/workflows/reusable-san.yml | 10 ------ .github/workflows/reusable-ubuntu.yml | 10 ------ .github/workflows/reusable-wasi.yml | 21 +---------- 5 files changed, 1 insertion(+), 97 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6aa99928278..a0f60c30ac8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -109,20 +109,10 @@ jobs: python-version: '3.x' - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}-${{ env.pythonLocation }} - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Add ccache to PATH run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Configure CPython run: | # Build Python with the libpython dynamic library @@ -278,11 +268,6 @@ jobs: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies @@ -304,10 +289,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Configure CPython run: ./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl="$OPENSSL_DIR" - name: Build CPython @@ -339,11 +320,6 @@ jobs: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies @@ -370,10 +346,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Configure CPython run: | ./configure CFLAGS="-fdiagnostics-format=json" \ @@ -479,10 +451,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Setup directory envs for out-of-tree builds run: | echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" @@ -493,11 +461,6 @@ jobs: run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | @@ -581,11 +544,6 @@ jobs: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies @@ -611,11 +569,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: Configure CPython run: ./configure --config-cache --with-address-sanitizer --without-pymalloc - name: Build CPython @@ -662,11 +615,6 @@ jobs: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Set build dir diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 3d310ae695b..d85c46b96f8 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -36,11 +36,6 @@ jobs: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} - name: Install Homebrew dependencies run: | brew install pkg-config openssl@3.0 xz gdbm tcl-tk@9 make diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index e6ff02e4838..7fe96d1b238 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -34,11 +34,6 @@ jobs: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.sanitizer }}-${{ inputs.config_hash }} - name: Install dependencies run: | sudo ./.github/workflows/posix-deps-apt.sh @@ -77,11 +72,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: Configure CPython run: >- ./configure diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 7f8b9fdf5d6..7b93b5f51b0 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -64,11 +64,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: Setup directory envs for out-of-tree builds run: | echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" @@ -79,11 +74,6 @@ jobs: run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} # `test_unpickle_module_race` writes to the source directory, which is diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 18feb564822..8f412288f53 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -42,11 +42,6 @@ jobs: mkdir "${WASI_SDK_PATH}" && \ curl -s -S --location "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-arm64-linux.tar.gz" | \ tar --strip-components 1 --directory "${WASI_SDK_PATH}" --extract --gunzip - - name: "Configure ccache action" - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: "Add ccache to PATH" run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: "Install Python" @@ -55,24 +50,10 @@ jobs: python-version: '3.x' - name: "Runner image version" run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: "Restore Python build config.cache" - uses: actions/cache@v4 - with: - path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache - # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python. - # Include the hash of `Tools/wasm/wasi/__main__.py` as it may change the environment variables. - # (Make sure to keep the key in sync with the other config.cache step below.) - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi/__main__.py') }}-${{ env.pythonLocation }} - name: "Configure build Python" run: python3 Tools/wasm/wasi configure-build-python -- --config-cache --with-pydebug - name: "Make build Python" - run: python3 Tools/wasm/wasi make-build-python - - name: "Restore host config.cache" - uses: actions/cache@v4 - with: - path: ${{ env.CROSS_BUILD_WASI }}/config.cache - # Should be kept in sync with the other config.cache step above. - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi/__main__.py') }}-${{ env.pythonLocation }} + run: python3 Tools/wasm/wasi.py make-build-python - name: "Configure host" # `--with-pydebug` inferred from configure-build-python run: python3 Tools/wasm/wasi configure-host -- --config-cache From 7906f4d96a8fffbee9f4d4991019595878ad54e9 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 12 Nov 2025 00:01:25 +0200 Subject: [PATCH 148/313] gh-132686: Add parameters inherit_class_doc and fallback_to_class_doc for inspect.getdoc() (GH-132691) --- Doc/library/inspect.rst | 17 +++- Doc/whatsnew/3.15.rst | 8 ++ Lib/inspect.py | 34 +++++-- Lib/pydoc.py | 92 +------------------ Lib/test/test_inspect/test_inspect.py | 27 ++++++ ...-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst | 2 + 6 files changed, 79 insertions(+), 101 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 2b3b294ff33..aff53b78c4a 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -619,17 +619,26 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Retrieving source code ---------------------- -.. function:: getdoc(object) +.. function:: getdoc(object, *, inherit_class_doc=True, fallback_to_class_doc=True) Get the documentation string for an object, cleaned up with :func:`cleandoc`. - If the documentation string for an object is not provided and the object is - a class, a method, a property or a descriptor, retrieve the documentation - string from the inheritance hierarchy. + If the documentation string for an object is not provided: + + * if the object is a class and *inherit_class_doc* is true (by default), + retrieve the documentation string from the inheritance hierarchy; + * if the object is a method, a property or a descriptor, retrieve + the documentation string from the inheritance hierarchy; + * otherwise, if *fallback_to_class_doc* is true (by default), retrieve + the documentation string from the class of the object. + Return ``None`` if the documentation string is invalid or missing. .. versionchanged:: 3.5 Documentation strings are now inherited if not overridden. + .. versionchanged:: next + Added parameters *inherit_class_doc* and *fallback_to_class_doc*. + .. function:: getcomments(object) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index ef18d36e4d4..ecab0d03e10 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -429,6 +429,14 @@ http.cookies (Contributed by Nick Burns and Senthil Kumaran in :gh:`92936`.) +inspect +------- + +* Add parameters *inherit_class_doc* and *fallback_to_class_doc* + for :func:`~inspect.getdoc`. + (Contributed by Serhiy Storchaka in :gh:`132686`.) + + locale ------ diff --git a/Lib/inspect.py b/Lib/inspect.py index bb22bab3040..bb17848b444 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -706,8 +706,8 @@ def _findclass(func): return None return cls -def _finddoc(obj): - if isclass(obj): +def _finddoc(obj, *, search_in_class=True): + if search_in_class and isclass(obj): for base in obj.__mro__: if base is not object: try: @@ -767,19 +767,37 @@ def _finddoc(obj): return doc return None -def getdoc(object): +def _getowndoc(obj): + """Get the documentation string for an object if it is not + inherited from its class.""" + try: + doc = object.__getattribute__(obj, '__doc__') + if doc is None: + return None + if obj is not type: + typedoc = type(obj).__doc__ + if isinstance(typedoc, str) and typedoc == doc: + return None + return doc + except AttributeError: + return None + +def getdoc(object, *, fallback_to_class_doc=True, inherit_class_doc=True): """Get the documentation string for an object. All tabs are expanded to spaces. To clean up docstrings that are indented to line up with blocks of code, any whitespace than can be uniformly removed from the second line onwards is removed.""" - try: - doc = object.__doc__ - except AttributeError: - return None + if fallback_to_class_doc: + try: + doc = object.__doc__ + except AttributeError: + return None + else: + doc = _getowndoc(object) if doc is None: try: - doc = _finddoc(object) + doc = _finddoc(object, search_in_class=inherit_class_doc) except (AttributeError, TypeError): return None if not isinstance(doc, str): diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 989fbd517d8..45ff5fca308 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -108,96 +108,10 @@ def pathdirs(): normdirs.append(normdir) return dirs -def _findclass(func): - cls = sys.modules.get(func.__module__) - if cls is None: - return None - for name in func.__qualname__.split('.')[:-1]: - cls = getattr(cls, name) - if not inspect.isclass(cls): - return None - return cls - -def _finddoc(obj): - if inspect.ismethod(obj): - name = obj.__func__.__name__ - self = obj.__self__ - if (inspect.isclass(self) and - getattr(getattr(self, name, None), '__func__') is obj.__func__): - # classmethod - cls = self - else: - cls = self.__class__ - elif inspect.isfunction(obj): - name = obj.__name__ - cls = _findclass(obj) - if cls is None or getattr(cls, name) is not obj: - return None - elif inspect.isbuiltin(obj): - name = obj.__name__ - self = obj.__self__ - if (inspect.isclass(self) and - self.__qualname__ + '.' + name == obj.__qualname__): - # classmethod - cls = self - else: - cls = self.__class__ - # Should be tested before isdatadescriptor(). - elif isinstance(obj, property): - name = obj.__name__ - cls = _findclass(obj.fget) - if cls is None or getattr(cls, name) is not obj: - return None - elif inspect.ismethoddescriptor(obj) or inspect.isdatadescriptor(obj): - name = obj.__name__ - cls = obj.__objclass__ - if getattr(cls, name) is not obj: - return None - if inspect.ismemberdescriptor(obj): - slots = getattr(cls, '__slots__', None) - if isinstance(slots, dict) and name in slots: - return slots[name] - else: - return None - for base in cls.__mro__: - try: - doc = _getowndoc(getattr(base, name)) - except AttributeError: - continue - if doc is not None: - return doc - return None - -def _getowndoc(obj): - """Get the documentation string for an object if it is not - inherited from its class.""" - try: - doc = object.__getattribute__(obj, '__doc__') - if doc is None: - return None - if obj is not type: - typedoc = type(obj).__doc__ - if isinstance(typedoc, str) and typedoc == doc: - return None - return doc - except AttributeError: - return None - def _getdoc(object): - """Get the documentation string for an object. - - All tabs are expanded to spaces. To clean up docstrings that are - indented to line up with blocks of code, any whitespace than can be - uniformly removed from the second line onwards is removed.""" - doc = _getowndoc(object) - if doc is None: - try: - doc = _finddoc(object) - except (AttributeError, TypeError): - return None - if not isinstance(doc, str): - return None - return inspect.cleandoc(doc) + return inspect.getdoc(object, + fallback_to_class_doc=False, + inherit_class_doc=False) def getdoc(object): """Get the doc string or comments for an object.""" diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index d42f2dbff99..24fd4a2fa62 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -688,10 +688,37 @@ def test_getdoc_inherited(self): self.assertEqual(inspect.getdoc(mod.FesteringGob.contradiction), 'The automatic gainsaying.') + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_getdoc_inherited_class_doc(self): + class A: + """Common base class""" + class B(A): + pass + + a = A() + self.assertEqual(inspect.getdoc(A), 'Common base class') + self.assertEqual(inspect.getdoc(A, inherit_class_doc=False), + 'Common base class') + self.assertEqual(inspect.getdoc(a), 'Common base class') + self.assertIsNone(inspect.getdoc(a, fallback_to_class_doc=False)) + a.__doc__ = 'Instance' + self.assertEqual(inspect.getdoc(a, fallback_to_class_doc=False), + 'Instance') + + b = B() + self.assertEqual(inspect.getdoc(B), 'Common base class') + self.assertIsNone(inspect.getdoc(B, inherit_class_doc=False)) + self.assertIsNone(inspect.getdoc(b)) + self.assertIsNone(inspect.getdoc(b, fallback_to_class_doc=False)) + b.__doc__ = 'Instance' + self.assertEqual(inspect.getdoc(b, fallback_to_class_doc=False), 'Instance') + @unittest.skipIf(MISSING_C_DOCSTRINGS, "test requires docstrings") def test_finddoc(self): finddoc = inspect._finddoc self.assertEqual(finddoc(int), int.__doc__) + self.assertIsNone(finddoc(int, search_in_class=False)) self.assertEqual(finddoc(int.to_bytes), int.to_bytes.__doc__) self.assertEqual(finddoc(int().to_bytes), int.to_bytes.__doc__) self.assertEqual(finddoc(int.from_bytes), int.from_bytes.__doc__) diff --git a/Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst b/Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst new file mode 100644 index 00000000000..d0c8e2d705c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst @@ -0,0 +1,2 @@ +Add parameters *inherit_class_doc* and *fallback_to_class_doc* for +:func:`inspect.getdoc`. From 9e7340cd3b5531784291088b504882cfb4d4c78c Mon Sep 17 00:00:00 2001 From: J Berg Date: Tue, 11 Nov 2025 22:09:58 +0000 Subject: [PATCH 149/313] gh-139462: Make the ProcessPoolExecutor BrokenProcessPool exception report which child process terminated (GH-139486) Report which process terminated as cause of BPE --- Doc/whatsnew/3.15.rst | 10 ++++++++++ Lib/concurrent/futures/process.py | 18 ++++++++++++++++-- .../test_process_pool.py | 15 +++++++++++++++ ...5-10-02-22-29-00.gh-issue-139462.VZXUHe.rst | 3 +++ 4 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index ecab0d03e10..c543b6e6c2a 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -369,6 +369,16 @@ collections.abc :mod:`!collections.abc` module. +concurrent.futures +------------------ + +* Improved error reporting when a child process in a + :class:`concurrent.futures.ProcessPoolExecutor` terminates abruptly. + The resulting traceback will now tell you the PID and exit code of the + terminated process. + (Contributed by Jonathan Berg in :gh:`139486`.) + + dataclasses ----------- diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index a14650bf5fa..a42afa68efc 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -474,9 +474,23 @@ def _terminate_broken(self, cause): bpe = BrokenProcessPool("A process in the process pool was " "terminated abruptly while the future was " "running or pending.") + cause_str = None if cause is not None: - bpe.__cause__ = _RemoteTraceback( - f"\n'''\n{''.join(cause)}'''") + cause_str = ''.join(cause) + else: + # No cause known, so report any processes that have + # terminated with nonzero exit codes, e.g. from a + # segfault. Multiple may terminate simultaneously, + # so include all of them in the traceback. + errors = [] + for p in self.processes.values(): + if p.exitcode is not None and p.exitcode != 0: + errors.append(f"Process {p.pid} terminated abruptly " + f"with exit code {p.exitcode}") + if errors: + cause_str = "\n".join(errors) + if cause_str: + bpe.__cause__ = _RemoteTraceback(f"\n'''\n{cause_str}'''") # Mark pending tasks as failed. for work_id, work_item in self.pending_work_items.items(): diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py index 9685f980119..731419a48bd 100644 --- a/Lib/test/test_concurrent_futures/test_process_pool.py +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -106,6 +106,21 @@ def test_traceback(self): self.assertIn('raise RuntimeError(123) # some comment', f1.getvalue()) + def test_traceback_when_child_process_terminates_abruptly(self): + # gh-139462 enhancement - BrokenProcessPool exceptions + # should describe which process terminated. + exit_code = 99 + with self.executor_type(max_workers=1) as executor: + future = executor.submit(os._exit, exit_code) + with self.assertRaises(BrokenProcessPool) as bpe: + future.result() + + cause = bpe.exception.__cause__ + self.assertIsInstance(cause, futures.process._RemoteTraceback) + self.assertIn( + f"terminated abruptly with exit code {exit_code}", cause.tb + ) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @hashlib_helper.requires_hashdigest('md5') def test_ressources_gced_in_workers(self): diff --git a/Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst b/Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst new file mode 100644 index 00000000000..390a6124386 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst @@ -0,0 +1,3 @@ +When a child process in a :class:`concurrent.futures.ProcessPoolExecutor` +terminates abruptly, the resulting traceback will now tell you the PID +and exit code of the terminated process. Contributed by Jonathan Berg. From 4359706ac8d5589fc37e2f1460a0d07a2319df15 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 12 Nov 2025 00:27:13 +0200 Subject: [PATCH 150/313] gh-120950: Fix overflow in math.log() with large int-like argument (GH-121011) Handling of arbitrary large int-like argument is now consistent with handling arbitrary large int arguments. --- Lib/test/test_math.py | 59 ++++++++++++++ ...-06-26-16-16-43.gh-issue-121011.qW54eh.rst | 2 + Modules/mathmodule.c | 80 ++++++++++++------- 3 files changed, 111 insertions(+), 30 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index ddeb8ad7cd6..68f41a2e620 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -189,6 +189,22 @@ def __init__(self, value): def __index__(self): return self.value +class IndexableFloatLike: + def __init__(self, float_value, index_value): + self.float_value = float_value + self.index_value = index_value + + def __float__(self): + if isinstance(self.float_value, BaseException): + raise self.float_value + return self.float_value + + def __index__(self): + if isinstance(self.index_value, BaseException): + raise self.index_value + return self.index_value + + class BadDescr: def __get__(self, obj, objtype=None): raise ValueError @@ -1192,13 +1208,32 @@ def testLog(self): self.ftest('log(10**40, 10**20)', math.log(10**40, 10**20), 2) self.ftest('log(10**1000)', math.log(10**1000), 2302.5850929940457) + self.ftest('log(10**2000, 10**1000)', math.log(10**2000, 10**1000), 2) + self.ftest('log(MyIndexable(32), MyIndexable(2))', + math.log(MyIndexable(32), MyIndexable(2)), 5) + self.ftest('log(MyIndexable(10**1000))', + math.log(MyIndexable(10**1000)), + 2302.5850929940457) + self.ftest('log(MyIndexable(10**2000), MyIndexable(10**1000))', + math.log(MyIndexable(10**2000), MyIndexable(10**1000)), + 2) + self.assertRaises(ValueError, math.log, 0.0) + self.assertRaises(ValueError, math.log, 0) + self.assertRaises(ValueError, math.log, MyIndexable(0)) self.assertRaises(ValueError, math.log, -1.5) + self.assertRaises(ValueError, math.log, -1) + self.assertRaises(ValueError, math.log, MyIndexable(-1)) self.assertRaises(ValueError, math.log, -10**1000) + self.assertRaises(ValueError, math.log, MyIndexable(-10**1000)) self.assertRaises(ValueError, math.log, 10, -10) self.assertRaises(ValueError, math.log, NINF) self.assertEqual(math.log(INF), INF) self.assertTrue(math.isnan(math.log(NAN))) + self.assertEqual(math.log(IndexableFloatLike(math.e, 10**1000)), 1.0) + self.assertAlmostEqual(math.log(IndexableFloatLike(OverflowError(), 10**1000)), + 2302.5850929940457) + def testLog1p(self): self.assertRaises(TypeError, math.log1p) for n in [2, 2**90, 2**300]: @@ -1214,16 +1249,28 @@ def testLog2(self): self.assertEqual(math.log2(1), 0.0) self.assertEqual(math.log2(2), 1.0) self.assertEqual(math.log2(4), 2.0) + self.assertEqual(math.log2(MyIndexable(4)), 2.0) # Large integer values self.assertEqual(math.log2(2**1023), 1023.0) self.assertEqual(math.log2(2**1024), 1024.0) self.assertEqual(math.log2(2**2000), 2000.0) + self.assertEqual(math.log2(MyIndexable(2**2000)), 2000.0) + self.assertRaises(ValueError, math.log2, 0.0) + self.assertRaises(ValueError, math.log2, 0) + self.assertRaises(ValueError, math.log2, MyIndexable(0)) self.assertRaises(ValueError, math.log2, -1.5) + self.assertRaises(ValueError, math.log2, -1) + self.assertRaises(ValueError, math.log2, MyIndexable(-1)) + self.assertRaises(ValueError, math.log2, -2**2000) + self.assertRaises(ValueError, math.log2, MyIndexable(-2**2000)) self.assertRaises(ValueError, math.log2, NINF) self.assertTrue(math.isnan(math.log2(NAN))) + self.assertEqual(math.log2(IndexableFloatLike(8.0, 2**2000)), 3.0) + self.assertEqual(math.log2(IndexableFloatLike(OverflowError(), 2**2000)), 2000.0) + @requires_IEEE_754 # log2() is not accurate enough on Mac OS X Tiger (10.4) @support.requires_mac_ver(10, 5) @@ -1239,12 +1286,24 @@ def testLog10(self): self.ftest('log10(1)', math.log10(1), 0) self.ftest('log10(10)', math.log10(10), 1) self.ftest('log10(10**1000)', math.log10(10**1000), 1000.0) + self.ftest('log10(MyIndexable(10))', math.log10(MyIndexable(10)), 1) + self.ftest('log10(MyIndexable(10**1000))', + math.log10(MyIndexable(10**1000)), 1000.0) + self.assertRaises(ValueError, math.log10, 0.0) + self.assertRaises(ValueError, math.log10, 0) + self.assertRaises(ValueError, math.log10, MyIndexable(0)) self.assertRaises(ValueError, math.log10, -1.5) + self.assertRaises(ValueError, math.log10, -1) + self.assertRaises(ValueError, math.log10, MyIndexable(-1)) self.assertRaises(ValueError, math.log10, -10**1000) + self.assertRaises(ValueError, math.log10, MyIndexable(-10**1000)) self.assertRaises(ValueError, math.log10, NINF) self.assertEqual(math.log(INF), INF) self.assertTrue(math.isnan(math.log10(NAN))) + self.assertEqual(math.log10(IndexableFloatLike(100.0, 10**1000)), 2.0) + self.assertEqual(math.log10(IndexableFloatLike(OverflowError(), 10**1000)), 1000.0) + @support.bigmemtest(2**32, memuse=0.2) def test_log_huge_integer(self, size): v = 1 << size diff --git a/Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst b/Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst new file mode 100644 index 00000000000..aee7fe2bcb5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst @@ -0,0 +1,2 @@ +:func:`math.log` now supports arbitrary large integer-like arguments in the +same way as arbitrary large integer arguments. diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 82846843cfb..de1886451ed 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -57,6 +57,7 @@ raised for division by zero and mod by zero. #endif #include "Python.h" +#include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_bitutils.h" // _Py_bit_length() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_import.h" // _PyImport_SetModuleString() @@ -1577,44 +1578,63 @@ math_modf_impl(PyObject *module, double x) However, intermediate overflow is possible for an int if the number of bits in that int is larger than PY_SSIZE_T_MAX. */ +static PyObject* +loghelper_int(PyObject* arg, double (*func)(double)) +{ + /* If it is int, do it ourselves. */ + double x, result; + int64_t e; + + /* Negative or zero inputs give a ValueError. */ + if (!_PyLong_IsPositive((PyLongObject *)arg)) { + PyErr_SetString(PyExc_ValueError, + "expected a positive input"); + return NULL; + } + + x = PyLong_AsDouble(arg); + if (x == -1.0 && PyErr_Occurred()) { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return NULL; + /* Here the conversion to double overflowed, but it's possible + to compute the log anyway. Clear the exception and continue. */ + PyErr_Clear(); + x = _PyLong_Frexp((PyLongObject *)arg, &e); + assert(!PyErr_Occurred()); + /* Value is ~= x * 2**e, so the log ~= log(x) + log(2) * e. */ + result = fma(func(2.0), (double)e, func(x)); + } + else + /* Successfully converted x to a double. */ + result = func(x); + return PyFloat_FromDouble(result); +} + static PyObject* loghelper(PyObject* arg, double (*func)(double)) { /* If it is int, do it ourselves. */ if (PyLong_Check(arg)) { - double x, result; - int64_t e; - - /* Negative or zero inputs give a ValueError. */ - if (!_PyLong_IsPositive((PyLongObject *)arg)) { - /* The input can be an arbitrary large integer, so we - don't include it's value in the error message. */ - PyErr_SetString(PyExc_ValueError, - "expected a positive input"); + return loghelper_int(arg, func); + } + /* Else let libm handle it by itself. */ + PyObject *res = math_1(arg, func, 0, "expected a positive input, got %s"); + if (res == NULL && + PyErr_ExceptionMatches(PyExc_OverflowError) && + PyIndex_Check(arg)) + { + /* Here the conversion to double overflowed, but it's possible + to compute the log anyway. Clear the exception, convert to + integer and continue. */ + PyErr_Clear(); + arg = _PyNumber_Index(arg); + if (arg == NULL) { return NULL; } - - x = PyLong_AsDouble(arg); - if (x == -1.0 && PyErr_Occurred()) { - if (!PyErr_ExceptionMatches(PyExc_OverflowError)) - return NULL; - /* Here the conversion to double overflowed, but it's possible - to compute the log anyway. Clear the exception and continue. */ - PyErr_Clear(); - x = _PyLong_Frexp((PyLongObject *)arg, &e); - assert(e >= 0); - assert(!PyErr_Occurred()); - /* Value is ~= x * 2**e, so the log ~= log(x) + log(2) * e. */ - result = fma(func(2.0), (double)e, func(x)); - } - else - /* Successfully converted x to a double. */ - result = func(x); - return PyFloat_FromDouble(result); + res = loghelper_int(arg, func); + Py_DECREF(arg); } - - /* Else let libm handle it by itself. */ - return math_1(arg, func, 0, "expected a positive input, got %s"); + return res; } From f5c2a41f9a6b3be95c5be9dbae0a4a3342d356dc Mon Sep 17 00:00:00 2001 From: yihong Date: Wed, 12 Nov 2025 07:47:57 +0800 Subject: [PATCH 151/313] gh-138775: fix handle `python -m base64` stdin correct with EOF signal (GH-138776) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: handle stdin correct with EOF single. * fix: flollow the comments when pipe stdin use buffer * Apply suggestions from code review * fix: apply review comments in Lib/base64.py * fix: address comments * Reword comment and NEWS entry. --------- Signed-off-by: yihong0618 Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> Co-authored-by: Peter Bierma Co-authored-by: Gregory P. Smith --- Lib/base64.py | 9 ++++++++- .../2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst diff --git a/Lib/base64.py b/Lib/base64.py index cfc57626c40..f95132a4274 100644 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -604,7 +604,14 @@ def main(): with open(args[0], 'rb') as f: func(f, sys.stdout.buffer) else: - func(sys.stdin.buffer, sys.stdout.buffer) + if sys.stdin.isatty(): + # gh-138775: read terminal input data all at once to detect EOF + import io + data = sys.stdin.buffer.read() + buffer = io.BytesIO(data) + else: + buffer = sys.stdin.buffer + func(buffer, sys.stdout.buffer) if __name__ == '__main__': diff --git a/Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst b/Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst new file mode 100644 index 00000000000..455c1a9925a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst @@ -0,0 +1,2 @@ +Use of ``python -m`` with :mod:`base64` has been fixed to detect input from a +terminal so that it properly notices EOF. From 0d7b48a8f5de5c1c6d57e1cf7194b6fb222d92e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Wed, 12 Nov 2025 01:03:14 +0100 Subject: [PATCH 152/313] gh-137952: update `csv.Sniffer().has_header()` docs to describe the actual off-by-onish behavior (GH-137953) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * checks 21, not 20 * Say "header" instead of "first row" to disambiguate per review. --------- Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Co-authored-by: Maurycy Pawłowski-Wieroński --- Doc/library/csv.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 3ea7cd210f7..4a033d823e6 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -295,8 +295,8 @@ The :mod:`csv` module defines the following classes: - the second through n-th rows contain strings where at least one value's length differs from that of the putative header of that column. - Twenty rows after the first row are sampled; if more than half of columns + - rows meet the criteria, :const:`True` is returned. + Twenty-one rows after the header are sampled; if more than half of the + columns + rows meet the criteria, :const:`True` is returned. .. note:: From 0e88be6f55f35ab045e57f9f869b893c15dcc099 Mon Sep 17 00:00:00 2001 From: Jan-Eric Nitschke <47750513+JanEricNitschke@users.noreply.github.com> Date: Wed, 12 Nov 2025 01:32:26 +0100 Subject: [PATCH 153/313] gh-138621: Increase test coverage for csv.DictReader and csv.Sniffer (GH-138622) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Increase test coverage for csv.DictReader and csv.Sniffer Previously there were no tests for the DictReader fieldnames setter, the case where a StopIteration was encountered when trying to determine the fieldnames from the content or the case where Sniffer could not find a delimiter. * Revert whitespace change to comment * Add a test that csv.Sniffer.has_header checks up to 20 rows * Replace name and age with letter and offset Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> * Address review comment --------- Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> Co-authored-by: Gregory P. Smith <68491+gpshead@users.noreply.github.com> --- Lib/test/test_csv.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 6be6a7ae222..df79840088a 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -918,6 +918,14 @@ def test_dict_reader_fieldnames_accepts_list(self): reader = csv.DictReader(f, fieldnames) self.assertEqual(reader.fieldnames, fieldnames) + def test_dict_reader_set_fieldnames(self): + fieldnames = ["a", "b", "c"] + f = StringIO() + reader = csv.DictReader(f) + self.assertIsNone(reader.fieldnames) + reader.fieldnames = fieldnames + self.assertEqual(reader.fieldnames, fieldnames) + def test_dict_writer_fieldnames_rejects_iter(self): fieldnames = ["a", "b", "c"] f = StringIO() @@ -933,6 +941,7 @@ def test_dict_writer_fieldnames_accepts_list(self): def test_dict_reader_fieldnames_is_optional(self): f = StringIO() reader = csv.DictReader(f, fieldnames=None) + self.assertIsNone(reader.fieldnames) def test_read_dict_fields(self): with TemporaryFile("w+", encoding="utf-8") as fileobj: @@ -1353,6 +1362,19 @@ class TestSniffer(unittest.TestCase): ghi\0jkl """ + sample15 = "\n\n\n" + sample16 = "abc\ndef\nghi" + + sample17 = ["letter,offset"] + sample17.extend(f"{chr(ord('a') + i)},{i}" for i in range(20)) + sample17.append("v,twenty_one") # 'u' was skipped + sample17 = '\n'.join(sample17) + + sample18 = ["letter,offset"] + sample18.extend(f"{chr(ord('a') + i)},{i}" for i in range(21)) + sample18.append("v,twenty_one") # 'u' was not skipped + sample18 = '\n'.join(sample18) + def test_issue43625(self): sniffer = csv.Sniffer() self.assertTrue(sniffer.has_header(self.sample12)) @@ -1374,6 +1396,11 @@ def test_has_header_regex_special_delimiter(self): self.assertIs(sniffer.has_header(self.sample8), False) self.assertIs(sniffer.has_header(self.header2 + self.sample8), True) + def test_has_header_checks_20_rows(self): + sniffer = csv.Sniffer() + self.assertFalse(sniffer.has_header(self.sample17)) + self.assertTrue(sniffer.has_header(self.sample18)) + def test_guess_quote_and_delimiter(self): sniffer = csv.Sniffer() for header in (";'123;4';", "'123;4';", ";'123;4'", "'123;4'"): @@ -1423,6 +1450,10 @@ def test_delimiters(self): self.assertEqual(dialect.quotechar, "'") dialect = sniffer.sniff(self.sample14) self.assertEqual(dialect.delimiter, '\0') + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, self.sample15) + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, self.sample16) def test_doublequote(self): sniffer = csv.Sniffer() From df6676549cd67c7b83111c6fce7c546270604aa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Cabello=20Jim=C3=A9nez?= <33024690+acabelloj@users.noreply.github.com> Date: Wed, 12 Nov 2025 01:36:43 +0100 Subject: [PATCH 154/313] gh-137928: remove redundant size validation in multiprocessing.heap (GH-137929) remove redundant size check, malloc does it --------- Co-authored-by: Gregory P. Smith --- Lib/multiprocessing/heap.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Lib/multiprocessing/heap.py b/Lib/multiprocessing/heap.py index 6217dfe1268..5c835648395 100644 --- a/Lib/multiprocessing/heap.py +++ b/Lib/multiprocessing/heap.py @@ -324,10 +324,6 @@ class BufferWrapper(object): _heap = Heap() def __init__(self, size): - if size < 0: - raise ValueError("Size {0:n} out of range".format(size)) - if sys.maxsize <= size: - raise OverflowError("Size {0:n} too large".format(size)) block = BufferWrapper._heap.malloc(size) self._state = (block, size) util.Finalize(self, BufferWrapper._heap.free, args=(block,)) From 9ce99c6c1901705238e4cb3ce81eb6f499e7b4f4 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 12 Nov 2025 00:53:21 +0000 Subject: [PATCH 155/313] GH-137618: Require Python 3.10 to Python 3.15 for PYTHON_FOR_REGEN (GH-137619) * Require Python 3.11 to Python 3.15 for PYTHON_FOR_REGEN * NEWS * keep allowing python 3.10 --------- Co-authored-by: Gregory P. Smith --- .../next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst | 2 ++ configure | 2 +- configure.ac | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst diff --git a/Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst b/Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst new file mode 100644 index 00000000000..0b56c4c8f68 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst @@ -0,0 +1,2 @@ +``PYTHON_FOR_REGEN`` now requires Python 3.10 to Python 3.15. +Patch by Adam Turner. diff --git a/configure b/configure index 8463b5b5e4a..eeb24c1d844 100755 --- a/configure +++ b/configure @@ -3818,7 +3818,7 @@ fi -for ac_prog in python$PACKAGE_VERSION python3.13 python3.12 python3.11 python3.10 python3 python +for ac_prog in python$PACKAGE_VERSION python3.15 python3.14 python3.13 python3.12 python3.11 python3.10 python3 python do # Extract the first word of "$ac_prog", so it can be a program name with args. set dummy $ac_prog; ac_word=$2 diff --git a/configure.ac b/configure.ac index df94ae25e63..92adc44da0d 100644 --- a/configure.ac +++ b/configure.ac @@ -205,7 +205,7 @@ AC_SUBST([FREEZE_MODULE_DEPS]) AC_SUBST([PYTHON_FOR_BUILD_DEPS]) AC_CHECK_PROGS([PYTHON_FOR_REGEN], - [python$PACKAGE_VERSION python3.13 python3.12 python3.11 python3.10 python3 python], + [python$PACKAGE_VERSION python3.15 python3.14 python3.13 python3.12 python3.11 python3.10 python3 python], [python3]) AC_SUBST([PYTHON_FOR_REGEN]) From fbebca289d811669fc1980e3a135325b8542a846 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 12 Nov 2025 09:59:48 +0500 Subject: [PATCH 156/313] GH-116946: eliminate the need for the GC in the `_thread.lock` and `_thread.RLock` (#141268) --- Modules/_threadmodule.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index cc8277c5783..0e22c7bd386 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -41,6 +41,7 @@ typedef struct { typedef struct { PyObject_HEAD PyMutex lock; + PyObject *weakreflist; /* List of weak references */ } lockobject; #define lockobject_CAST(op) ((lockobject *)(op)) @@ -48,6 +49,7 @@ typedef struct { typedef struct { PyObject_HEAD _PyRecursiveMutex lock; + PyObject *weakreflist; /* List of weak references */ } rlockobject; #define rlockobject_CAST(op) ((rlockobject *)(op)) @@ -767,7 +769,6 @@ static PyType_Spec ThreadHandle_Type_spec = { static void lock_dealloc(PyObject *self) { - PyObject_GC_UnTrack(self); PyObject_ClearWeakRefs(self); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); @@ -999,6 +1000,10 @@ lock_new_impl(PyTypeObject *type) return (PyObject *)self; } +static PyMemberDef lock_members[] = { + {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(lockobject, weakreflist), Py_READONLY}, + {NULL} +}; static PyMethodDef lock_methods[] = { _THREAD_LOCK_ACQUIRE_LOCK_METHODDEF @@ -1034,8 +1039,8 @@ static PyType_Slot lock_type_slots[] = { {Py_tp_dealloc, lock_dealloc}, {Py_tp_repr, lock_repr}, {Py_tp_doc, (void *)lock_doc}, + {Py_tp_members, lock_members}, {Py_tp_methods, lock_methods}, - {Py_tp_traverse, _PyObject_VisitType}, {Py_tp_new, lock_new}, {0, 0} }; @@ -1043,8 +1048,7 @@ static PyType_Slot lock_type_slots[] = { static PyType_Spec lock_type_spec = { .name = "_thread.lock", .basicsize = sizeof(lockobject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_WEAKREF), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE), .slots = lock_type_slots, }; @@ -1059,7 +1063,6 @@ rlock_locked_impl(rlockobject *self) static void rlock_dealloc(PyObject *self) { - PyObject_GC_UnTrack(self); PyObject_ClearWeakRefs(self); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); @@ -1319,6 +1322,11 @@ _thread_RLock__at_fork_reinit_impl(rlockobject *self) #endif /* HAVE_FORK */ +static PyMemberDef rlock_members[] = { + {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(rlockobject, weakreflist), Py_READONLY}, + {NULL} +}; + static PyMethodDef rlock_methods[] = { _THREAD_RLOCK_ACQUIRE_METHODDEF _THREAD_RLOCK_RELEASE_METHODDEF @@ -1339,10 +1347,10 @@ static PyMethodDef rlock_methods[] = { static PyType_Slot rlock_type_slots[] = { {Py_tp_dealloc, rlock_dealloc}, {Py_tp_repr, rlock_repr}, + {Py_tp_members, rlock_members}, {Py_tp_methods, rlock_methods}, {Py_tp_alloc, PyType_GenericAlloc}, {Py_tp_new, rlock_new}, - {Py_tp_traverse, _PyObject_VisitType}, {0, 0}, }; @@ -1350,7 +1358,7 @@ static PyType_Spec rlock_type_spec = { .name = "_thread.RLock", .basicsize = sizeof(rlockobject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_WEAKREF), + Py_TPFLAGS_IMMUTABLETYPE), .slots = rlock_type_slots, }; From ef474cfafbdf3aa383fb1334a7ab95cef9834ced Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 12 Nov 2025 10:47:38 +0530 Subject: [PATCH 157/313] gh-103847: fix cancellation safety of `asyncio.create_subprocess_exec` (#140805) --- Lib/asyncio/base_subprocess.py | 11 +++++ Lib/test/test_asyncio/test_subprocess.py | 40 ++++++++++++++++++- ...-10-31-13-57-55.gh-issue-103847.VM7TnW.rst | 1 + 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index d40af422e61..321a4e5d5d1 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -26,6 +26,7 @@ def __init__(self, loop, protocol, args, shell, self._pending_calls = collections.deque() self._pipes = {} self._finished = False + self._pipes_connected = False if stdin == subprocess.PIPE: self._pipes[0] = None @@ -213,6 +214,7 @@ async def _connect_pipes(self, waiter): else: if waiter is not None and not waiter.cancelled(): waiter.set_result(None) + self._pipes_connected = True def _call(self, cb, *data): if self._pending_calls is not None: @@ -256,6 +258,15 @@ def _try_finish(self): assert not self._finished if self._returncode is None: return + if not self._pipes_connected: + # self._pipes_connected can be False if not all pipes were connected + # because either the process failed to start or the self._connect_pipes task + # got cancelled. In this broken state we consider all pipes disconnected and + # to avoid hanging forever in self._wait as otherwise _exit_waiters + # would never be woken up, we wake them up here. + for waiter in self._exit_waiters: + if not waiter.cancelled(): + waiter.set_result(self._returncode) if all(p is not None and p.disconnected for p in self._pipes.values()): self._finished = True diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 3a17c169c34..bf301740741 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -11,7 +11,7 @@ from asyncio import subprocess from test.test_asyncio import utils as test_utils from test import support -from test.support import os_helper +from test.support import os_helper, warnings_helper, gc_collect if not support.has_subprocess_support: raise unittest.SkipTest("test module requires subprocess") @@ -879,6 +879,44 @@ async def main(): self.loop.run_until_complete(main()) + @warnings_helper.ignore_warnings(category=ResourceWarning) + def test_subprocess_read_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_read_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED, stderr=asyncio.subprocess.PIPE) + + asyncio.run(main()) + gc_collect() + + @warnings_helper.ignore_warnings(category=ResourceWarning) + def test_subprocess_write_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_write_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED, stdin=asyncio.subprocess.PIPE) + + asyncio.run(main()) + gc_collect() + + @warnings_helper.ignore_warnings(category=ResourceWarning) + def test_subprocess_read_write_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_read_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + loop.connect_write_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec( + *PROGRAM_BLOCKED, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + asyncio.run(main()) + gc_collect() if sys.platform != 'win32': # Unix diff --git a/Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst b/Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst new file mode 100644 index 00000000000..e14af7d9708 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst @@ -0,0 +1 @@ +Fix hang when cancelling process created by :func:`asyncio.create_subprocess_exec` or :func:`asyncio.create_subprocess_shell`. Patch by Kumar Aditya. From f1b7961ccfa050e9c80622fff1b3cdada46f9aab Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 12 Nov 2025 12:51:43 +0530 Subject: [PATCH 158/313] GH-116946: revert eliminate the need for the GC in the `_thread.lock` and `_thread.RLock` (#141448) Revert "GH-116946: eliminate the need for the GC in the `_thread.lock` and `_thread.RLock` (#141268)" This reverts commit fbebca289d811669fc1980e3a135325b8542a846. --- Modules/_threadmodule.c | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 0e22c7bd386..cc8277c5783 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -41,7 +41,6 @@ typedef struct { typedef struct { PyObject_HEAD PyMutex lock; - PyObject *weakreflist; /* List of weak references */ } lockobject; #define lockobject_CAST(op) ((lockobject *)(op)) @@ -49,7 +48,6 @@ typedef struct { typedef struct { PyObject_HEAD _PyRecursiveMutex lock; - PyObject *weakreflist; /* List of weak references */ } rlockobject; #define rlockobject_CAST(op) ((rlockobject *)(op)) @@ -769,6 +767,7 @@ static PyType_Spec ThreadHandle_Type_spec = { static void lock_dealloc(PyObject *self) { + PyObject_GC_UnTrack(self); PyObject_ClearWeakRefs(self); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); @@ -1000,10 +999,6 @@ lock_new_impl(PyTypeObject *type) return (PyObject *)self; } -static PyMemberDef lock_members[] = { - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(lockobject, weakreflist), Py_READONLY}, - {NULL} -}; static PyMethodDef lock_methods[] = { _THREAD_LOCK_ACQUIRE_LOCK_METHODDEF @@ -1039,8 +1034,8 @@ static PyType_Slot lock_type_slots[] = { {Py_tp_dealloc, lock_dealloc}, {Py_tp_repr, lock_repr}, {Py_tp_doc, (void *)lock_doc}, - {Py_tp_members, lock_members}, {Py_tp_methods, lock_methods}, + {Py_tp_traverse, _PyObject_VisitType}, {Py_tp_new, lock_new}, {0, 0} }; @@ -1048,7 +1043,8 @@ static PyType_Slot lock_type_slots[] = { static PyType_Spec lock_type_spec = { .name = "_thread.lock", .basicsize = sizeof(lockobject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_WEAKREF), .slots = lock_type_slots, }; @@ -1063,6 +1059,7 @@ rlock_locked_impl(rlockobject *self) static void rlock_dealloc(PyObject *self) { + PyObject_GC_UnTrack(self); PyObject_ClearWeakRefs(self); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); @@ -1322,11 +1319,6 @@ _thread_RLock__at_fork_reinit_impl(rlockobject *self) #endif /* HAVE_FORK */ -static PyMemberDef rlock_members[] = { - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(rlockobject, weakreflist), Py_READONLY}, - {NULL} -}; - static PyMethodDef rlock_methods[] = { _THREAD_RLOCK_ACQUIRE_METHODDEF _THREAD_RLOCK_RELEASE_METHODDEF @@ -1347,10 +1339,10 @@ static PyMethodDef rlock_methods[] = { static PyType_Slot rlock_type_slots[] = { {Py_tp_dealloc, rlock_dealloc}, {Py_tp_repr, rlock_repr}, - {Py_tp_members, rlock_members}, {Py_tp_methods, rlock_methods}, {Py_tp_alloc, PyType_GenericAlloc}, {Py_tp_new, rlock_new}, + {Py_tp_traverse, _PyObject_VisitType}, {0, 0}, }; @@ -1358,7 +1350,7 @@ static PyType_Spec rlock_type_spec = { .name = "_thread.RLock", .basicsize = sizeof(rlockobject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_WEAKREF), .slots = rlock_type_slots, }; From 35908265b09ac39b67116bfdfe8a053be09e6d8f Mon Sep 17 00:00:00 2001 From: Mark Byrne <31762852+mbyrnepr2@users.noreply.github.com> Date: Wed, 12 Nov 2025 09:20:55 +0100 Subject: [PATCH 159/313] gh-75593: Add support of bytes and path-like paths in wave.open() (GH-140951) --- Doc/library/wave.rst | 9 ++++++-- Lib/test/test_wave.py | 22 +++++++++++++++++++ Lib/wave.py | 5 +++-- ...5-11-04-12-16-13.gh-issue-75593.EFVhKR.rst | 1 + 4 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst diff --git a/Doc/library/wave.rst b/Doc/library/wave.rst index a3f5bfd5e2f..7ff2c97992c 100644 --- a/Doc/library/wave.rst +++ b/Doc/library/wave.rst @@ -25,8 +25,9 @@ The :mod:`wave` module defines the following function and exception: .. function:: open(file, mode=None) - If *file* is a string, open the file by that name, otherwise treat it as a - file-like object. *mode* can be: + If *file* is a string, a :term:`path-like object` or a + :term:`bytes-like object` open the file by that name, otherwise treat it as + a file-like object. *mode* can be: ``'rb'`` Read only mode. @@ -52,6 +53,10 @@ The :mod:`wave` module defines the following function and exception: .. versionchanged:: 3.4 Added support for unseekable files. + .. versionchanged:: 3.15 + Added support for :term:`path-like objects ` + and :term:`bytes-like objects `. + .. exception:: Error An error raised when something is impossible because it violates the WAV diff --git a/Lib/test/test_wave.py b/Lib/test/test_wave.py index 226b1aa84bd..4c21f165537 100644 --- a/Lib/test/test_wave.py +++ b/Lib/test/test_wave.py @@ -1,9 +1,11 @@ import unittest from test import audiotests from test import support +from test.support.os_helper import FakePath import io import os import struct +import tempfile import sys import wave @@ -206,5 +208,25 @@ def test_open_in_write_raises(self): self.assertIsNone(cm.unraisable) +class WaveOpen(unittest.TestCase): + def test_open_pathlike(self): + """It is possible to use `wave.read` and `wave.write` with a path-like object""" + with tempfile.NamedTemporaryFile(delete_on_close=False) as fp: + cases = ( + FakePath(fp.name), + FakePath(os.fsencode(fp.name)), + os.fsencode(fp.name), + ) + for fake_path in cases: + with self.subTest(fake_path): + with wave.open(fake_path, 'wb') as f: + f.setnchannels(1) + f.setsampwidth(2) + f.setframerate(44100) + + with wave.open(fake_path, 'rb') as f: + pass + + if __name__ == '__main__': unittest.main() diff --git a/Lib/wave.py b/Lib/wave.py index 5af745e2217..056bd6aab7f 100644 --- a/Lib/wave.py +++ b/Lib/wave.py @@ -69,6 +69,7 @@ from collections import namedtuple import builtins +import os import struct import sys @@ -274,7 +275,7 @@ def initfp(self, file): def __init__(self, f): self._i_opened_the_file = None - if isinstance(f, str): + if isinstance(f, (bytes, str, os.PathLike)): f = builtins.open(f, 'rb') self._i_opened_the_file = f # else, assume it is an open file object already @@ -431,7 +432,7 @@ class Wave_write: def __init__(self, f): self._i_opened_the_file = None - if isinstance(f, str): + if isinstance(f, (bytes, str, os.PathLike)): f = builtins.open(f, 'wb') self._i_opened_the_file = f try: diff --git a/Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst b/Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst new file mode 100644 index 00000000000..9a31af9c110 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst @@ -0,0 +1 @@ +Add support of :term:`path-like objects ` and :term:`bytes-like objects ` in :func:`wave.open`. From 909f76dab91f028edd2ae7bd589d3975996de9e1 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 12 Nov 2025 09:42:56 +0100 Subject: [PATCH 160/313] gh-141376: Rename _AsyncioDebug to _Py_AsyncioDebug (GH-141391) --- Modules/_asynciomodule.c | 4 ++-- Tools/c-analyzer/cpython/ignored.tsv | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 1f58b1fb350..9b2b7011244 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -119,7 +119,7 @@ typedef struct _Py_AsyncioModuleDebugOffsets { } asyncio_thread_state; } Py_AsyncioModuleDebugOffsets; -GENERATE_DEBUG_SECTION(AsyncioDebug, Py_AsyncioModuleDebugOffsets _AsyncioDebug) +GENERATE_DEBUG_SECTION(AsyncioDebug, Py_AsyncioModuleDebugOffsets _Py_AsyncioDebug) = {.asyncio_task_object = { .size = sizeof(TaskObj), .task_name = offsetof(TaskObj, task_name), @@ -4338,7 +4338,7 @@ module_init(asyncio_state *state) goto fail; } - state->debug_offsets = &_AsyncioDebug; + state->debug_offsets = &_Py_AsyncioDebug; Py_DECREF(module); return 0; diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 8b73189fb07..11a3cd794ff 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -56,7 +56,7 @@ Python/pyhash.c - _Py_HashSecret - Python/parking_lot.c - buckets - ## data needed for introspecting asyncio state from debuggers and profilers -Modules/_asynciomodule.c - _AsyncioDebug - +Modules/_asynciomodule.c - _Py_AsyncioDebug - ################################## From 6f988b08d122e44848e89c04ad1e10c25d072cc7 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 12 Nov 2025 01:37:48 -0800 Subject: [PATCH 161/313] gh-85524: Raise "UnsupportedOperation" on FileIO.readall (#141214) io.UnsupportedOperation is a subclass of OSError and recommended by io.IOBase for this case; matches other read methods on io.FileIO. --- Lib/test/test_io/test_general.py | 1 + .../2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst | 3 +++ Modules/_io/clinic/fileio.c.h | 14 +++++++++----- Modules/_io/fileio.c | 13 ++++++++++--- 4 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst diff --git a/Lib/test/test_io/test_general.py b/Lib/test/test_io/test_general.py index a1cdd6876c2..f0677b01ea5 100644 --- a/Lib/test/test_io/test_general.py +++ b/Lib/test/test_io/test_general.py @@ -125,6 +125,7 @@ def test_invalid_operations(self): self.assertRaises(exc, fp.readline) with self.open(os_helper.TESTFN, "wb", buffering=0) as fp: self.assertRaises(exc, fp.read) + self.assertRaises(exc, fp.readall) self.assertRaises(exc, fp.readline) with self.open(os_helper.TESTFN, "rb", buffering=0) as fp: self.assertRaises(exc, fp.write, b"blah") diff --git a/Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst b/Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst new file mode 100644 index 00000000000..3e4fd1a5897 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst @@ -0,0 +1,3 @@ +Update ``io.FileIO.readall``, an implementation of :meth:`io.RawIOBase.readall`, +to follow :class:`io.IOBase` guidelines and raise :exc:`io.UnsupportedOperation` +when a file is in "w" mode rather than :exc:`OSError` diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index 04870b1c890..96c31ce8d6f 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -277,15 +277,19 @@ PyDoc_STRVAR(_io_FileIO_readall__doc__, "data is available (EAGAIN is returned before bytes are read) returns None."); #define _IO_FILEIO_READALL_METHODDEF \ - {"readall", (PyCFunction)_io_FileIO_readall, METH_NOARGS, _io_FileIO_readall__doc__}, + {"readall", _PyCFunction_CAST(_io_FileIO_readall), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_readall__doc__}, static PyObject * -_io_FileIO_readall_impl(fileio *self); +_io_FileIO_readall_impl(fileio *self, PyTypeObject *cls); static PyObject * -_io_FileIO_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) +_io_FileIO_readall(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _io_FileIO_readall_impl((fileio *)self); + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "readall() takes no arguments"); + return NULL; + } + return _io_FileIO_readall_impl((fileio *)self, cls); } PyDoc_STRVAR(_io_FileIO_read__doc__, @@ -543,4 +547,4 @@ _io_FileIO_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=1902fac9e39358aa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2e48f3df2f189170 input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 2544ff4ea91..5d7741fdd83 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -728,6 +728,9 @@ new_buffersize(fileio *self, size_t currentsize) @permit_long_docstring_body _io.FileIO.readall + cls: defining_class + / + Read all data from the file, returned as bytes. Reads until either there is an error or read() returns size 0 (indicates EOF). @@ -738,8 +741,8 @@ data is available (EAGAIN is returned before bytes are read) returns None. [clinic start generated code]*/ static PyObject * -_io_FileIO_readall_impl(fileio *self) -/*[clinic end generated code: output=faa0292b213b4022 input=10d8b2ec403302dc]*/ +_io_FileIO_readall_impl(fileio *self, PyTypeObject *cls) +/*[clinic end generated code: output=d546737ec895c462 input=cecda40bf9961299]*/ { Py_off_t pos, end; PyBytesWriter *writer; @@ -750,6 +753,10 @@ _io_FileIO_readall_impl(fileio *self) if (self->fd < 0) { return err_closed(); } + if (!self->readable) { + _PyIO_State *state = get_io_state_by_cls(cls); + return err_mode(state, "reading"); + } if (self->stat_atopen != NULL && self->stat_atopen->st_size < _PY_READ_MAX) { end = (Py_off_t)self->stat_atopen->st_size; @@ -873,7 +880,7 @@ _io_FileIO_read_impl(fileio *self, PyTypeObject *cls, Py_ssize_t size) } if (size < 0) - return _io_FileIO_readall_impl(self); + return _io_FileIO_readall_impl(self, cls); if (size > _PY_READ_MAX) { size = _PY_READ_MAX; From 20f53df07d42c495a08c73a3d54b8dd9098a62f0 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 12 Nov 2025 12:50:44 +0300 Subject: [PATCH 162/313] gh-141370: document undefined behavior of Py_ABS() (GH-141439) --- Doc/c-api/intro.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 6e1a9dcb355..c76cc2f70ec 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -121,6 +121,10 @@ complete listing. Return the absolute value of ``x``. + If the result cannot be represented (for example, if ``x`` has + :c:macro:`!INT_MIN` value for :c:expr:`int` type), the behavior is + undefined. + .. versionadded:: 3.3 .. c:macro:: Py_ALWAYS_INLINE From 7d54374f9c7d91e0ef90c4ad84baf10073cf1d8a Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 12 Nov 2025 01:57:05 -0800 Subject: [PATCH 163/313] gh-141311: Avoid assertion in BytesIO.readinto() (GH-141333) Fix error in assertion which causes failure if pos is equal to PY_SSIZE_T_MAX. Fix undefined behavior in read() and readinto() if pos is larger that the size of the underlying buffer. --- Lib/test/test_io/test_memoryio.py | 14 ++++++++++++++ ...025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst | 2 ++ Modules/_io/bytesio.c | 16 +++++++++++++--- 3 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst diff --git a/Lib/test/test_io/test_memoryio.py b/Lib/test/test_io/test_memoryio.py index 63998a86c45..bb023735e21 100644 --- a/Lib/test/test_io/test_memoryio.py +++ b/Lib/test/test_io/test_memoryio.py @@ -54,6 +54,12 @@ def testSeek(self): self.assertEqual(buf[3:], bytesIo.read()) self.assertRaises(TypeError, bytesIo.seek, 0.0) + self.assertEqual(sys.maxsize, bytesIo.seek(sys.maxsize)) + self.assertEqual(self.EOF, bytesIo.read(4)) + + self.assertEqual(sys.maxsize - 2, bytesIo.seek(sys.maxsize - 2)) + self.assertEqual(self.EOF, bytesIo.read(4)) + def testTell(self): buf = self.buftype("1234567890") bytesIo = self.ioclass(buf) @@ -552,6 +558,14 @@ def test_relative_seek(self): memio.seek(1, 1) self.assertEqual(memio.read(), buf[1:]) + def test_issue141311(self): + memio = self.ioclass() + # Seek allows PY_SSIZE_T_MAX, read should handle that. + # Past end of buffer read should always return 0 (EOF). + self.assertEqual(sys.maxsize, memio.seek(sys.maxsize)) + buf = bytearray(2) + self.assertEqual(0, memio.readinto(buf)) + def test_unicode(self): memio = self.ioclass() diff --git a/Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst b/Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst new file mode 100644 index 00000000000..bb425ce5df3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst @@ -0,0 +1,2 @@ +Fix assertion failure in :func:`!io.BytesIO.readinto` and undefined behavior +arising when read position is above capcity in :class:`io.BytesIO`. diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index d6bfb93177c..96611823ab6 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -436,6 +436,13 @@ read_bytes_lock_held(bytesio *self, Py_ssize_t size) return Py_NewRef(self->buf); } + /* gh-141311: Avoid undefined behavior when self->pos (limit PY_SSIZE_T_MAX) + is beyond the size of self->buf. Assert above validates size is always in + bounds. When self->pos is out of bounds calling code sets size to 0. */ + if (size == 0) { + return PyBytes_FromStringAndSize(NULL, 0); + } + output = PyBytes_AS_STRING(self->buf) + self->pos; self->pos += size; return PyBytes_FromStringAndSize(output, size); @@ -609,11 +616,14 @@ _io_BytesIO_readinto_impl(bytesio *self, Py_buffer *buffer) n = self->string_size - self->pos; if (len > n) { len = n; - if (len < 0) - len = 0; + if (len < 0) { + /* gh-141311: Avoid undefined behavior when self->pos (limit + PY_SSIZE_T_MAX) points beyond the size of self->buf. */ + return PyLong_FromSsize_t(0); + } } - assert(self->pos + len < PY_SSIZE_T_MAX); + assert(self->pos + len <= PY_SSIZE_T_MAX); assert(len >= 0); memcpy(buffer->buf, PyBytes_AS_STRING(self->buf) + self->pos, len); self->pos += len; From 23d85a2a3fb029172ea15c6e596f64f8c2868ed3 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 12 Nov 2025 13:06:29 +0300 Subject: [PATCH 164/313] gh-141042: fix sNaN's packing for mixed floating-point formats (#141107) --- Lib/test/test_capi/test_float.py | 54 +++++++++++++++---- ...-11-06-06-28-14.gh-issue-141042.brOioJ.rst | 3 ++ Objects/floatobject.c | 16 ++++-- 3 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst diff --git a/Lib/test/test_capi/test_float.py b/Lib/test/test_capi/test_float.py index 983b991b4f1..df7017e6436 100644 --- a/Lib/test/test_capi/test_float.py +++ b/Lib/test/test_capi/test_float.py @@ -29,6 +29,23 @@ NAN = float("nan") +def make_nan(size, sign, quiet, payload=None): + if size == 8: + payload_mask = 0x7ffffffffffff + i = (sign << 63) + (0x7ff << 52) + (quiet << 51) + elif size == 4: + payload_mask = 0x3fffff + i = (sign << 31) + (0xff << 23) + (quiet << 22) + elif size == 2: + payload_mask = 0x1ff + i = (sign << 15) + (0x1f << 10) + (quiet << 9) + else: + raise ValueError("size must be either 2, 4, or 8") + if payload is None: + payload = random.randint(not quiet, payload_mask) + return i + payload + + class CAPIFloatTest(unittest.TestCase): def test_check(self): # Test PyFloat_Check() @@ -202,16 +219,7 @@ def test_pack_unpack_roundtrip_for_nans(self): # HP PA RISC uses 0 for quiet, see: # https://en.wikipedia.org/wiki/NaN#Encoding signaling = 1 - quiet = int(not signaling) - if size == 8: - payload = random.randint(signaling, 0x7ffffffffffff) - i = (sign << 63) + (0x7ff << 52) + (quiet << 51) + payload - elif size == 4: - payload = random.randint(signaling, 0x3fffff) - i = (sign << 31) + (0xff << 23) + (quiet << 22) + payload - elif size == 2: - payload = random.randint(signaling, 0x1ff) - i = (sign << 15) + (0x1f << 10) + (quiet << 9) + payload + i = make_nan(size, sign, not signaling) data = bytes.fromhex(f'{i:x}') for endian in (BIG_ENDIAN, LITTLE_ENDIAN): with self.subTest(data=data, size=size, endian=endian): @@ -221,6 +229,32 @@ def test_pack_unpack_roundtrip_for_nans(self): self.assertTrue(math.isnan(value)) self.assertEqual(data1, data2) + @unittest.skipUnless(HAVE_IEEE_754, "requires IEEE 754") + @unittest.skipUnless(sys.maxsize != 2147483647, "requires 64-bit mode") + def test_pack_unpack_nans_for_different_formats(self): + pack = _testcapi.float_pack + unpack = _testcapi.float_unpack + + for endian in (BIG_ENDIAN, LITTLE_ENDIAN): + with self.subTest(endian=endian): + byteorder = "big" if endian == BIG_ENDIAN else "little" + + # Convert sNaN to qNaN, if payload got truncated + data = make_nan(8, 0, False, 0x80001).to_bytes(8, byteorder) + snan_low = unpack(data, endian) + qnan4 = make_nan(4, 0, True, 0).to_bytes(4, byteorder) + qnan2 = make_nan(2, 0, True, 0).to_bytes(2, byteorder) + self.assertEqual(pack(4, snan_low, endian), qnan4) + self.assertEqual(pack(2, snan_low, endian), qnan2) + + # Preserve NaN type, if payload not truncated + data = make_nan(8, 0, False, 0x80000000001).to_bytes(8, byteorder) + snan_high = unpack(data, endian) + snan4 = make_nan(4, 0, False, 16384).to_bytes(4, byteorder) + snan2 = make_nan(2, 0, False, 2).to_bytes(2, byteorder) + self.assertEqual(pack(4, snan_high, endian), snan4) + self.assertEqual(pack(2, snan_high, endian), snan2) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst b/Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst new file mode 100644 index 00000000000..22a1aa1f405 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst @@ -0,0 +1,3 @@ +Make qNaN in :c:func:`PyFloat_Pack2` and :c:func:`PyFloat_Pack4`, if while +conversion to a narrower precision floating-point format --- the remaining +after truncation payload will be zero. Patch by Sergey B Kirpichev. diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 1fefb12803e..ef613efe4e7 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -2030,6 +2030,10 @@ PyFloat_Pack2(double x, char *data, int le) memcpy(&v, &x, sizeof(v)); v &= 0xffc0000000000ULL; bits = (unsigned short)(v >> 42); /* NaN's type & payload */ + /* set qNaN if no payload */ + if (!bits) { + bits |= (1<<9); + } } else { sign = (x < 0.0); @@ -2202,16 +2206,16 @@ PyFloat_Pack4(double x, char *data, int le) if ((v & (1ULL << 51)) == 0) { uint32_t u32; memcpy(&u32, &y, 4); - u32 &= ~(1 << 22); /* make sNaN */ + /* if have payload, make sNaN */ + if (u32 & 0x3fffff) { + u32 &= ~(1 << 22); + } memcpy(&y, &u32, 4); } #else uint32_t u32; memcpy(&u32, &y, 4); - if ((v & (1ULL << 51)) == 0) { - u32 &= ~(1 << 22); - } /* Workaround RISC-V: "If a NaN value is converted to a * different floating-point type, the result is the * canonical NaN of the new type". The canonical NaN here @@ -2222,6 +2226,10 @@ PyFloat_Pack4(double x, char *data, int le) /* add payload */ u32 -= (u32 & 0x3fffff); u32 += (uint32_t)((v & 0x7ffffffffffffULL) >> 29); + /* if have payload, make sNaN */ + if ((v & (1ULL << 51)) == 0 && (u32 & 0x3fffff)) { + u32 &= ~(1 << 22); + } memcpy(&y, &u32, 4); #endif From 70748bdbea872a84dd8eadad9b48c73e218d2e1f Mon Sep 17 00:00:00 2001 From: Jacob Austin Lincoln <99031153+lincolnj1@users.noreply.github.com> Date: Wed, 12 Nov 2025 02:07:21 -0800 Subject: [PATCH 165/313] gh-131116: Fix inspect.getdoc() to work with cached_property objects (GH-131165) --- Doc/library/inspect.rst | 3 ++ Lib/inspect.py | 6 +++ Lib/test/test_inspect/inspect_fodder3.py | 39 +++++++++++++++++++ Lib/test/test_inspect/test_inspect.py | 20 ++++++++++ ...-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst | 2 + 5 files changed, 70 insertions(+) create mode 100644 Lib/test/test_inspect/inspect_fodder3.py create mode 100644 Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index aff53b78c4a..13a352cbdb2 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -639,6 +639,9 @@ Retrieving source code .. versionchanged:: next Added parameters *inherit_class_doc* and *fallback_to_class_doc*. + Documentation strings on :class:`~functools.cached_property` + objects are now inherited if not overriden. + .. function:: getcomments(object) diff --git a/Lib/inspect.py b/Lib/inspect.py index bb17848b444..8e7511b3af0 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -747,6 +747,12 @@ def _finddoc(obj, *, search_in_class=True): cls = _findclass(obj.fget) if cls is None or getattr(cls, name) is not obj: return None + # Should be tested before ismethoddescriptor() + elif isinstance(obj, functools.cached_property): + name = obj.attrname + cls = _findclass(obj.func) + if cls is None or getattr(cls, name) is not obj: + return None elif ismethoddescriptor(obj) or isdatadescriptor(obj): name = obj.__name__ cls = obj.__objclass__ diff --git a/Lib/test/test_inspect/inspect_fodder3.py b/Lib/test/test_inspect/inspect_fodder3.py new file mode 100644 index 00000000000..ea2481edf93 --- /dev/null +++ b/Lib/test/test_inspect/inspect_fodder3.py @@ -0,0 +1,39 @@ +from functools import cached_property + +# docstring in parent, inherited in child +class ParentInheritDoc: + @cached_property + def foo(self): + """docstring for foo defined in parent""" + +class ChildInheritDoc(ParentInheritDoc): + pass + +class ChildInheritDefineDoc(ParentInheritDoc): + @cached_property + def foo(self): + pass + +# Redefine foo as something other than cached_property +class ChildPropertyFoo(ParentInheritDoc): + @property + def foo(self): + """docstring for the property foo""" + +class ChildMethodFoo(ParentInheritDoc): + def foo(self): + """docstring for the method foo""" + +# docstring in child but not parent +class ParentNoDoc: + @cached_property + def foo(self): + pass + +class ChildNoDoc(ParentNoDoc): + pass + +class ChildDefineDoc(ParentNoDoc): + @cached_property + def foo(self): + """docstring for foo defined in child""" diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 24fd4a2fa62..dd3b7d9c5b4 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -46,6 +46,7 @@ from test.test_inspect import inspect_fodder as mod from test.test_inspect import inspect_fodder2 as mod2 +from test.test_inspect import inspect_fodder3 as mod3 from test.test_inspect import inspect_stringized_annotations from test.test_inspect import inspect_deferred_annotations @@ -714,6 +715,25 @@ class B(A): b.__doc__ = 'Instance' self.assertEqual(inspect.getdoc(b, fallback_to_class_doc=False), 'Instance') + def test_getdoc_inherited_cached_property(self): + doc = inspect.getdoc(mod3.ParentInheritDoc.foo) + self.assertEqual(doc, 'docstring for foo defined in parent') + self.assertEqual(inspect.getdoc(mod3.ChildInheritDoc.foo), doc) + self.assertEqual(inspect.getdoc(mod3.ChildInheritDefineDoc.foo), doc) + + def test_getdoc_redefine_cached_property_as_other(self): + self.assertEqual(inspect.getdoc(mod3.ChildPropertyFoo.foo), + 'docstring for the property foo') + self.assertEqual(inspect.getdoc(mod3.ChildMethodFoo.foo), + 'docstring for the method foo') + + def test_getdoc_define_cached_property(self): + self.assertEqual(inspect.getdoc(mod3.ChildDefineDoc.foo), + 'docstring for foo defined in child') + + def test_getdoc_nodoc_inherited(self): + self.assertIsNone(inspect.getdoc(mod3.ChildNoDoc.foo)) + @unittest.skipIf(MISSING_C_DOCSTRINGS, "test requires docstrings") def test_finddoc(self): finddoc = inspect._finddoc diff --git a/Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst b/Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst new file mode 100644 index 00000000000..f5e60ab6e8c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst @@ -0,0 +1,2 @@ +:func:`inspect.getdoc` now correctly returns an inherited docstring on +:class:`~functools.cached_property` objects if none is given in a subclass. From c6f3dd6a506a9bb1808c070e5ef5cf345a3bedc8 Mon Sep 17 00:00:00 2001 From: Rani Pinchuk <33353578+rani-pinchuk@users.noreply.github.com> Date: Wed, 12 Nov 2025 13:35:01 +0100 Subject: [PATCH 166/313] gh-98896: resource_tracker: use json&base64 to allow arbitrary shared memory names (GH-138473) --- Lib/multiprocessing/resource_tracker.py | 60 ++++++++++++++++--- Lib/test/_test_multiprocessing.py | 43 +++++++++++++ ...5-09-03-20-18-39.gh-issue-98896.tjez89.rst | 2 + 3 files changed, 97 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 38fcaed48fa..b0f9099f4a5 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -15,6 +15,7 @@ # this resource tracker process, "killall python" would probably leave unlinked # resources. +import base64 import os import signal import sys @@ -22,6 +23,8 @@ import warnings from collections import deque +import json + from . import spawn from . import util @@ -196,6 +199,17 @@ def _launch(self): finally: os.close(r) + def _make_probe_message(self): + """Return a JSON-encoded probe message.""" + return ( + json.dumps( + {"cmd": "PROBE", "rtype": "noop"}, + ensure_ascii=True, + separators=(",", ":"), + ) + + "\n" + ).encode("ascii") + def _ensure_running_and_write(self, msg=None): with self._lock: if self._lock._recursion_count() > 1: @@ -207,7 +221,7 @@ def _ensure_running_and_write(self, msg=None): if self._fd is not None: # resource tracker was launched before, is it still running? if msg is None: - to_send = b'PROBE:0:noop\n' + to_send = self._make_probe_message() else: to_send = msg try: @@ -234,7 +248,7 @@ def _check_alive(self): try: # We cannot use send here as it calls ensure_running, creating # a cycle. - os.write(self._fd, b'PROBE:0:noop\n') + os.write(self._fd, self._make_probe_message()) except OSError: return False else: @@ -253,11 +267,25 @@ def _write(self, msg): assert nbytes == len(msg), f"{nbytes=} != {len(msg)=}" def _send(self, cmd, name, rtype): - msg = f"{cmd}:{name}:{rtype}\n".encode("ascii") - if len(msg) > 512: - # posix guarantees that writes to a pipe of less than PIPE_BUF - # bytes are atomic, and that PIPE_BUF >= 512 - raise ValueError('msg too long') + # POSIX guarantees that writes to a pipe of less than PIPE_BUF (512 on Linux) + # bytes are atomic. Therefore, we want the message to be shorter than 512 bytes. + # POSIX shm_open() and sem_open() require the name, including its leading slash, + # to be at most NAME_MAX bytes (255 on Linux) + # With json.dump(..., ensure_ascii=True) every non-ASCII byte becomes a 6-char + # escape like \uDC80. + # As we want the overall message to be kept atomic and therefore smaller than 512, + # we encode encode the raw name bytes with URL-safe Base64 - so a 255 long name + # will not exceed 340 bytes. + b = name.encode('utf-8', 'surrogateescape') + if len(b) > 255: + raise ValueError('shared memory name too long (max 255 bytes)') + b64 = base64.urlsafe_b64encode(b).decode('ascii') + + payload = {"cmd": cmd, "rtype": rtype, "base64_name": b64} + msg = (json.dumps(payload, ensure_ascii=True, separators=(",", ":")) + "\n").encode("ascii") + + # The entire JSON message is guaranteed < PIPE_BUF (512 bytes) by construction. + assert len(msg) <= 512, f"internal error: message too long ({len(msg)} bytes)" self._ensure_running_and_write(msg) @@ -290,7 +318,23 @@ def main(fd): with open(fd, 'rb') as f: for line in f: try: - cmd, name, rtype = line.strip().decode('ascii').split(':') + try: + obj = json.loads(line.decode('ascii')) + except Exception as e: + raise ValueError("malformed resource_tracker message: %r" % (line,)) from e + + cmd = obj["cmd"] + rtype = obj["rtype"] + b64 = obj.get("base64_name", "") + + if not isinstance(cmd, str) or not isinstance(rtype, str) or not isinstance(b64, str): + raise ValueError("malformed resource_tracker fields: %r" % (obj,)) + + try: + name = base64.urlsafe_b64decode(b64).decode('utf-8', 'surrogateescape') + except ValueError as e: + raise ValueError("malformed resource_tracker base64_name: %r" % (b64,)) from e + cleanup_func = _CLEANUP_FUNCS.get(rtype, None) if cleanup_func is None: raise ValueError( diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 850744e47d0..0f9c5c22225 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -7364,3 +7364,46 @@ def test_forkpty(self): res = assert_python_failure("-c", code, PYTHONWARNINGS='error') self.assertIn(b'DeprecationWarning', res.err) self.assertIn(b'is multi-threaded, use of forkpty() may lead to deadlocks in the child', res.err) + +@unittest.skipUnless(HAS_SHMEM, "requires multiprocessing.shared_memory") +class TestSharedMemoryNames(unittest.TestCase): + def test_that_shared_memory_name_with_colons_has_no_resource_tracker_errors(self): + # Test script that creates and cleans up shared memory with colon in name + test_script = textwrap.dedent(""" + import sys + from multiprocessing import shared_memory + import time + + # Test various patterns of colons in names + test_names = [ + "a:b", + "a:b:c", + "test:name:with:many:colons", + ":starts:with:colon", + "ends:with:colon:", + "::double::colons::", + "name\\nwithnewline", + "name-with-trailing-newline\\n", + "\\nname-starts-with-newline", + "colons:and\\nnewlines:mix", + "multi\\nline\\nname", + ] + + for name in test_names: + try: + shm = shared_memory.SharedMemory(create=True, size=100, name=name) + shm.buf[:5] = b'hello' # Write something to the shared memory + shm.close() + shm.unlink() + + except Exception as e: + print(f"Error with name '{name}': {e}", file=sys.stderr) + sys.exit(1) + + print("SUCCESS") + """) + + rc, out, err = assert_python_ok("-c", test_script) + self.assertIn(b"SUCCESS", out) + self.assertNotIn(b"traceback", err.lower(), err) + self.assertNotIn(b"resource_tracker.py", err, err) diff --git a/Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst b/Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst new file mode 100644 index 00000000000..6831499c0af --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst @@ -0,0 +1,2 @@ +Fix a failure in multiprocessing resource_tracker when SharedMemory names contain colons. +Patch by Rani Pinchuk. From e2026731f5680022bd016b8b5ca5841c82e9574c Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 12 Nov 2025 15:44:49 +0300 Subject: [PATCH 167/313] gh-141004: soft-deprecate Py_INFINITY macro (#141033) Co-authored-by: Victor Stinner --- Doc/c-api/conversion.rst | 2 +- Doc/c-api/float.rst | 7 +++++-- Doc/whatsnew/3.14.rst | 2 +- Doc/whatsnew/3.15.rst | 4 ++++ Include/floatobject.h | 16 +++++++-------- Include/internal/pycore_pymath.h | 6 +++--- Include/pymath.h | 3 ++- ...-11-05-04-38-16.gh-issue-141004.rJL43P.rst | 1 + Modules/cmathmodule.c | 6 +++--- Modules/mathmodule.c | 20 +++++++++---------- Objects/complexobject.c | 8 ++++---- Objects/floatobject.c | 2 +- Python/pystrtod.c | 4 ++-- 13 files changed, 45 insertions(+), 36 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index 533e5460da8..a18bbf4e0e3 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -105,7 +105,7 @@ The following functions provide locale-independent string to number conversions. If ``s`` represents a value that is too large to store in a float (for example, ``"1e500"`` is such a string on many platforms) then - if ``overflow_exception`` is ``NULL`` return ``Py_INFINITY`` (with + if ``overflow_exception`` is ``NULL`` return :c:macro:`!INFINITY` (with an appropriate sign) and don't set any exception. Otherwise, ``overflow_exception`` must point to a Python exception object; raise that exception and return ``-1.0``. In both cases, set diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index eae4792af7d..b6020533a2b 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -83,8 +83,11 @@ Floating-Point Objects This macro expands a to constant expression of type :c:expr:`double`, that represents the positive infinity. - On most platforms, this is equivalent to the :c:macro:`!INFINITY` macro from - the C11 standard ```` header. + It is equivalent to the :c:macro:`!INFINITY` macro from the C11 standard + ```` header. + + .. deprecated:: 3.15 + The macro is soft deprecated. .. c:macro:: Py_NAN diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 1a2fbda0c4c..9459b73bcb5 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -3045,7 +3045,7 @@ Deprecated C APIs ----------------- * The :c:macro:`!Py_HUGE_VAL` macro is now :term:`soft deprecated`. - Use :c:macro:`!Py_INFINITY` instead. + Use :c:macro:`!INFINITY` instead. (Contributed by Sergey B Kirpichev in :gh:`120026`.) * The :c:macro:`!Py_IS_NAN`, :c:macro:`!Py_IS_INFINITY`, diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index c543b6e6c2a..f0fd49c9033 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1095,6 +1095,10 @@ Deprecated C APIs since 3.15 and will be removed in 3.17. (Contributed by Nikita Sobolev in :gh:`136355`.) +* :c:macro:`!Py_INFINITY` macro is :term:`soft deprecated`, + use the C11 standard ```` :c:macro:`!INFINITY` instead. + (Contributed by Sergey B Kirpichev in :gh:`141004`.) + * :c:macro:`!Py_MATH_El` and :c:macro:`!Py_MATH_PIl` are deprecated since 3.15 and will be removed in 3.20. (Contributed by Sergey B Kirpichev in :gh:`141004`.) diff --git a/Include/floatobject.h b/Include/floatobject.h index 4d24a76edd5..814337b070a 100644 --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -18,14 +18,14 @@ PyAPI_DATA(PyTypeObject) PyFloat_Type; #define Py_RETURN_NAN return PyFloat_FromDouble(Py_NAN) -#define Py_RETURN_INF(sign) \ - do { \ - if (copysign(1., sign) == 1.) { \ - return PyFloat_FromDouble(Py_INFINITY); \ - } \ - else { \ - return PyFloat_FromDouble(-Py_INFINITY); \ - } \ +#define Py_RETURN_INF(sign) \ + do { \ + if (copysign(1., sign) == 1.) { \ + return PyFloat_FromDouble(INFINITY); \ + } \ + else { \ + return PyFloat_FromDouble(-INFINITY); \ + } \ } while(0) PyAPI_FUNC(double) PyFloat_GetMax(void); diff --git a/Include/internal/pycore_pymath.h b/Include/internal/pycore_pymath.h index eea8996ba68..4fcac3aab8b 100644 --- a/Include/internal/pycore_pymath.h +++ b/Include/internal/pycore_pymath.h @@ -33,7 +33,7 @@ extern "C" { static inline void _Py_ADJUST_ERANGE1(double x) { if (errno == 0) { - if (x == Py_INFINITY || x == -Py_INFINITY) { + if (x == INFINITY || x == -INFINITY) { errno = ERANGE; } } @@ -44,8 +44,8 @@ static inline void _Py_ADJUST_ERANGE1(double x) static inline void _Py_ADJUST_ERANGE2(double x, double y) { - if (x == Py_INFINITY || x == -Py_INFINITY || - y == Py_INFINITY || y == -Py_INFINITY) + if (x == INFINITY || x == -INFINITY || + y == INFINITY || y == -INFINITY) { if (errno == 0) { errno = ERANGE; diff --git a/Include/pymath.h b/Include/pymath.h index 0f9f0f3b299..7cfe441365d 100644 --- a/Include/pymath.h +++ b/Include/pymath.h @@ -45,13 +45,14 @@ #define Py_IS_FINITE(X) isfinite(X) // Py_INFINITY: Value that evaluates to a positive double infinity. +// Soft deprecated since Python 3.15, use INFINITY instead. #ifndef Py_INFINITY # define Py_INFINITY ((double)INFINITY) #endif /* Py_HUGE_VAL should always be the same as Py_INFINITY. But historically * this was not reliable and Python did not require IEEE floats and C99 - * conformity. The macro was soft deprecated in Python 3.14, use Py_INFINITY instead. + * conformity. The macro was soft deprecated in Python 3.14, use INFINITY instead. */ #ifndef Py_HUGE_VAL # define Py_HUGE_VAL HUGE_VAL diff --git a/Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst b/Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst new file mode 100644 index 00000000000..a054f8eda6f --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst @@ -0,0 +1 @@ +The :c:macro:`!Py_INFINITY` macro is :term:`soft deprecated`. diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index a4ea5557a6a..aee3e4f343d 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -150,7 +150,7 @@ special_type(double d) #define P14 0.25*Py_MATH_PI #define P12 0.5*Py_MATH_PI #define P34 0.75*Py_MATH_PI -#define INF Py_INFINITY +#define INF INFINITY #define N Py_NAN #define U -9.5426319407711027e33 /* unlikely value, used as placeholder */ @@ -1186,11 +1186,11 @@ cmath_exec(PyObject *mod) if (PyModule_Add(mod, "tau", PyFloat_FromDouble(Py_MATH_TAU)) < 0) { return -1; } - if (PyModule_Add(mod, "inf", PyFloat_FromDouble(Py_INFINITY)) < 0) { + if (PyModule_Add(mod, "inf", PyFloat_FromDouble(INFINITY)) < 0) { return -1; } - Py_complex infj = {0.0, Py_INFINITY}; + Py_complex infj = {0.0, INFINITY}; if (PyModule_Add(mod, "infj", PyComplex_FromCComplex(infj)) < 0) { return -1; } diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index de1886451ed..11c46c987e1 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -395,7 +395,7 @@ m_tgamma(double x) if (x == 0.0) { errno = EDOM; /* tgamma(+-0.0) = +-inf, divide-by-zero */ - return copysign(Py_INFINITY, x); + return copysign(INFINITY, x); } /* integer arguments */ @@ -426,7 +426,7 @@ m_tgamma(double x) } else { errno = ERANGE; - return Py_INFINITY; + return INFINITY; } } @@ -490,14 +490,14 @@ m_lgamma(double x) if (isnan(x)) return x; /* lgamma(nan) = nan */ else - return Py_INFINITY; /* lgamma(+-inf) = +inf */ + return INFINITY; /* lgamma(+-inf) = +inf */ } /* integer arguments */ if (x == floor(x) && x <= 2.0) { if (x <= 0.0) { errno = EDOM; /* lgamma(n) = inf, divide-by-zero for */ - return Py_INFINITY; /* integers n <= 0 */ + return INFINITY; /* integers n <= 0 */ } else { return 0.0; /* lgamma(1) = lgamma(2) = 0.0 */ @@ -633,7 +633,7 @@ m_log(double x) return log(x); errno = EDOM; if (x == 0.0) - return -Py_INFINITY; /* log(0) = -inf */ + return -INFINITY; /* log(0) = -inf */ else return Py_NAN; /* log(-ve) = nan */ } @@ -676,7 +676,7 @@ m_log2(double x) } else if (x == 0.0) { errno = EDOM; - return -Py_INFINITY; /* log2(0) = -inf, divide-by-zero */ + return -INFINITY; /* log2(0) = -inf, divide-by-zero */ } else { errno = EDOM; @@ -692,7 +692,7 @@ m_log10(double x) return log10(x); errno = EDOM; if (x == 0.0) - return -Py_INFINITY; /* log10(0) = -inf */ + return -INFINITY; /* log10(0) = -inf */ else return Py_NAN; /* log10(-ve) = nan */ } @@ -1500,7 +1500,7 @@ math_ldexp_impl(PyObject *module, double x, PyObject *i) errno = 0; } else if (exp > INT_MAX) { /* overflow */ - r = copysign(Py_INFINITY, x); + r = copysign(INFINITY, x); errno = ERANGE; } else if (exp < INT_MIN) { /* underflow to +-0 */ @@ -2983,7 +2983,7 @@ math_ulp_impl(PyObject *module, double x) if (isinf(x)) { return x; } - double inf = Py_INFINITY; + double inf = INFINITY; double x2 = nextafter(x, inf); if (isinf(x2)) { /* special case: x is the largest positive representable float */ @@ -3007,7 +3007,7 @@ math_exec(PyObject *module) if (PyModule_Add(module, "tau", PyFloat_FromDouble(Py_MATH_TAU)) < 0) { return -1; } - if (PyModule_Add(module, "inf", PyFloat_FromDouble(Py_INFINITY)) < 0) { + if (PyModule_Add(module, "inf", PyFloat_FromDouble(INFINITY)) < 0) { return -1; } if (PyModule_Add(module, "nan", PyFloat_FromDouble(fabs(Py_NAN))) < 0) { diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 6247376a0e6..3612c2699a5 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -139,8 +139,8 @@ _Py_c_prod(Py_complex z, Py_complex w) recalc = 1; } if (recalc) { - r.real = Py_INFINITY*(a*c - b*d); - r.imag = Py_INFINITY*(a*d + b*c); + r.real = INFINITY*(a*c - b*d); + r.imag = INFINITY*(a*d + b*c); } } @@ -229,8 +229,8 @@ _Py_c_quot(Py_complex a, Py_complex b) { const double x = copysign(isinf(a.real) ? 1.0 : 0.0, a.real); const double y = copysign(isinf(a.imag) ? 1.0 : 0.0, a.imag); - r.real = Py_INFINITY * (x*b.real + y*b.imag); - r.imag = Py_INFINITY * (y*b.real - x*b.imag); + r.real = INFINITY * (x*b.real + y*b.imag); + r.imag = INFINITY * (y*b.real - x*b.imag); } else if ((isinf(abs_breal) || isinf(abs_bimag)) && isfinite(a.real) && isfinite(a.imag)) diff --git a/Objects/floatobject.c b/Objects/floatobject.c index ef613efe4e7..78006783c6e 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -2415,7 +2415,7 @@ PyFloat_Unpack2(const char *data, int le) if (e == 0x1f) { if (f == 0) { /* Infinity */ - return sign ? -Py_INFINITY : Py_INFINITY; + return sign ? -INFINITY : INFINITY; } else { /* NaN */ diff --git a/Python/pystrtod.c b/Python/pystrtod.c index 7b74f613ed5..e8aca939d1f 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -43,7 +43,7 @@ _Py_parse_inf_or_nan(const char *p, char **endptr) s += 3; if (case_insensitive_match(s, "inity")) s += 5; - retval = negate ? -Py_INFINITY : Py_INFINITY; + retval = negate ? -INFINITY : INFINITY; } else if (case_insensitive_match(s, "nan")) { s += 3; @@ -286,7 +286,7 @@ _PyOS_ascii_strtod(const char *nptr, char **endptr) string, -1.0 is returned and again ValueError is raised. On overflow (e.g., when trying to convert '1e500' on an IEEE 754 machine), - if overflow_exception is NULL then +-Py_INFINITY is returned, and no Python + if overflow_exception is NULL then +-INFINITY is returned, and no Python exception is raised. Otherwise, overflow_exception should point to a Python exception, this exception will be raised, -1.0 will be returned, and *endptr will point just past the end of the converted value. From f963864cb54c2e7364b2c850485c6bf25479f6f2 Mon Sep 17 00:00:00 2001 From: yihong Date: Wed, 12 Nov 2025 20:45:43 +0800 Subject: [PATCH 168/313] gh-141464: a typo in profiling sampling when can not run warning in linux (#141465) --- Lib/profiling/sampling/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/profiling/sampling/__main__.py b/Lib/profiling/sampling/__main__.py index a76ca62e2cd..cd1425b8b9c 100644 --- a/Lib/profiling/sampling/__main__.py +++ b/Lib/profiling/sampling/__main__.py @@ -15,7 +15,7 @@ """ LINUX_PERMISSION_ERROR = """ -🔒 Tachyon was unable to acess process memory. This could be because tachyon +🔒 Tachyon was unable to access process memory. This could be because tachyon has insufficient privileges (the required capability is CAP_SYS_PTRACE). Unprivileged processes cannot trace processes that they cannot send signals to or those running set-user-ID/set-group-ID programs, for security reasons. From 88aeff8eabefdc13b6fb29edb3cde618f743a034 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Wed, 12 Nov 2025 14:22:01 +0000 Subject: [PATCH 169/313] gh-87710: Update mime type for ``.ai`` (#141239) --- Doc/whatsnew/3.15.rst | 4 +++- Lib/mimetypes.py | 2 +- Lib/test/test_mimetypes.py | 1 + .../Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst | 1 + 4 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index f0fd49c9033..c6089f63dee 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -472,7 +472,9 @@ mimetypes * Add ``application/node`` MIME type for ``.cjs`` extension. (Contributed by John Franey in :gh:`140937`.) * Add ``application/toml``. (Contributed by Gil Forcada in :gh:`139959`.) * Rename ``application/x-texinfo`` to ``application/texinfo``. - (Contributed by Charlie Lin in :gh:`140165`) + (Contributed by Charlie Lin in :gh:`140165`.) +* Changed the MIME type for ``.ai`` files to ``application/pdf``. + (Contributed by Stan Ulbrych in :gh:`141239`.) mmap diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index d6896fc4042..42477713c78 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -497,9 +497,9 @@ def _default_mime_types(): '.oda' : 'application/oda', '.ogx' : 'application/ogg', '.pdf' : 'application/pdf', + '.ai' : 'application/pdf', '.p7c' : 'application/pkcs7-mime', '.ps' : 'application/postscript', - '.ai' : 'application/postscript', '.eps' : 'application/postscript', '.texi' : 'application/texinfo', '.texinfo': 'application/texinfo', diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index 746984ec0ca..73414498359 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -229,6 +229,7 @@ def check_extensions(): ("application/octet-stream", ".bin"), ("application/gzip", ".gz"), ("application/ogg", ".ogx"), + ("application/pdf", ".pdf"), ("application/postscript", ".ps"), ("application/texinfo", ".texi"), ("application/toml", ".toml"), diff --git a/Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst b/Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst new file mode 100644 index 00000000000..62073280e32 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst @@ -0,0 +1 @@ +:mod:`mimetypes`: Update mime type for ``.ai`` files to ``application/pdf``. From 2ac738d325a6934e39fecb097f43d4d4ed97a2b9 Mon Sep 17 00:00:00 2001 From: M Bussonnier Date: Wed, 12 Nov 2025 16:20:08 +0100 Subject: [PATCH 170/313] gh-132657: add regression test for `PySet_Contains` with unhashable type (#141411) --- Modules/_testlimitedcapi/set.c | 47 ++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/Modules/_testlimitedcapi/set.c b/Modules/_testlimitedcapi/set.c index 35da5fa5f00..34ed6b1d60b 100644 --- a/Modules/_testlimitedcapi/set.c +++ b/Modules/_testlimitedcapi/set.c @@ -155,6 +155,51 @@ test_frozenset_add_in_capi(PyObject *self, PyObject *Py_UNUSED(obj)) return NULL; } +static PyObject * +test_set_contains_does_not_convert_unhashable_key(PyObject *self, PyObject *Py_UNUSED(obj)) +{ + // See https://docs.python.org/3/c-api/set.html#c.PySet_Contains + PyObject *outer_set = PySet_New(NULL); + + PyObject *needle = PySet_New(NULL); + if (needle == NULL) { + Py_DECREF(outer_set); + return NULL; + } + + PyObject *num = PyLong_FromLong(42); + if (num == NULL) { + Py_DECREF(outer_set); + Py_DECREF(needle); + return NULL; + } + + if (PySet_Add(needle, num) < 0) { + Py_DECREF(outer_set); + Py_DECREF(needle); + Py_DECREF(num); + return NULL; + } + + int result = PySet_Contains(outer_set, needle); + + Py_DECREF(num); + Py_DECREF(needle); + Py_DECREF(outer_set); + + if (result < 0) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_Clear(); + Py_RETURN_NONE; + } + return NULL; + } + + PyErr_SetString(PyExc_AssertionError, + "PySet_Contains should have raised TypeError for unhashable key"); + return NULL; +} + static PyMethodDef test_methods[] = { {"set_check", set_check, METH_O}, {"set_checkexact", set_checkexact, METH_O}, @@ -174,6 +219,8 @@ static PyMethodDef test_methods[] = { {"set_clear", set_clear, METH_O}, {"test_frozenset_add_in_capi", test_frozenset_add_in_capi, METH_NOARGS}, + {"test_set_contains_does_not_convert_unhashable_key", + test_set_contains_does_not_convert_unhashable_key, METH_NOARGS}, {NULL}, }; From f1330b35b8eb43904dfed0656acde80c08d63176 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Wed, 12 Nov 2025 16:37:54 +0000 Subject: [PATCH 171/313] gh-141004: Document `Py_MATH_{E, PI, TAU}` constants (#141373) --- Doc/c-api/float.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index b6020533a2b..79de5daaa90 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -99,6 +99,11 @@ Floating-Point Objects the C11 standard ```` header. +.. c:macro:: Py_MATH_E + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.e` constant. + + .. c:macro:: Py_MATH_El High precision (long double) definition of :data:`~math.e` constant. @@ -106,6 +111,11 @@ Floating-Point Objects .. deprecated-removed:: 3.15 3.20 +.. c:macro:: Py_MATH_PI + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.pi` constant. + + .. c:macro:: Py_MATH_PIl High precision (long double) definition of :data:`~math.pi` constant. @@ -113,6 +123,13 @@ Floating-Point Objects .. deprecated-removed:: 3.15 3.20 +.. c:macro:: Py_MATH_TAU + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.tau` constant. + + .. versionadded:: 3.6 + + .. c:macro:: Py_RETURN_NAN Return :data:`math.nan` from a function. From 9cd5427d9619b96db20d0347a136b3d331af71ae Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 12 Nov 2025 11:38:17 -0500 Subject: [PATCH 172/313] gh-141004: Document `PyType_SUPPORTS_WEAKREFS` (GH-141408) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/type.rst | 26 ++++++++++++++++++++++++++ Doc/c-api/weakref.rst | 8 ++++++++ 2 files changed, 34 insertions(+) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 29ffeb7c483..b608f815160 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -195,12 +195,14 @@ Type Objects before initialization) and should be paired with :c:func:`PyObject_Free` in :c:member:`~PyTypeObject.tp_free`. + .. c:function:: PyObject* PyType_GenericNew(PyTypeObject *type, PyObject *args, PyObject *kwds) Generic handler for the :c:member:`~PyTypeObject.tp_new` slot of a type object. Creates a new instance using the type's :c:member:`~PyTypeObject.tp_alloc` slot and returns the resulting object. + .. c:function:: int PyType_Ready(PyTypeObject *type) Finalize a type object. This should be called on all type objects to finish @@ -217,6 +219,7 @@ Type Objects GC protocol itself by at least implementing the :c:member:`~PyTypeObject.tp_traverse` handle. + .. c:function:: PyObject* PyType_GetName(PyTypeObject *type) Return the type's name. Equivalent to getting the type's @@ -224,6 +227,7 @@ Type Objects .. versionadded:: 3.11 + .. c:function:: PyObject* PyType_GetQualName(PyTypeObject *type) Return the type's qualified name. Equivalent to getting the @@ -239,6 +243,7 @@ Type Objects .. versionadded:: 3.13 + .. c:function:: PyObject* PyType_GetModuleName(PyTypeObject *type) Return the type's module name. Equivalent to getting the @@ -246,6 +251,7 @@ Type Objects .. versionadded:: 3.13 + .. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) Return the function pointer stored in the given slot. If the @@ -262,6 +268,7 @@ Type Objects :c:func:`PyType_GetSlot` can now accept all types. Previously, it was limited to :ref:`heap types `. + .. c:function:: PyObject* PyType_GetModule(PyTypeObject *type) Return the module object associated with the given type when the type was @@ -281,6 +288,7 @@ Type Objects .. versionadded:: 3.9 + .. c:function:: void* PyType_GetModuleState(PyTypeObject *type) Return the state of the module object associated with the given type. @@ -295,6 +303,7 @@ Type Objects .. versionadded:: 3.9 + .. c:function:: PyObject* PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def) Find the first superclass whose module was created from @@ -314,6 +323,7 @@ Type Objects .. versionadded:: 3.11 + .. c:function:: int PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) Find the first superclass in *type*'s :term:`method resolution order` whose @@ -332,6 +342,7 @@ Type Objects .. versionadded:: 3.14 + .. c:function:: int PyUnstable_Type_AssignVersionTag(PyTypeObject *type) Attempt to assign a version tag to the given type. @@ -342,6 +353,16 @@ Type Objects .. versionadded:: 3.12 +.. c:function:: int PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) + + Return true if instances of *type* support creating weak references, false + otherwise. This function always succeeds. *type* must not be ``NULL``. + + .. seealso:: + * :ref:`weakrefobjects` + * :py:mod:`weakref` + + Creating Heap-Allocated Types ............................. @@ -390,6 +411,7 @@ The following functions and structs are used to create .. versionadded:: 3.12 + .. c:function:: PyObject* PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) Equivalent to ``PyType_FromMetaclass(NULL, module, spec, bases)``. @@ -416,6 +438,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, bases)``. @@ -437,6 +460,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, NULL)``. @@ -457,6 +481,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: int PyType_Freeze(PyTypeObject *type) Make a type immutable: set the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag. @@ -628,6 +653,7 @@ The following functions and structs are used to create * :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` rather than ``NULL``) + .. c:macro:: Py_tp_token A :c:member:`~PyType_Slot.slot` that records a static memory layout ID diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index 39e4febd3ef..db6ae0a9d4e 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -45,6 +45,10 @@ as much as it can. weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + .. seealso:: + :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly + referenceable. + .. c:function:: PyObject* PyWeakref_NewProxy(PyObject *ob, PyObject *callback) @@ -57,6 +61,10 @@ as much as it can. is not a weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + .. seealso:: + :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly + referenceable. + .. c:function:: int PyWeakref_GetRef(PyObject *ref, PyObject **pobj) From d162c427904e232fec52d8da759caa1bfa4c01b5 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Wed, 12 Nov 2025 10:09:25 -0800 Subject: [PATCH 173/313] GH-140479: Update JIT builds to use LLVM 21 (#140973) --- .github/workflows/jit.yml | 8 ++++---- ...-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst | 1 + PCbuild/get_externals.bat | 4 ++-- Tools/jit/README.md | 20 +++++++++---------- Tools/jit/_llvm.py | 4 ++-- 5 files changed, 19 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 69d900091a3..62325250bd3 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -68,7 +68,7 @@ jobs: - true - false llvm: - - 20 + - 21 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 @@ -138,7 +138,7 @@ jobs: fail-fast: false matrix: llvm: - - 20 + - 21 steps: - uses: actions/checkout@v4 with: @@ -166,7 +166,7 @@ jobs: fail-fast: false matrix: llvm: - - 20 + - 21 steps: - uses: actions/checkout@v4 with: @@ -193,7 +193,7 @@ jobs: fail-fast: false matrix: llvm: - - 20 + - 21 steps: - uses: actions/checkout@v4 with: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst new file mode 100644 index 00000000000..0a615ed1311 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst @@ -0,0 +1 @@ +Update JIT compilation to use LLVM 21 at build time. diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 115203cecc8..9d02e2121cc 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -82,7 +82,7 @@ if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.18 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 -if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-20.1.8.0 +if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-21.1.4.0 for %%b in (%binaries%) do ( if exist "%EXTERNALS_DIR%\%%b" ( @@ -92,7 +92,7 @@ for %%b in (%binaries%) do ( git clone --depth 1 https://github.com/%ORG%/cpython-bin-deps --branch %%b "%EXTERNALS_DIR%\%%b" ) else ( echo.Fetching %%b... - if "%%b"=="llvm-20.1.8.0" ( + if "%%b"=="llvm-21.1.4.0" ( %PYTHON% -E "%PCBUILD%\get_external.py" --release --organization %ORG% --externals-dir "%EXTERNALS_DIR%" %%b ) else ( %PYTHON% -E "%PCBUILD%\get_external.py" --binary --organization %ORG% --externals-dir "%EXTERNALS_DIR%" %%b diff --git a/Tools/jit/README.md b/Tools/jit/README.md index d83b09aab59..dd7deb7b256 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -9,32 +9,32 @@ ## Installing LLVM The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). -LLVM version 20 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. +LLVM version 21 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. It's easy to install all of the required tools: ### Linux -Install LLVM 20 on Ubuntu/Debian: +Install LLVM 21 on Ubuntu/Debian: ```sh wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh -sudo ./llvm.sh 20 +sudo ./llvm.sh 21 ``` -Install LLVM 20 on Fedora Linux 40 or newer: +Install LLVM 21 on Fedora Linux 40 or newer: ```sh -sudo dnf install 'clang(major) = 20' 'llvm(major) = 20' +sudo dnf install 'clang(major) = 21' 'llvm(major) = 21' ``` ### macOS -Install LLVM 20 with [Homebrew](https://brew.sh): +Install LLVM 21 with [Homebrew](https://brew.sh): ```sh -brew install llvm@20 +brew install llvm@21 ``` Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them. @@ -43,18 +43,18 @@ ### Windows LLVM is downloaded automatically (along with other external binary dependencies) by `PCbuild\build.bat`. -Otherwise, you can install LLVM 20 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=20), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** +Otherwise, you can install LLVM 21 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=21), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** Alternatively, you can use [chocolatey](https://chocolatey.org): ```sh -choco install llvm --version=20.1.8 +choco install llvm --version=21.1.0 ``` ### Dev Containers If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no -need to install LLVM as the Fedora 42 base image includes LLVM 20 out of the box. +need to install LLVM as the Fedora 43 base image includes LLVM 21 out of the box. ## Building diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index f1b0ad3f5db..0b9cb5192f1 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -11,8 +11,8 @@ import _targets -_LLVM_VERSION = "20" -_EXTERNALS_LLVM_TAG = "llvm-20.1.8.0" +_LLVM_VERSION = "21" +_EXTERNALS_LLVM_TAG = "llvm-21.1.4.0" _P = typing.ParamSpec("_P") _R = typing.TypeVar("_R") From fbcac799518e0cb29fcf5f84ed1fa001010b9073 Mon Sep 17 00:00:00 2001 From: Bob Kline Date: Wed, 12 Nov 2025 13:25:23 -0500 Subject: [PATCH 174/313] gh-141412: Use reliable target URL for urllib example (GH-141428) The endpoint used for demonstrating reading URLs is no longer stable. This change substitutes a target over which we have more control. --- Doc/tutorial/stdlib.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 49a3e370a4c..342c1a00193 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -183,13 +183,13 @@ protocols. Two of the simplest are :mod:`urllib.request` for retrieving data from URLs and :mod:`smtplib` for sending mail:: >>> from urllib.request import urlopen - >>> with urlopen('http://worldtimeapi.org/api/timezone/etc/UTC.txt') as response: + >>> with urlopen('https://docs.python.org/3/') as response: ... for line in response: ... line = line.decode() # Convert bytes to a str - ... if line.startswith('datetime'): + ... if 'updated' in line: ... print(line.rstrip()) # Remove trailing newline ... - datetime: 2022-01-01T01:36:47.689215+00:00 + Last updated on Nov 11, 2025 (20:11 UTC). >>> import smtplib >>> server = smtplib.SMTP('localhost') From 1f381a579cc50aa82838de84c2294b4979586bd9 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Wed, 12 Nov 2025 10:26:50 -0800 Subject: [PATCH 175/313] Add details about JIT build infrastructure and updating dependencies to `Tools/jit` (#141167) --- Tools/jit/README.md | 3 +++ Tools/jit/jit_infra.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 Tools/jit/jit_infra.md diff --git a/Tools/jit/README.md b/Tools/jit/README.md index dd7deb7b256..c70c0c47d94 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -66,6 +66,9 @@ ## Building The JIT can also be enabled or disabled using the `PYTHON_JIT` environment variable, even on builds where it is enabled or disabled by default. More details about configuring CPython with the JIT and optional values for `--enable-experimental-jit` can be found [here](https://docs.python.org/dev/using/configure.html#cmdoption-enable-experimental-jit). +## Miscellaneous +If you're looking for information on how to update the JIT build dependencies, see [JIT Build Infrastructure](jit_infra.md). + [^pep-744]: [PEP 744](https://peps.python.org/pep-0744/) [^why-llvm]: Clang is specifically needed because it's the only C compiler with support for guaranteed tail calls (`musttail`), which are required by CPython's continuation-passing-style approach to JIT compilation. Since LLVM also includes other functionalities we need (namely, object file parsing and disassembly), it's convenient to only support one toolchain at this time. diff --git a/Tools/jit/jit_infra.md b/Tools/jit/jit_infra.md new file mode 100644 index 00000000000..1a954755611 --- /dev/null +++ b/Tools/jit/jit_infra.md @@ -0,0 +1,28 @@ +# JIT Build Infrastructure + +This document includes details about the intricacies of the JIT build infrastructure. + +## Updating LLVM + +When we update LLVM, we need to also update the LLVM release artifact for Windows builds. This is because Windows builds automatically pull prebuilt LLVM binaries in our pipelines (e.g. notice that `.github/workflows/jit.yml` does not explicitly download LLVM or build it from source). + +To update the LLVM release artifact for Windows builds, follow these steps: +1. Go to the [LLVM releases page](https://github.com/llvm/llvm-project/releases). +1. Download x86_64 Windows artifact for the desired LLVM version (e.g. `clang+llvm-21.1.4-x86_64-pc-windows-msvc.tar.xz`). +1. Extract and repackage the tarball with the correct directory structure. For example: + ```bash + tar -xf clang+llvm-21.1.4-x86_64-pc-windows-msvc.tar.xz + mv clang+llvm-21.1.4-x86_64-pc-windows-msvc llvm-21.1.4.0 + tar -cf - llvm-21.1.4.0 | pv | xz > llvm-21.1.4.0.tar.xz + ``` + The tarball must contain a top-level directory named `llvm-{version}.0/`. +1. Go to [cpython-bin-deps](https://github.com/python/cpython-bin-deps). +1. Create a new release with the updated LLVM artifact. + - Create a new tag to match the LLVM version (e.g. `llvm-21.1.4.0`). + - Specify the release title (e.g. `LLVM 21.1.4 for x86_64 Windows`). + - Upload the asset (you can leave all other fields the same). + +### Other notes +- You must make sure that the name of the artifact matches exactly what is expected in `Tools/jit/_llvm.py` and `PCbuild/get_externals.py`. +- We don't need multiple release artifacts for each architecture because LLVM can cross-compile for different architectures on Windows; x86_64 is sufficient. +- You must have permissions to create releases in the `cpython-bin-deps` repository. If you don't have permissions, you should contact one of the organization admins. \ No newline at end of file From 35ed3e4cedc8aef3936da81a6b64e90374532b13 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Wed, 12 Nov 2025 22:04:02 +0300 Subject: [PATCH 176/313] gh-140936: Fix JIT assertion crash at finalization if some generator is alive (GH-140969) --- Lib/test/test_capi/test_opt.py | 19 +++++++++++++++++++ Python/optimizer.c | 8 +++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 4e94f62d35e..e65556fb28f 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -2660,6 +2660,25 @@ def f(): f() + def test_interpreter_finalization_with_generator_alive(self): + script_helper.assert_python_ok("-c", textwrap.dedent(""" + import sys + t = tuple(range(%d)) + def simple_for(): + for x in t: + x + + def gen(): + try: + yield + except: + simple_for() + + sys.settrace(lambda *args: None) + simple_for() + g = gen() + next(g) + """ % _testinternalcapi.SPECIALIZATION_THRESHOLD)) def global_identity(x): diff --git a/Python/optimizer.c b/Python/optimizer.c index f44f8a9614b..3b7e2dafab8 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -118,7 +118,13 @@ _PyOptimizer_Optimize( { _PyStackRef *stack_pointer = frame->stackpointer; PyInterpreterState *interp = _PyInterpreterState_GET(); - assert(interp->jit); + if (!interp->jit) { + // gh-140936: It is possible that interp->jit will become false during + // interpreter finalization. However, the specialized JUMP_BACKWARD_JIT + // instruction may still be present. In this case, we should + // return immediately without optimization. + return 0; + } assert(!interp->compiling); #ifndef Py_GIL_DISABLED interp->compiling = true; From 558936bec1f1e0f8346063a8cb2b2782d085178e Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Thu, 13 Nov 2025 05:41:26 +0800 Subject: [PATCH 177/313] gh-141442: Add escaping to iOS testbed arguments (#141443) Xcode concatenates the test argument array, losing quoting in the process. --- Apple/testbed/__main__.py | 3 ++- .../Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst diff --git a/Apple/testbed/__main__.py b/Apple/testbed/__main__.py index 42eb60a4c8d..49974cb1428 100644 --- a/Apple/testbed/__main__.py +++ b/Apple/testbed/__main__.py @@ -2,6 +2,7 @@ import json import os import re +import shlex import shutil import subprocess import sys @@ -252,7 +253,7 @@ def update_test_plan(testbed_path, platform, args): test_plan = json.load(f) test_plan["defaultOptions"]["commandLineArgumentEntries"] = [ - {"argument": arg} for arg in args + {"argument": shlex.quote(arg)} for arg in args ] with test_plan_path.open("w", encoding="utf-8") as f: diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst b/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst new file mode 100644 index 00000000000..073c070413f --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst @@ -0,0 +1 @@ +The iOS testbed now correctly handles test arguments that contain spaces. From dc0987080ed66c662e8e0b24cdb8c179817bd697 Mon Sep 17 00:00:00 2001 From: Michael Cho Date: Wed, 12 Nov 2025 17:16:58 -0500 Subject: [PATCH 178/313] gh-124111: Fix TCL 9 thread detection (GH-128103) --- .../Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst | 2 ++ Modules/_tkinter.c | 4 ++++ 2 files changed, 6 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst diff --git a/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst b/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst new file mode 100644 index 00000000000..8436cd2415d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst @@ -0,0 +1,2 @@ +Updated Tcl threading configuration in :mod:`_tkinter` to assume that +threads are always available in Tcl 9 and later. diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index c0ed8977d8f..8cea7b59fe7 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -575,8 +575,12 @@ Tkapp_New(const char *screenName, const char *className, v->interp = Tcl_CreateInterp(); v->wantobjects = wantobjects; +#if TCL_MAJOR_VERSION >= 9 + v->threaded = 1; +#else v->threaded = Tcl_GetVar2Ex(v->interp, "tcl_platform", "threaded", TCL_GLOBAL_ONLY) != NULL; +#endif v->thread_id = Tcl_GetCurrentThread(); v->dispatching = 0; v->trace = NULL; From 26b7df2430cd5a9ee772bfa6ee03a73bd0b11619 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 12 Nov 2025 17:52:56 -0500 Subject: [PATCH 179/313] gh-141004: Document `PyRun_InteractiveOneObject` (GH-141405) --- Doc/c-api/veryhigh.rst | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 916c616dfee..3b07b5fbed5 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -100,6 +100,20 @@ the same library that the Python runtime is using. Otherwise, Python may not handle script file with LF line ending correctly. +.. c:function:: int PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags) + + Read and execute a single statement from a file associated with an + interactive device according to the *flags* argument. The user will be + prompted using ``sys.ps1`` and ``sys.ps2``. *filename* must be a Python + :class:`str` object. + + Returns ``0`` when the input was + executed successfully, ``-1`` if there was an exception, or an error code + from the :file:`errcode.h` include file distributed as part of Python if + there was a parse error. (Note that :file:`errcode.h` is not included by + :file:`Python.h`, so must be included specifically if needed.) + + .. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_InteractiveOneFlags` below, @@ -108,17 +122,10 @@ the same library that the Python runtime is using. .. c:function:: int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) - Read and execute a single statement from a file associated with an - interactive device according to the *flags* argument. The user will be - prompted using ``sys.ps1`` and ``sys.ps2``. *filename* is decoded from the + Similar to :c:func:`PyRun_InteractiveOneObject`, but *filename* is a + :c:expr:`const char*`, which is decoded from the :term:`filesystem encoding and error handler`. - Returns ``0`` when the input was - executed successfully, ``-1`` if there was an exception, or an error code - from the :file:`errcode.h` include file distributed as part of Python if - there was a parse error. (Note that :file:`errcode.h` is not included by - :file:`Python.h`, so must be included specifically if needed.) - .. c:function:: int PyRun_InteractiveLoop(FILE *fp, const char *filename) From 781cc68c3c814e46e6a74c3a6a32e0f9f8f7eb11 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" <68491+gpshead@users.noreply.github.com> Date: Wed, 12 Nov 2025 18:15:16 -0800 Subject: [PATCH 180/313] gh-137109: refactor warning about threads when forking (#141438) * gh-137109: refactor warning about threads when forking This splits the OS API specific functionality to get the number of threads out from the fallback Python method and warning raising code itself. This way the OS APIs can be queried before we've run `os.register_at_fork(after_in_parent=...)` registered functions which themselves may (re)start threads that would otherwise be detected. This is best effort. If the OS APIs are either unavailable or fail, the warning generating code still falls back to looking at the Python threading state after the CPython interpreter world has been restarted and the after_in_parent calls have been made. The common case for most Linux and macOS environments should work today. This also lines up with the existing TODO refactoring, we may choose to expose this API to get the number of OS threads in the `os` module in the future. * NEWS entry * avoid "function-prototype" compiler warning? --- ...-11-12-01-49-03.gh-issue-137109.D6sq2B.rst | 5 + Modules/posixmodule.c | 103 ++++++++++-------- 2 files changed, 65 insertions(+), 43 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst diff --git a/Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst b/Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst new file mode 100644 index 00000000000..32f4e39f6d5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst @@ -0,0 +1,5 @@ +The :mod:`os.fork` and related forking APIs will no longer warn in the +common case where Linux or macOS platform APIs return the number of threads +in a process and find the answer to be 1 even when a +:func:`os.register_at_fork` ``after_in_parent=`` callback (re)starts a +thread. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 6390f1fc5fe..fc609b2707c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8431,53 +8431,19 @@ os_register_at_fork_impl(PyObject *module, PyObject *before, // running in the process. Best effort, silent if unable to count threads. // Constraint: Quick. Never overcounts. Never leaves an error set. // -// This should only be called from the parent process after +// This MUST only be called from the parent process after // PyOS_AfterFork_Parent(). static int -warn_about_fork_with_threads(const char* name) +warn_about_fork_with_threads( + const char* name, // Name of the API to use in the warning message. + const Py_ssize_t num_os_threads // Only trusted when >= 1. +) { // It's not safe to issue the warning while the world is stopped, because // other threads might be holding locks that we need, which would deadlock. assert(!_PyRuntime.stoptheworld.world_stopped); - // TODO: Consider making an `os` module API to return the current number - // of threads in the process. That'd presumably use this platform code but - // raise an error rather than using the inaccurate fallback. - Py_ssize_t num_python_threads = 0; -#if defined(__APPLE__) && defined(HAVE_GETPID) - mach_port_t macos_self = mach_task_self(); - mach_port_t macos_task; - if (task_for_pid(macos_self, getpid(), &macos_task) == KERN_SUCCESS) { - thread_array_t macos_threads; - mach_msg_type_number_t macos_n_threads; - if (task_threads(macos_task, &macos_threads, - &macos_n_threads) == KERN_SUCCESS) { - num_python_threads = macos_n_threads; - } - } -#elif defined(__linux__) - // Linux /proc/self/stat 20th field is the number of threads. - FILE* proc_stat = fopen("/proc/self/stat", "r"); - if (proc_stat) { - size_t n; - // Size chosen arbitrarily. ~60% more bytes than a 20th column index - // observed on the author's workstation. - char stat_line[160]; - n = fread(&stat_line, 1, 159, proc_stat); - stat_line[n] = '\0'; - fclose(proc_stat); - - char *saveptr = NULL; - char *field = strtok_r(stat_line, " ", &saveptr); - unsigned int idx; - for (idx = 19; idx && field; --idx) { - field = strtok_r(NULL, " ", &saveptr); - } - if (idx == 0 && field) { // found the 20th field - num_python_threads = atoi(field); // 0 on error - } - } -#endif + Py_ssize_t num_python_threads = num_os_threads; if (num_python_threads <= 0) { // Fall back to just the number our threading module knows about. // An incomplete view of the world, but better than nothing. @@ -8530,6 +8496,51 @@ warn_about_fork_with_threads(const char* name) } return 0; } + +// If this returns <= 0, we were unable to successfully use any OS APIs. +// Returns a positive number of threads otherwise. +static Py_ssize_t get_number_of_os_threads(void) +{ + // TODO: Consider making an `os` module API to return the current number + // of threads in the process. That'd presumably use this platform code but + // raise an error rather than using the inaccurate fallback. + Py_ssize_t num_python_threads = 0; +#if defined(__APPLE__) && defined(HAVE_GETPID) + mach_port_t macos_self = mach_task_self(); + mach_port_t macos_task; + if (task_for_pid(macos_self, getpid(), &macos_task) == KERN_SUCCESS) { + thread_array_t macos_threads; + mach_msg_type_number_t macos_n_threads; + if (task_threads(macos_task, &macos_threads, + &macos_n_threads) == KERN_SUCCESS) { + num_python_threads = macos_n_threads; + } + } +#elif defined(__linux__) + // Linux /proc/self/stat 20th field is the number of threads. + FILE* proc_stat = fopen("/proc/self/stat", "r"); + if (proc_stat) { + size_t n; + // Size chosen arbitrarily. ~60% more bytes than a 20th column index + // observed on the author's workstation. + char stat_line[160]; + n = fread(&stat_line, 1, 159, proc_stat); + stat_line[n] = '\0'; + fclose(proc_stat); + + char *saveptr = NULL; + char *field = strtok_r(stat_line, " ", &saveptr); + unsigned int idx; + for (idx = 19; idx && field; --idx) { + field = strtok_r(NULL, " ", &saveptr); + } + if (idx == 0 && field) { // found the 20th field + num_python_threads = atoi(field); // 0 on error + } + } +#endif + return num_python_threads; +} #endif // HAVE_FORK1 || HAVE_FORKPTY || HAVE_FORK #ifdef HAVE_FORK1 @@ -8564,10 +8575,12 @@ os_fork1_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + // Called before AfterFork_Parent in case those hooks start threads. + Py_ssize_t num_os_threads = get_number_of_os_threads(); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. - if (warn_about_fork_with_threads("fork1") < 0) { + if (warn_about_fork_with_threads("fork1", num_os_threads) < 0) { return NULL; } } @@ -8615,10 +8628,12 @@ os_fork_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + // Called before AfterFork_Parent in case those hooks start threads. + Py_ssize_t num_os_threads = get_number_of_os_threads(); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. - if (warn_about_fork_with_threads("fork") < 0) + if (warn_about_fork_with_threads("fork", num_os_threads) < 0) return NULL; } if (pid == -1) { @@ -9476,6 +9491,8 @@ os_forkpty_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + // Called before AfterFork_Parent in case those hooks start threads. + Py_ssize_t num_os_threads = get_number_of_os_threads(); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); /* set O_CLOEXEC on master_fd */ @@ -9485,7 +9502,7 @@ os_forkpty_impl(PyObject *module) } // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. - if (warn_about_fork_with_threads("forkpty") < 0) + if (warn_about_fork_with_threads("forkpty", num_os_threads) < 0) return NULL; } if (pid == -1) { From 63548b36998e7f7cd5c7c28b53b348a93f836737 Mon Sep 17 00:00:00 2001 From: Shamil Date: Thu, 13 Nov 2025 14:01:31 +0300 Subject: [PATCH 181/313] gh-140260: fix data race in `_struct` module initialization with subinterpreters (#140909) --- Lib/test/test_struct.py | 17 ++++ ...-11-02-15-28-33.gh-issue-140260.JNzlGz.rst | 2 + Modules/_struct.c | 91 ++++++++++--------- Tools/c-analyzer/cpython/ignored.tsv | 1 + 4 files changed, 70 insertions(+), 41 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 75c76a36ee9..cceecdd526c 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -800,6 +800,23 @@ def test_c_complex_round_trip(self): round_trip = struct.unpack(f, struct.pack(f, z))[0] self.assertComplexesAreIdentical(z, round_trip) + @unittest.skipIf( + support.is_android or support.is_apple_mobile, + "Subinterpreters are not supported on Android and iOS" + ) + def test_endian_table_init_subinterpreters(self): + # Verify that the _struct extension module can be initialized + # concurrently in subinterpreters (gh-140260). + try: + from concurrent.futures import InterpreterPoolExecutor + except ImportError: + raise unittest.SkipTest("InterpreterPoolExecutor not available") + + code = "import struct" + with InterpreterPoolExecutor(max_workers=5) as executor: + results = executor.map(exec, [code] * 5) + self.assertListEqual(list(results), [None] * 5) + class UnpackIteratorTest(unittest.TestCase): """ diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst new file mode 100644 index 00000000000..96bf9b51e48 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst @@ -0,0 +1,2 @@ +Fix :mod:`struct` data race in endian table initialization with +subinterpreters. Patch by Shamil Abdulaev. diff --git a/Modules/_struct.c b/Modules/_struct.c index f09252e82c3..2acb3df3a30 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -9,6 +9,7 @@ #include "Python.h" #include "pycore_bytesobject.h" // _PyBytesWriter +#include "pycore_lock.h" // _PyOnceFlag_CallOnce() #include "pycore_long.h" // _PyLong_AsByteArray() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() @@ -1505,6 +1506,53 @@ static formatdef lilendian_table[] = { {0} }; +/* Ensure endian table optimization happens exactly once across all interpreters */ +static _PyOnceFlag endian_tables_init_once = {0}; + +static int +init_endian_tables(void *Py_UNUSED(arg)) +{ + const formatdef *native = native_table; + formatdef *other, *ptr; +#if PY_LITTLE_ENDIAN + other = lilendian_table; +#else + other = bigendian_table; +#endif + /* Scan through the native table, find a matching + entry in the endian table and swap in the + native implementations whenever possible + (64-bit platforms may not have "standard" sizes) */ + while (native->format != '\0' && other->format != '\0') { + ptr = other; + while (ptr->format != '\0') { + if (ptr->format == native->format) { + /* Match faster when formats are + listed in the same order */ + if (ptr == other) + other++; + /* Only use the trick if the + size matches */ + if (ptr->size != native->size) + break; + /* Skip float and double, could be + "unknown" float format */ + if (ptr->format == 'd' || ptr->format == 'f') + break; + /* Skip _Bool, semantics are different for standard size */ + if (ptr->format == '?') + break; + ptr->pack = native->pack; + ptr->unpack = native->unpack; + break; + } + ptr++; + } + native++; + } + return 0; +} + static const formatdef * whichtable(const char **pfmt) @@ -2710,47 +2758,8 @@ _structmodule_exec(PyObject *m) return -1; } - /* Check endian and swap in faster functions */ - { - const formatdef *native = native_table; - formatdef *other, *ptr; -#if PY_LITTLE_ENDIAN - other = lilendian_table; -#else - other = bigendian_table; -#endif - /* Scan through the native table, find a matching - entry in the endian table and swap in the - native implementations whenever possible - (64-bit platforms may not have "standard" sizes) */ - while (native->format != '\0' && other->format != '\0') { - ptr = other; - while (ptr->format != '\0') { - if (ptr->format == native->format) { - /* Match faster when formats are - listed in the same order */ - if (ptr == other) - other++; - /* Only use the trick if the - size matches */ - if (ptr->size != native->size) - break; - /* Skip float and double, could be - "unknown" float format */ - if (ptr->format == 'd' || ptr->format == 'f') - break; - /* Skip _Bool, semantics are different for standard size */ - if (ptr->format == '?') - break; - ptr->pack = native->pack; - ptr->unpack = native->unpack; - break; - } - ptr++; - } - native++; - } - } + /* init cannot fail */ + (void)_PyOnceFlag_CallOnce(&endian_tables_init_once, init_endian_tables, NULL); /* Add some symbolic constants to the module */ state->StructError = PyErr_NewException("struct.error", NULL, NULL); diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 11a3cd794ff..4621ad250f4 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -24,6 +24,7 @@ Modules/posixmodule.c os_dup2_impl dup3_works - ## guards around resource init Python/thread_pthread.h PyThread__init_thread lib_initialized - +Modules/_struct.c - endian_tables_init_once - ##----------------------- ## other values (not Python-specific) From d8e6bdc0d083f4e76ac49574544555ad91257592 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Nov 2025 13:21:32 +0200 Subject: [PATCH 182/313] gh-135801: Add the module parameter to compile() etc (GH-139652) Many functions related to compiling or parsing Python code, such as compile(), ast.parse(), symtable.symtable(), and importlib.abc.InspectLoader.source_to_code() now allow to pass the module name used when filtering syntax warnings. --- Doc/library/ast.rst | 7 ++- Doc/library/functions.rst | 11 +++- Doc/library/importlib.rst | 10 +++- Doc/library/symtable.rst | 8 ++- Doc/whatsnew/3.15.rst | 7 +++ Include/internal/pycore_compile.h | 9 ++- Include/internal/pycore_parser.h | 3 +- Include/internal/pycore_pyerrors.h | 3 +- Include/internal/pycore_pythonrun.h | 6 ++ Include/internal/pycore_symtable.h | 3 +- Lib/ast.py | 5 +- Lib/importlib/_bootstrap_external.py | 9 +-- Lib/importlib/abc.py | 11 ++-- Lib/modulefinder.py | 2 +- Lib/profiling/sampling/_sync_coordinator.py | 2 +- Lib/profiling/tracing/__init__.py | 2 +- Lib/runpy.py | 6 +- Lib/symtable.py | 4 +- Lib/test/test_ast/test_ast.py | 10 ++++ Lib/test/test_builtin.py | 3 +- Lib/test/test_cmd_line_script.py | 23 ++++++-- Lib/test/test_compile.py | 10 ++++ Lib/test/test_import/__init__.py | 15 +---- Lib/test/test_runpy.py | 43 ++++++++++++++ Lib/test/test_symtable.py | 10 ++++ Lib/test/test_zipimport_support.py | 23 ++++++++ Lib/zipimport.py | 6 +- ...-10-06-14-19-47.gh-issue-135801.OhxEZS.rst | 6 ++ Modules/clinic/symtablemodule.c.h | 58 ++++++++++++++++--- Modules/symtablemodule.c | 19 +++++- Parser/lexer/state.c | 2 + Parser/lexer/state.h | 1 + Parser/peg_api.c | 6 +- Parser/pegen.c | 8 ++- Parser/pegen.h | 2 +- Parser/string_parser.c | 2 +- Parser/tokenizer/helpers.c | 4 +- Programs/_freeze_module.py | 2 +- Programs/freeze_test_frozenmain.py | 2 +- Python/ast_preprocess.c | 8 ++- Python/bltinmodule.c | 25 ++++++-- Python/clinic/bltinmodule.c.h | 26 ++++++--- Python/compile.c | 30 ++++++---- Python/errors.c | 7 ++- Python/pythonrun.c | 38 ++++++++++-- Python/symtable.c | 4 +- .../peg_extension/peg_extension.c | 4 +- 47 files changed, 390 insertions(+), 115 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 49462167217..0ea3c3c59a6 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2205,10 +2205,10 @@ Async and await Apart from the node classes, the :mod:`ast` module defines these utility functions and classes for traversing abstract syntax trees: -.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1) +.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1, module=None) Parse the source into an AST node. Equivalent to ``compile(source, - filename, mode, flags=FLAGS_VALUE, optimize=optimize)``, + filename, mode, flags=FLAGS_VALUE, optimize=optimize, module=module)``, where ``FLAGS_VALUE`` is ``ast.PyCF_ONLY_AST`` if ``optimize <= 0`` and ``ast.PyCF_OPTIMIZED_AST`` otherwise. @@ -2261,6 +2261,9 @@ and classes for traversing abstract syntax trees: The minimum supported version for ``feature_version`` is now ``(3, 7)``. The ``optimize`` argument was added. + .. versionadded:: next + Added the *module* parameter. + .. function:: unparse(ast_obj) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index e9879397555..3257daf89d3 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -292,7 +292,9 @@ are always available. They are listed here in alphabetical order. :func:`property`. -.. function:: compile(source, filename, mode, flags=0, dont_inherit=False, optimize=-1) +.. function:: compile(source, filename, mode, flags=0, \ + dont_inherit=False, optimize=-1, \ + *, module=None) Compile the *source* into a code or AST object. Code objects can be executed by :func:`exec` or :func:`eval`. *source* can either be a normal string, a @@ -334,6 +336,10 @@ are always available. They are listed here in alphabetical order. ``__debug__`` is true), ``1`` (asserts are removed, ``__debug__`` is false) or ``2`` (docstrings are removed too). + The optional argument *module* specifies the module name. + It is needed to unambiguous :ref:`filter ` syntax warnings + by module name. + This function raises :exc:`SyntaxError` if the compiled source is invalid, and :exc:`ValueError` if the source contains null bytes. @@ -371,6 +377,9 @@ are always available. They are listed here in alphabetical order. ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable support for top-level ``await``, ``async for``, and ``async with``. + .. versionadded:: next + Added the *module* parameter. + .. class:: complex(number=0, /) complex(string, /) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 602a7100a12..03ba23b6216 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -459,7 +459,7 @@ ABC hierarchy:: .. versionchanged:: 3.4 Raises :exc:`ImportError` instead of :exc:`NotImplementedError`. - .. staticmethod:: source_to_code(data, path='') + .. staticmethod:: source_to_code(data, path='', fullname=None) Create a code object from Python source. @@ -471,11 +471,19 @@ ABC hierarchy:: With the subsequent code object one can execute it in a module by running ``exec(code, module.__dict__)``. + The optional argument *fullname* specifies the module name. + It is needed to unambiguous :ref:`filter ` syntax + warnings by module name. + .. versionadded:: 3.4 .. versionchanged:: 3.5 Made the method static. + .. versionadded:: next + Added the *fullname* parameter. + + .. method:: exec_module(module) Implementation of :meth:`Loader.exec_module`. diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 54e19af4bd6..c0d9e79197d 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -21,11 +21,17 @@ tables. Generating Symbol Tables ------------------------ -.. function:: symtable(code, filename, compile_type) +.. function:: symtable(code, filename, compile_type, *, module=None) Return the toplevel :class:`SymbolTable` for the Python source *code*. *filename* is the name of the file containing the code. *compile_type* is like the *mode* argument to :func:`compile`. + The optional argument *module* specifies the module name. + It is needed to unambiguous :ref:`filter ` syntax warnings + by module name. + + .. versionadded:: next + Added the *module* parameter. Examining Symbol Tables diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index c6089f63dee..3cb766978a7 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -307,6 +307,13 @@ Other language changes not only integers or floats, although this does not improve precision. (Contributed by Serhiy Storchaka in :gh:`67795`.) +* Many functions related to compiling or parsing Python code, such as + :func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, + and :func:`importlib.abc.InspectLoader.source_to_code`, now allow to pass + the module name. It is needed to unambiguous :ref:`filter ` + syntax warnings by module name. + (Contributed by Serhiy Storchaka in :gh:`135801`.) + New modules =========== diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 1c60834fa20..527141b54d0 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -32,7 +32,8 @@ PyAPI_FUNC(PyCodeObject*) _PyAST_Compile( PyObject *filename, PyCompilerFlags *flags, int optimize, - struct _arena *arena); + struct _arena *arena, + PyObject *module); /* AST preprocessing */ extern int _PyCompile_AstPreprocess( @@ -41,7 +42,8 @@ extern int _PyCompile_AstPreprocess( PyCompilerFlags *flags, int optimize, struct _arena *arena, - int syntax_check_only); + int syntax_check_only, + PyObject *module); extern int _PyAST_Preprocess( struct _mod *, @@ -50,7 +52,8 @@ extern int _PyAST_Preprocess( int optimize, int ff_features, int syntax_check_only, - int enable_warnings); + int enable_warnings, + PyObject *module); typedef struct { diff --git a/Include/internal/pycore_parser.h b/Include/internal/pycore_parser.h index 2885dee63dc..2c46f59ab7d 100644 --- a/Include/internal/pycore_parser.h +++ b/Include/internal/pycore_parser.h @@ -48,7 +48,8 @@ extern struct _mod* _PyParser_ASTFromString( PyObject* filename, int mode, PyCompilerFlags *flags, - PyArena *arena); + PyArena *arena, + PyObject *module); extern struct _mod* _PyParser_ASTFromFile( FILE *fp, diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 2c2048f7e12..f80808fcc8c 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -123,7 +123,8 @@ extern void _PyErr_SetNone(PyThreadState *tstate, PyObject *exception); extern PyObject* _PyErr_NoMemory(PyThreadState *tstate); extern int _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_offset, - int end_lineno, int end_col_offset); + int end_lineno, int end_col_offset, + PyObject *module); extern void _PyErr_RaiseSyntaxError(PyObject *msg, PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset); diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h index c2832098ddb..f954f1b63ef 100644 --- a/Include/internal/pycore_pythonrun.h +++ b/Include/internal/pycore_pythonrun.h @@ -33,6 +33,12 @@ extern const char* _Py_SourceAsString( PyCompilerFlags *cf, PyObject **cmd_copy); +extern PyObject * _Py_CompileStringObjectWithModule( + const char *str, + PyObject *filename, int start, + PyCompilerFlags *flags, int optimize, + PyObject *module); + /* Stack size, in "pointers". This must be large enough, so * no two calls to check recursion depth are more than this far diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 98099b4a497..9dbfa913219 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -188,7 +188,8 @@ extern struct symtable* _Py_SymtableStringObjectFlags( const char *str, PyObject *filename, int start, - PyCompilerFlags *flags); + PyCompilerFlags *flags, + PyObject *module); int _PyFuture_FromAST( struct _mod * mod, diff --git a/Lib/ast.py b/Lib/ast.py index 983ac1710d0..d9743ba7ab4 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -24,7 +24,7 @@ def parse(source, filename='', mode='exec', *, - type_comments=False, feature_version=None, optimize=-1): + type_comments=False, feature_version=None, optimize=-1, module=None): """ Parse the source into an AST node. Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). @@ -44,7 +44,8 @@ def parse(source, filename='', mode='exec', *, feature_version = minor # Else it should be an int giving the minor version for 3.x. return compile(source, filename, mode, flags, - _feature_version=feature_version, optimize=optimize) + _feature_version=feature_version, optimize=optimize, + module=module) def literal_eval(node_or_string): diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 035ae0fcae1..4ab0e79ea6e 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -819,13 +819,14 @@ def get_source(self, fullname): name=fullname) from exc return decode_source(source_bytes) - def source_to_code(self, data, path, *, _optimize=-1): + def source_to_code(self, data, path, fullname=None, *, _optimize=-1): """Return the code object compiled from source. The 'data' argument can be any object type that compile() supports. """ return _bootstrap._call_with_frames_removed(compile, data, path, 'exec', - dont_inherit=True, optimize=_optimize) + dont_inherit=True, optimize=_optimize, + module=fullname) def get_code(self, fullname): """Concrete implementation of InspectLoader.get_code. @@ -894,7 +895,7 @@ def get_code(self, fullname): source_path=source_path) if source_bytes is None: source_bytes = self.get_data(source_path) - code_object = self.source_to_code(source_bytes, source_path) + code_object = self.source_to_code(source_bytes, source_path, fullname) _bootstrap._verbose_message('code object from {}', source_path) if (not sys.dont_write_bytecode and bytecode_path is not None and source_mtime is not None): @@ -1186,7 +1187,7 @@ def get_source(self, fullname): return '' def get_code(self, fullname): - return compile('', '', 'exec', dont_inherit=True) + return compile('', '', 'exec', dont_inherit=True, module=fullname) def create_module(self, spec): """Use default semantics for module creation.""" diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 1e47495f65f..5c13432b5bd 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -108,7 +108,7 @@ def get_code(self, fullname): source = self.get_source(fullname) if source is None: return None - return self.source_to_code(source) + return self.source_to_code(source, '', fullname) @abc.abstractmethod def get_source(self, fullname): @@ -120,12 +120,12 @@ def get_source(self, fullname): raise ImportError @staticmethod - def source_to_code(data, path=''): + def source_to_code(data, path='', fullname=None): """Compile 'data' into a code object. The 'data' argument can be anything that compile() can handle. The'path' argument should be where the data was retrieved (when applicable).""" - return compile(data, path, 'exec', dont_inherit=True) + return compile(data, path, 'exec', dont_inherit=True, module=fullname) exec_module = _bootstrap_external._LoaderBasics.exec_module load_module = _bootstrap_external._LoaderBasics.load_module @@ -163,9 +163,8 @@ def get_code(self, fullname): try: path = self.get_filename(fullname) except ImportError: - return self.source_to_code(source) - else: - return self.source_to_code(source, path) + path = '' + return self.source_to_code(source, path, fullname) _register( ExecutionLoader, diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index ac478ee7f51..b115d99ab30 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -334,7 +334,7 @@ def load_module(self, fqname, fp, pathname, file_info): self.msgout(2, "load_module ->", m) return m if type == _PY_SOURCE: - co = compile(fp.read(), pathname, 'exec') + co = compile(fp.read(), pathname, 'exec', module=fqname) elif type == _PY_COMPILED: try: data = fp.read() diff --git a/Lib/profiling/sampling/_sync_coordinator.py b/Lib/profiling/sampling/_sync_coordinator.py index 8716e654104..adb040e89cc 100644 --- a/Lib/profiling/sampling/_sync_coordinator.py +++ b/Lib/profiling/sampling/_sync_coordinator.py @@ -182,7 +182,7 @@ def _execute_script(script_path: str, script_args: List[str], cwd: str) -> None: try: # Compile and execute the script - code = compile(source_code, script_path, 'exec') + code = compile(source_code, script_path, 'exec', module='__main__') exec(code, {'__name__': '__main__', '__file__': script_path}) except SyntaxError as e: raise TargetError(f"Syntax error in script {script_path}: {e}") from e diff --git a/Lib/profiling/tracing/__init__.py b/Lib/profiling/tracing/__init__.py index 2dc7ea92c8c..a6b8edf7216 100644 --- a/Lib/profiling/tracing/__init__.py +++ b/Lib/profiling/tracing/__init__.py @@ -185,7 +185,7 @@ def main(): progname = args[0] sys.path.insert(0, os.path.dirname(progname)) with io.open_code(progname) as fp: - code = compile(fp.read(), progname, 'exec') + code = compile(fp.read(), progname, 'exec', module='__main__') spec = importlib.machinery.ModuleSpec(name='__main__', loader=None, origin=progname) module = importlib.util.module_from_spec(spec) diff --git a/Lib/runpy.py b/Lib/runpy.py index ef54d3282ee..f072498f6cb 100644 --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -247,7 +247,7 @@ def _get_main_module_details(error=ImportError): sys.modules[main_name] = saved_main -def _get_code_from_file(fname): +def _get_code_from_file(fname, module): # Check for a compiled file first from pkgutil import read_code code_path = os.path.abspath(fname) @@ -256,7 +256,7 @@ def _get_code_from_file(fname): if code is None: # That didn't work, so try it as normal source code with io.open_code(code_path) as f: - code = compile(f.read(), fname, 'exec') + code = compile(f.read(), fname, 'exec', module=module) return code def run_path(path_name, init_globals=None, run_name=None): @@ -283,7 +283,7 @@ def run_path(path_name, init_globals=None, run_name=None): if isinstance(importer, type(None)): # Not a valid sys.path entry, so run the code directly # execfile() doesn't help as we want to allow compiled files - code = _get_code_from_file(path_name) + code = _get_code_from_file(path_name, run_name) return _run_module_code(code, init_globals, run_name, pkg_name=pkg_name, script_name=path_name) else: diff --git a/Lib/symtable.py b/Lib/symtable.py index 77475c3ffd9..4c832e68f94 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -17,13 +17,13 @@ __all__ = ["symtable", "SymbolTableType", "SymbolTable", "Class", "Function", "Symbol"] -def symtable(code, filename, compile_type): +def symtable(code, filename, compile_type, *, module=None): """ Return the toplevel *SymbolTable* for the source code. *filename* is the name of the file with the code and *compile_type* is the *compile()* mode argument. """ - top = _symtable.symtable(code, filename, compile_type) + top = _symtable.symtable(code, filename, compile_type, module=module) return _newSymbolTable(top, filename) class SymbolTableFactory: diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 551de5851da..fb4a441ca64 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -1083,6 +1083,16 @@ def test_filter_syntax_warnings_by_module(self): self.assertEqual(wm.filename, '') self.assertIs(wm.category, SyntaxWarning) + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.module\z') + warnings.filterwarnings('error', module=r'') + ast.parse(source, filename, module='package.module') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + class CopyTests(unittest.TestCase): """Test copying and pickling AST nodes.""" diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index fba46af6617..ce60a5d095d 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1103,7 +1103,8 @@ def test_exec_filter_syntax_warnings_by_module(self): with warnings.catch_warnings(record=True) as wlog: warnings.simplefilter('error') - warnings.filterwarnings('always', module=r'\z') + warnings.filterwarnings('always', module=r'package.module\z') + warnings.filterwarnings('error', module=r'') exec(source, {'__name__': 'package.module', '__file__': filename}) self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) for wm in wlog: diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index f8115cc8300..cc1a625a509 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -814,15 +814,26 @@ def test_filter_syntax_warnings_by_module(self): filename = support.findfile('test_import/data/syntax_warnings.py') rc, out, err = assert_python_ok( '-Werror', - '-Walways:::test.test_import.data.syntax_warnings', + '-Walways:::__main__', + '-Werror:::test.test_import.data.syntax_warnings', + '-Werror:::syntax_warnings', filename) self.assertEqual(err.count(b': SyntaxWarning: '), 6) - rc, out, err = assert_python_ok( - '-Werror', - '-Walways:::syntax_warnings', - filename) - self.assertEqual(err.count(b': SyntaxWarning: '), 6) + def test_zipfile_run_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with open(filename, 'rb') as f: + source = f.read() + with os_helper.temp_dir() as script_dir: + zip_name, _ = make_zip_pkg( + script_dir, 'test_zip', 'test_pkg', '__main__', source) + rc, out, err = assert_python_ok( + '-Werror', + '-Walways:::__main__', + '-Werror:::test_pkg.__main__', + os.path.join(zip_name, 'test_pkg') + ) + self.assertEqual(err.count(b': SyntaxWarning: '), 12) def tearDownModule(): diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 9c2364491fe..30f21875b22 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1759,6 +1759,16 @@ def test_filter_syntax_warnings_by_module(self): self.assertEqual(wm.filename, filename) self.assertIs(wm.category, SyntaxWarning) + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.module\z') + warnings.filterwarnings('error', module=module_re) + compile(source, filename, 'exec', module='package.module') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + @support.subTests('src', [ textwrap.dedent(""" def f(): diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index e87d8b7e7bb..fe669bb04df 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1259,20 +1259,7 @@ def test_filter_syntax_warnings_by_module(self): warnings.catch_warnings(record=True) as wlog): warnings.simplefilter('error') warnings.filterwarnings('always', module=module_re) - import test.test_import.data.syntax_warnings - self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) - filename = test.test_import.data.syntax_warnings.__file__ - for wm in wlog: - self.assertEqual(wm.filename, filename) - self.assertIs(wm.category, SyntaxWarning) - - module_re = r'syntax_warnings\z' - unload('test.test_import.data.syntax_warnings') - with (os_helper.temp_dir() as tmpdir, - temporary_pycache_prefix(tmpdir), - warnings.catch_warnings(record=True) as wlog): - warnings.simplefilter('error') - warnings.filterwarnings('always', module=module_re) + warnings.filterwarnings('error', module='syntax_warnings') import test.test_import.data.syntax_warnings self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) filename = test.test_import.data.syntax_warnings.__file__ diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index a2a07c04f58..cc76b72b963 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -20,9 +20,11 @@ requires_subprocess, verbose, ) +from test import support from test.support.import_helper import forget, make_legacy_pyc, unload from test.support.os_helper import create_empty_file, temp_dir, FakePath from test.support.script_helper import make_script, make_zip_script +from test.test_importlib.util import temporary_pycache_prefix import runpy @@ -763,6 +765,47 @@ def test_encoding(self): result = run_path(filename) self.assertEqual(result['s'], "non-ASCII: h\xe9") + def test_run_module_filter_syntax_warnings_by_module(self): + module_re = r'test\.test_import\.data\.syntax_warnings\z' + with (temp_dir() as tmpdir, + temporary_pycache_prefix(tmpdir), + warnings.catch_warnings(record=True) as wlog): + warnings.simplefilter('error') + warnings.filterwarnings('always', module=module_re) + warnings.filterwarnings('error', module='syntax_warnings') + ns = run_module('test.test_import.data.syntax_warnings') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + filename = ns['__file__'] + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + + def test_run_path_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'\z') + warnings.filterwarnings('error', module='test') + warnings.filterwarnings('error', module='syntax_warnings') + warnings.filterwarnings('error', + module=r'test\.test_import\.data\.syntax_warnings') + run_path(filename) + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.script\z') + warnings.filterwarnings('error', module='') + warnings.filterwarnings('error', module='test') + warnings.filterwarnings('error', module='syntax_warnings') + warnings.filterwarnings('error', + module=r'test\.test_import\.data\.syntax_warnings') + run_path(filename, run_name='package.script') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + @force_not_colorized_test_class class TestExit(unittest.TestCase): diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index ef2c00e04b8..094ab8f573e 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -601,6 +601,16 @@ def test_filter_syntax_warnings_by_module(self): self.assertEqual(wm.filename, filename) self.assertIs(wm.category, SyntaxWarning) + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.module\z') + warnings.filterwarnings('error', module=module_re) + symtable.symtable(source, filename, 'exec', module='package.module') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + class ComprehensionTests(unittest.TestCase): def get_identifiers_recursive(self, st, res): diff --git a/Lib/test/test_zipimport_support.py b/Lib/test/test_zipimport_support.py index ae8a8c99762..2b28f46149b 100644 --- a/Lib/test/test_zipimport_support.py +++ b/Lib/test/test_zipimport_support.py @@ -13,9 +13,12 @@ import inspect import linecache import unittest +import warnings +from test import support from test.support import os_helper from test.support.script_helper import (spawn_python, kill_python, assert_python_ok, make_script, make_zip_script) +from test.support import import_helper verbose = test.support.verbose @@ -236,6 +239,26 @@ def f(): # bdb/pdb applies normcase to its filename before displaying self.assertIn(os.path.normcase(run_name.encode('utf-8')), data) + def test_import_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with (os_helper.temp_dir() as tmpdir, + import_helper.DirsOnSysPath()): + zip_name, _ = make_zip_script(tmpdir, "test_zip", + filename, 'test_pkg/test_mod.py') + sys.path.insert(0, zip_name) + import_helper.unload('test_pkg.test_mod') + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'test_pkg\.test_mod\z') + warnings.filterwarnings('error', module='test_mod') + import test_pkg.test_mod + self.assertEqual(sorted(wm.lineno for wm in wlog), + sorted([4, 7, 10, 13, 14, 21]*2)) + filename = test_pkg.test_mod.__file__ + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + def tearDownModule(): test.support.reap_children() diff --git a/Lib/zipimport.py b/Lib/zipimport.py index 340a7e07112..19279d1c2be 100644 --- a/Lib/zipimport.py +++ b/Lib/zipimport.py @@ -742,9 +742,9 @@ def _normalize_line_endings(source): # Given a string buffer containing Python source code, compile it # and return a code object. -def _compile_source(pathname, source): +def _compile_source(pathname, source, module): source = _normalize_line_endings(source) - return compile(source, pathname, 'exec', dont_inherit=True) + return compile(source, pathname, 'exec', dont_inherit=True, module=module) # Convert the date/time values found in the Zip archive to a value # that's compatible with the time stamp stored in .pyc files. @@ -815,7 +815,7 @@ def _get_module_code(self, fullname): except ImportError as exc: import_error = exc else: - code = _compile_source(modpath, data) + code = _compile_source(modpath, data, fullname) if code is None: # bad magic number or non-matching mtime # in byte code, try next diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst new file mode 100644 index 00000000000..96226a7c525 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst @@ -0,0 +1,6 @@ +Many functions related to compiling or parsing Python code, such as +:func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, and +:func:`importlib.abc.InspectLoader.source_to_code` now allow to specify +the module name. +It is needed to unambiguous :ref:`filter ` syntax warnings +by module name. diff --git a/Modules/clinic/symtablemodule.c.h b/Modules/clinic/symtablemodule.c.h index bd55d77c540..65352593f94 100644 --- a/Modules/clinic/symtablemodule.c.h +++ b/Modules/clinic/symtablemodule.c.h @@ -2,30 +2,67 @@ preserve [clinic start generated code]*/ -#include "pycore_modsupport.h" // _PyArg_CheckPositional() +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_symtable_symtable__doc__, -"symtable($module, source, filename, startstr, /)\n" +"symtable($module, source, filename, startstr, /, *, module=None)\n" "--\n" "\n" "Return symbol and scope dictionaries used internally by compiler."); #define _SYMTABLE_SYMTABLE_METHODDEF \ - {"symtable", _PyCFunction_CAST(_symtable_symtable), METH_FASTCALL, _symtable_symtable__doc__}, + {"symtable", _PyCFunction_CAST(_symtable_symtable), METH_FASTCALL|METH_KEYWORDS, _symtable_symtable__doc__}, static PyObject * _symtable_symtable_impl(PyObject *module, PyObject *source, - PyObject *filename, const char *startstr); + PyObject *filename, const char *startstr, + PyObject *modname); static PyObject * -_symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +_symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(module), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "", "", "module", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "symtable", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; PyObject *source; PyObject *filename = NULL; const char *startstr; + PyObject *modname = Py_None; - if (!_PyArg_CheckPositional("symtable", nargs, 3, 3)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { goto exit; } source = args[0]; @@ -45,7 +82,12 @@ _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - return_value = _symtable_symtable_impl(module, source, filename, startstr); + if (!noptargs) { + goto skip_optional_kwonly; + } + modname = args[3]; +skip_optional_kwonly: + return_value = _symtable_symtable_impl(module, source, filename, startstr, modname); exit: /* Cleanup for filename */ @@ -53,4 +95,4 @@ exit: return return_value; } -/*[clinic end generated code: output=7a8545d9a1efe837 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0137be60c487c841 input=a9049054013a1b77]*/ diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index d353f406831..a24927a9db6 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -16,14 +16,17 @@ _symtable.symtable filename: unicode_fs_decoded startstr: str / + * + module as modname: object = None Return symbol and scope dictionaries used internally by compiler. [clinic start generated code]*/ static PyObject * _symtable_symtable_impl(PyObject *module, PyObject *source, - PyObject *filename, const char *startstr) -/*[clinic end generated code: output=59eb0d5fc7285ac4 input=436ffff90d02e4f6]*/ + PyObject *filename, const char *startstr, + PyObject *modname) +/*[clinic end generated code: output=235ec5a87a9ce178 input=fbf9adaa33c7070d]*/ { struct symtable *st; PyObject *t; @@ -50,7 +53,17 @@ _symtable_symtable_impl(PyObject *module, PyObject *source, Py_XDECREF(source_copy); return NULL; } - st = _Py_SymtableStringObjectFlags(str, filename, start, &cf); + if (modname == Py_None) { + modname = NULL; + } + else if (!PyUnicode_Check(modname)) { + PyErr_Format(PyExc_TypeError, + "symtable() argument 'module' must be str or None, not %T", + modname); + Py_XDECREF(source_copy); + return NULL; + } + st = _Py_SymtableStringObjectFlags(str, filename, start, &cf, modname); Py_XDECREF(source_copy); if (st == NULL) { return NULL; diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 2de9004fe08..3663dc3eb7f 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -43,6 +43,7 @@ _PyTokenizer_tok_new(void) tok->encoding = NULL; tok->cont_line = 0; tok->filename = NULL; + tok->module = NULL; tok->decoding_readline = NULL; tok->decoding_buffer = NULL; tok->readline = NULL; @@ -91,6 +92,7 @@ _PyTokenizer_Free(struct tok_state *tok) Py_XDECREF(tok->decoding_buffer); Py_XDECREF(tok->readline); Py_XDECREF(tok->filename); + Py_XDECREF(tok->module); if ((tok->readline != NULL || tok->fp != NULL ) && tok->buf != NULL) { PyMem_Free(tok->buf); } diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 877127125a7..9cd196a114c 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -102,6 +102,7 @@ struct tok_state { int parenlinenostack[MAXLEVEL]; int parencolstack[MAXLEVEL]; PyObject *filename; + PyObject *module; /* Stuff for checking on different tab sizes */ int altindstack[MAXINDENT]; /* Stack of alternate indents */ /* Stuff for PEP 0263 */ diff --git a/Parser/peg_api.c b/Parser/peg_api.c index d4acc3e4935..e30ca0453bd 100644 --- a/Parser/peg_api.c +++ b/Parser/peg_api.c @@ -4,13 +4,15 @@ mod_ty _PyParser_ASTFromString(const char *str, PyObject* filename, int mode, - PyCompilerFlags *flags, PyArena *arena) + PyCompilerFlags *flags, PyArena *arena, + PyObject *module) { if (PySys_Audit("compile", "yO", str, filename) < 0) { return NULL; } - mod_ty result = _PyPegen_run_parser_from_string(str, mode, filename, flags, arena); + mod_ty result = _PyPegen_run_parser_from_string(str, mode, filename, flags, + arena, module); return result; } diff --git a/Parser/pegen.c b/Parser/pegen.c index 70493031656..a38e973b3f6 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -1010,6 +1010,11 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena // From here on we need to clean up even if there's an error mod_ty result = NULL; + tok->module = PyUnicode_FromString("__main__"); + if (tok->module == NULL) { + goto error; + } + int parser_flags = compute_parser_flags(flags); Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, PY_MINOR_VERSION, errcode, NULL, arena); @@ -1036,7 +1041,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena mod_ty _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filename_ob, - PyCompilerFlags *flags, PyArena *arena) + PyCompilerFlags *flags, PyArena *arena, PyObject *module) { int exec_input = start_rule == Py_file_input; @@ -1054,6 +1059,7 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen } // This transfers the ownership to the tokenizer tok->filename = Py_NewRef(filename_ob); + tok->module = Py_XNewRef(module); // We need to clear up from here on mod_ty result = NULL; diff --git a/Parser/pegen.h b/Parser/pegen.h index 6b49b3537a0..b8f887608b1 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -378,7 +378,7 @@ mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char const char *, const char *, PyCompilerFlags *, int *, PyObject **, PyArena *); void *_PyPegen_run_parser(Parser *); -mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *); +mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *, PyObject *); asdl_stmt_seq *_PyPegen_interactive_exit(Parser *); // Generated function in parse.c - function definition in python.gram diff --git a/Parser/string_parser.c b/Parser/string_parser.c index ebe68989d1a..b164dfbc81a 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -88,7 +88,7 @@ warn_invalid_escape_sequence(Parser *p, const char* buffer, const char *first_in } if (PyErr_WarnExplicitObject(category, msg, p->tok->filename, - lineno, NULL, NULL) < 0) { + lineno, p->tok->module, NULL) < 0) { if (PyErr_ExceptionMatches(category)) { /* Replace the Syntax/DeprecationWarning exception with a SyntaxError to get a more accurate error report */ diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index e5e2eed2d34..a03531a7441 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -127,7 +127,7 @@ _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_inval } if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, tok->filename, - tok->lineno, NULL, NULL) < 0) { + tok->lineno, tok->module, NULL) < 0) { Py_DECREF(msg); if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { @@ -166,7 +166,7 @@ _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char * } if (PyErr_WarnExplicitObject(category, errmsg, tok->filename, - tok->lineno, NULL, NULL) < 0) { + tok->lineno, tok->module, NULL) < 0) { if (PyErr_ExceptionMatches(category)) { /* Replace the DeprecationWarning exception with a SyntaxError to get a more accurate error report */ diff --git a/Programs/_freeze_module.py b/Programs/_freeze_module.py index ba638eef6c4..62274e4aa9c 100644 --- a/Programs/_freeze_module.py +++ b/Programs/_freeze_module.py @@ -23,7 +23,7 @@ def read_text(inpath: str) -> bytes: def compile_and_marshal(name: str, text: bytes) -> bytes: filename = f"" # exec == Py_file_input - code = compile(text, filename, "exec", optimize=0, dont_inherit=True) + code = compile(text, filename, "exec", optimize=0, dont_inherit=True, module=name) return marshal.dumps(code) diff --git a/Programs/freeze_test_frozenmain.py b/Programs/freeze_test_frozenmain.py index 848fc31b3d6..1a986bbac2a 100644 --- a/Programs/freeze_test_frozenmain.py +++ b/Programs/freeze_test_frozenmain.py @@ -24,7 +24,7 @@ def dump(fp, filename, name): with tokenize.open(filename) as source_fp: source = source_fp.read() - code = compile(source, code_filename, 'exec') + code = compile(source, code_filename, 'exec', module=name) data = marshal.dumps(code) writecode(fp, name, data) diff --git a/Python/ast_preprocess.c b/Python/ast_preprocess.c index fe6fd9479d1..d45435257cc 100644 --- a/Python/ast_preprocess.c +++ b/Python/ast_preprocess.c @@ -16,6 +16,7 @@ typedef struct { typedef struct { PyObject *filename; + PyObject *module; int optimize; int ff_features; int syntax_check_only; @@ -71,7 +72,8 @@ control_flow_in_finally_warning(const char *kw, stmt_ty n, _PyASTPreprocessState } int ret = _PyErr_EmitSyntaxWarning(msg, state->filename, n->lineno, n->col_offset + 1, n->end_lineno, - n->end_col_offset + 1); + n->end_col_offset + 1, + state->module); Py_DECREF(msg); return ret < 0 ? 0 : 1; } @@ -969,11 +971,13 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTPreprocessState *st int _PyAST_Preprocess(mod_ty mod, PyArena *arena, PyObject *filename, int optimize, - int ff_features, int syntax_check_only, int enable_warnings) + int ff_features, int syntax_check_only, int enable_warnings, + PyObject *module) { _PyASTPreprocessState state; memset(&state, 0, sizeof(_PyASTPreprocessState)); state.filename = filename; + state.module = module; state.optimize = optimize; state.ff_features = ff_features; state.syntax_check_only = syntax_check_only; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index f6fadd936bb..c2d780ac9b9 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -751,6 +751,7 @@ compile as builtin_compile dont_inherit: bool = False optimize: int = -1 * + module as modname: object = None _feature_version as feature_version: int = -1 Compile source into a code object that can be executed by exec() or eval(). @@ -770,8 +771,8 @@ in addition to any features explicitly specified. static PyObject * builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, const char *mode, int flags, int dont_inherit, - int optimize, int feature_version) -/*[clinic end generated code: output=b0c09c84f116d3d7 input=8f0069edbdac381b]*/ + int optimize, PyObject *modname, int feature_version) +/*[clinic end generated code: output=9a0dce1945917a86 input=ddeae1e0253459dc]*/ { PyObject *source_copy; const char *str; @@ -800,6 +801,15 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, "compile(): invalid optimize value"); goto error; } + if (modname == Py_None) { + modname = NULL; + } + else if (!PyUnicode_Check(modname)) { + PyErr_Format(PyExc_TypeError, + "compile() argument 'module' must be str or None, not %T", + modname); + goto error; + } if (!dont_inherit) { PyEval_MergeCompilerFlags(&cf); @@ -845,8 +855,9 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, goto error; } int syntax_check_only = ((flags & PyCF_OPTIMIZED_AST) == PyCF_ONLY_AST); /* unoptiomized AST */ - if (_PyCompile_AstPreprocess(mod, filename, &cf, optimize, - arena, syntax_check_only) < 0) { + if (_PyCompile_AstPreprocess(mod, filename, &cf, optimize, arena, + syntax_check_only, modname) < 0) + { _PyArena_Free(arena); goto error; } @@ -859,7 +870,7 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, goto error; } result = (PyObject*)_PyAST_Compile(mod, filename, - &cf, optimize, arena); + &cf, optimize, arena, modname); } _PyArena_Free(arena); goto finally; @@ -877,7 +888,9 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, tstate->suppress_co_const_immortalization++; #endif - result = Py_CompileStringObject(str, filename, start[compile_mode], &cf, optimize); + result = _Py_CompileStringObjectWithModule(str, filename, + start[compile_mode], &cf, + optimize, modname); #ifdef Py_GIL_DISABLED tstate->suppress_co_const_immortalization--; diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index adb82f45c25..f08e5847abe 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -238,7 +238,8 @@ PyDoc_STRVAR(builtin_chr__doc__, PyDoc_STRVAR(builtin_compile__doc__, "compile($module, /, source, filename, mode, flags=0,\n" -" dont_inherit=False, optimize=-1, *, _feature_version=-1)\n" +" dont_inherit=False, optimize=-1, *, module=None,\n" +" _feature_version=-1)\n" "--\n" "\n" "Compile source into a code object that can be executed by exec() or eval().\n" @@ -260,7 +261,7 @@ PyDoc_STRVAR(builtin_compile__doc__, static PyObject * builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, const char *mode, int flags, int dont_inherit, - int optimize, int feature_version); + int optimize, PyObject *modname, int feature_version); static PyObject * builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -268,7 +269,7 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 7 + #define NUM_KEYWORDS 8 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -277,7 +278,7 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(source), &_Py_ID(filename), &_Py_ID(mode), &_Py_ID(flags), &_Py_ID(dont_inherit), &_Py_ID(optimize), &_Py_ID(_feature_version), }, + .ob_item = { &_Py_ID(source), &_Py_ID(filename), &_Py_ID(mode), &_Py_ID(flags), &_Py_ID(dont_inherit), &_Py_ID(optimize), &_Py_ID(module), &_Py_ID(_feature_version), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -286,14 +287,14 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"source", "filename", "mode", "flags", "dont_inherit", "optimize", "_feature_version", NULL}; + static const char * const _keywords[] = {"source", "filename", "mode", "flags", "dont_inherit", "optimize", "module", "_feature_version", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "compile", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[7]; + PyObject *argsbuf[8]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; PyObject *source; PyObject *filename = NULL; @@ -301,6 +302,7 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj int flags = 0; int dont_inherit = 0; int optimize = -1; + PyObject *modname = Py_None; int feature_version = -1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, @@ -359,12 +361,18 @@ skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - feature_version = PyLong_AsInt(args[6]); + if (args[6]) { + modname = args[6]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + feature_version = PyLong_AsInt(args[7]); if (feature_version == -1 && PyErr_Occurred()) { goto exit; } skip_optional_kwonly: - return_value = builtin_compile_impl(module, source, filename, mode, flags, dont_inherit, optimize, feature_version); + return_value = builtin_compile_impl(module, source, filename, mode, flags, dont_inherit, optimize, modname, feature_version); exit: /* Cleanup for filename */ @@ -1277,4 +1285,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=7eada753dc2e046f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=06500bcc9a341e68 input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index e2f1c7e8eb5..6951c98500d 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -104,11 +104,13 @@ typedef struct _PyCompiler { * (including instructions for nested code objects) */ int c_disable_warning; + PyObject *c_module; } compiler; static int compiler_setup(compiler *c, mod_ty mod, PyObject *filename, - PyCompilerFlags *flags, int optimize, PyArena *arena) + PyCompilerFlags *flags, int optimize, PyArena *arena, + PyObject *module) { PyCompilerFlags local_flags = _PyCompilerFlags_INIT; @@ -126,6 +128,7 @@ compiler_setup(compiler *c, mod_ty mod, PyObject *filename, if (!_PyFuture_FromAST(mod, filename, &c->c_future)) { return ERROR; } + c->c_module = Py_XNewRef(module); if (!flags) { flags = &local_flags; } @@ -136,7 +139,9 @@ compiler_setup(compiler *c, mod_ty mod, PyObject *filename, c->c_optimize = (optimize == -1) ? _Py_GetConfig()->optimization_level : optimize; c->c_save_nested_seqs = false; - if (!_PyAST_Preprocess(mod, arena, filename, c->c_optimize, merged, 0, 1)) { + if (!_PyAST_Preprocess(mod, arena, filename, c->c_optimize, merged, + 0, 1, module)) + { return ERROR; } c->c_st = _PySymtable_Build(mod, filename, &c->c_future); @@ -156,6 +161,7 @@ compiler_free(compiler *c) _PySymtable_Free(c->c_st); } Py_XDECREF(c->c_filename); + Py_XDECREF(c->c_module); Py_XDECREF(c->c_const_cache); Py_XDECREF(c->c_stack); PyMem_Free(c); @@ -163,13 +169,13 @@ compiler_free(compiler *c) static compiler* new_compiler(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, - int optimize, PyArena *arena) + int optimize, PyArena *arena, PyObject *module) { compiler *c = PyMem_Calloc(1, sizeof(compiler)); if (c == NULL) { return NULL; } - if (compiler_setup(c, mod, filename, pflags, optimize, arena) < 0) { + if (compiler_setup(c, mod, filename, pflags, optimize, arena, module) < 0) { compiler_free(c); return NULL; } @@ -1221,7 +1227,8 @@ _PyCompile_Warn(compiler *c, location loc, const char *format, ...) return ERROR; } int ret = _PyErr_EmitSyntaxWarning(msg, c->c_filename, loc.lineno, loc.col_offset + 1, - loc.end_lineno, loc.end_col_offset + 1); + loc.end_lineno, loc.end_col_offset + 1, + c->c_module); Py_DECREF(msg); return ret; } @@ -1476,10 +1483,10 @@ _PyCompile_OptimizeAndAssemble(compiler *c, int addNone) PyCodeObject * _PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, - int optimize, PyArena *arena) + int optimize, PyArena *arena, PyObject *module) { assert(!PyErr_Occurred()); - compiler *c = new_compiler(mod, filename, pflags, optimize, arena); + compiler *c = new_compiler(mod, filename, pflags, optimize, arena, module); if (c == NULL) { return NULL; } @@ -1492,7 +1499,8 @@ _PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, int _PyCompile_AstPreprocess(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, - int optimize, PyArena *arena, int no_const_folding) + int optimize, PyArena *arena, int no_const_folding, + PyObject *module) { _PyFutureFeatures future; if (!_PyFuture_FromAST(mod, filename, &future)) { @@ -1502,7 +1510,9 @@ _PyCompile_AstPreprocess(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, if (optimize == -1) { optimize = _Py_GetConfig()->optimization_level; } - if (!_PyAST_Preprocess(mod, arena, filename, optimize, flags, no_const_folding, 0)) { + if (!_PyAST_Preprocess(mod, arena, filename, optimize, flags, + no_const_folding, 0, module)) + { return -1; } return 0; @@ -1627,7 +1637,7 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, return NULL; } - compiler *c = new_compiler(mod, filename, pflags, optimize, arena); + compiler *c = new_compiler(mod, filename, pflags, optimize, arena, NULL); if (c == NULL) { _PyArena_Free(arena); return NULL; diff --git a/Python/errors.c b/Python/errors.c index 9fe95cec0ab..5c6ac48371a 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -1960,10 +1960,11 @@ _PyErr_RaiseSyntaxError(PyObject *msg, PyObject *filename, int lineno, int col_o */ int _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_offset, - int end_lineno, int end_col_offset) + int end_lineno, int end_col_offset, + PyObject *module) { - if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, - filename, lineno, NULL, NULL) < 0) + if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, filename, lineno, + module, NULL) < 0) { if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { /* Replace the SyntaxWarning exception with a SyntaxError diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 45211e1b075..49ce0a97d47 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1252,12 +1252,19 @@ _PyRun_StringFlagsWithName(const char *str, PyObject* name, int start, } else { name = &_Py_STR(anon_string); } + PyObject *module = NULL; + if (globals && PyDict_GetItemStringRef(globals, "__name__", &module) < 0) { + goto done; + } - mod = _PyParser_ASTFromString(str, name, start, flags, arena); + mod = _PyParser_ASTFromString(str, name, start, flags, arena, module); + Py_XDECREF(module); - if (mod != NULL) { + if (mod != NULL) { ret = run_mod(mod, name, globals, locals, flags, arena, source, generate_new_source); } + +done: Py_XDECREF(source); _PyArena_Free(arena); return ret; @@ -1407,8 +1414,17 @@ run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals, return NULL; } } + PyObject *module = NULL; + if (globals && PyDict_GetItemStringRef(globals, "__name__", &module) < 0) { + if (interactive_src) { + Py_DECREF(interactive_filename); + } + return NULL; + } - PyCodeObject *co = _PyAST_Compile(mod, interactive_filename, flags, -1, arena); + PyCodeObject *co = _PyAST_Compile(mod, interactive_filename, flags, -1, + arena, module); + Py_XDECREF(module); if (co == NULL) { if (interactive_src) { Py_DECREF(interactive_filename); @@ -1507,6 +1523,14 @@ run_pyc_file(FILE *fp, PyObject *globals, PyObject *locals, PyObject * Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize) +{ + return _Py_CompileStringObjectWithModule(str, filename, start, + flags, optimize, NULL); +} + +PyObject * +_Py_CompileStringObjectWithModule(const char *str, PyObject *filename, int start, + PyCompilerFlags *flags, int optimize, PyObject *module) { PyCodeObject *co; mod_ty mod; @@ -1514,14 +1538,16 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, if (arena == NULL) return NULL; - mod = _PyParser_ASTFromString(str, filename, start, flags, arena); + mod = _PyParser_ASTFromString(str, filename, start, flags, arena, module); if (mod == NULL) { _PyArena_Free(arena); return NULL; } if (flags && (flags->cf_flags & PyCF_ONLY_AST)) { int syntax_check_only = ((flags->cf_flags & PyCF_OPTIMIZED_AST) == PyCF_ONLY_AST); /* unoptiomized AST */ - if (_PyCompile_AstPreprocess(mod, filename, flags, optimize, arena, syntax_check_only) < 0) { + if (_PyCompile_AstPreprocess(mod, filename, flags, optimize, arena, + syntax_check_only, module) < 0) + { _PyArena_Free(arena); return NULL; } @@ -1529,7 +1555,7 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, _PyArena_Free(arena); return result; } - co = _PyAST_Compile(mod, filename, flags, optimize, arena); + co = _PyAST_Compile(mod, filename, flags, optimize, arena, module); _PyArena_Free(arena); return (PyObject *)co; } diff --git a/Python/symtable.c b/Python/symtable.c index bcd7365f8e1..29cf9190a4e 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -3137,7 +3137,7 @@ symtable_raise_if_not_coroutine(struct symtable *st, const char *msg, _Py_Source struct symtable * _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, - int start, PyCompilerFlags *flags) + int start, PyCompilerFlags *flags, PyObject *module) { struct symtable *st; mod_ty mod; @@ -3147,7 +3147,7 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, if (arena == NULL) return NULL; - mod = _PyParser_ASTFromString(str, filename, start, flags, arena); + mod = _PyParser_ASTFromString(str, filename, start, flags, arena, module); if (mod == NULL) { _PyArena_Free(arena); return NULL; diff --git a/Tools/peg_generator/peg_extension/peg_extension.c b/Tools/peg_generator/peg_extension/peg_extension.c index 1587d53d594..2fec5b05129 100644 --- a/Tools/peg_generator/peg_extension/peg_extension.c +++ b/Tools/peg_generator/peg_extension/peg_extension.c @@ -8,7 +8,7 @@ _build_return_object(mod_ty module, int mode, PyObject *filename_ob, PyArena *ar PyObject *result = NULL; if (mode == 2) { - result = (PyObject *)_PyAST_Compile(module, filename_ob, NULL, -1, arena); + result = (PyObject *)_PyAST_Compile(module, filename_ob, NULL, -1, arena, NULL); } else if (mode == 1) { result = PyAST_mod2obj(module); } else { @@ -93,7 +93,7 @@ parse_string(PyObject *self, PyObject *args, PyObject *kwds) PyCompilerFlags flags = _PyCompilerFlags_INIT; mod_ty res = _PyPegen_run_parser_from_string(the_string, Py_file_input, filename_ob, - &flags, arena); + &flags, arena, NULL); if (res == NULL) { goto error; } From 2fbd39666663cb5ca1c0e3021ce2e7bc72331020 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Nov 2025 13:37:01 +0200 Subject: [PATCH 183/313] gh-140601: Refactor ElementTree.iterparse() tests (GH-141499) Split existing tests on smaller methods and move them to separate class. Rename variable "content" to "it". Use BytesIO instead of StringIO. Add few more tests. --- Lib/test/test_xml_etree.py | 430 ++++++++++++++++++++----------------- 1 file changed, 228 insertions(+), 202 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index f65baa0cfae..25c084c8b9c 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -574,208 +574,6 @@ def test_parseliteral(self): self.assertEqual(len(ids), 1) self.assertEqual(ids["body"].tag, 'body') - def test_iterparse(self): - # Test iterparse interface. - - iterparse = ET.iterparse - - context = iterparse(SIMPLE_XMLFILE) - self.assertIsNone(context.root) - action, elem = next(context) - self.assertIsNone(context.root) - self.assertEqual((action, elem.tag), ('end', 'element')) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('end', 'element'), - ('end', 'empty-element'), - ('end', 'root'), - ]) - self.assertEqual(context.root.tag, 'root') - - context = iterparse(SIMPLE_NS_XMLFILE) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('end', '{namespace}element'), - ('end', '{namespace}element'), - ('end', '{namespace}empty-element'), - ('end', '{namespace}root'), - ]) - - with open(SIMPLE_XMLFILE, 'rb') as source: - context = iterparse(source) - action, elem = next(context) - self.assertEqual((action, elem.tag), ('end', 'element')) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('end', 'element'), - ('end', 'empty-element'), - ('end', 'root'), - ]) - self.assertEqual(context.root.tag, 'root') - - events = () - context = iterparse(SIMPLE_XMLFILE, events) - self.assertEqual([(action, elem.tag) for action, elem in context], []) - - events = () - context = iterparse(SIMPLE_XMLFILE, events=events) - self.assertEqual([(action, elem.tag) for action, elem in context], []) - - events = ("start", "end") - context = iterparse(SIMPLE_XMLFILE, events) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('start', 'root'), - ('start', 'element'), - ('end', 'element'), - ('start', 'element'), - ('end', 'element'), - ('start', 'empty-element'), - ('end', 'empty-element'), - ('end', 'root'), - ]) - - events = ("start", "end", "start-ns", "end-ns") - context = iterparse(SIMPLE_NS_XMLFILE, events) - self.assertEqual([(action, elem.tag) if action in ("start", "end") - else (action, elem) - for action, elem in context], [ - ('start-ns', ('', 'namespace')), - ('start', '{namespace}root'), - ('start', '{namespace}element'), - ('end', '{namespace}element'), - ('start', '{namespace}element'), - ('end', '{namespace}element'), - ('start', '{namespace}empty-element'), - ('end', '{namespace}empty-element'), - ('end', '{namespace}root'), - ('end-ns', None), - ]) - - events = ('start-ns', 'end-ns') - context = iterparse(io.StringIO(r""), events) - res = [action for action, elem in context] - self.assertEqual(res, ['start-ns', 'end-ns']) - - events = ("start", "end", "bogus") - with open(SIMPLE_XMLFILE, "rb") as f: - with self.assertRaises(ValueError) as cm: - iterparse(f, events) - self.assertFalse(f.closed) - self.assertEqual(str(cm.exception), "unknown event 'bogus'") - - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(ValueError) as cm: - iterparse(SIMPLE_XMLFILE, events) - self.assertEqual(str(cm.exception), "unknown event 'bogus'") - del cm - - source = io.BytesIO( - b"\n" - b"text\n") - events = ("start-ns",) - context = iterparse(source, events) - self.assertEqual([(action, elem) for action, elem in context], [ - ('start-ns', ('', 'http://\xe9ffbot.org/ns')), - ('start-ns', ('cl\xe9', 'http://effbot.org/ns')), - ]) - - source = io.StringIO("junk") - it = iterparse(source) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'document')) - with self.assertRaises(ET.ParseError) as cm: - next(it) - self.assertEqual(str(cm.exception), - 'junk after document element: line 1, column 12') - - self.addCleanup(os_helper.unlink, TESTFN) - with open(TESTFN, "wb") as f: - f.write(b"junk") - it = iterparse(TESTFN) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'document')) - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(ET.ParseError) as cm: - next(it) - self.assertEqual(str(cm.exception), - 'junk after document element: line 1, column 12') - del cm, it - - # Not exhausting the iterator still closes the resource (bpo-43292) - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - del it - - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - it.close() - del it - - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'element')) - del it, elem - - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - action, elem = next(it) - it.close() - self.assertEqual((action, elem.tag), ('end', 'element')) - del it, elem - - with self.assertRaises(FileNotFoundError): - iterparse("nonexistent") - - def test_iterparse_close(self): - iterparse = ET.iterparse - - it = iterparse(SIMPLE_XMLFILE) - it.close() - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - with open(SIMPLE_XMLFILE, 'rb') as source: - it = iterparse(source) - it.close() - self.assertFalse(source.closed) - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - it = iterparse(SIMPLE_XMLFILE) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'element')) - it.close() - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - with open(SIMPLE_XMLFILE, 'rb') as source: - it = iterparse(source) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'element')) - it.close() - self.assertFalse(source.closed) - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - it = iterparse(SIMPLE_XMLFILE) - list(it) - it.close() - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - with open(SIMPLE_XMLFILE, 'rb') as source: - it = iterparse(source) - list(it) - it.close() - self.assertFalse(source.closed) - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - def test_writefile(self): elem = ET.Element("tag") elem.text = "text" @@ -1499,6 +1297,234 @@ def test_attlist_default(self): {'{http://www.w3.org/XML/1998/namespace}lang': 'eng'}) +class IterparseTest(unittest.TestCase): + # Test iterparse interface. + + def test_basic(self): + iterparse = ET.iterparse + + it = iterparse(SIMPLE_XMLFILE) + self.assertIsNone(it.root) + action, elem = next(it) + self.assertIsNone(it.root) + self.assertEqual((action, elem.tag), ('end', 'element')) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('end', 'element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + self.assertEqual(it.root.tag, 'root') + it.close() + + it = iterparse(SIMPLE_NS_XMLFILE) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('end', '{namespace}element'), + ('end', '{namespace}element'), + ('end', '{namespace}empty-element'), + ('end', '{namespace}root'), + ]) + it.close() + + def test_external_file(self): + with open(SIMPLE_XMLFILE, 'rb') as source: + it = ET.iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('end', 'element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + self.assertEqual(it.root.tag, 'root') + + def test_events(self): + iterparse = ET.iterparse + + events = () + it = iterparse(SIMPLE_XMLFILE, events) + self.assertEqual([(action, elem.tag) for action, elem in it], []) + it.close() + + events = () + it = iterparse(SIMPLE_XMLFILE, events=events) + self.assertEqual([(action, elem.tag) for action, elem in it], []) + it.close() + + events = ("start", "end") + it = iterparse(SIMPLE_XMLFILE, events) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('start', 'root'), + ('start', 'element'), + ('end', 'element'), + ('start', 'element'), + ('end', 'element'), + ('start', 'empty-element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + it.close() + + def test_namespace_events(self): + iterparse = ET.iterparse + + events = ("start", "end", "start-ns", "end-ns") + it = iterparse(SIMPLE_NS_XMLFILE, events) + self.assertEqual([(action, elem.tag) if action in ("start", "end") + else (action, elem) + for action, elem in it], [ + ('start-ns', ('', 'namespace')), + ('start', '{namespace}root'), + ('start', '{namespace}element'), + ('end', '{namespace}element'), + ('start', '{namespace}element'), + ('end', '{namespace}element'), + ('start', '{namespace}empty-element'), + ('end', '{namespace}empty-element'), + ('end', '{namespace}root'), + ('end-ns', None), + ]) + it.close() + + events = ('start-ns', 'end-ns') + it = iterparse(io.BytesIO(br""), events) + res = [action for action, elem in it] + self.assertEqual(res, ['start-ns', 'end-ns']) + it.close() + + def test_unknown_events(self): + iterparse = ET.iterparse + + events = ("start", "end", "bogus") + with open(SIMPLE_XMLFILE, "rb") as f: + with self.assertRaises(ValueError) as cm: + iterparse(f, events) + self.assertFalse(f.closed) + self.assertEqual(str(cm.exception), "unknown event 'bogus'") + + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(ValueError) as cm: + iterparse(SIMPLE_XMLFILE, events) + self.assertEqual(str(cm.exception), "unknown event 'bogus'") + del cm + gc_collect() + + def test_non_utf8(self): + source = io.BytesIO( + b"\n" + b"text\n") + events = ("start-ns",) + it = ET.iterparse(source, events) + self.assertEqual([(action, elem) for action, elem in it], [ + ('start-ns', ('', 'http://\xe9ffbot.org/ns')), + ('start-ns', ('cl\xe9', 'http://effbot.org/ns')), + ]) + + def test_parsing_error(self): + source = io.BytesIO(b"junk") + it = ET.iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'document')) + with self.assertRaises(ET.ParseError) as cm: + next(it) + self.assertEqual(str(cm.exception), + 'junk after document element: line 1, column 12') + + def test_nonexistent_file(self): + with self.assertRaises(FileNotFoundError): + ET.iterparse("nonexistent") + + def test_resource_warnings_not_exhausted(self): + # Not exhausting the iterator still closes the underlying file (bpo-43292) + it = ET.iterparse(SIMPLE_XMLFILE) + with warnings_helper.check_no_resource_warning(self): + del it + gc_collect() + + it = ET.iterparse(SIMPLE_XMLFILE) + with warnings_helper.check_no_resource_warning(self): + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + del it, elem + gc_collect() + + def test_resource_warnings_failed_iteration(self): + self.addCleanup(os_helper.unlink, TESTFN) + with open(TESTFN, "wb") as f: + f.write(b"junk") + + it = ET.iterparse(TESTFN) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'document')) + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(ET.ParseError) as cm: + next(it) + self.assertEqual(str(cm.exception), + 'junk after document element: line 1, column 12') + del cm, it + gc_collect() + + def test_resource_warnings_exhausted(self): + it = ET.iterparse(SIMPLE_XMLFILE) + with warnings_helper.check_no_resource_warning(self): + list(it) + del it + gc_collect() + + def test_close_not_exhausted(self): + iterparse = ET.iterparse + + it = iterparse(SIMPLE_XMLFILE) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + it = iterparse(SIMPLE_XMLFILE) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + def test_close_exhausted(self): + iterparse = ET.iterparse + it = iterparse(SIMPLE_XMLFILE) + list(it) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + list(it) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + class XMLPullParserTest(unittest.TestCase): def _feed(self, parser, data, chunk_size=None, flush=False): From 732224e1139f7ed4fe0259a2dad900f84910949e Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Thu, 13 Nov 2025 05:19:44 -0800 Subject: [PATCH 184/313] gh-139871: Add `bytearray.take_bytes([n])` to efficiently extract `bytes` (GH-140128) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update `bytearray` to contain a `bytes` and provide a zero-copy path to "extract" the `bytes`. This allows making several code paths more efficient. This does not move any codepaths to make use of this new API. The documentation changes include common code patterns which can be made more efficient with this API. --- When just changing `bytearray` to contain `bytes` I ran pyperformance on a `--with-lto --enable-optimizations --with-static-libpython` build and don't see any major speedups or slowdowns with this; all seems to be in the noise of my machine (Generally changes under 5% or benchmarks that don't touch bytes/bytearray). Co-authored-by: Victor Stinner Co-authored-by: Maurycy Pawłowski-Wieroński <5383+maurycy@users.noreply.github.com> --- Doc/library/stdtypes.rst | 24 ++ Doc/whatsnew/3.15.rst | 80 ++++++ Include/cpython/bytearrayobject.h | 16 +- Include/internal/pycore_bytesobject.h | 8 + Lib/test/test_bytes.py | 81 ++++++ Lib/test/test_capi/test_bytearray.py | 5 +- Lib/test/test_sys.py | 2 +- ...-10-14-18-24-16.gh-issue-139871.SWtuUz.rst | 2 + Objects/bytearrayobject.c | 238 ++++++++++++------ Objects/bytesobject.c | 8 +- Objects/clinic/bytearrayobject.c.h | 39 ++- 11 files changed, 407 insertions(+), 96 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 97e7e08364e..c539345e598 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -3173,6 +3173,30 @@ objects. .. versionadded:: 3.14 + .. method:: take_bytes(n=None, /) + + Remove the first *n* bytes from the bytearray and return them as an immutable + :class:`bytes`. + By default (if *n* is ``None``), return all bytes and clear the bytearray. + + If *n* is negative, index from the end and take the first :func:`len` + plus *n* bytes. If *n* is out of bounds, raise :exc:`IndexError`. + + Taking less than the full length will leave remaining bytes in the + :class:`bytearray`, which requires a copy. If the remaining bytes should be + discarded, use :func:`~bytearray.resize` or :keyword:`del` to truncate + then :func:`~bytearray.take_bytes` without a size. + + .. impl-detail:: + + Taking all bytes is a zero-copy operation. + + .. versionadded:: next + + See the :ref:`What's New ` entry for + common code patterns which can be optimized with + :func:`bytearray.take_bytes`. + Since bytearray objects are sequences of integers (akin to a list), for a bytearray object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be a bytearray object of length 1. (This contrasts with text strings, where diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 3cb766978a7..d7c9a41eeb2 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -307,6 +307,86 @@ Other language changes not only integers or floats, although this does not improve precision. (Contributed by Serhiy Storchaka in :gh:`67795`.) +.. _whatsnew315-bytearray-take-bytes: + +* Added :meth:`bytearray.take_bytes(n=None, /) ` to take + bytes out of a :class:`bytearray` without copying. This enables optimizing code + which must return :class:`bytes` after working with a mutable buffer of bytes + such as data buffering, network protocol parsing, encoding, decoding, + and compression. Common code patterns which can be optimized with + :func:`~bytearray.take_bytes` are listed below. + + (Contributed by Cody Maloney in :gh:`139871`.) + + .. list-table:: Suggested Optimizing Refactors + :header-rows: 1 + + * - Description + - Old + - New + + * - Return :class:`bytes` after working with :class:`bytearray` + - .. code:: python + + def read() -> bytes: + buffer = bytearray(1024) + ... + return bytes(buffer) + + - .. code:: python + + def read() -> bytes: + buffer = bytearray(1024) + ... + return buffer.take_bytes() + + * - Empty a buffer getting the bytes + - .. code:: python + + buffer = bytearray(1024) + ... + data = bytes(buffer) + buffer.clear() + + - .. code:: python + + buffer = bytearray(1024) + ... + data = buffer.take_bytes() + + * - Split a buffer at a specific separator + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + data = bytes(buffer[:n + 1]) + del buffer[:n + 1] + assert data == b'abc' + assert buffer == bytearray(b'def') + + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + data = buffer.take_bytes(n + 1) + + * - Split a buffer at a specific separator; discard after the separator + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + data = bytes(buffer[:n]) + buffer.clear() + assert data == b'abc' + assert len(buffer) == 0 + + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + buffer.resize(n) + data = buffer.take_bytes() + * Many functions related to compiling or parsing Python code, such as :func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, and :func:`importlib.abc.InspectLoader.source_to_code`, now allow to pass diff --git a/Include/cpython/bytearrayobject.h b/Include/cpython/bytearrayobject.h index 4dddef713ce..1edd0820742 100644 --- a/Include/cpython/bytearrayobject.h +++ b/Include/cpython/bytearrayobject.h @@ -5,25 +5,25 @@ /* Object layout */ typedef struct { PyObject_VAR_HEAD - Py_ssize_t ob_alloc; /* How many bytes allocated in ob_bytes */ + /* How many bytes allocated in ob_bytes + + In the current implementation this is equivalent to Py_SIZE(ob_bytes_object). + The value is always loaded and stored atomically for thread safety. + There are API compatibilty concerns with removing so keeping for now. */ + Py_ssize_t ob_alloc; char *ob_bytes; /* Physical backing buffer */ char *ob_start; /* Logical start inside ob_bytes */ Py_ssize_t ob_exports; /* How many buffer exports */ + PyObject *ob_bytes_object; /* PyBytes for zero-copy bytes conversion */ } PyByteArrayObject; -PyAPI_DATA(char) _PyByteArray_empty_string[]; - /* Macros and static inline functions, trading safety for speed */ #define _PyByteArray_CAST(op) \ (assert(PyByteArray_Check(op)), _Py_CAST(PyByteArrayObject*, op)) static inline char* PyByteArray_AS_STRING(PyObject *op) { - PyByteArrayObject *self = _PyByteArray_CAST(op); - if (Py_SIZE(self)) { - return self->ob_start; - } - return _PyByteArray_empty_string; + return _PyByteArray_CAST(op)->ob_start; } #define PyByteArray_AS_STRING(self) PyByteArray_AS_STRING(_PyObject_CAST(self)) diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h index c7bc53b6073..8e8fa696ee0 100644 --- a/Include/internal/pycore_bytesobject.h +++ b/Include/internal/pycore_bytesobject.h @@ -60,6 +60,14 @@ PyAPI_FUNC(void) _PyBytes_Repeat(char* dest, Py_ssize_t len_dest, const char* src, Py_ssize_t len_src); +/* _PyBytesObject_SIZE gives the basic size of a bytes object; any memory allocation + for a bytes object of length n should request PyBytesObject_SIZE + n bytes. + + Using _PyBytesObject_SIZE instead of sizeof(PyBytesObject) saves + 3 or 7 bytes per bytes object allocation on a typical system. +*/ +#define _PyBytesObject_SIZE (offsetof(PyBytesObject, ob_sval) + 1) + /* --- PyBytesWriter ------------------------------------------------------ */ struct PyBytesWriter { diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index e012042159d..86898bfcab9 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1397,6 +1397,16 @@ def test_clear(self): b.append(ord('p')) self.assertEqual(b, b'p') + # Cleared object should be empty. + b = bytearray(b'abc') + b.clear() + self.assertEqual(b.__alloc__(), 0) + base_size = sys.getsizeof(bytearray()) + self.assertEqual(sys.getsizeof(b), base_size) + c = b.copy() + self.assertEqual(c.__alloc__(), 0) + self.assertEqual(sys.getsizeof(c), base_size) + def test_copy(self): b = bytearray(b'abc') bb = b.copy() @@ -1458,6 +1468,61 @@ def test_resize(self): self.assertRaises(MemoryError, bytearray().resize, sys.maxsize) self.assertRaises(MemoryError, bytearray(1000).resize, sys.maxsize) + def test_take_bytes(self): + ba = bytearray(b'ab') + self.assertEqual(ba.take_bytes(), b'ab') + self.assertEqual(len(ba), 0) + self.assertEqual(ba, bytearray(b'')) + self.assertEqual(ba.__alloc__(), 0) + base_size = sys.getsizeof(bytearray()) + self.assertEqual(sys.getsizeof(ba), base_size) + + # Positive and negative slicing. + ba = bytearray(b'abcdef') + self.assertEqual(ba.take_bytes(1), b'a') + self.assertEqual(ba, bytearray(b'bcdef')) + self.assertEqual(len(ba), 5) + self.assertEqual(ba.take_bytes(-5), b'') + self.assertEqual(ba, bytearray(b'bcdef')) + self.assertEqual(len(ba), 5) + self.assertEqual(ba.take_bytes(-3), b'bc') + self.assertEqual(ba, bytearray(b'def')) + self.assertEqual(len(ba), 3) + self.assertEqual(ba.take_bytes(3), b'def') + self.assertEqual(ba, bytearray(b'')) + self.assertEqual(len(ba), 0) + + # Take nothing from emptiness. + self.assertEqual(ba.take_bytes(0), b'') + self.assertEqual(ba.take_bytes(), b'') + self.assertEqual(ba.take_bytes(None), b'') + + # Out of bounds, bad take value. + self.assertRaises(IndexError, ba.take_bytes, -1) + self.assertRaises(TypeError, ba.take_bytes, 3.14) + ba = bytearray(b'abcdef') + self.assertRaises(IndexError, ba.take_bytes, 7) + + # Offset between physical and logical start (ob_bytes != ob_start). + ba = bytearray(b'abcde') + del ba[:2] + self.assertEqual(ba, bytearray(b'cde')) + self.assertEqual(ba.take_bytes(), b'cde') + + # Overallocation at end. + ba = bytearray(b'abcde') + del ba[-2:] + self.assertEqual(ba, bytearray(b'abc')) + self.assertEqual(ba.take_bytes(), b'abc') + ba = bytearray(b'abcde') + ba.resize(4) + self.assertEqual(ba.take_bytes(), b'abcd') + + # Take of a bytearray with references should fail. + ba = bytearray(b'abc') + with memoryview(ba) as mv: + self.assertRaises(BufferError, ba.take_bytes) + self.assertEqual(ba.take_bytes(), b'abc') def test_setitem(self): def setitem_as_mapping(b, i, val): @@ -2564,6 +2629,18 @@ def zfill(b, a): c = a.zfill(0x400000) assert not c or c[-1] not in (0xdd, 0xcd) + def take_bytes(b, a): # MODIFIES! + b.wait() + c = a.take_bytes() + assert not c or c[0] == 48 # '0' + + def take_bytes_n(b, a): # MODIFIES! + b.wait() + try: + c = a.take_bytes(10) + assert c == b'0123456789' + except IndexError: pass + def check(funcs, a=None, *args): if a is None: a = bytearray(b'0' * 0x400000) @@ -2625,6 +2702,10 @@ def check(funcs, a=None, *args): check([clear] + [startswith] * 10) check([clear] + [strip] * 10) + check([clear] + [take_bytes] * 10) + check([take_bytes_n] * 10, bytearray(b'0123456789' * 0x400)) + check([take_bytes_n] * 10, bytearray(b'0123456789' * 5)) + check([clear] + [contains] * 10) check([clear] + [subscript] * 10) check([clear2] + [ass_subscript2] * 10, None, bytearray(b'0' * 0x400000)) diff --git a/Lib/test/test_capi/test_bytearray.py b/Lib/test/test_capi/test_bytearray.py index 52565ea34c6..cb7ad8b2225 100644 --- a/Lib/test/test_capi/test_bytearray.py +++ b/Lib/test/test_capi/test_bytearray.py @@ -1,3 +1,4 @@ +import sys import unittest from test.support import import_helper @@ -55,7 +56,9 @@ def test_fromstringandsize(self): self.assertEqual(fromstringandsize(b'', 0), bytearray()) self.assertEqual(fromstringandsize(NULL, 0), bytearray()) self.assertEqual(len(fromstringandsize(NULL, 3)), 3) - self.assertRaises(MemoryError, fromstringandsize, NULL, PY_SSIZE_T_MAX) + self.assertRaises(OverflowError, fromstringandsize, NULL, PY_SSIZE_T_MAX) + self.assertRaises(OverflowError, fromstringandsize, NULL, + PY_SSIZE_T_MAX-sys.getsizeof(b'') + 1) self.assertRaises(SystemError, fromstringandsize, b'abc', -1) self.assertRaises(SystemError, fromstringandsize, b'abc', PY_SSIZE_T_MIN) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 3ceed019ac4..9d3248d972e 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1583,7 +1583,7 @@ def test_objecttypes(self): samples = [b'', b'u'*100000] for sample in samples: x = bytearray(sample) - check(x, vsize('n2Pi') + x.__alloc__()) + check(x, vsize('n2PiP') + x.__alloc__()) # bytearray_iterator check(iter(bytearray()), size('nP')) # bytes diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst new file mode 100644 index 00000000000..d4b8578afe3 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst @@ -0,0 +1,2 @@ +Update :class:`bytearray` to use a :class:`bytes` under the hood as its buffer +and add :func:`bytearray.take_bytes` to take it out. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index a73bfff340c..99bfdec89f6 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -17,8 +17,8 @@ class bytearray "PyByteArrayObject *" "&PyByteArray_Type" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=5535b77c37a119e0]*/ -/* For PyByteArray_AS_STRING(). */ -char _PyByteArray_empty_string[] = ""; +/* Max number of bytes a bytearray can contain */ +#define PyByteArray_SIZE_MAX ((Py_ssize_t)(PY_SSIZE_T_MAX - _PyBytesObject_SIZE)) /* Helpers */ @@ -43,6 +43,14 @@ _getbytevalue(PyObject* arg, int *value) return 1; } +static void +bytearray_reinit_from_bytes(PyByteArrayObject *self, Py_ssize_t size, + Py_ssize_t alloc) { + self->ob_bytes = self->ob_start = PyBytes_AS_STRING(self->ob_bytes_object); + Py_SET_SIZE(self, size); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->ob_alloc, alloc); +} + static int bytearray_getbuffer_lock_held(PyObject *self, Py_buffer *view, int flags) { @@ -127,7 +135,6 @@ PyObject * PyByteArray_FromStringAndSize(const char *bytes, Py_ssize_t size) { PyByteArrayObject *new; - Py_ssize_t alloc; if (size < 0) { PyErr_SetString(PyExc_SystemError, @@ -135,35 +142,32 @@ PyByteArray_FromStringAndSize(const char *bytes, Py_ssize_t size) return NULL; } - /* Prevent buffer overflow when setting alloc to size+1. */ - if (size == PY_SSIZE_T_MAX) { - return PyErr_NoMemory(); - } - new = PyObject_New(PyByteArrayObject, &PyByteArray_Type); - if (new == NULL) + if (new == NULL) { return NULL; + } - if (size == 0) { - new->ob_bytes = NULL; - alloc = 0; - } - else { - alloc = size + 1; - new->ob_bytes = PyMem_Malloc(alloc); - if (new->ob_bytes == NULL) { - Py_DECREF(new); - return PyErr_NoMemory(); - } - if (bytes != NULL && size > 0) - memcpy(new->ob_bytes, bytes, size); - new->ob_bytes[size] = '\0'; /* Trailing null byte */ - } - Py_SET_SIZE(new, size); - new->ob_alloc = alloc; - new->ob_start = new->ob_bytes; + /* Fill values used in bytearray_dealloc. + + In an optimized build the memory isn't zeroed and ob_exports would be + uninitialized when when PyBytes_FromStringAndSize errored leading to + intermittent test failures. */ new->ob_exports = 0; + /* Optimization: size=0 bytearray should not allocate space + + PyBytes_FromStringAndSize returns the empty bytes global when size=0 so + no allocation occurs. */ + new->ob_bytes_object = PyBytes_FromStringAndSize(NULL, size); + if (new->ob_bytes_object == NULL) { + Py_DECREF(new); + return NULL; + } + bytearray_reinit_from_bytes(new, size, size); + if (bytes != NULL && size > 0) { + memcpy(new->ob_bytes, bytes, size); + } + return (PyObject *)new; } @@ -189,7 +193,6 @@ static int bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); - void *sval; PyByteArrayObject *obj = ((PyByteArrayObject *)self); /* All computations are done unsigned to avoid integer overflows (see issue #22335). */ @@ -214,16 +217,17 @@ bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size) return -1; } - if (size + logical_offset + 1 <= alloc) { + if (size + logical_offset <= alloc) { /* Current buffer is large enough to host the requested size, decide on a strategy. */ if (size < alloc / 2) { /* Major downsize; resize down to exact size */ - alloc = size + 1; + alloc = size; } else { /* Minor downsize; quick exit */ Py_SET_SIZE(self, size); + /* Add mid-buffer null; end provided by bytes. */ PyByteArray_AS_STRING(self)[size] = '\0'; /* Trailing null */ return 0; } @@ -236,38 +240,36 @@ bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size) } else { /* Major upsize; resize up to exact size */ - alloc = size + 1; + alloc = size; } } - if (alloc > PY_SSIZE_T_MAX) { + if (alloc > PyByteArray_SIZE_MAX) { PyErr_NoMemory(); return -1; } + /* Re-align data to the start of the allocation. */ if (logical_offset > 0) { - sval = PyMem_Malloc(alloc); - if (sval == NULL) { - PyErr_NoMemory(); - return -1; - } - memcpy(sval, PyByteArray_AS_STRING(self), - Py_MIN((size_t)requested_size, (size_t)Py_SIZE(self))); - PyMem_Free(obj->ob_bytes); - } - else { - sval = PyMem_Realloc(obj->ob_bytes, alloc); - if (sval == NULL) { - PyErr_NoMemory(); - return -1; - } + /* optimization tradeoff: This is faster than a new allocation when + the number of bytes being removed in a resize is small; for large + size changes it may be better to just make a new bytes object as + _PyBytes_Resize will do a malloc + memcpy internally. */ + memmove(obj->ob_bytes, obj->ob_start, + Py_MIN(requested_size, Py_SIZE(self))); } - obj->ob_bytes = obj->ob_start = sval; - Py_SET_SIZE(self, size); - FT_ATOMIC_STORE_SSIZE_RELAXED(obj->ob_alloc, alloc); - obj->ob_bytes[size] = '\0'; /* Trailing null byte */ + int ret = _PyBytes_Resize(&obj->ob_bytes_object, alloc); + if (ret == -1) { + obj->ob_bytes_object = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + size = alloc = 0; + } + bytearray_reinit_from_bytes(obj, size, alloc); + if (alloc != size) { + /* Add mid-buffer null; end provided by bytes. */ + obj->ob_bytes[size] = '\0'; + } - return 0; + return ret; } int @@ -295,7 +297,7 @@ PyByteArray_Concat(PyObject *a, PyObject *b) goto done; } - if (va.len > PY_SSIZE_T_MAX - vb.len) { + if (va.len > PyByteArray_SIZE_MAX - vb.len) { PyErr_NoMemory(); goto done; } @@ -339,7 +341,7 @@ bytearray_iconcat_lock_held(PyObject *op, PyObject *other) } Py_ssize_t size = Py_SIZE(self); - if (size > PY_SSIZE_T_MAX - vo.len) { + if (size > PyByteArray_SIZE_MAX - vo.len) { PyBuffer_Release(&vo); return PyErr_NoMemory(); } @@ -373,7 +375,7 @@ bytearray_repeat_lock_held(PyObject *op, Py_ssize_t count) count = 0; } const Py_ssize_t mysize = Py_SIZE(self); - if (count > 0 && mysize > PY_SSIZE_T_MAX / count) { + if (count > 0 && mysize > PyByteArray_SIZE_MAX / count) { return PyErr_NoMemory(); } Py_ssize_t size = mysize * count; @@ -409,7 +411,7 @@ bytearray_irepeat_lock_held(PyObject *op, Py_ssize_t count) } const Py_ssize_t mysize = Py_SIZE(self); - if (count > 0 && mysize > PY_SSIZE_T_MAX / count) { + if (count > 0 && mysize > PyByteArray_SIZE_MAX / count) { return PyErr_NoMemory(); } const Py_ssize_t size = mysize * count; @@ -585,7 +587,7 @@ bytearray_setslice_linear(PyByteArrayObject *self, buf = PyByteArray_AS_STRING(self); } else if (growth > 0) { - if (Py_SIZE(self) > (Py_ssize_t)PY_SSIZE_T_MAX - growth) { + if (Py_SIZE(self) > PyByteArray_SIZE_MAX - growth) { PyErr_NoMemory(); return -1; } @@ -899,6 +901,13 @@ bytearray___init___impl(PyByteArrayObject *self, PyObject *arg, PyObject *it; PyObject *(*iternext)(PyObject *); + /* First __init__; set ob_bytes_object so ob_bytes is always non-null. */ + if (self->ob_bytes_object == NULL) { + self->ob_bytes_object = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + bytearray_reinit_from_bytes(self, 0, 0); + self->ob_exports = 0; + } + if (Py_SIZE(self) != 0) { /* Empty previous contents (yes, do this first of all!) */ if (PyByteArray_Resize((PyObject *)self, 0) < 0) @@ -1169,9 +1178,7 @@ bytearray_dealloc(PyObject *op) "deallocated bytearray object has exported buffers"); PyErr_Print(); } - if (self->ob_bytes != 0) { - PyMem_Free(self->ob_bytes); - } + Py_XDECREF(self->ob_bytes_object); Py_TYPE(self)->tp_free((PyObject *)self); } @@ -1491,6 +1498,82 @@ bytearray_resize_impl(PyByteArrayObject *self, Py_ssize_t size) } +/*[clinic input] +@critical_section +bytearray.take_bytes + n: object = None + Bytes to take, negative indexes from end. None indicates all bytes. + / +Take *n* bytes from the bytearray and return them as a bytes object. +[clinic start generated code]*/ + +static PyObject * +bytearray_take_bytes_impl(PyByteArrayObject *self, PyObject *n) +/*[clinic end generated code: output=3147fbc0bbbe8d94 input=b15b5172cdc6deda]*/ +{ + Py_ssize_t to_take; + Py_ssize_t size = Py_SIZE(self); + if (Py_IsNone(n)) { + to_take = size; + } + // Integer index, from start (zero, positive) or end (negative). + else if (_PyIndex_Check(n)) { + to_take = PyNumber_AsSsize_t(n, PyExc_IndexError); + if (to_take == -1 && PyErr_Occurred()) { + return NULL; + } + if (to_take < 0) { + to_take += size; + } + } + else { + PyErr_SetString(PyExc_TypeError, "n must be an integer or None"); + return NULL; + } + + if (to_take < 0 || to_take > size) { + PyErr_Format(PyExc_IndexError, + "can't take %zd bytes outside size %zd", + to_take, size); + return NULL; + } + + // Exports may change the contents. No mutable bytes allowed. + if (!_canresize(self)) { + return NULL; + } + + if (to_take == 0 || size == 0) { + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + } + + // Copy remaining bytes to a new bytes. + Py_ssize_t remaining_length = size - to_take; + PyObject *remaining = PyBytes_FromStringAndSize(self->ob_start + to_take, + remaining_length); + if (remaining == NULL) { + return NULL; + } + + // If the bytes are offset inside the buffer must first align. + if (self->ob_start != self->ob_bytes) { + memmove(self->ob_bytes, self->ob_start, to_take); + self->ob_start = self->ob_bytes; + } + + if (_PyBytes_Resize(&self->ob_bytes_object, to_take) == -1) { + Py_DECREF(remaining); + return NULL; + } + + // Point the bytearray towards the buffer with the remaining data. + PyObject *result = self->ob_bytes_object; + self->ob_bytes_object = remaining; + bytearray_reinit_from_bytes(self, remaining_length, remaining_length); + return result; +} + + /*[clinic input] @critical_section bytearray.translate @@ -1868,11 +1951,6 @@ bytearray_insert_impl(PyByteArrayObject *self, Py_ssize_t index, int item) Py_ssize_t n = Py_SIZE(self); char *buf; - if (n == PY_SSIZE_T_MAX) { - PyErr_SetString(PyExc_OverflowError, - "cannot add more objects to bytearray"); - return NULL; - } if (bytearray_resize_lock_held((PyObject *)self, n + 1) < 0) return NULL; buf = PyByteArray_AS_STRING(self); @@ -1987,11 +2065,6 @@ bytearray_append_impl(PyByteArrayObject *self, int item) { Py_ssize_t n = Py_SIZE(self); - if (n == PY_SSIZE_T_MAX) { - PyErr_SetString(PyExc_OverflowError, - "cannot add more objects to bytearray"); - return NULL; - } if (bytearray_resize_lock_held((PyObject *)self, n + 1) < 0) return NULL; @@ -2099,16 +2172,16 @@ bytearray_extend_impl(PyByteArrayObject *self, PyObject *iterable_of_ints) if (len >= buf_size) { Py_ssize_t addition; - if (len == PY_SSIZE_T_MAX) { + if (len == PyByteArray_SIZE_MAX) { Py_DECREF(it); Py_DECREF(bytearray_obj); return PyErr_NoMemory(); } addition = len >> 1; - if (addition > PY_SSIZE_T_MAX - len - 1) - buf_size = PY_SSIZE_T_MAX; + if (addition > PyByteArray_SIZE_MAX - len) + buf_size = PyByteArray_SIZE_MAX; else - buf_size = len + addition + 1; + buf_size = len + addition; if (bytearray_resize_lock_held((PyObject *)bytearray_obj, buf_size) < 0) { Py_DECREF(it); Py_DECREF(bytearray_obj); @@ -2405,7 +2478,11 @@ static PyObject * bytearray_alloc(PyObject *op, PyObject *Py_UNUSED(ignored)) { PyByteArrayObject *self = _PyByteArray_CAST(op); - return PyLong_FromSsize_t(FT_ATOMIC_LOAD_SSIZE_RELAXED(self->ob_alloc)); + Py_ssize_t alloc = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->ob_alloc); + if (alloc > 0) { + alloc += _PyBytesObject_SIZE; + } + return PyLong_FromSsize_t(alloc); } /*[clinic input] @@ -2601,9 +2678,13 @@ static PyObject * bytearray_sizeof_impl(PyByteArrayObject *self) /*[clinic end generated code: output=738abdd17951c427 input=e27320fd98a4bc5a]*/ { - size_t res = _PyObject_SIZE(Py_TYPE(self)); - res += (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(self->ob_alloc) * sizeof(char); - return PyLong_FromSize_t(res); + Py_ssize_t res = _PyObject_SIZE(Py_TYPE(self)); + Py_ssize_t alloc = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->ob_alloc); + if (alloc > 0) { + res += _PyBytesObject_SIZE + alloc; + } + + return PyLong_FromSsize_t(res); } static PySequenceMethods bytearray_as_sequence = { @@ -2686,6 +2767,7 @@ static PyMethodDef bytearray_methods[] = { BYTEARRAY_STARTSWITH_METHODDEF BYTEARRAY_STRIP_METHODDEF {"swapcase", bytearray_swapcase, METH_NOARGS, _Py_swapcase__doc__}, + BYTEARRAY_TAKE_BYTES_METHODDEF {"title", bytearray_title, METH_NOARGS, _Py_title__doc__}, BYTEARRAY_TRANSLATE_METHODDEF {"upper", bytearray_upper, METH_NOARGS, _Py_upper__doc__}, diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 2b9513abe91..2b0925017f2 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -25,13 +25,7 @@ class bytes "PyBytesObject *" "&PyBytes_Type" #include "clinic/bytesobject.c.h" -/* PyBytesObject_SIZE gives the basic size of a bytes object; any memory allocation - for a bytes object of length n should request PyBytesObject_SIZE + n bytes. - - Using PyBytesObject_SIZE instead of sizeof(PyBytesObject) saves - 3 or 7 bytes per bytes object allocation on a typical system. -*/ -#define PyBytesObject_SIZE (offsetof(PyBytesObject, ob_sval) + 1) +#define PyBytesObject_SIZE _PyBytesObject_SIZE /* Forward declaration */ static void* _PyBytesWriter_ResizeAndUpdatePointer(PyBytesWriter *writer, diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h index 6f13865177d..be704ccf68f 100644 --- a/Objects/clinic/bytearrayobject.c.h +++ b/Objects/clinic/bytearrayobject.c.h @@ -631,6 +631,43 @@ exit: return return_value; } +PyDoc_STRVAR(bytearray_take_bytes__doc__, +"take_bytes($self, n=None, /)\n" +"--\n" +"\n" +"Take *n* bytes from the bytearray and return them as a bytes object.\n" +"\n" +" n\n" +" Bytes to take, negative indexes from end. None indicates all bytes."); + +#define BYTEARRAY_TAKE_BYTES_METHODDEF \ + {"take_bytes", _PyCFunction_CAST(bytearray_take_bytes), METH_FASTCALL, bytearray_take_bytes__doc__}, + +static PyObject * +bytearray_take_bytes_impl(PyByteArrayObject *self, PyObject *n); + +static PyObject * +bytearray_take_bytes(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *n = Py_None; + + if (!_PyArg_CheckPositional("take_bytes", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + n = args[0]; +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = bytearray_take_bytes_impl((PyByteArrayObject *)self, n); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + PyDoc_STRVAR(bytearray_translate__doc__, "translate($self, table, /, delete=b\'\')\n" "--\n" @@ -1796,4 +1833,4 @@ bytearray_sizeof(PyObject *self, PyObject *Py_UNUSED(ignored)) { return bytearray_sizeof_impl((PyByteArrayObject *)self); } -/*[clinic end generated code: output=fdfe41139c91e409 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5eddefde2a001ceb input=a9049054013a1b77]*/ From c2470b39fa21f355f811419f1b3d1c776c36fb10 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Thu, 13 Nov 2025 17:44:40 +0300 Subject: [PATCH 185/313] gh-137959: Fix `TIER1_TO_TIER2` macro name in JIT InternalDocs (GH-141496) JIT InternalDocs fix --- InternalDocs/jit.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/InternalDocs/jit.md b/InternalDocs/jit.md index 09585380737..1740b22b85f 100644 --- a/InternalDocs/jit.md +++ b/InternalDocs/jit.md @@ -53,7 +53,7 @@ ## The micro-op optimizer ## The JIT interpreter After a `JUMP_BACKWARD` instruction invokes the uop optimizer to create a uop -executor, it transfers control to this executor via the `GOTO_TIER_TWO` macro. +executor, it transfers control to this executor via the `TIER1_TO_TIER2` macro. CPython implements two executors. Here we describe the JIT interpreter, which is the simpler of them and is therefore useful for debugging and analyzing From f72768f30e6ed9253eb3b6374b4395dfcaf4842a Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 13 Nov 2025 10:02:21 -0500 Subject: [PATCH 186/313] gh-141004: Document C APIs for dictionary keys, values, and items (GH-141009) Co-authored-by: Petr Viktorin --- Doc/c-api/dict.rst | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 246ce5391e1..b7f201811aa 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -431,3 +431,49 @@ Dictionary Objects it before returning. .. versionadded:: 3.12 + + +Dictionary View Objects +^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:function:: int PyDictViewSet_Check(PyObject *op) + + Return true if *op* is a view of a set inside a dictionary. This is currently + equivalent to :c:expr:`PyDictKeys_Check(op) || PyDictItems_Check(op)`. This + function always succeeds. + + +.. c:var:: PyTypeObject PyDictKeys_Type + + Type object for a view of dictionary keys. In Python, this is the type of + the object returned by :meth:`dict.keys`. + + +.. c:function:: int PyDictKeys_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary keys view. This function + always succeeds. + + +.. c:var:: PyTypeObject PyDictValues_Type + + Type object for a view of dictionary values. In Python, this is the type of + the object returned by :meth:`dict.values`. + + +.. c:function:: int PyDictValues_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary values view. This function + always succeeds. + + +.. c:var:: PyTypeObject PyDictItems_Type + + Type object for a view of dictionary items. In Python, this is the type of + the object returned by :meth:`dict.items`. + + +.. c:function:: int PyDictItems_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary items view. This function + always succeeds. From d7862e9b1bd8f82e41c4f2c4dad31e15707d856f Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 13 Nov 2025 10:07:57 -0500 Subject: [PATCH 187/313] gh-141004: Document `PyCode_Optimize` (GH-141378) --- Doc/c-api/code.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index c9741b61254..45f5e83adc4 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -211,6 +211,17 @@ bound into a function. .. versionadded:: 3.12 +.. c:function:: PyObject *PyCode_Optimize(PyObject *code, PyObject *consts, PyObject *names, PyObject *lnotab_obj) + + This is a :term:`soft deprecated` function that does nothing. + + Prior to Python 3.10, this function would perform basic optimizations to a + code object. + + .. versionchanged:: 3.10 + This function now does nothing. + + .. _c_codeobject_flags: Code Object Flags From b99db92dde38b17c3fba3b5db76a383ceddfce49 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 13 Nov 2025 17:30:50 +0100 Subject: [PATCH 188/313] gh-139653: Add PyUnstable_ThreadState_SetStackProtection() (#139668) Add PyUnstable_ThreadState_SetStackProtection() and PyUnstable_ThreadState_ResetStackProtection() functions to set the stack base address and stack size of a Python thread state. Co-authored-by: Petr Viktorin --- Doc/c-api/exceptions.rst | 3 + Doc/c-api/init.rst | 37 +++++++++ Doc/whatsnew/3.15.rst | 6 ++ Include/cpython/pystate.h | 12 +++ Include/internal/pycore_pythonrun.h | 6 ++ Include/internal/pycore_tstate.h | 4 + ...-10-06-22-17-47.gh-issue-139653.6-1MOd.rst | 4 + Modules/_testinternalcapi.c | 54 +++++++++++++ Python/ceval.c | 77 +++++++++++++++++-- Python/pystate.c | 3 + 10 files changed, 199 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 5241533e112..0ee595a07ac 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -976,6 +976,9 @@ because the :ref:`call protocol ` takes care of recursion handling. be concatenated to the :exc:`RecursionError` message caused by the recursion depth limit. + .. seealso:: + The :c:func:`PyUnstable_ThreadState_SetStackProtection` function. + .. versionchanged:: 3.9 This function is now also available in the :ref:`limited API `. diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 49ffeab5585..18ee1611807 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1366,6 +1366,43 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. versionadded:: 3.11 +.. c:function:: int PyUnstable_ThreadState_SetStackProtection(PyThreadState *tstate, void *stack_start_addr, size_t stack_size) + + Set the stack protection start address and stack protection size + of a Python thread state. + + On success, return ``0``. + On failure, set an exception and return ``-1``. + + CPython implements :ref:`recursion control ` for C code by raising + :py:exc:`RecursionError` when it notices that the machine execution stack is close + to overflow. See for example the :c:func:`Py_EnterRecursiveCall` function. + For this, it needs to know the location of the current thread's stack, which it + normally gets from the operating system. + When the stack is changed, for example using context switching techniques like the + Boost library's ``boost::context``, you must call + :c:func:`~PyUnstable_ThreadState_SetStackProtection` to inform CPython of the change. + + Call :c:func:`~PyUnstable_ThreadState_SetStackProtection` either before + or after changing the stack. + Do not call any other Python C API between the call and the stack + change. + + See :c:func:`PyUnstable_ThreadState_ResetStackProtection` for undoing this operation. + + .. versionadded:: next + + +.. c:function:: void PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) + + Reset the stack protection start address and stack protection size + of a Python thread state to the operating system defaults. + + See :c:func:`PyUnstable_ThreadState_SetStackProtection` for an explanation. + + .. versionadded:: next + + .. c:function:: PyInterpreterState* PyInterpreterState_Get(void) Get the current interpreter. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index d7c9a41eeb2..b360ad964cf 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1066,6 +1066,12 @@ New features * Add :c:func:`PyTuple_FromArray` to create a :class:`tuple` from an array. (Contributed by Victor Stinner in :gh:`111489`.) +* Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and + :c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set + the stack protection base address and stack protection size of a Python + thread state. + (Contributed by Victor Stinner in :gh:`139653`.) + Changed C APIs -------------- diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index dd2ea1202b3..c53abe43ebe 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -276,6 +276,18 @@ PyAPI_FUNC(int) PyGILState_Check(void); */ PyAPI_FUNC(PyObject*) _PyThread_CurrentFrames(void); +// Set the stack protection start address and stack protection size +// of a Python thread state +PyAPI_FUNC(int) PyUnstable_ThreadState_SetStackProtection( + PyThreadState *tstate, + void *stack_start_addr, // Stack start address + size_t stack_size); // Stack size (in bytes) + +// Reset the stack protection start address and stack protection size +// of a Python thread state +PyAPI_FUNC(void) PyUnstable_ThreadState_ResetStackProtection( + PyThreadState *tstate); + /* Routines for advanced debuggers, requested by David Beazley. Don't use unless you know what you are doing! */ PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Main(void); diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h index f954f1b63ef..04a557e1204 100644 --- a/Include/internal/pycore_pythonrun.h +++ b/Include/internal/pycore_pythonrun.h @@ -60,6 +60,12 @@ extern PyObject * _Py_CompileStringObjectWithModule( # define _PyOS_STACK_MARGIN_SHIFT (_PyOS_LOG2_STACK_MARGIN + 2) #endif +#ifdef _Py_THREAD_SANITIZER +# define _PyOS_MIN_STACK_SIZE (_PyOS_STACK_MARGIN_BYTES * 6) +#else +# define _PyOS_MIN_STACK_SIZE (_PyOS_STACK_MARGIN_BYTES * 3) +#endif + #ifdef __cplusplus } diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index 29ebdfd7e01..a44c523e202 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -37,6 +37,10 @@ typedef struct _PyThreadStateImpl { uintptr_t c_stack_soft_limit; uintptr_t c_stack_hard_limit; + // PyUnstable_ThreadState_ResetStackProtection() values + uintptr_t c_stack_init_base; + uintptr_t c_stack_init_top; + PyObject *asyncio_running_loop; // Strong reference PyObject *asyncio_running_task; // Strong reference diff --git a/Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst b/Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst new file mode 100644 index 00000000000..cd3d5262fa0 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst @@ -0,0 +1,4 @@ +Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and +:c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set the +stack protection base address and stack protection size of a Python thread +state. Patch by Victor Stinner. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index dede05960d7..6514ca7f3cd 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2446,6 +2446,58 @@ module_get_gc_hooks(PyObject *self, PyObject *arg) return result; } + +static void +check_threadstate_set_stack_protection(PyThreadState *tstate, + void *start, size_t size) +{ + assert(PyUnstable_ThreadState_SetStackProtection(tstate, start, size) == 0); + assert(!PyErr_Occurred()); + + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(ts->c_stack_top == (uintptr_t)start + size); + assert(ts->c_stack_hard_limit <= ts->c_stack_soft_limit); + assert(ts->c_stack_soft_limit < ts->c_stack_top); +} + + +static PyObject * +test_threadstate_set_stack_protection(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyThreadState *tstate = PyThreadState_GET(); + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(!PyErr_Occurred()); + + uintptr_t init_base = ts->c_stack_init_base; + size_t init_top = ts->c_stack_init_top; + + // Test the minimum stack size + size_t size = _PyOS_MIN_STACK_SIZE; + void *start = (void*)(_Py_get_machine_stack_pointer() - size); + check_threadstate_set_stack_protection(tstate, start, size); + + // Test a larger size + size = 7654321; + assert(size > _PyOS_MIN_STACK_SIZE); + start = (void*)(_Py_get_machine_stack_pointer() - size); + check_threadstate_set_stack_protection(tstate, start, size); + + // Test invalid size (too small) + size = 5; + start = (void*)(_Py_get_machine_stack_pointer() - size); + assert(PyUnstable_ThreadState_SetStackProtection(tstate, start, size) == -1); + assert(PyErr_ExceptionMatches(PyExc_ValueError)); + PyErr_Clear(); + + // Test PyUnstable_ThreadState_ResetStackProtection() + PyUnstable_ThreadState_ResetStackProtection(tstate); + assert(ts->c_stack_init_base == init_base); + assert(ts->c_stack_init_top == init_top); + + Py_RETURN_NONE; +} + + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -2556,6 +2608,8 @@ static PyMethodDef module_functions[] = { {"simple_pending_call", simple_pending_call, METH_O}, {"set_vectorcall_nop", set_vectorcall_nop, METH_O}, {"module_get_gc_hooks", module_get_gc_hooks, METH_O}, + {"test_threadstate_set_stack_protection", + test_threadstate_set_stack_protection, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Python/ceval.c b/Python/ceval.c index 43e8ee71206..07d21575e3a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -443,7 +443,7 @@ int pthread_attr_destroy(pthread_attr_t *a) #endif static void -hardware_stack_limits(uintptr_t *top, uintptr_t *base) +hardware_stack_limits(uintptr_t *base, uintptr_t *top) { #ifdef WIN32 ULONG_PTR low, high; @@ -486,23 +486,86 @@ hardware_stack_limits(uintptr_t *top, uintptr_t *base) #endif } -void -_Py_InitializeRecursionLimits(PyThreadState *tstate) +static void +tstate_set_stack(PyThreadState *tstate, + uintptr_t base, uintptr_t top) { - uintptr_t top; - uintptr_t base; - hardware_stack_limits(&top, &base); + assert(base < top); + assert((top - base) >= _PyOS_MIN_STACK_SIZE); + #ifdef _Py_THREAD_SANITIZER // Thread sanitizer crashes if we use more than half the stack. uintptr_t stacksize = top - base; - base += stacksize/2; + base += stacksize / 2; #endif _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; _tstate->c_stack_top = top; _tstate->c_stack_hard_limit = base + _PyOS_STACK_MARGIN_BYTES; _tstate->c_stack_soft_limit = base + _PyOS_STACK_MARGIN_BYTES * 2; + +#ifndef NDEBUG + // Sanity checks + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(ts->c_stack_hard_limit <= ts->c_stack_soft_limit); + assert(ts->c_stack_soft_limit < ts->c_stack_top); +#endif } + +void +_Py_InitializeRecursionLimits(PyThreadState *tstate) +{ + uintptr_t base, top; + hardware_stack_limits(&base, &top); + assert(top != 0); + + tstate_set_stack(tstate, base, top); + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + ts->c_stack_init_base = base; + ts->c_stack_init_top = top; + + // Test the stack pointer +#if !defined(NDEBUG) && !defined(__wasi__) + uintptr_t here_addr = _Py_get_machine_stack_pointer(); + assert(ts->c_stack_soft_limit < here_addr); + assert(here_addr < ts->c_stack_top); +#endif +} + + +int +PyUnstable_ThreadState_SetStackProtection(PyThreadState *tstate, + void *stack_start_addr, size_t stack_size) +{ + if (stack_size < _PyOS_MIN_STACK_SIZE) { + PyErr_Format(PyExc_ValueError, + "stack_size must be at least %zu bytes", + _PyOS_MIN_STACK_SIZE); + return -1; + } + + uintptr_t base = (uintptr_t)stack_start_addr; + uintptr_t top = base + stack_size; + tstate_set_stack(tstate, base, top); + return 0; +} + + +void +PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) +{ + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + if (ts->c_stack_init_top != 0) { + tstate_set_stack(tstate, + ts->c_stack_init_base, + ts->c_stack_init_top); + return; + } + + _Py_InitializeRecursionLimits(tstate); +} + + /* The function _Py_EnterRecursiveCallTstate() only calls _Py_CheckRecursiveCall() if the recursion_depth reaches recursion_limit. */ int diff --git a/Python/pystate.c b/Python/pystate.c index cf251c120d7..341c680a403 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1495,6 +1495,9 @@ init_threadstate(_PyThreadStateImpl *_tstate, _tstate->c_stack_top = 0; _tstate->c_stack_hard_limit = 0; + _tstate->c_stack_init_base = 0; + _tstate->c_stack_init_top = 0; + _tstate->asyncio_running_loop = NULL; _tstate->asyncio_running_task = NULL; From b2b68d40f887c8a9583a9b48babc40f25bc5e0e2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Nov 2025 19:48:52 +0200 Subject: [PATCH 189/313] gh-140873: Add support of non-descriptor callables in functools.singledispatchmethod() (GH-140884) --- Doc/library/functools.rst | 5 ++- Doc/whatsnew/3.15.rst | 8 +++++ Lib/functools.py | 5 ++- Lib/test/test_functools.py | 35 ++++++++++++++++++- ...-11-01-14-44-09.gh-issue-140873.kfuc9B.rst | 2 ++ 5 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 1d9ac328f32..b2e2e11c0dc 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -672,7 +672,7 @@ The :mod:`functools` module defines the following functions: dispatch>` :term:`generic function`. To define a generic method, decorate it with the ``@singledispatchmethod`` - decorator. When defining a function using ``@singledispatchmethod``, note + decorator. When defining a method using ``@singledispatchmethod``, note that the dispatch happens on the type of the first non-*self* or non-*cls* argument:: @@ -716,6 +716,9 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.8 + .. versionchanged:: next + Added support of non-:term:`descriptor` callables. + .. function:: update_wrapper(wrapper, wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index b360ad964cf..895616e3049 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -498,6 +498,14 @@ difflib (Contributed by Jiahao Li in :gh:`134580`.) +functools +--------- + +* :func:`~functools.singledispatchmethod` now supports non-:term:`descriptor` + callables. + (Contributed by Serhiy Storchaka in :gh:`140873`.) + + hashlib ------- diff --git a/Lib/functools.py b/Lib/functools.py index a92844ba722..8063eb5ffc3 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -1083,7 +1083,10 @@ def __call__(self, /, *args, **kwargs): 'singledispatchmethod method') raise TypeError(f'{funcname} requires at least ' '1 positional argument') - return self._dispatch(args[0].__class__).__get__(self._obj, self._cls)(*args, **kwargs) + method = self._dispatch(args[0].__class__) + if hasattr(method, "__get__"): + method = method.__get__(self._obj, self._cls) + return method(*args, **kwargs) def __getattr__(self, name): # Resolve these attributes lazily to speed up creation of diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index ce9e7f6d57d..090926fd8d8 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -2785,7 +2785,7 @@ class Slot: @functools.singledispatchmethod @classmethod def go(cls, item, arg): - pass + return item - arg @go.register @classmethod @@ -2794,7 +2794,9 @@ def _(cls, item: int, arg): s = Slot() self.assertEqual(s.go(1, 1), 2) + self.assertEqual(s.go(1.5, 1), 0.5) self.assertEqual(Slot.go(1, 1), 2) + self.assertEqual(Slot.go(1.5, 1), 0.5) def test_staticmethod_slotted_class(self): class A: @@ -3485,6 +3487,37 @@ def _(item, arg: bytes) -> str: self.assertEqual(str(Signature.from_callable(A.static_func)), '(item, arg: int) -> str') + def test_method_non_descriptor(self): + class Callable: + def __init__(self, value): + self.value = value + def __call__(self, arg): + return self.value, arg + + class A: + t = functools.singledispatchmethod(Callable('general')) + t.register(int, Callable('special')) + + @functools.singledispatchmethod + def u(self, arg): + return 'general', arg + u.register(int, Callable('special')) + + v = functools.singledispatchmethod(Callable('general')) + @v.register(int) + def _(self, arg): + return 'special', arg + + a = A() + self.assertEqual(a.t(0), ('special', 0)) + self.assertEqual(a.t(2.5), ('general', 2.5)) + self.assertEqual(A.t(0), ('special', 0)) + self.assertEqual(A.t(2.5), ('general', 2.5)) + self.assertEqual(a.u(0), ('special', 0)) + self.assertEqual(a.u(2.5), ('general', 2.5)) + self.assertEqual(a.v(0), ('special', 0)) + self.assertEqual(a.v(2.5), ('general', 2.5)) + class CachedCostItem: _cost = 1 diff --git a/Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst b/Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst new file mode 100644 index 00000000000..e1505764064 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst @@ -0,0 +1,2 @@ +Add support of non-:term:`descriptor` callables in +:func:`functools.singledispatchmethod`. From 196f1519cd2d8134d7643536f13f2b2844bea65d Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Thu, 13 Nov 2025 17:58:47 +0000 Subject: [PATCH 190/313] gh-141004: Document `PyErr_RangedSyntaxLocationObject` (#141521) PyErr_RangedSyntaxLocationObject --- Doc/c-api/exceptions.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 0ee595a07ac..d7fe9e2c9ec 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -309,6 +309,14 @@ For convenience, some of these functions will always return a .. versionadded:: 3.4 +.. c:function:: void PyErr_RangedSyntaxLocationObject(PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset) + + Similar to :c:func:`PyErr_SyntaxLocationObject`, but also sets the + *end_lineno* and *end_col_offset* information for the current exception. + + .. versionadded:: 3.10 + + .. c:function:: void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string From 4fa80ce74c6d9f5159bdc5ec3596a194f0391e21 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Fri, 14 Nov 2025 02:08:32 +0800 Subject: [PATCH 191/313] gh-139109: A new tracing JIT compiler frontend for CPython (GH-140310) This PR changes the current JIT model from trace projection to trace recording. Benchmarking: better pyperformance (about 1.7% overall) geomean versus current https://raw.githubusercontent.com/facebookexperimental/free-threading-benchmarking/refs/heads/main/results/bm-20251108-3.15.0a1%2B-7e2bc1d-JIT/bm-20251108-vultr-x86_64-Fidget%252dSpinner-tracing_jit-3.15.0a1%2B-7e2bc1d-vs-base.svg, 100% faster Richards on the most improved benchmark versus the current JIT. Slowdown of about 10-15% on the worst benchmark versus the current JIT. **Note: the fastest version isn't the one merged, as it relies on fixing bugs in the specializing interpreter, which is left to another PR**. The speedup in the merged version is about 1.1%. https://raw.githubusercontent.com/facebookexperimental/free-threading-benchmarking/refs/heads/main/results/bm-20251112-3.15.0a1%2B-f8a764a-JIT/bm-20251112-vultr-x86_64-Fidget%252dSpinner-tracing_jit-3.15.0a1%2B-f8a764a-vs-base.svg Stats: 50% more uops executed, 30% more traces entered the last time we ran them. It also suggests our trace lengths for a real trace recording JIT are too short, as a lot of trace too long aborts https://github.com/facebookexperimental/free-threading-benchmarking/blob/main/results/bm-20251023-3.15.0a1%2B-eb73378-CLANG%2CJIT/bm-20251023-vultr-x86_64-Fidget%252dSpinner-tracing_jit-3.15.0a1%2B-eb73378-pystats-vs-base.md . This new JIT frontend is already able to record/execute significantly more instructions than the previous JIT frontend. In this PR, we are now able to record through custom dunders, simple object creation, generators, etc. None of these were done by the old JIT frontend. Some custom dunders uops were discovered to be broken as part of this work gh-140277 The optimizer stack space check is disabled, as it's no longer valid to deal with underflow. Pros: * Ignoring the generated tracer code as it's automatically created, this is only additional 1k lines of code. The maintenance burden is handled by the DSL and code generator. * `optimizer.c` is now significantly simpler, as we don't have to do strange things to recover the bytecode from a trace. * The new JIT frontend is able to handle a lot more control-flow than the old one. * Tracing is very low overhead. We use the tail calling interpreter/computed goto interpreter to switch between tracing mode and non-tracing mode. I call this mechanism dual dispatch, as we have two dispatch tables dispatching to each other. Specialization is still enabled while tracing. * Better handling of polymorphism. We leverage the specializing interpreter for this. Cons: * (For now) requires tail calling interpreter or computed gotos. This means no Windows JIT for now :(. Not to fret, tail calling is coming soon to Windows though https://github.com/python/cpython/pull/139962 Design: * After each instruction, the `record_previous_inst` function/label is executed. This does as the name suggests. * The tracing interpreter lowers bytecode to uops directly so that it can obtain "fresh" values at the point of lowering. * The tracing version behaves nearly identical to the normal interpreter, in fact it even has specialization! This allows it to run without much of a slowdown when tracing. The actual cost of tracing is only a function call and writes to memory. * The tracing interpreter uses the specializing interpreter's deopt to naturally form the side exit chains. This allows it to side exit chain effectively, without repeating much code. We force a re-specializing when tracing a deopt. * The tracing interpreter can even handle goto errors/exceptions, but I chose to disable them for now as it's not tested. * Because we do not share interpreter dispatch, there is should be no significant slowdown to the original specializing interpreter on tailcall and computed got with JIT disabled. With JIT enabled, there might be a slowdown in the form of the JIT trying to trace. * Things that could have dynamic instruction pointer effects are guarded on. The guard deopts to a new instruction --- `_DYNAMIC_EXIT`. --- .github/workflows/jit.yml | 26 +- Include/cpython/pystats.h | 2 + Include/internal/pycore_backoff.h | 17 +- Include/internal/pycore_ceval.h | 2 + Include/internal/pycore_interp_structs.h | 4 +- Include/internal/pycore_opcode_metadata.h | 71 +- Include/internal/pycore_optimizer.h | 41 +- Include/internal/pycore_tstate.h | 39 +- Include/internal/pycore_uop.h | 12 +- Include/internal/pycore_uop_ids.h | 389 +++--- Include/internal/pycore_uop_metadata.h | 38 +- Lib/test/test_ast/test_ast.py | 4 +- Lib/test/test_capi/test_opt.py | 65 +- Lib/test/test_sys.py | 5 +- ...-10-18-21-50-44.gh-issue-139109.9QQOzN.rst | 1 + Modules/_testinternalcapi.c | 3 +- Objects/codeobject.c | 1 + Objects/frameobject.c | 6 +- Objects/funcobject.c | 6 +- Python/bytecodes.c | 194 ++- Python/ceval.c | 55 +- Python/ceval_macros.h | 67 +- Python/executor_cases.c.h | 139 ++- Python/generated_cases.c.h | 104 +- Python/instrumentation.c | 2 + Python/jit.c | 2 +- Python/opcode_targets.h | 526 +++++++- Python/optimizer.c | 1063 +++++++++-------- Python/optimizer_analysis.c | 54 +- Python/optimizer_bytecodes.c | 137 ++- Python/optimizer_cases.c.h | 153 ++- Python/optimizer_symbols.c | 44 +- Python/pystate.c | 27 +- Tools/c-analyzer/cpython/ignored.tsv | 1 + Tools/cases_generator/analyzer.py | 58 + Tools/cases_generator/generators_common.py | 17 +- .../opcode_metadata_generator.py | 4 +- Tools/cases_generator/target_generator.py | 26 +- Tools/cases_generator/tier2_generator.py | 54 +- .../cases_generator/uop_metadata_generator.py | 4 +- Tools/jit/template.c | 11 +- 41 files changed, 2409 insertions(+), 1065 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 62325250bd3..3349eb04242 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -57,9 +57,10 @@ jobs: fail-fast: false matrix: target: - - i686-pc-windows-msvc/msvc - - x86_64-pc-windows-msvc/msvc - - aarch64-pc-windows-msvc/msvc +# To re-enable later when we support these. +# - i686-pc-windows-msvc/msvc +# - x86_64-pc-windows-msvc/msvc +# - aarch64-pc-windows-msvc/msvc - x86_64-apple-darwin/clang - aarch64-apple-darwin/clang - x86_64-unknown-linux-gnu/gcc @@ -70,15 +71,16 @@ jobs: llvm: - 21 include: - - target: i686-pc-windows-msvc/msvc - architecture: Win32 - runner: windows-2022 - - target: x86_64-pc-windows-msvc/msvc - architecture: x64 - runner: windows-2022 - - target: aarch64-pc-windows-msvc/msvc - architecture: ARM64 - runner: windows-11-arm +# To re-enable later when we support these. +# - target: i686-pc-windows-msvc/msvc +# architecture: Win32 +# runner: windows-2022 +# - target: x86_64-pc-windows-msvc/msvc +# architecture: x64 +# runner: windows-2022 +# - target: aarch64-pc-windows-msvc/msvc +# architecture: ARM64 +# runner: windows-11-arm - target: x86_64-apple-darwin/clang architecture: x86_64 runner: macos-15-intel diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index d0a925a3055..1c94603c08b 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -150,6 +150,8 @@ typedef struct _optimization_stats { uint64_t optimized_trace_length_hist[_Py_UOP_HIST_SIZE]; uint64_t optimizer_attempts; uint64_t optimizer_successes; + uint64_t optimizer_contradiction; + uint64_t optimizer_frame_overflow; uint64_t optimizer_failure_reason_no_memory; uint64_t remove_globals_builtins_changed; uint64_t remove_globals_incorrect_keys; diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index 454c8dde031..71066f1bd9f 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -95,11 +95,24 @@ backoff_counter_triggers(_Py_BackoffCounter counter) return counter.value_and_backoff < UNREACHABLE_BACKOFF; } +static inline _Py_BackoffCounter +trigger_backoff_counter(void) +{ + _Py_BackoffCounter result; + result.value_and_backoff = 0; + return result; +} + // Initial JUMP_BACKWARD counter. // Must be larger than ADAPTIVE_COOLDOWN_VALUE, otherwise when JIT code is // invalidated we may construct a new trace before the bytecode has properly // re-specialized: -#define JUMP_BACKWARD_INITIAL_VALUE 4095 +// Note: this should be a prime number-1. This increases the likelihood of +// finding a "good" loop iteration to trace. +// For example, 4095 does not work for the nqueens benchmark on pyperformance +// as we always end up tracing the loop iteration's +// exhaustion iteration. Which aborts our current tracer. +#define JUMP_BACKWARD_INITIAL_VALUE 4000 #define JUMP_BACKWARD_INITIAL_BACKOFF 12 static inline _Py_BackoffCounter initial_jump_backoff_counter(void) @@ -112,7 +125,7 @@ initial_jump_backoff_counter(void) * Must be larger than ADAPTIVE_COOLDOWN_VALUE, * otherwise when a side exit warms up we may construct * a new trace before the Tier 1 code has properly re-specialized. */ -#define SIDE_EXIT_INITIAL_VALUE 4095 +#define SIDE_EXIT_INITIAL_VALUE 4000 #define SIDE_EXIT_INITIAL_BACKOFF 12 static inline _Py_BackoffCounter diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index fe72a0123eb..33b9fd053f7 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -392,6 +392,8 @@ _PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyInterpreterFrame #define SPECIAL___AEXIT__ 3 #define SPECIAL_MAX 3 +PyAPI_DATA(const _Py_CODEUNIT *) _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR; + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index e8cbe9d894e..9e4504479cd 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -14,8 +14,6 @@ extern "C" { #include "pycore_structs.h" // PyHamtObject #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_typedefs.h" // _PyRuntimeState -#include "pycore_uop.h" // struct _PyUOpInstruction - #define CODE_MAX_WATCHERS 8 #define CONTEXT_MAX_WATCHERS 8 @@ -934,10 +932,10 @@ struct _is { PyObject *common_consts[NUM_COMMON_CONSTANTS]; bool jit; bool compiling; - struct _PyUOpInstruction *jit_uop_buffer; struct _PyExecutorObject *executor_list_head; struct _PyExecutorObject *executor_deletion_list_head; struct _PyExecutorObject *cold_executor; + struct _PyExecutorObject *cold_dynamic_executor; int executor_deletion_list_remaining_capacity; size_t executor_creation_counter; _rare_events rare_events; diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index bd6b84ec7fd..548627dc798 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1031,6 +1031,8 @@ enum InstructionFormat { #define HAS_ERROR_NO_POP_FLAG (4096) #define HAS_NO_SAVE_IP_FLAG (8192) #define HAS_PERIODIC_FLAG (16384) +#define HAS_UNPREDICTABLE_JUMP_FLAG (32768) +#define HAS_NEEDS_GUARD_IP_FLAG (65536) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -1046,6 +1048,8 @@ enum InstructionFormat { #define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) #define OPCODE_HAS_NO_SAVE_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NO_SAVE_IP_FLAG)) #define OPCODE_HAS_PERIODIC(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PERIODIC_FLAG)) +#define OPCODE_HAS_UNPREDICTABLE_JUMP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_UNPREDICTABLE_JUMP_FLAG)) +#define OPCODE_HAS_NEEDS_GUARD_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NEEDS_GUARD_IP_FLAG)) #define OPARG_SIMPLE 0 #define OPARG_CACHE_1 1 @@ -1062,7 +1066,7 @@ enum InstructionFormat { struct opcode_metadata { uint8_t valid_entry; uint8_t instr_format; - uint16_t flags; + uint32_t flags; }; extern const struct opcode_metadata _PyOpcode_opcode_metadata[267]; @@ -1077,7 +1081,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG }, + [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_LIST_SLICE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, @@ -1094,22 +1098,22 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BUILD_TEMPLATE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [CACHE] = { true, INSTR_FMT_IX, 0 }, - [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_ISINSTANCE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_KW_NON_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_LEN] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1117,8 +1121,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_NON_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, @@ -1143,7 +1147,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1151,11 +1155,11 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG }, + [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1164,13 +1168,13 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1183,8 +1187,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1197,7 +1201,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, @@ -1205,7 +1209,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1253,10 +1257,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [RESERVED] = { true, INSTR_FMT_IX, 0 }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1292,7 +1296,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [ANNOTATIONS_PLACEHOLDER] = { true, -1, HAS_PURE_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1406,6 +1410,9 @@ _PyOpcode_macro_expansion[256] = { [IMPORT_FROM] = { .nuops = 1, .uops = { { _IMPORT_FROM, OPARG_SIMPLE, 0 } } }, [IMPORT_NAME] = { .nuops = 1, .uops = { { _IMPORT_NAME, OPARG_SIMPLE, 0 } } }, [IS_OP] = { .nuops = 1, .uops = { { _IS_OP, OPARG_SIMPLE, 0 } } }, + [JUMP_BACKWARD] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 1 } } }, + [JUMP_BACKWARD_NO_INTERRUPT] = { .nuops = 1, .uops = { { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 0 } } }, + [JUMP_BACKWARD_NO_JIT] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 1 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { _LIST_APPEND, OPARG_SIMPLE, 0 } } }, [LIST_EXTEND] = { .nuops = 1, .uops = { { _LIST_EXTEND, OPARG_SIMPLE, 0 } } }, [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, OPARG_SIMPLE, 8 } } }, diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 8ed5436eb68..653285a2c6b 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -21,14 +21,6 @@ typedef struct _PyExecutorLinkListNode { } _PyExecutorLinkListNode; -/* Bloom filter with m = 256 - * https://en.wikipedia.org/wiki/Bloom_filter */ -#define _Py_BLOOM_FILTER_WORDS 8 - -typedef struct { - uint32_t bits[_Py_BLOOM_FILTER_WORDS]; -} _PyBloomFilter; - typedef struct { uint8_t opcode; uint8_t oparg; @@ -44,7 +36,9 @@ typedef struct { typedef struct _PyExitData { uint32_t target; - uint16_t index; + uint16_t index:14; + uint16_t is_dynamic:1; + uint16_t is_control_flow:1; _Py_BackoffCounter temperature; struct _PyExecutorObject *executor; } _PyExitData; @@ -94,9 +88,8 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); // This value is arbitrary and was not optimized. #define JIT_CLEANUP_THRESHOLD 1000 -#define TRACE_STACK_SIZE 5 - -int _Py_uop_analyze_and_optimize(_PyInterpreterFrame *frame, +int _Py_uop_analyze_and_optimize( + PyFunctionObject *func, _PyUOpInstruction *trace, int trace_len, int curr_stackentries, _PyBloomFilter *dependencies); @@ -130,7 +123,7 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) #define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * 5) // Need extras for root frame and for overflow frame (see TRACE_STACK_PUSH()) -#define MAX_ABSTRACT_FRAME_DEPTH (TRACE_STACK_SIZE + 2) +#define MAX_ABSTRACT_FRAME_DEPTH (16) // The maximum number of side exits that we can take before requiring forward // progress (and inserting a new ENTER_EXECUTOR instruction). In practice, this @@ -258,6 +251,7 @@ struct _Py_UOpsAbstractFrame { int stack_len; int locals_len; PyFunctionObject *func; + PyCodeObject *code; JitOptRef *stack_pointer; JitOptRef *stack; @@ -333,11 +327,11 @@ extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( int curr_stackentries, JitOptRef *args, int arg_len); -extern int _Py_uop_frame_pop(JitOptContext *ctx); +extern int _Py_uop_frame_pop(JitOptContext *ctx, PyCodeObject *co, int curr_stackentries); PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); -PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyExecutorObject **exec_ptr, int chain_depth); +PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, PyThreadState *tstate); static inline _PyExecutorObject *_PyExecutor_FromExit(_PyExitData *exit) { @@ -346,6 +340,7 @@ static inline _PyExecutorObject *_PyExecutor_FromExit(_PyExitData *exit) } extern _PyExecutorObject *_PyExecutor_GetColdExecutor(void); +extern _PyExecutorObject *_PyExecutor_GetColdDynamicExecutor(void); PyAPI_FUNC(void) _PyExecutor_ClearExit(_PyExitData *exit); @@ -354,7 +349,9 @@ static inline int is_terminator(const _PyUOpInstruction *uop) int opcode = uop->opcode; return ( opcode == _EXIT_TRACE || - opcode == _JUMP_TO_TOP + opcode == _DEOPT || + opcode == _JUMP_TO_TOP || + opcode == _DYNAMIC_EXIT ); } @@ -365,6 +362,18 @@ PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); #endif +int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, bool stop_tracing); + +int +_PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *start_instr, + _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, + int oparg); + +void _PyJit_FinalizeTracing(PyThreadState *tstate); + +void _PyJit_Tracer_InvalidateDependency(PyThreadState *old_tstate, void *obj); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index a44c523e202..50048801b2e 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -12,7 +12,8 @@ extern "C" { #include "pycore_freelist_state.h" // struct _Py_freelists #include "pycore_mimalloc.h" // struct _mimalloc_thread_state #include "pycore_qsbr.h" // struct qsbr - +#include "pycore_uop.h" // struct _PyUOpInstruction +#include "pycore_structs.h" #ifdef Py_GIL_DISABLED struct _gc_thread_state { @@ -21,6 +22,38 @@ struct _gc_thread_state { }; #endif +#if _Py_TIER2 +typedef struct _PyJitTracerInitialState { + int stack_depth; + int chain_depth; + struct _PyExitData *exit; + PyCodeObject *code; // Strong + PyFunctionObject *func; // Strong + _Py_CODEUNIT *start_instr; + _Py_CODEUNIT *close_loop_instr; + _Py_CODEUNIT *jump_backward_instr; +} _PyJitTracerInitialState; + +typedef struct _PyJitTracerPreviousState { + bool dependencies_still_valid; + bool instr_is_super; + int code_max_size; + int code_curr_size; + int instr_oparg; + int instr_stacklevel; + _Py_CODEUNIT *instr; + PyCodeObject *instr_code; // Strong + struct _PyInterpreterFrame *instr_frame; + _PyBloomFilter dependencies; +} _PyJitTracerPreviousState; + +typedef struct _PyJitTracerState { + _PyUOpInstruction *code_buffer; + _PyJitTracerInitialState initial_state; + _PyJitTracerPreviousState prev_state; +} _PyJitTracerState; +#endif + // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The // PyThreadState fields are exposed as part of the C API, although most fields // are intended to be private. The _PyThreadStateImpl fields not exposed. @@ -85,7 +118,9 @@ typedef struct _PyThreadStateImpl { #if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) Py_ssize_t reftotal; // this thread's total refcount operations #endif - +#if _Py_TIER2 + _PyJitTracerState jit_tracer_state; +#endif } _PyThreadStateImpl; #ifdef __cplusplus diff --git a/Include/internal/pycore_uop.h b/Include/internal/pycore_uop.h index 4abefd3b95d..4e1b15af42c 100644 --- a/Include/internal/pycore_uop.h +++ b/Include/internal/pycore_uop.h @@ -35,10 +35,18 @@ typedef struct _PyUOpInstruction{ #endif } _PyUOpInstruction; -// This is the length of the trace we project initially. -#define UOP_MAX_TRACE_LENGTH 1200 +// This is the length of the trace we translate initially. +#define UOP_MAX_TRACE_LENGTH 3000 #define UOP_BUFFER_SIZE (UOP_MAX_TRACE_LENGTH * sizeof(_PyUOpInstruction)) +/* Bloom filter with m = 256 + * https://en.wikipedia.org/wiki/Bloom_filter */ +#define _Py_BLOOM_FILTER_WORDS 8 + +typedef struct { + uint32_t bits[_Py_BLOOM_FILTER_WORDS]; +} _PyBloomFilter; + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index ff1d75c0cb1..7a33a5b84fd 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -81,101 +81,107 @@ extern "C" { #define _CHECK_STACK_SPACE 357 #define _CHECK_STACK_SPACE_OPERAND 358 #define _CHECK_VALIDITY 359 -#define _COLD_EXIT 360 -#define _COMPARE_OP 361 -#define _COMPARE_OP_FLOAT 362 -#define _COMPARE_OP_INT 363 -#define _COMPARE_OP_STR 364 -#define _CONTAINS_OP 365 -#define _CONTAINS_OP_DICT 366 -#define _CONTAINS_OP_SET 367 +#define _COLD_DYNAMIC_EXIT 360 +#define _COLD_EXIT 361 +#define _COMPARE_OP 362 +#define _COMPARE_OP_FLOAT 363 +#define _COMPARE_OP_INT 364 +#define _COMPARE_OP_STR 365 +#define _CONTAINS_OP 366 +#define _CONTAINS_OP_DICT 367 +#define _CONTAINS_OP_SET 368 #define _CONVERT_VALUE CONVERT_VALUE -#define _COPY 368 -#define _COPY_1 369 -#define _COPY_2 370 -#define _COPY_3 371 +#define _COPY 369 +#define _COPY_1 370 +#define _COPY_2 371 +#define _COPY_3 372 #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 372 +#define _CREATE_INIT_FRAME 373 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 373 +#define _DEOPT 374 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 374 -#define _DO_CALL_FUNCTION_EX 375 -#define _DO_CALL_KW 376 +#define _DO_CALL 375 +#define _DO_CALL_FUNCTION_EX 376 +#define _DO_CALL_KW 377 +#define _DYNAMIC_EXIT 378 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 377 +#define _ERROR_POP_N 379 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 378 -#define _EXPAND_METHOD_KW 379 -#define _FATAL_ERROR 380 +#define _EXPAND_METHOD 380 +#define _EXPAND_METHOD_KW 381 +#define _FATAL_ERROR 382 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 381 -#define _FOR_ITER_GEN_FRAME 382 -#define _FOR_ITER_TIER_TWO 383 +#define _FOR_ITER 383 +#define _FOR_ITER_GEN_FRAME 384 +#define _FOR_ITER_TIER_TWO 385 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 384 -#define _GUARD_CALLABLE_ISINSTANCE 385 -#define _GUARD_CALLABLE_LEN 386 -#define _GUARD_CALLABLE_LIST_APPEND 387 -#define _GUARD_CALLABLE_STR_1 388 -#define _GUARD_CALLABLE_TUPLE_1 389 -#define _GUARD_CALLABLE_TYPE_1 390 -#define _GUARD_DORV_NO_DICT 391 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 392 -#define _GUARD_GLOBALS_VERSION 393 -#define _GUARD_IS_FALSE_POP 394 -#define _GUARD_IS_NONE_POP 395 -#define _GUARD_IS_NOT_NONE_POP 396 -#define _GUARD_IS_TRUE_POP 397 -#define _GUARD_KEYS_VERSION 398 -#define _GUARD_NOS_DICT 399 -#define _GUARD_NOS_FLOAT 400 -#define _GUARD_NOS_INT 401 -#define _GUARD_NOS_LIST 402 -#define _GUARD_NOS_NOT_NULL 403 -#define _GUARD_NOS_NULL 404 -#define _GUARD_NOS_OVERFLOWED 405 -#define _GUARD_NOS_TUPLE 406 -#define _GUARD_NOS_UNICODE 407 -#define _GUARD_NOT_EXHAUSTED_LIST 408 -#define _GUARD_NOT_EXHAUSTED_RANGE 409 -#define _GUARD_NOT_EXHAUSTED_TUPLE 410 -#define _GUARD_THIRD_NULL 411 -#define _GUARD_TOS_ANY_SET 412 -#define _GUARD_TOS_DICT 413 -#define _GUARD_TOS_FLOAT 414 -#define _GUARD_TOS_INT 415 -#define _GUARD_TOS_LIST 416 -#define _GUARD_TOS_OVERFLOWED 417 -#define _GUARD_TOS_SLICE 418 -#define _GUARD_TOS_TUPLE 419 -#define _GUARD_TOS_UNICODE 420 -#define _GUARD_TYPE_VERSION 421 -#define _GUARD_TYPE_VERSION_AND_LOCK 422 -#define _HANDLE_PENDING_AND_DEOPT 423 +#define _GUARD_BINARY_OP_EXTEND 386 +#define _GUARD_CALLABLE_ISINSTANCE 387 +#define _GUARD_CALLABLE_LEN 388 +#define _GUARD_CALLABLE_LIST_APPEND 389 +#define _GUARD_CALLABLE_STR_1 390 +#define _GUARD_CALLABLE_TUPLE_1 391 +#define _GUARD_CALLABLE_TYPE_1 392 +#define _GUARD_DORV_NO_DICT 393 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 394 +#define _GUARD_GLOBALS_VERSION 395 +#define _GUARD_IP_RETURN_GENERATOR 396 +#define _GUARD_IP_RETURN_VALUE 397 +#define _GUARD_IP_YIELD_VALUE 398 +#define _GUARD_IP__PUSH_FRAME 399 +#define _GUARD_IS_FALSE_POP 400 +#define _GUARD_IS_NONE_POP 401 +#define _GUARD_IS_NOT_NONE_POP 402 +#define _GUARD_IS_TRUE_POP 403 +#define _GUARD_KEYS_VERSION 404 +#define _GUARD_NOS_DICT 405 +#define _GUARD_NOS_FLOAT 406 +#define _GUARD_NOS_INT 407 +#define _GUARD_NOS_LIST 408 +#define _GUARD_NOS_NOT_NULL 409 +#define _GUARD_NOS_NULL 410 +#define _GUARD_NOS_OVERFLOWED 411 +#define _GUARD_NOS_TUPLE 412 +#define _GUARD_NOS_UNICODE 413 +#define _GUARD_NOT_EXHAUSTED_LIST 414 +#define _GUARD_NOT_EXHAUSTED_RANGE 415 +#define _GUARD_NOT_EXHAUSTED_TUPLE 416 +#define _GUARD_THIRD_NULL 417 +#define _GUARD_TOS_ANY_SET 418 +#define _GUARD_TOS_DICT 419 +#define _GUARD_TOS_FLOAT 420 +#define _GUARD_TOS_INT 421 +#define _GUARD_TOS_LIST 422 +#define _GUARD_TOS_OVERFLOWED 423 +#define _GUARD_TOS_SLICE 424 +#define _GUARD_TOS_TUPLE 425 +#define _GUARD_TOS_UNICODE 426 +#define _GUARD_TYPE_VERSION 427 +#define _GUARD_TYPE_VERSION_AND_LOCK 428 +#define _HANDLE_PENDING_AND_DEOPT 429 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 424 -#define _INIT_CALL_PY_EXACT_ARGS 425 -#define _INIT_CALL_PY_EXACT_ARGS_0 426 -#define _INIT_CALL_PY_EXACT_ARGS_1 427 -#define _INIT_CALL_PY_EXACT_ARGS_2 428 -#define _INIT_CALL_PY_EXACT_ARGS_3 429 -#define _INIT_CALL_PY_EXACT_ARGS_4 430 -#define _INSERT_NULL 431 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 430 +#define _INIT_CALL_PY_EXACT_ARGS 431 +#define _INIT_CALL_PY_EXACT_ARGS_0 432 +#define _INIT_CALL_PY_EXACT_ARGS_1 433 +#define _INIT_CALL_PY_EXACT_ARGS_2 434 +#define _INIT_CALL_PY_EXACT_ARGS_3 435 +#define _INIT_CALL_PY_EXACT_ARGS_4 436 +#define _INSERT_NULL 437 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -185,177 +191,178 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 432 +#define _IS_NONE 438 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 433 -#define _ITER_CHECK_RANGE 434 -#define _ITER_CHECK_TUPLE 435 -#define _ITER_JUMP_LIST 436 -#define _ITER_JUMP_RANGE 437 -#define _ITER_JUMP_TUPLE 438 -#define _ITER_NEXT_LIST 439 -#define _ITER_NEXT_LIST_TIER_TWO 440 -#define _ITER_NEXT_RANGE 441 -#define _ITER_NEXT_TUPLE 442 -#define _JUMP_TO_TOP 443 +#define _ITER_CHECK_LIST 439 +#define _ITER_CHECK_RANGE 440 +#define _ITER_CHECK_TUPLE 441 +#define _ITER_JUMP_LIST 442 +#define _ITER_JUMP_RANGE 443 +#define _ITER_JUMP_TUPLE 444 +#define _ITER_NEXT_LIST 445 +#define _ITER_NEXT_LIST_TIER_TWO 446 +#define _ITER_NEXT_RANGE 447 +#define _ITER_NEXT_TUPLE 448 +#define _JUMP_BACKWARD_NO_INTERRUPT JUMP_BACKWARD_NO_INTERRUPT +#define _JUMP_TO_TOP 449 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 444 -#define _LOAD_ATTR_CLASS 445 +#define _LOAD_ATTR 450 +#define _LOAD_ATTR_CLASS 451 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 446 -#define _LOAD_ATTR_METHOD_LAZY_DICT 447 -#define _LOAD_ATTR_METHOD_NO_DICT 448 -#define _LOAD_ATTR_METHOD_WITH_VALUES 449 -#define _LOAD_ATTR_MODULE 450 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 451 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 452 -#define _LOAD_ATTR_PROPERTY_FRAME 453 -#define _LOAD_ATTR_SLOT 454 -#define _LOAD_ATTR_WITH_HINT 455 +#define _LOAD_ATTR_INSTANCE_VALUE 452 +#define _LOAD_ATTR_METHOD_LAZY_DICT 453 +#define _LOAD_ATTR_METHOD_NO_DICT 454 +#define _LOAD_ATTR_METHOD_WITH_VALUES 455 +#define _LOAD_ATTR_MODULE 456 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 457 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 458 +#define _LOAD_ATTR_PROPERTY_FRAME 459 +#define _LOAD_ATTR_SLOT 460 +#define _LOAD_ATTR_WITH_HINT 461 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 456 +#define _LOAD_BYTECODE 462 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 457 -#define _LOAD_CONST_INLINE_BORROW 458 -#define _LOAD_CONST_UNDER_INLINE 459 -#define _LOAD_CONST_UNDER_INLINE_BORROW 460 +#define _LOAD_CONST_INLINE 463 +#define _LOAD_CONST_INLINE_BORROW 464 +#define _LOAD_CONST_UNDER_INLINE 465 +#define _LOAD_CONST_UNDER_INLINE_BORROW 466 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 461 -#define _LOAD_FAST_0 462 -#define _LOAD_FAST_1 463 -#define _LOAD_FAST_2 464 -#define _LOAD_FAST_3 465 -#define _LOAD_FAST_4 466 -#define _LOAD_FAST_5 467 -#define _LOAD_FAST_6 468 -#define _LOAD_FAST_7 469 +#define _LOAD_FAST 467 +#define _LOAD_FAST_0 468 +#define _LOAD_FAST_1 469 +#define _LOAD_FAST_2 470 +#define _LOAD_FAST_3 471 +#define _LOAD_FAST_4 472 +#define _LOAD_FAST_5 473 +#define _LOAD_FAST_6 474 +#define _LOAD_FAST_7 475 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 470 -#define _LOAD_FAST_BORROW_0 471 -#define _LOAD_FAST_BORROW_1 472 -#define _LOAD_FAST_BORROW_2 473 -#define _LOAD_FAST_BORROW_3 474 -#define _LOAD_FAST_BORROW_4 475 -#define _LOAD_FAST_BORROW_5 476 -#define _LOAD_FAST_BORROW_6 477 -#define _LOAD_FAST_BORROW_7 478 +#define _LOAD_FAST_BORROW 476 +#define _LOAD_FAST_BORROW_0 477 +#define _LOAD_FAST_BORROW_1 478 +#define _LOAD_FAST_BORROW_2 479 +#define _LOAD_FAST_BORROW_3 480 +#define _LOAD_FAST_BORROW_4 481 +#define _LOAD_FAST_BORROW_5 482 +#define _LOAD_FAST_BORROW_6 483 +#define _LOAD_FAST_BORROW_7 484 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 479 -#define _LOAD_GLOBAL_BUILTINS 480 -#define _LOAD_GLOBAL_MODULE 481 +#define _LOAD_GLOBAL 485 +#define _LOAD_GLOBAL_BUILTINS 486 +#define _LOAD_GLOBAL_MODULE 487 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 482 -#define _LOAD_SMALL_INT_0 483 -#define _LOAD_SMALL_INT_1 484 -#define _LOAD_SMALL_INT_2 485 -#define _LOAD_SMALL_INT_3 486 -#define _LOAD_SPECIAL 487 +#define _LOAD_SMALL_INT 488 +#define _LOAD_SMALL_INT_0 489 +#define _LOAD_SMALL_INT_1 490 +#define _LOAD_SMALL_INT_2 491 +#define _LOAD_SMALL_INT_3 492 +#define _LOAD_SPECIAL 493 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 488 +#define _MAKE_CALLARGS_A_TUPLE 494 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 489 +#define _MAKE_WARM 495 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 490 -#define _MAYBE_EXPAND_METHOD_KW 491 -#define _MONITOR_CALL 492 -#define _MONITOR_CALL_KW 493 -#define _MONITOR_JUMP_BACKWARD 494 -#define _MONITOR_RESUME 495 +#define _MAYBE_EXPAND_METHOD 496 +#define _MAYBE_EXPAND_METHOD_KW 497 +#define _MONITOR_CALL 498 +#define _MONITOR_CALL_KW 499 +#define _MONITOR_JUMP_BACKWARD 500 +#define _MONITOR_RESUME 501 #define _NOP NOP -#define _POP_CALL 496 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 497 -#define _POP_CALL_ONE 498 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 499 -#define _POP_CALL_TWO 500 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 501 +#define _POP_CALL 502 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 503 +#define _POP_CALL_ONE 504 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 505 +#define _POP_CALL_TWO 506 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 507 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 502 -#define _POP_JUMP_IF_TRUE 503 +#define _POP_JUMP_IF_FALSE 508 +#define _POP_JUMP_IF_TRUE 509 #define _POP_TOP POP_TOP -#define _POP_TOP_FLOAT 504 -#define _POP_TOP_INT 505 -#define _POP_TOP_LOAD_CONST_INLINE 506 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 507 -#define _POP_TOP_NOP 508 -#define _POP_TOP_UNICODE 509 -#define _POP_TWO 510 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 511 +#define _POP_TOP_FLOAT 510 +#define _POP_TOP_INT 511 +#define _POP_TOP_LOAD_CONST_INLINE 512 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 513 +#define _POP_TOP_NOP 514 +#define _POP_TOP_UNICODE 515 +#define _POP_TWO 516 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 517 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 512 +#define _PUSH_FRAME 518 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 513 -#define _PY_FRAME_GENERAL 514 -#define _PY_FRAME_KW 515 -#define _QUICKEN_RESUME 516 -#define _REPLACE_WITH_TRUE 517 +#define _PUSH_NULL_CONDITIONAL 519 +#define _PY_FRAME_GENERAL 520 +#define _PY_FRAME_KW 521 +#define _QUICKEN_RESUME 522 +#define _REPLACE_WITH_TRUE 523 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 518 -#define _SEND 519 -#define _SEND_GEN_FRAME 520 +#define _SAVE_RETURN_OFFSET 524 +#define _SEND 525 +#define _SEND_GEN_FRAME 526 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 521 -#define _STORE_ATTR 522 -#define _STORE_ATTR_INSTANCE_VALUE 523 -#define _STORE_ATTR_SLOT 524 -#define _STORE_ATTR_WITH_HINT 525 +#define _START_EXECUTOR 527 +#define _STORE_ATTR 528 +#define _STORE_ATTR_INSTANCE_VALUE 529 +#define _STORE_ATTR_SLOT 530 +#define _STORE_ATTR_WITH_HINT 531 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 526 -#define _STORE_FAST_0 527 -#define _STORE_FAST_1 528 -#define _STORE_FAST_2 529 -#define _STORE_FAST_3 530 -#define _STORE_FAST_4 531 -#define _STORE_FAST_5 532 -#define _STORE_FAST_6 533 -#define _STORE_FAST_7 534 +#define _STORE_FAST 532 +#define _STORE_FAST_0 533 +#define _STORE_FAST_1 534 +#define _STORE_FAST_2 535 +#define _STORE_FAST_3 536 +#define _STORE_FAST_4 537 +#define _STORE_FAST_5 538 +#define _STORE_FAST_6 539 +#define _STORE_FAST_7 540 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 535 -#define _STORE_SUBSCR 536 -#define _STORE_SUBSCR_DICT 537 -#define _STORE_SUBSCR_LIST_INT 538 -#define _SWAP 539 -#define _SWAP_2 540 -#define _SWAP_3 541 -#define _TIER2_RESUME_CHECK 542 -#define _TO_BOOL 543 +#define _STORE_SLICE 541 +#define _STORE_SUBSCR 542 +#define _STORE_SUBSCR_DICT 543 +#define _STORE_SUBSCR_LIST_INT 544 +#define _SWAP 545 +#define _SWAP_2 546 +#define _SWAP_3 547 +#define _TIER2_RESUME_CHECK 548 +#define _TO_BOOL 549 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 544 +#define _TO_BOOL_LIST 550 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 545 +#define _TO_BOOL_STR 551 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 546 -#define _UNPACK_SEQUENCE_LIST 547 -#define _UNPACK_SEQUENCE_TUPLE 548 -#define _UNPACK_SEQUENCE_TWO_TUPLE 549 +#define _UNPACK_SEQUENCE 552 +#define _UNPACK_SEQUENCE_LIST 553 +#define _UNPACK_SEQUENCE_TUPLE 554 +#define _UNPACK_SEQUENCE_TWO_TUPLE 555 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 549 +#define MAX_UOP_ID 555 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 12487719969..d5a3c362d87 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -11,7 +11,7 @@ extern "C" { #include #include "pycore_uop_ids.h" -extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1]; +extern const uint32_t _PyUop_Flags[MAX_UOP_ID+1]; typedef struct _rep_range { uint8_t start; uint8_t stop; } ReplicationRange; extern const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1]; extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; @@ -19,7 +19,7 @@ extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; extern int _PyUop_num_popped(int opcode, int oparg); #ifdef NEED_OPCODE_METADATA -const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { +const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_NOP] = HAS_PURE_FLAG, [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -128,12 +128,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_RETURN_VALUE] = HAS_ESCAPES_FLAG, + [_RETURN_VALUE] = HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SEND_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_YIELD_VALUE] = HAS_ARG_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -256,7 +256,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_PURE_FLAG, - [_PUSH_FRAME] = 0, + [_PUSH_FRAME] = HAS_NEEDS_GUARD_IP_FLAG, [_GUARD_NOS_NULL] = HAS_DEOPT_FLAG, [_GUARD_NOS_NOT_NULL] = HAS_EXIT_FLAG, [_GUARD_THIRD_NULL] = HAS_DEOPT_FLAG, @@ -293,7 +293,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MAKE_CALLARGS_A_TUPLE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG, - [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -315,6 +315,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_STACK_SPACE_OPERAND] = HAS_DEOPT_FLAG, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, [_EXIT_TRACE] = HAS_ESCAPES_FLAG, + [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, [_POP_TOP_LOAD_CONST_INLINE] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, @@ -336,7 +337,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_HANDLE_PENDING_AND_DEOPT] = HAS_ESCAPES_FLAG, [_ERROR_POP_N] = HAS_ARG_FLAG, [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, - [_COLD_EXIT] = HAS_ESCAPES_FLAG, + [_COLD_EXIT] = 0, + [_COLD_DYNAMIC_EXIT] = 0, + [_GUARD_IP__PUSH_FRAME] = HAS_EXIT_FLAG, + [_GUARD_IP_YIELD_VALUE] = HAS_EXIT_FLAG, + [_GUARD_IP_RETURN_VALUE] = HAS_EXIT_FLAG, + [_GUARD_IP_RETURN_GENERATOR] = HAS_EXIT_FLAG, }; const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = { @@ -419,6 +425,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", [_CHECK_STACK_SPACE_OPERAND] = "_CHECK_STACK_SPACE_OPERAND", [_CHECK_VALIDITY] = "_CHECK_VALIDITY", + [_COLD_DYNAMIC_EXIT] = "_COLD_DYNAMIC_EXIT", [_COLD_EXIT] = "_COLD_EXIT", [_COMPARE_OP] = "_COMPARE_OP", [_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT", @@ -443,6 +450,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_DEOPT] = "_DEOPT", [_DICT_MERGE] = "_DICT_MERGE", [_DICT_UPDATE] = "_DICT_UPDATE", + [_DYNAMIC_EXIT] = "_DYNAMIC_EXIT", [_END_FOR] = "_END_FOR", [_END_SEND] = "_END_SEND", [_ERROR_POP_N] = "_ERROR_POP_N", @@ -471,6 +479,10 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT", [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", + [_GUARD_IP_RETURN_GENERATOR] = "_GUARD_IP_RETURN_GENERATOR", + [_GUARD_IP_RETURN_VALUE] = "_GUARD_IP_RETURN_VALUE", + [_GUARD_IP_YIELD_VALUE] = "_GUARD_IP_YIELD_VALUE", + [_GUARD_IP__PUSH_FRAME] = "_GUARD_IP__PUSH_FRAME", [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", @@ -1261,6 +1273,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _EXIT_TRACE: return 0; + case _DYNAMIC_EXIT: + return 0; case _CHECK_VALIDITY: return 0; case _LOAD_CONST_INLINE: @@ -1305,6 +1319,16 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _COLD_EXIT: return 0; + case _COLD_DYNAMIC_EXIT: + return 0; + case _GUARD_IP__PUSH_FRAME: + return 0; + case _GUARD_IP_YIELD_VALUE: + return 0; + case _GUARD_IP_RETURN_VALUE: + return 0; + case _GUARD_IP_RETURN_GENERATOR: + return 0; default: return -1; } diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index fb4a441ca64..608ffdfad12 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -3057,8 +3057,8 @@ def test_source_segment_missing_info(self): class NodeTransformerTests(ASTTestMixin, unittest.TestCase): def assertASTTransformation(self, transformer_class, - initial_code, expected_code): - initial_ast = ast.parse(dedent(initial_code)) + code, expected_code): + initial_ast = ast.parse(dedent(code)) expected_ast = ast.parse(dedent(expected_code)) transformer = transformer_class() diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index e65556fb28f..f06c6cbda29 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -422,32 +422,6 @@ def testfunc(n, m): uops = get_opnames(ex) self.assertIn("_FOR_ITER_TIER_TWO", uops) - def test_confidence_score(self): - def testfunc(n): - bits = 0 - for i in range(n): - if i & 0x01: - bits += 1 - if i & 0x02: - bits += 1 - if i&0x04: - bits += 1 - if i&0x08: - bits += 1 - if i&0x10: - bits += 1 - return bits - - x = testfunc(TIER2_THRESHOLD * 2) - - self.assertEqual(x, TIER2_THRESHOLD * 5) - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - ops = list(iter_opnames(ex)) - #Since branch is 50/50 the trace could go either way. - count = ops.count("_GUARD_IS_TRUE_POP") + ops.count("_GUARD_IS_FALSE_POP") - self.assertLessEqual(count, 2) - @requires_specialization @unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") @@ -847,38 +821,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_unicode_count), 1) self.assertIn("_COMPARE_OP_STR", uops) - def test_type_inconsistency(self): - ns = {} - src = textwrap.dedent(""" - def testfunc(n): - for i in range(n): - x = _test_global + _test_global - """) - exec(src, ns, ns) - testfunc = ns['testfunc'] - ns['_test_global'] = 0 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNone(ex) - ns['_test_global'] = 1 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertNotIn("_GUARD_TOS_INT", uops) - self.assertNotIn("_GUARD_NOS_INT", uops) - self.assertNotIn("_BINARY_OP_ADD_INT", uops) - self.assertNotIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops) - # Try again, but between the runs, set the global to a float. - # This should result in no executor the second time. - ns = {} - exec(src, ns, ns) - testfunc = ns['testfunc'] - ns['_test_global'] = 0 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNone(ex) - ns['_test_global'] = 3.14 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNone(ex) - + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_sequential(self): def dummy12(x): return x - 1 @@ -907,6 +850,7 @@ def testfunc(n): largest_stack = _testinternalcapi.get_co_framesize(dummy13.__code__) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_nested(self): def dummy12(x): return x + 3 @@ -937,6 +881,7 @@ def testfunc(n): ) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_several_calls(self): def dummy12(x): return x + 3 @@ -972,6 +917,7 @@ def testfunc(n): ) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_several_calls_different_order(self): # same as `several_calls` but with top-level calls reversed def dummy12(x): @@ -1008,6 +954,7 @@ def testfunc(n): ) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_complex(self): def dummy0(x): return x @@ -1057,6 +1004,7 @@ def testfunc(n): ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands ) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_large_framesize(self): # Create a function with a large framesize. This ensures _CHECK_STACK_SPACE is # actually doing its job. Note that the resulting trace hits @@ -1118,6 +1066,7 @@ def testfunc(n): ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands ) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_recursion(self): def dummy15(x): while x > 0: diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 9d3248d972e..798f58737b1 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -2253,9 +2253,10 @@ def frame_2_jit(expected: bool) -> None: def frame_3_jit() -> None: # JITs just before the last loop: - for i in range(_testinternalcapi.TIER2_THRESHOLD + 1): + # 1 extra iteration for tracing. + for i in range(_testinternalcapi.TIER2_THRESHOLD + 2): # Careful, doing this in the reverse order breaks tracing: - expected = {enabled} and i == _testinternalcapi.TIER2_THRESHOLD + expected = {enabled} and i >= _testinternalcapi.TIER2_THRESHOLD + 1 assert sys._jit.is_active() is expected frame_2_jit(expected) assert sys._jit.is_active() is expected diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst new file mode 100644 index 00000000000..40b9d19ee42 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst @@ -0,0 +1 @@ +A new tracing frontend for the JIT compiler has been implemented. Patch by Ken Jin. Design for CPython by Ken Jin, Mark Shannon and Brandt Bucher. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 6514ca7f3cd..89e558b0fe8 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2661,7 +2661,8 @@ module_exec(PyObject *module) } if (PyModule_Add(module, "TIER2_THRESHOLD", - PyLong_FromLong(JUMP_BACKWARD_INITIAL_VALUE + 1)) < 0) { + // + 1 more due to one loop spent on tracing. + PyLong_FromLong(JUMP_BACKWARD_INITIAL_VALUE + 2)) < 0) { return 1; } diff --git a/Objects/codeobject.c b/Objects/codeobject.c index fc3f5d9dde0..3aea2038fd1 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -2432,6 +2432,7 @@ code_dealloc(PyObject *self) PyMem_Free(co_extra); } #ifdef _Py_TIER2 + _PyJit_Tracer_InvalidateDependency(tstate, self); if (co->co_executors != NULL) { clear_executors(co); } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 0cae3703d1d..b652973600c 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -17,6 +17,7 @@ #include "frameobject.h" // PyFrameLocalsProxyObject #include "opcode.h" // EXTENDED_ARG +#include "pycore_optimizer.h" #include "clinic/frameobject.c.h" @@ -260,7 +261,10 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) return -1; } - _Py_Executors_InvalidateDependency(PyInterpreterState_Get(), co, 1); +#if _Py_TIER2 + _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), co, 1); + _PyJit_Tracer_InvalidateDependency(_PyThreadState_GET(), co); +#endif _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); _PyStackRef oldvalue = fast[i]; diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 43198aaf8a7..b659ac80233 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -11,7 +11,7 @@ #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_stats.h" #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() - +#include "pycore_optimizer.h" // _PyJit_Tracer_InvalidateDependency static const char * func_event_name(PyFunction_WatchEvent event) { @@ -1151,6 +1151,10 @@ func_dealloc(PyObject *self) if (_PyObject_ResurrectEnd(self)) { return; } +#if _Py_TIER2 + _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), self, 1); + _PyJit_Tracer_InvalidateDependency(_PyThreadState_GET(), self); +#endif _PyObject_GC_UNTRACK(op); FT_CLEAR_WEAKREFS(self, op->func_weakreflist); (void)func_clear((PyObject*)op); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6ebd9ebdfce..2c798855a71 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2938,8 +2938,8 @@ dummy_func( JUMP_BACKWARD_JIT, }; - tier1 op(_SPECIALIZE_JUMP_BACKWARD, (--)) { - #if ENABLE_SPECIALIZATION_FT + specializing tier1 op(_SPECIALIZE_JUMP_BACKWARD, (--)) { + #if ENABLE_SPECIALIZATION if (this_instr->op.code == JUMP_BACKWARD) { uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); @@ -2953,25 +2953,21 @@ dummy_func( tier1 op(_JIT, (--)) { #ifdef _Py_TIER2 _Py_BackoffCounter counter = this_instr[1].counter; - if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { - _Py_CODEUNIT *start = this_instr; - /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ + if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && + this_instr->op.code == JUMP_BACKWARD_JIT && + next_instr->op.code != ENTER_EXECUTOR) { + /* Back up over EXTENDED_ARGs so executor is inserted at the correct place */ + _Py_CODEUNIT *insert_exec_at = this_instr; while (oparg > 255) { oparg >>= 8; - start--; + insert_exec_at--; } - _PyExecutorObject *executor; - int optimized = _PyOptimizer_Optimize(frame, start, &executor, 0); - if (optimized <= 0) { - this_instr[1].counter = restart_backoff_counter(counter); - ERROR_IF(optimized < 0); + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg); + if (succ) { + ENTER_TRACING(); } else { - this_instr[1].counter = initial_jump_backoff_counter(); - assert(tstate->current_executor == NULL); - assert(executor != tstate->interp->cold_executor); - tstate->jit_exit = NULL; - TIER1_TO_TIER2(executor); + this_instr[1].counter = restart_backoff_counter(counter); } } else { @@ -3017,6 +3013,10 @@ dummy_func( tier1 inst(ENTER_EXECUTOR, (--)) { #ifdef _Py_TIER2 + if (IS_JIT_TRACING()) { + next_instr = this_instr; + goto stop_tracing; + } PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); @@ -3078,7 +3078,7 @@ dummy_func( macro(POP_JUMP_IF_NOT_NONE) = unused/1 + _IS_NONE + _POP_JUMP_IF_FALSE; - tier1 inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { + replaced inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. @@ -5245,19 +5245,40 @@ dummy_func( tier2 op(_EXIT_TRACE, (exit_p/4 --)) { _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) - _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (frame->lltrace >= 2) { + if (frame->lltrace >= 3) { printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); + printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code], exit->is_control_flow); + } + #endif + tstate->jit_exit = exit; + TIER2_TO_TIER2(exit->executor); + } + + tier2 op(_DYNAMIC_EXIT, (exit_p/4 --)) { + #if defined(Py_DEBUG) && !defined(_Py_JIT) + _PyExitData *exit = (_PyExitData *)exit_p; + _Py_CODEUNIT *target = frame->instr_ptr; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 3) { + printf("DYNAMIC EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); printf(", exit %tu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); } - #endif - tstate->jit_exit = exit; - TIER2_TO_TIER2(exit->executor); + #endif + // Disabled for now (gh-139109) as it slows down dynamic code tremendously. + // Compile and jump to the cold dynamic executors in the future. + GOTO_TIER_ONE(frame->instr_ptr); } tier2 op(_CHECK_VALIDITY, (--)) { @@ -5369,7 +5390,8 @@ dummy_func( } tier2 op(_DEOPT, (--)) { - GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } tier2 op(_HANDLE_PENDING_AND_DEOPT, (--)) { @@ -5399,32 +5421,76 @@ dummy_func( tier2 op(_COLD_EXIT, ( -- )) { _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); + assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; _Py_BackoffCounter temperature = exit->temperature; - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target); - } _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); + assert(tstate->jit_exit == exit); + exit->executor = executor; + TIER2_TO_TIER2(exit->executor); } else { + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(target); + } _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); - int chain_depth = previous_executor->vm_data.chain_depth + 1; - int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); - if (optimized <= 0) { - exit->temperature = restart_backoff_counter(temperature); - GOTO_TIER_ONE(optimized < 0 ? NULL : target); + // For control-flow guards, we don't want to increase the chain depth, as those don't actually + // represent deopts but rather just normal programs! + int chain_depth = previous_executor->vm_data.chain_depth + !exit->is_control_flow; + // Note: it's safe to use target->op.arg here instead of the oparg given by EXTENDED_ARG. + // The invariant in the optimizer is the deopt target always points back to the first EXTENDED_ARG. + // So setting it to anything else is wrong. + int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg); + exit->temperature = restart_backoff_counter(exit->temperature); + if (succ) { + GOTO_TIER_ONE_CONTINUE_TRACING(target); } - exit->temperature = initial_temperature_backoff_counter(); + GOTO_TIER_ONE(target); + } + } + + tier2 op(_COLD_DYNAMIC_EXIT, ( -- )) { + // TODO (gh-139109): This should be similar to _COLD_EXIT in the future. + _Py_CODEUNIT *target = frame->instr_ptr; + GOTO_TIER_ONE(target); + } + + tier2 op(_GUARD_IP__PUSH_FRAME, (ip/4 --)) { + _Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(_PUSH_FRAME); + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += IP_OFFSET_OF(_PUSH_FRAME); + EXIT_IF(true); + } + } + + tier2 op(_GUARD_IP_YIELD_VALUE, (ip/4 --)) { + _Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(YIELD_VALUE); + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += IP_OFFSET_OF(YIELD_VALUE); + EXIT_IF(true); + } + } + + tier2 op(_GUARD_IP_RETURN_VALUE, (ip/4 --)) { + _Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(RETURN_VALUE); + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += IP_OFFSET_OF(RETURN_VALUE); + EXIT_IF(true); + } + } + + tier2 op(_GUARD_IP_RETURN_GENERATOR, (ip/4 --)) { + _Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(RETURN_GENERATOR); + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += IP_OFFSET_OF(RETURN_GENERATOR); + EXIT_IF(true); } - assert(tstate->jit_exit == exit); - exit->executor = executor; - TIER2_TO_TIER2(exit->executor); } label(pop_2_error) { @@ -5571,6 +5637,62 @@ dummy_func( DISPATCH(); } + label(record_previous_inst) { +#if _Py_TIER2 + assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; + bool stop_tracing = (opcode == WITH_EXCEPT_START || + opcode == RERAISE || opcode == CLEANUP_THROW || + opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing); + if (full) { + LEAVE_TRACING(); + int err = stop_tracing_and_jit(tstate, frame); + ERROR_IF(err < 0); + DISPATCH_GOTO_NON_TRACING(); + } + // Super instructions. Instruction deopted. There's a mismatch in what the stack expects + // in the optimizer. So we have to reflect in the trace correctly. + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if ((_tstate->jit_tracer_state.prev_state.instr->op.code == CALL_LIST_APPEND && + opcode == POP_TOP) || + (_tstate->jit_tracer_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + opcode == STORE_FAST)) { + _tstate->jit_tracer_state.prev_state.instr_is_super = true; + } + else { + _tstate->jit_tracer_state.prev_state.instr = next_instr; + } + PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + if (_tstate->jit_tracer_state.prev_state.instr_code != (PyCodeObject *)prev_code) { + Py_SETREF(_tstate->jit_tracer_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); + } + + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; + _tstate->jit_tracer_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + (&next_instr[1])->counter = trigger_backoff_counter(); + } + DISPATCH_GOTO_NON_TRACING(); +#else + Py_FatalError("JIT label executed in non-jit build."); +#endif + } + + label(stop_tracing) { +#if _Py_TIER2 + assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; + _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, true); + LEAVE_TRACING(); + int err = stop_tracing_and_jit(tstate, frame); + ERROR_IF(err < 0); + DISPATCH_GOTO_NON_TRACING(); +#else + Py_FatalError("JIT label executed in non-jit build."); +#endif + } // END BYTECODES // diff --git a/Python/ceval.c b/Python/ceval.c index 07d21575e3a..b76c9ec2811 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1004,6 +1004,8 @@ static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { { .op.code = RESUME, .op.arg = RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START } }; +const _Py_CODEUNIT *_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR = (_Py_CODEUNIT*)&_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS; + #ifdef Py_DEBUG extern void _PyUOpPrint(const _PyUOpInstruction *uop); #endif @@ -1051,6 +1053,43 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) } } +#if _Py_TIER2 +// 0 for success, -1 for error. +static int +stop_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) +{ + int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); + int err = 0; + if (!_PyErr_Occurred(tstate) && !_is_sys_tracing) { + err = _PyOptimizer_Optimize(frame, tstate); + } + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + // Deal with backoffs + _PyExitData *exit = _tstate->jit_tracer_state.initial_state.exit; + if (exit == NULL) { + // We hold a strong reference to the code object, so the instruction won't be freed. + if (err <= 0) { + _Py_BackoffCounter counter = _tstate->jit_tracer_state.initial_state.jump_backward_instr[1].counter; + _tstate->jit_tracer_state.initial_state.jump_backward_instr[1].counter = restart_backoff_counter(counter); + } + else { + _tstate->jit_tracer_state.initial_state.jump_backward_instr[1].counter = initial_jump_backoff_counter(); + } + } + else { + // Likewise, we hold a strong reference to the executor containing this exit, so the exit is guaranteed + // to be valid to access. + if (err <= 0) { + exit->temperature = restart_backoff_counter(exit->temperature); + } + else { + exit->temperature = initial_temperature_backoff_counter(); + } + } + _PyJit_FinalizeTracing(tstate); + return err; +} +#endif /* _PyEval_EvalFrameDefault is too large to optimize for speed with PGO on MSVC. */ @@ -1180,9 +1219,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TAIL_CALL_INTERP # if Py_STATS - return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, 0, lastopcode); + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0, lastopcode); # else - return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, 0); + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0); # endif #else goto error; @@ -1191,9 +1230,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #if _Py_TAIL_CALL_INTERP # if Py_STATS - return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_table, 0, lastopcode); + return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_handler_table, 0, lastopcode); # else - return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_table, 0); + return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_handler_table, 0); # endif #else goto start_frame; @@ -1235,7 +1274,9 @@ _PyTier2Interpreter( tier2_start: next_uop = current_executor->trace; - assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); + assert(next_uop->opcode == _START_EXECUTOR || + next_uop->opcode == _COLD_EXIT || + next_uop->opcode == _COLD_DYNAMIC_EXIT); #undef LOAD_IP #define LOAD_IP(UNUSED) (void)0 @@ -1259,7 +1300,9 @@ _PyTier2Interpreter( uint64_t trace_uop_execution_counter = 0; #endif - assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); + assert(next_uop->opcode == _START_EXECUTOR || + next_uop->opcode == _COLD_EXIT || + next_uop->opcode == _COLD_DYNAMIC_EXIT); tier2_dispatch: for (;;) { uopcode = next_uop->opcode; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index afdcbc563b2..05a2760671e 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -93,11 +93,19 @@ # define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) Py_PRESERVE_NONE_CC typedef PyObject* (*py_tail_call_funcptr)(TAIL_CALL_PARAMS); +# define DISPATCH_TABLE_VAR instruction_funcptr_table +# define DISPATCH_TABLE instruction_funcptr_handler_table +# define TRACING_DISPATCH_TABLE instruction_funcptr_tracing_table # define TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_##op(TAIL_CALL_PARAMS) + # define DISPATCH_GOTO() \ do { \ Py_MUSTTAIL return (((py_tail_call_funcptr *)instruction_funcptr_table)[opcode])(TAIL_CALL_ARGS); \ } while (0) +# define DISPATCH_GOTO_NON_TRACING() \ + do { \ + Py_MUSTTAIL return (((py_tail_call_funcptr *)DISPATCH_TABLE)[opcode])(TAIL_CALL_ARGS); \ + } while (0) # define JUMP_TO_LABEL(name) \ do { \ Py_MUSTTAIL return (_TAIL_CALL_##name)(TAIL_CALL_ARGS); \ @@ -115,19 +123,36 @@ # endif # define LABEL(name) TARGET(name) #elif USE_COMPUTED_GOTOS +# define DISPATCH_TABLE_VAR opcode_targets +# define DISPATCH_TABLE opcode_targets_table +# define TRACING_DISPATCH_TABLE opcode_tracing_targets_table # define TARGET(op) TARGET_##op: # define DISPATCH_GOTO() goto *opcode_targets[opcode] +# define DISPATCH_GOTO_NON_TRACING() goto *DISPATCH_TABLE[opcode]; # define JUMP_TO_LABEL(name) goto name; # define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; # define LABEL(name) name: #else # define TARGET(op) case op: TARGET_##op: # define DISPATCH_GOTO() goto dispatch_opcode +# define DISPATCH_GOTO_NON_TRACING() goto dispatch_opcode # define JUMP_TO_LABEL(name) goto name; # define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; # define LABEL(name) name: #endif +#if (_Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS) && _Py_TIER2 +# define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) +# define ENTER_TRACING() \ + DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; +# define LEAVE_TRACING() \ + DISPATCH_TABLE_VAR = DISPATCH_TABLE; +#else +# define IS_JIT_TRACING() (0) +# define ENTER_TRACING() +# define LEAVE_TRACING() +#endif + /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ #ifdef Py_DEBUG #define PRE_DISPATCH_GOTO() if (frame->lltrace >= 5) { \ @@ -164,11 +189,19 @@ do { \ DISPATCH_GOTO(); \ } +#define DISPATCH_NON_TRACING() \ + { \ + assert(frame->stackpointer == NULL); \ + NEXTOPARG(); \ + PRE_DISPATCH_GOTO(); \ + DISPATCH_GOTO_NON_TRACING(); \ + } + #define DISPATCH_SAME_OPARG() \ { \ opcode = next_instr->op.code; \ PRE_DISPATCH_GOTO(); \ - DISPATCH_GOTO(); \ + DISPATCH_GOTO_NON_TRACING(); \ } #define DISPATCH_INLINED(NEW_FRAME) \ @@ -280,6 +313,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* This takes a uint16_t instead of a _Py_BackoffCounter, * because it is used directly on the cache entry in generated code, * which is always an integral type. */ +// Force re-specialization when tracing a side exit to get good side exits. #define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \ backoff_counter_triggers(forge_backoff_counter((COUNTER))) @@ -366,12 +400,19 @@ do { \ next_instr = _Py_jit_entry((EXECUTOR), frame, stack_pointer, tstate); \ frame = tstate->current_frame; \ stack_pointer = _PyFrame_GetStackPointer(frame); \ + int keep_tracing_bit = (uintptr_t)next_instr & 1; \ + next_instr = (_Py_CODEUNIT *)(((uintptr_t)next_instr) & (~1)); \ if (next_instr == NULL) { \ /* gh-140104: The exception handler expects frame->instr_ptr to after this_instr, not this_instr! */ \ next_instr = frame->instr_ptr + 1; \ JUMP_TO_LABEL(error); \ } \ + if (keep_tracing_bit) { \ + assert(((_PyThreadStateImpl *)tstate)->jit_tracer_state.prev_state.code_curr_size == 2); \ + ENTER_TRACING(); \ + DISPATCH_NON_TRACING(); \ + } \ DISPATCH(); \ } while (0) @@ -382,13 +423,23 @@ do { \ goto tier2_start; \ } while (0) -#define GOTO_TIER_ONE(TARGET) \ - do \ - { \ - tstate->current_executor = NULL; \ - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - return TARGET; \ +#define GOTO_TIER_ONE_SETUP \ + tstate->current_executor = NULL; \ + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \ + _PyFrame_SetStackPointer(frame, stack_pointer); + +#define GOTO_TIER_ONE(TARGET) \ + do \ + { \ + GOTO_TIER_ONE_SETUP \ + return (_Py_CODEUNIT *)(TARGET); \ + } while (0) + +#define GOTO_TIER_ONE_CONTINUE_TRACING(TARGET) \ + do \ + { \ + GOTO_TIER_ONE_SETUP \ + return (_Py_CODEUNIT *)(((uintptr_t)(TARGET))| 1); \ } while (0) #define CURRENT_OPARG() (next_uop[-1].oparg) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 9ce0a9f8a4d..7ba2e9d0d92 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4189,6 +4189,8 @@ break; } + /* _JUMP_BACKWARD_NO_INTERRUPT is not a viable micro-op for tier 2 because it is replaced */ + case _GET_LEN: { _PyStackRef obj; _PyStackRef len; @@ -7108,12 +7110,36 @@ PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) - _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (frame->lltrace >= 2) { + if (frame->lltrace >= 3) { _PyFrame_SetStackPointer(frame, stack_pointer); printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); + printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code], exit->is_control_flow); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + #endif + tstate->jit_exit = exit; + TIER2_TO_TIER2(exit->executor); + break; + } + + case _DYNAMIC_EXIT: { + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + #if defined(Py_DEBUG) && !defined(_Py_JIT) + _PyExitData *exit = (_PyExitData *)exit_p; + _Py_CODEUNIT *target = frame->instr_ptr; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 3) { + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("DYNAMIC EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); printf(", exit %tu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), @@ -7121,8 +7147,8 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } #endif - tstate->jit_exit = exit; - TIER2_TO_TIER2(exit->executor); + + GOTO_TIER_ONE(frame->instr_ptr); break; } @@ -7419,7 +7445,8 @@ } case _DEOPT: { - GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); break; } @@ -7460,37 +7487,101 @@ case _COLD_EXIT: { _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); + assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; _Py_BackoffCounter temperature = exit->temperature; - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target); - } _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); + assert(tstate->jit_exit == exit); + exit->executor = executor; + TIER2_TO_TIER2(exit->executor); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(tstate->current_executor == (PyObject *)previous_executor); - int chain_depth = previous_executor->vm_data.chain_depth + 1; - _PyFrame_SetStackPointer(frame, stack_pointer); - int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (optimized <= 0) { - exit->temperature = restart_backoff_counter(temperature); - GOTO_TIER_ONE(optimized < 0 ? NULL : target); + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(target); } - exit->temperature = initial_temperature_backoff_counter(); + _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); + assert(tstate->current_executor == (PyObject *)previous_executor); + int chain_depth = previous_executor->vm_data.chain_depth + !exit->is_control_flow; + int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg); + exit->temperature = restart_backoff_counter(exit->temperature); + if (succ) { + GOTO_TIER_ONE_CONTINUE_TRACING(target); + } + GOTO_TIER_ONE(target); } - assert(tstate->jit_exit == exit); - exit->executor = executor; - TIER2_TO_TIER2(exit->executor); break; } + case _COLD_DYNAMIC_EXIT: { + _Py_CODEUNIT *target = frame->instr_ptr; + GOTO_TIER_ONE(target); + break; + } + + case _GUARD_IP__PUSH_FRAME: { + #define OFFSET_OF__PUSH_FRAME ((0)) + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF__PUSH_FRAME; + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #undef OFFSET_OF__PUSH_FRAME + break; + } + + case _GUARD_IP_YIELD_VALUE: { + #define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND)) + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF_YIELD_VALUE; + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #undef OFFSET_OF_YIELD_VALUE + break; + } + + case _GUARD_IP_RETURN_VALUE: { + #define OFFSET_OF_RETURN_VALUE ((frame->return_offset)) + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF_RETURN_VALUE; + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #undef OFFSET_OF_RETURN_VALUE + break; + } + + case _GUARD_IP_RETURN_GENERATOR: { + #define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset)) + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR; + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #undef OFFSET_OF_RETURN_GENERATOR + break; + } + + #undef TIER_TWO diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 79328a7b725..a984da6dc91 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5476,6 +5476,10 @@ INSTRUCTION_STATS(ENTER_EXECUTOR); opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 + if (IS_JIT_TRACING()) { + next_instr = this_instr; + JUMP_TO_LABEL(stop_tracing); + } PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); @@ -7589,7 +7593,7 @@ /* Skip 1 cache entry */ // _SPECIALIZE_JUMP_BACKWARD { - #if ENABLE_SPECIALIZATION_FT + #if ENABLE_SPECIALIZATION if (this_instr->op.code == JUMP_BACKWARD) { uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); @@ -7645,30 +7649,20 @@ { #ifdef _Py_TIER2 _Py_BackoffCounter counter = this_instr[1].counter; - if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { - _Py_CODEUNIT *start = this_instr; + if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && + this_instr->op.code == JUMP_BACKWARD_JIT && + next_instr->op.code != ENTER_EXECUTOR) { + _Py_CODEUNIT *insert_exec_at = this_instr; while (oparg > 255) { oparg >>= 8; - start--; + insert_exec_at--; } - _PyExecutorObject *executor; - _PyFrame_SetStackPointer(frame, stack_pointer); - int optimized = _PyOptimizer_Optimize(frame, start, &executor, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (optimized <= 0) { - this_instr[1].counter = restart_backoff_counter(counter); - if (optimized < 0) { - JUMP_TO_LABEL(error); - } + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg); + if (succ) { + ENTER_TRACING(); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - this_instr[1].counter = initial_jump_backoff_counter(); - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(tstate->current_executor == NULL); - assert(executor != tstate->interp->cold_executor); - tstate->jit_exit = NULL; - TIER1_TO_TIER2(executor); + this_instr[1].counter = restart_backoff_counter(counter); } } else { @@ -12265,5 +12259,75 @@ JUMP_TO_LABEL(error); DISPATCH(); } + LABEL(record_previous_inst) + { + #if _Py_TIER2 + assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; + bool stop_tracing = (opcode == WITH_EXCEPT_START || + opcode == RERAISE || opcode == CLEANUP_THROW || + opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); + _PyFrame_SetStackPointer(frame, stack_pointer); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (full) { + LEAVE_TRACING(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = stop_tracing_and_jit(tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH_GOTO_NON_TRACING(); + } + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if ((_tstate->jit_tracer_state.prev_state.instr->op.code == CALL_LIST_APPEND && + opcode == POP_TOP) || + (_tstate->jit_tracer_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + opcode == STORE_FAST)) { + _tstate->jit_tracer_state.prev_state.instr_is_super = true; + } + else { + _tstate->jit_tracer_state.prev_state.instr = next_instr; + } + PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + if (_tstate->jit_tracer_state.prev_state.instr_code != (PyCodeObject *)prev_code) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_SETREF(_tstate->jit_tracer_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; + _tstate->jit_tracer_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + (&next_instr[1])->counter = trigger_backoff_counter(); + } + DISPATCH_GOTO_NON_TRACING(); + #else + Py_FatalError("JIT label executed in non-jit build."); + #endif + } + + LABEL(stop_tracing) + { + #if _Py_TIER2 + assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, true); + stack_pointer = _PyFrame_GetStackPointer(frame); + LEAVE_TRACING(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = stop_tracing_and_jit(tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH_GOTO_NON_TRACING(); + #else + Py_FatalError("JIT label executed in non-jit build."); + #endif + } + /* END LABELS */ #undef TIER_ONE diff --git a/Python/instrumentation.c b/Python/instrumentation.c index b4b2bc5dc69..81e46a331e0 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -18,6 +18,7 @@ #include "pycore_tuple.h" // _PyTuple_FromArraySteal() #include "opcode_ids.h" +#include "pycore_optimizer.h" /* Uncomment this to dump debugging output when assertions fail */ @@ -1785,6 +1786,7 @@ force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) _PyCode_Clear_Executors(code); } _Py_Executors_InvalidateDependency(interp, code, 1); + _PyJit_Tracer_InvalidateDependency(PyThreadState_GET(), code); #endif int code_len = (int)Py_SIZE(code); /* Exit early to avoid creating instrumentation diff --git a/Python/jit.c b/Python/jit.c index 279e1ce6a0d..7ab0f8ddd43 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -604,7 +604,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz unsigned char *code = memory; state.trampolines.mem = memory + code_size; unsigned char *data = memory + code_size + state.trampolines.size + code_padding; - assert(trace[0].opcode == _START_EXECUTOR || trace[0].opcode == _COLD_EXIT); + assert(trace[0].opcode == _START_EXECUTOR || trace[0].opcode == _COLD_EXIT || trace[0].opcode == _COLD_DYNAMIC_EXIT); for (size_t i = 0; i < length; i++) { const _PyUOpInstruction *instruction = &trace[i]; group = &stencil_groups[instruction->opcode]; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 6dd443e1655..1b9196503b5 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -257,8 +257,270 @@ static void *opcode_targets_table[256] = { &&TARGET_INSTRUMENTED_LINE, &&TARGET_ENTER_EXECUTOR, }; +#if _Py_TIER2 +static void *opcode_tracing_targets_table[256] = { + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, +}; +#endif #else /* _Py_TAIL_CALL_INTERP */ -static py_tail_call_funcptr instruction_funcptr_table[256]; +static py_tail_call_funcptr instruction_funcptr_handler_table[256]; + +static py_tail_call_funcptr instruction_funcptr_tracing_table[256]; Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); @@ -266,6 +528,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_record_previous_inst(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); @@ -503,7 +767,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) JUMP_TO_LABEL(error); } -static py_tail_call_funcptr instruction_funcptr_table[256] = { +static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [BINARY_OP] = _TAIL_CALL_BINARY_OP, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, @@ -761,4 +1025,262 @@ static py_tail_call_funcptr instruction_funcptr_table[256] = { [232] = _TAIL_CALL_UNKNOWN_OPCODE, [233] = _TAIL_CALL_UNKNOWN_OPCODE, }; +static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { + [BINARY_OP] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_ADD_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_EXTEND] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_SLICE] = _TAIL_CALL_record_previous_inst, + [BUILD_INTERPOLATION] = _TAIL_CALL_record_previous_inst, + [BUILD_LIST] = _TAIL_CALL_record_previous_inst, + [BUILD_MAP] = _TAIL_CALL_record_previous_inst, + [BUILD_SET] = _TAIL_CALL_record_previous_inst, + [BUILD_SLICE] = _TAIL_CALL_record_previous_inst, + [BUILD_STRING] = _TAIL_CALL_record_previous_inst, + [BUILD_TEMPLATE] = _TAIL_CALL_record_previous_inst, + [BUILD_TUPLE] = _TAIL_CALL_record_previous_inst, + [CACHE] = _TAIL_CALL_record_previous_inst, + [CALL] = _TAIL_CALL_record_previous_inst, + [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_record_previous_inst, + [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_record_previous_inst, + [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_record_previous_inst, + [CALL_BUILTIN_CLASS] = _TAIL_CALL_record_previous_inst, + [CALL_BUILTIN_FAST] = _TAIL_CALL_record_previous_inst, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_record_previous_inst, + [CALL_BUILTIN_O] = _TAIL_CALL_record_previous_inst, + [CALL_FUNCTION_EX] = _TAIL_CALL_record_previous_inst, + [CALL_INTRINSIC_1] = _TAIL_CALL_record_previous_inst, + [CALL_INTRINSIC_2] = _TAIL_CALL_record_previous_inst, + [CALL_ISINSTANCE] = _TAIL_CALL_record_previous_inst, + [CALL_KW] = _TAIL_CALL_record_previous_inst, + [CALL_KW_BOUND_METHOD] = _TAIL_CALL_record_previous_inst, + [CALL_KW_NON_PY] = _TAIL_CALL_record_previous_inst, + [CALL_KW_PY] = _TAIL_CALL_record_previous_inst, + [CALL_LEN] = _TAIL_CALL_record_previous_inst, + [CALL_LIST_APPEND] = _TAIL_CALL_record_previous_inst, + [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_record_previous_inst, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_record_previous_inst, + [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_record_previous_inst, + [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_record_previous_inst, + [CALL_NON_PY_GENERAL] = _TAIL_CALL_record_previous_inst, + [CALL_PY_EXACT_ARGS] = _TAIL_CALL_record_previous_inst, + [CALL_PY_GENERAL] = _TAIL_CALL_record_previous_inst, + [CALL_STR_1] = _TAIL_CALL_record_previous_inst, + [CALL_TUPLE_1] = _TAIL_CALL_record_previous_inst, + [CALL_TYPE_1] = _TAIL_CALL_record_previous_inst, + [CHECK_EG_MATCH] = _TAIL_CALL_record_previous_inst, + [CHECK_EXC_MATCH] = _TAIL_CALL_record_previous_inst, + [CLEANUP_THROW] = _TAIL_CALL_record_previous_inst, + [COMPARE_OP] = _TAIL_CALL_record_previous_inst, + [COMPARE_OP_FLOAT] = _TAIL_CALL_record_previous_inst, + [COMPARE_OP_INT] = _TAIL_CALL_record_previous_inst, + [COMPARE_OP_STR] = _TAIL_CALL_record_previous_inst, + [CONTAINS_OP] = _TAIL_CALL_record_previous_inst, + [CONTAINS_OP_DICT] = _TAIL_CALL_record_previous_inst, + [CONTAINS_OP_SET] = _TAIL_CALL_record_previous_inst, + [CONVERT_VALUE] = _TAIL_CALL_record_previous_inst, + [COPY] = _TAIL_CALL_record_previous_inst, + [COPY_FREE_VARS] = _TAIL_CALL_record_previous_inst, + [DELETE_ATTR] = _TAIL_CALL_record_previous_inst, + [DELETE_DEREF] = _TAIL_CALL_record_previous_inst, + [DELETE_FAST] = _TAIL_CALL_record_previous_inst, + [DELETE_GLOBAL] = _TAIL_CALL_record_previous_inst, + [DELETE_NAME] = _TAIL_CALL_record_previous_inst, + [DELETE_SUBSCR] = _TAIL_CALL_record_previous_inst, + [DICT_MERGE] = _TAIL_CALL_record_previous_inst, + [DICT_UPDATE] = _TAIL_CALL_record_previous_inst, + [END_ASYNC_FOR] = _TAIL_CALL_record_previous_inst, + [END_FOR] = _TAIL_CALL_record_previous_inst, + [END_SEND] = _TAIL_CALL_record_previous_inst, + [ENTER_EXECUTOR] = _TAIL_CALL_record_previous_inst, + [EXIT_INIT_CHECK] = _TAIL_CALL_record_previous_inst, + [EXTENDED_ARG] = _TAIL_CALL_record_previous_inst, + [FORMAT_SIMPLE] = _TAIL_CALL_record_previous_inst, + [FORMAT_WITH_SPEC] = _TAIL_CALL_record_previous_inst, + [FOR_ITER] = _TAIL_CALL_record_previous_inst, + [FOR_ITER_GEN] = _TAIL_CALL_record_previous_inst, + [FOR_ITER_LIST] = _TAIL_CALL_record_previous_inst, + [FOR_ITER_RANGE] = _TAIL_CALL_record_previous_inst, + [FOR_ITER_TUPLE] = _TAIL_CALL_record_previous_inst, + [GET_AITER] = _TAIL_CALL_record_previous_inst, + [GET_ANEXT] = _TAIL_CALL_record_previous_inst, + [GET_AWAITABLE] = _TAIL_CALL_record_previous_inst, + [GET_ITER] = _TAIL_CALL_record_previous_inst, + [GET_LEN] = _TAIL_CALL_record_previous_inst, + [GET_YIELD_FROM_ITER] = _TAIL_CALL_record_previous_inst, + [IMPORT_FROM] = _TAIL_CALL_record_previous_inst, + [IMPORT_NAME] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_CALL] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_CALL_KW] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_END_FOR] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_END_SEND] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_LINE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_POP_ITER] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_RESUME] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_record_previous_inst, + [INTERPRETER_EXIT] = _TAIL_CALL_record_previous_inst, + [IS_OP] = _TAIL_CALL_record_previous_inst, + [JUMP_BACKWARD] = _TAIL_CALL_record_previous_inst, + [JUMP_BACKWARD_JIT] = _TAIL_CALL_record_previous_inst, + [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_record_previous_inst, + [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_record_previous_inst, + [JUMP_FORWARD] = _TAIL_CALL_record_previous_inst, + [LIST_APPEND] = _TAIL_CALL_record_previous_inst, + [LIST_EXTEND] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_CLASS] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_MODULE] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_PROPERTY] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_SLOT] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_record_previous_inst, + [LOAD_BUILD_CLASS] = _TAIL_CALL_record_previous_inst, + [LOAD_COMMON_CONSTANT] = _TAIL_CALL_record_previous_inst, + [LOAD_CONST] = _TAIL_CALL_record_previous_inst, + [LOAD_DEREF] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST_BORROW] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST_CHECK] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_record_previous_inst, + [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_record_previous_inst, + [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_record_previous_inst, + [LOAD_GLOBAL] = _TAIL_CALL_record_previous_inst, + [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_record_previous_inst, + [LOAD_GLOBAL_MODULE] = _TAIL_CALL_record_previous_inst, + [LOAD_LOCALS] = _TAIL_CALL_record_previous_inst, + [LOAD_NAME] = _TAIL_CALL_record_previous_inst, + [LOAD_SMALL_INT] = _TAIL_CALL_record_previous_inst, + [LOAD_SPECIAL] = _TAIL_CALL_record_previous_inst, + [LOAD_SUPER_ATTR] = _TAIL_CALL_record_previous_inst, + [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_record_previous_inst, + [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_record_previous_inst, + [MAKE_CELL] = _TAIL_CALL_record_previous_inst, + [MAKE_FUNCTION] = _TAIL_CALL_record_previous_inst, + [MAP_ADD] = _TAIL_CALL_record_previous_inst, + [MATCH_CLASS] = _TAIL_CALL_record_previous_inst, + [MATCH_KEYS] = _TAIL_CALL_record_previous_inst, + [MATCH_MAPPING] = _TAIL_CALL_record_previous_inst, + [MATCH_SEQUENCE] = _TAIL_CALL_record_previous_inst, + [NOP] = _TAIL_CALL_record_previous_inst, + [NOT_TAKEN] = _TAIL_CALL_record_previous_inst, + [POP_EXCEPT] = _TAIL_CALL_record_previous_inst, + [POP_ITER] = _TAIL_CALL_record_previous_inst, + [POP_JUMP_IF_FALSE] = _TAIL_CALL_record_previous_inst, + [POP_JUMP_IF_NONE] = _TAIL_CALL_record_previous_inst, + [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_record_previous_inst, + [POP_JUMP_IF_TRUE] = _TAIL_CALL_record_previous_inst, + [POP_TOP] = _TAIL_CALL_record_previous_inst, + [PUSH_EXC_INFO] = _TAIL_CALL_record_previous_inst, + [PUSH_NULL] = _TAIL_CALL_record_previous_inst, + [RAISE_VARARGS] = _TAIL_CALL_record_previous_inst, + [RERAISE] = _TAIL_CALL_record_previous_inst, + [RESERVED] = _TAIL_CALL_record_previous_inst, + [RESUME] = _TAIL_CALL_record_previous_inst, + [RESUME_CHECK] = _TAIL_CALL_record_previous_inst, + [RETURN_GENERATOR] = _TAIL_CALL_record_previous_inst, + [RETURN_VALUE] = _TAIL_CALL_record_previous_inst, + [SEND] = _TAIL_CALL_record_previous_inst, + [SEND_GEN] = _TAIL_CALL_record_previous_inst, + [SETUP_ANNOTATIONS] = _TAIL_CALL_record_previous_inst, + [SET_ADD] = _TAIL_CALL_record_previous_inst, + [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_record_previous_inst, + [SET_UPDATE] = _TAIL_CALL_record_previous_inst, + [STORE_ATTR] = _TAIL_CALL_record_previous_inst, + [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_record_previous_inst, + [STORE_ATTR_SLOT] = _TAIL_CALL_record_previous_inst, + [STORE_ATTR_WITH_HINT] = _TAIL_CALL_record_previous_inst, + [STORE_DEREF] = _TAIL_CALL_record_previous_inst, + [STORE_FAST] = _TAIL_CALL_record_previous_inst, + [STORE_FAST_LOAD_FAST] = _TAIL_CALL_record_previous_inst, + [STORE_FAST_STORE_FAST] = _TAIL_CALL_record_previous_inst, + [STORE_GLOBAL] = _TAIL_CALL_record_previous_inst, + [STORE_NAME] = _TAIL_CALL_record_previous_inst, + [STORE_SLICE] = _TAIL_CALL_record_previous_inst, + [STORE_SUBSCR] = _TAIL_CALL_record_previous_inst, + [STORE_SUBSCR_DICT] = _TAIL_CALL_record_previous_inst, + [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_record_previous_inst, + [SWAP] = _TAIL_CALL_record_previous_inst, + [TO_BOOL] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_BOOL] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_INT] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_LIST] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_NONE] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_STR] = _TAIL_CALL_record_previous_inst, + [UNARY_INVERT] = _TAIL_CALL_record_previous_inst, + [UNARY_NEGATIVE] = _TAIL_CALL_record_previous_inst, + [UNARY_NOT] = _TAIL_CALL_record_previous_inst, + [UNPACK_EX] = _TAIL_CALL_record_previous_inst, + [UNPACK_SEQUENCE] = _TAIL_CALL_record_previous_inst, + [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_record_previous_inst, + [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_record_previous_inst, + [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_record_previous_inst, + [WITH_EXCEPT_START] = _TAIL_CALL_record_previous_inst, + [YIELD_VALUE] = _TAIL_CALL_record_previous_inst, + [121] = _TAIL_CALL_UNKNOWN_OPCODE, + [122] = _TAIL_CALL_UNKNOWN_OPCODE, + [123] = _TAIL_CALL_UNKNOWN_OPCODE, + [124] = _TAIL_CALL_UNKNOWN_OPCODE, + [125] = _TAIL_CALL_UNKNOWN_OPCODE, + [126] = _TAIL_CALL_UNKNOWN_OPCODE, + [127] = _TAIL_CALL_UNKNOWN_OPCODE, + [210] = _TAIL_CALL_UNKNOWN_OPCODE, + [211] = _TAIL_CALL_UNKNOWN_OPCODE, + [212] = _TAIL_CALL_UNKNOWN_OPCODE, + [213] = _TAIL_CALL_UNKNOWN_OPCODE, + [214] = _TAIL_CALL_UNKNOWN_OPCODE, + [215] = _TAIL_CALL_UNKNOWN_OPCODE, + [216] = _TAIL_CALL_UNKNOWN_OPCODE, + [217] = _TAIL_CALL_UNKNOWN_OPCODE, + [218] = _TAIL_CALL_UNKNOWN_OPCODE, + [219] = _TAIL_CALL_UNKNOWN_OPCODE, + [220] = _TAIL_CALL_UNKNOWN_OPCODE, + [221] = _TAIL_CALL_UNKNOWN_OPCODE, + [222] = _TAIL_CALL_UNKNOWN_OPCODE, + [223] = _TAIL_CALL_UNKNOWN_OPCODE, + [224] = _TAIL_CALL_UNKNOWN_OPCODE, + [225] = _TAIL_CALL_UNKNOWN_OPCODE, + [226] = _TAIL_CALL_UNKNOWN_OPCODE, + [227] = _TAIL_CALL_UNKNOWN_OPCODE, + [228] = _TAIL_CALL_UNKNOWN_OPCODE, + [229] = _TAIL_CALL_UNKNOWN_OPCODE, + [230] = _TAIL_CALL_UNKNOWN_OPCODE, + [231] = _TAIL_CALL_UNKNOWN_OPCODE, + [232] = _TAIL_CALL_UNKNOWN_OPCODE, + [233] = _TAIL_CALL_UNKNOWN_OPCODE, +}; #endif /* _Py_TAIL_CALL_INTERP */ diff --git a/Python/optimizer.c b/Python/optimizer.c index 3b7e2dafab8..65007a256d0 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -29,11 +29,24 @@ #define MAX_EXECUTORS_SIZE 256 +// Trace too short, no progress: +// _START_EXECUTOR +// _MAKE_WARM +// _CHECK_VALIDITY +// _SET_IP +// is 4-5 instructions. +#define CODE_SIZE_NO_PROGRESS 5 +// We start with _START_EXECUTOR, _MAKE_WARM +#define CODE_SIZE_EMPTY 2 + #define _PyExecutorObject_CAST(op) ((_PyExecutorObject *)(op)) static bool has_space_for_executor(PyCodeObject *code, _Py_CODEUNIT *instr) { + if (code == (PyCodeObject *)&_Py_InitCleanup) { + return false; + } if (instr->op.code == ENTER_EXECUTOR) { return true; } @@ -100,11 +113,11 @@ insert_executor(PyCodeObject *code, _Py_CODEUNIT *instr, int index, _PyExecutorO } static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies); +make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies, int chain_depth); static int -uop_optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, - _PyExecutorObject **exec_ptr, int curr_stackentries, +uop_optimize(_PyInterpreterFrame *frame, PyThreadState *tstate, + _PyExecutorObject **exec_ptr, bool progress_needed); /* Returns 1 if optimized, 0 if not optimized, and -1 for an error. @@ -113,10 +126,10 @@ uop_optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, // gh-137573: inlining this function causes stack overflows Py_NO_INLINE int _PyOptimizer_Optimize( - _PyInterpreterFrame *frame, _Py_CODEUNIT *start, - _PyExecutorObject **executor_ptr, int chain_depth) + _PyInterpreterFrame *frame, PyThreadState *tstate) { - _PyStackRef *stack_pointer = frame->stackpointer; + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + int chain_depth = _tstate->jit_tracer_state.initial_state.chain_depth; PyInterpreterState *interp = _PyInterpreterState_GET(); if (!interp->jit) { // gh-140936: It is possible that interp->jit will become false during @@ -126,7 +139,9 @@ _PyOptimizer_Optimize( return 0; } assert(!interp->compiling); + assert(_tstate->jit_tracer_state.initial_state.stack_depth >= 0); #ifndef Py_GIL_DISABLED + assert(_tstate->jit_tracer_state.initial_state.func != NULL); interp->compiling = true; // The first executor in a chain and the MAX_CHAIN_DEPTH'th executor *must* // make progress in order to avoid infinite loops or excessively-long @@ -134,18 +149,24 @@ _PyOptimizer_Optimize( // this is true, since a deopt won't infinitely re-enter the executor: chain_depth %= MAX_CHAIN_DEPTH; bool progress_needed = chain_depth == 0; - PyCodeObject *code = _PyFrame_GetCode(frame); - assert(PyCode_Check(code)); + PyCodeObject *code = (PyCodeObject *)_tstate->jit_tracer_state.initial_state.code; + _Py_CODEUNIT *start = _tstate->jit_tracer_state.initial_state.start_instr; if (progress_needed && !has_space_for_executor(code, start)) { interp->compiling = false; return 0; } - int err = uop_optimize(frame, start, executor_ptr, (int)(stack_pointer - _PyFrame_Stackbase(frame)), progress_needed); + // One of our dependencies while tracing was invalidated. Not worth compiling. + if (!_tstate->jit_tracer_state.prev_state.dependencies_still_valid) { + interp->compiling = false; + return 0; + } + _PyExecutorObject *executor; + int err = uop_optimize(frame, tstate, &executor, progress_needed); if (err <= 0) { interp->compiling = false; return err; } - assert(*executor_ptr != NULL); + assert(executor != NULL); if (progress_needed) { int index = get_index_for_executor(code, start); if (index < 0) { @@ -155,17 +176,21 @@ _PyOptimizer_Optimize( * If an optimizer has already produced an executor, * it might get confused by the executor disappearing, * but there is not much we can do about that here. */ - Py_DECREF(*executor_ptr); + Py_DECREF(executor); interp->compiling = false; return 0; } - insert_executor(code, start, index, *executor_ptr); + insert_executor(code, start, index, executor); } else { - (*executor_ptr)->vm_data.code = NULL; + executor->vm_data.code = NULL; } - (*executor_ptr)->vm_data.chain_depth = chain_depth; - assert((*executor_ptr)->vm_data.valid); + _PyExitData *exit = _tstate->jit_tracer_state.initial_state.exit; + if (exit != NULL) { + exit->executor = executor; + } + executor->vm_data.chain_depth = chain_depth; + assert(executor->vm_data.valid); interp->compiling = false; return 1; #else @@ -474,6 +499,14 @@ BRANCH_TO_GUARD[4][2] = { [POP_JUMP_IF_NOT_NONE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_NOT_NONE_POP, }; +static const uint16_t +guard_ip_uop[MAX_UOP_ID + 1] = { + [_PUSH_FRAME] = _GUARD_IP__PUSH_FRAME, + [_RETURN_GENERATOR] = _GUARD_IP_RETURN_GENERATOR, + [_RETURN_VALUE] = _GUARD_IP_RETURN_VALUE, + [_YIELD_VALUE] = _GUARD_IP_YIELD_VALUE, +}; + #define CONFIDENCE_RANGE 1000 #define CONFIDENCE_CUTOFF 333 @@ -530,64 +563,19 @@ add_to_trace( DPRINTF(2, "No room for %s (need %d, got %d)\n", \ (opname), (n), max_length - trace_length); \ OPT_STAT_INC(trace_too_long); \ - goto done; \ + goto full; \ } -// Reserve space for N uops, plus 3 for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE -#define RESERVE(needed) RESERVE_RAW((needed) + 3, _PyUOpName(opcode)) -// Trace stack operations (used by _PUSH_FRAME, _RETURN_VALUE) -#define TRACE_STACK_PUSH() \ - if (trace_stack_depth >= TRACE_STACK_SIZE) { \ - DPRINTF(2, "Trace stack overflow\n"); \ - OPT_STAT_INC(trace_stack_overflow); \ - return 0; \ - } \ - assert(func == NULL || func->func_code == (PyObject *)code); \ - trace_stack[trace_stack_depth].func = func; \ - trace_stack[trace_stack_depth].code = code; \ - trace_stack[trace_stack_depth].instr = instr; \ - trace_stack_depth++; -#define TRACE_STACK_POP() \ - if (trace_stack_depth <= 0) { \ - Py_FatalError("Trace stack underflow\n"); \ - } \ - trace_stack_depth--; \ - func = trace_stack[trace_stack_depth].func; \ - code = trace_stack[trace_stack_depth].code; \ - assert(func == NULL || func->func_code == (PyObject *)code); \ - instr = trace_stack[trace_stack_depth].instr; - -/* Returns the length of the trace on success, - * 0 if it failed to produce a worthwhile trace, - * and -1 on an error. +/* Returns 1 on success (added to trace), 0 on trace end. */ -static int -translate_bytecode_to_trace( +int +_PyJit_translate_single_bytecode_to_trace( + PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, - _PyUOpInstruction *trace, - int buffer_size, - _PyBloomFilter *dependencies, bool progress_needed) + _Py_CODEUNIT *next_instr, + bool stop_tracing) { - bool first = true; - PyCodeObject *code = _PyFrame_GetCode(frame); - PyFunctionObject *func = _PyFrame_GetFunction(frame); - assert(PyFunction_Check(func)); - PyCodeObject *initial_code = code; - _Py_BloomFilter_Add(dependencies, initial_code); - _Py_CODEUNIT *initial_instr = instr; - int trace_length = 0; - // Leave space for possible trailing _EXIT_TRACE - int max_length = buffer_size-2; - struct { - PyFunctionObject *func; - PyCodeObject *code; - _Py_CODEUNIT *instr; - } trace_stack[TRACE_STACK_SIZE]; - int trace_stack_depth = 0; - int confidence = CONFIDENCE_RANGE; // Adjusted by branch instructions - bool jump_seen = false; #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -596,410 +584,468 @@ translate_bytecode_to_trace( lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } #endif + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + PyCodeObject *old_code = _tstate->jit_tracer_state.prev_state.instr_code; + bool progress_needed = (_tstate->jit_tracer_state.initial_state.chain_depth % MAX_CHAIN_DEPTH) == 0; + _PyBloomFilter *dependencies = &_tstate->jit_tracer_state.prev_state.dependencies; + int trace_length = _tstate->jit_tracer_state.prev_state.code_curr_size; + _PyUOpInstruction *trace = _tstate->jit_tracer_state.code_buffer; + int max_length = _tstate->jit_tracer_state.prev_state.code_max_size; - DPRINTF(2, - "Optimizing %s (%s:%d) at byte offset %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code)); - ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)instr, INSTR_IP(instr, code)); - ADD_TO_TRACE(_MAKE_WARM, 0, 0, 0); + _Py_CODEUNIT *this_instr = _tstate->jit_tracer_state.prev_state.instr; + _Py_CODEUNIT *target_instr = this_instr; uint32_t target = 0; - for (;;) { - target = INSTR_IP(instr, code); - // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT - max_length-=2; - uint32_t opcode = instr->op.code; - uint32_t oparg = instr->op.arg; + target = Py_IsNone((PyObject *)old_code) + ? (int)(target_instr - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR) + : INSTR_IP(target_instr, old_code); - if (!first && instr == initial_instr) { - // We have looped around to the start: - RESERVE(1); - ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); + // Rewind EXTENDED_ARG so that we see the whole thing. + // We must point to the first EXTENDED_ARG when deopting. + int oparg = _tstate->jit_tracer_state.prev_state.instr_oparg; + int opcode = this_instr->op.code; + int rewind_oparg = oparg; + while (rewind_oparg > 255) { + rewind_oparg >>= 8; + target--; + } + + int old_stack_level = _tstate->jit_tracer_state.prev_state.instr_stacklevel; + + // Strange control-flow + bool has_dynamic_jump_taken = OPCODE_HAS_UNPREDICTABLE_JUMP(opcode) && + (next_instr != this_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]); + + /* Special case the first instruction, + * so that we can guarantee forward progress */ + if (progress_needed && _tstate->jit_tracer_state.prev_state.code_curr_size < CODE_SIZE_NO_PROGRESS) { + if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { + opcode = _PyOpcode_Deopt[opcode]; + } + assert(!OPCODE_HAS_EXIT(opcode)); + assert(!OPCODE_HAS_DEOPT(opcode)); + } + + bool needs_guard_ip = OPCODE_HAS_NEEDS_GUARD_IP(opcode); + if (has_dynamic_jump_taken && !needs_guard_ip) { + DPRINTF(2, "Unsupported: dynamic jump taken %s\n", _PyOpcode_OpName[opcode]); + goto unsupported; + } + + int is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); + if (is_sys_tracing) { + goto full; + } + + if (stop_tracing) { + ADD_TO_TRACE(_DEOPT, 0, 0, target); + goto done; + } + + DPRINTF(2, "%p %d: %s(%d) %d %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, needs_guard_ip, old_stack_level); + +#ifdef Py_DEBUG + if (oparg > 255) { + assert(_Py_GetBaseCodeUnit(old_code, target).op.code == EXTENDED_ARG); + } +#endif + + // Skip over super instructions. + if (_tstate->jit_tracer_state.prev_state.instr_is_super) { + _tstate->jit_tracer_state.prev_state.instr_is_super = false; + return 1; + } + + if (opcode == ENTER_EXECUTOR) { + goto full; + } + + if (!_tstate->jit_tracer_state.prev_state.dependencies_still_valid) { + goto done; + } + + // This happens when a recursive call happens that we can't trace. Such as Python -> C -> Python calls + // If we haven't guarded the IP, then it's untraceable. + if (frame != _tstate->jit_tracer_state.prev_state.instr_frame && !needs_guard_ip) { + DPRINTF(2, "Unsupported: unguardable jump taken\n"); + goto unsupported; + } + + if (oparg > 0xFFFF) { + DPRINTF(2, "Unsupported: oparg too large\n"); + goto unsupported; + } + + // TODO (gh-140277): The constituent use one extra stack slot. So we need to check for headroom. + if (opcode == BINARY_OP_SUBSCR_GETITEM && old_stack_level + 1 > old_code->co_stacksize) { + unsupported: + { + // Rewind to previous instruction and replace with _EXIT_TRACE. + _PyUOpInstruction *curr = &trace[trace_length-1]; + while (curr->opcode != _SET_IP && trace_length > 2) { + trace_length--; + curr = &trace[trace_length-1]; + } + assert(curr->opcode == _SET_IP || trace_length == 2); + if (curr->opcode == _SET_IP) { + int32_t old_target = (int32_t)uop_get_target(curr); + curr++; + trace_length++; + curr->opcode = _EXIT_TRACE; + curr->format = UOP_FORMAT_TARGET; + curr->target = old_target; + } goto done; } + } - DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); + if (opcode == NOP) { + return 1; + } - if (opcode == EXTENDED_ARG) { - instr++; - opcode = instr->op.code; - oparg = (oparg << 8) | instr->op.arg; - if (opcode == EXTENDED_ARG) { - instr--; + if (opcode == JUMP_FORWARD) { + return 1; + } + + if (opcode == EXTENDED_ARG) { + return 1; + } + + // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT + max_length -= 2; + + const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; + + assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); + assert(!_PyErr_Occurred(tstate)); + + + if (OPCODE_HAS_EXIT(opcode)) { + // Make space for side exit and final _EXIT_TRACE: + max_length--; + } + if (OPCODE_HAS_ERROR(opcode)) { + // Make space for error stub and final _EXIT_TRACE: + max_length--; + } + + // _GUARD_IP leads to an exit. + max_length -= needs_guard_ip; + + RESERVE_RAW(expansion->nuops + needs_guard_ip + 2 + (!OPCODE_HAS_NO_SAVE_IP(opcode)), "uop and various checks"); + + ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); + + if (!OPCODE_HAS_NO_SAVE_IP(opcode)) { + ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target); + } + + // Can be NULL for the entry frame. + if (old_code != NULL) { + _Py_BloomFilter_Add(dependencies, old_code); + } + + switch (opcode) { + case POP_JUMP_IF_NONE: + case POP_JUMP_IF_NOT_NONE: + case POP_JUMP_IF_FALSE: + case POP_JUMP_IF_TRUE: + { + _Py_CODEUNIT *computed_next_instr_without_modifiers = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + _Py_CODEUNIT *computed_next_instr = computed_next_instr_without_modifiers + (computed_next_instr_without_modifiers->op.code == NOT_TAKEN); + _Py_CODEUNIT *computed_jump_instr = computed_next_instr_without_modifiers + oparg; + assert(next_instr == computed_next_instr || next_instr == computed_jump_instr); + int jump_happened = computed_jump_instr == next_instr; + assert(jump_happened == (target_instr[1].cache & 1)); + uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_happened]; + ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(jump_happened ? computed_next_instr : computed_jump_instr, old_code)); + break; + } + case JUMP_BACKWARD_JIT: + // This is possible as the JIT might have re-activated after it was disabled + case JUMP_BACKWARD_NO_JIT: + case JUMP_BACKWARD: + ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); + _Py_FALLTHROUGH; + case JUMP_BACKWARD_NO_INTERRUPT: + { + if ((next_instr != _tstate->jit_tracer_state.initial_state.close_loop_instr) && + (next_instr != _tstate->jit_tracer_state.initial_state.start_instr) && + _tstate->jit_tracer_state.prev_state.code_curr_size > CODE_SIZE_NO_PROGRESS && + // For side exits, we don't want to terminate them early. + _tstate->jit_tracer_state.initial_state.exit == NULL && + // These are coroutines, and we want to unroll those usually. + opcode != JUMP_BACKWARD_NO_INTERRUPT) { + // We encountered a JUMP_BACKWARD but not to the top of our own loop. + // We don't want to continue tracing as we might get stuck in the + // inner loop. Instead, end the trace where the executor of the + // inner loop might start and let the traces rejoin. + OPT_STAT_INC(inner_loop); + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + trace[trace_length-1].operand1 = true; // is_control_flow + DPRINTF(2, "JUMP_BACKWARD not to top ends trace %p %p %p\n", next_instr, + _tstate->jit_tracer_state.initial_state.close_loop_instr, _tstate->jit_tracer_state.initial_state.start_instr); goto done; } - } - if (opcode == ENTER_EXECUTOR) { - // We have a couple of options here. We *could* peek "underneath" - // this executor and continue tracing, which could give us a longer, - // more optimizeable trace (at the expense of lots of duplicated - // tier two code). Instead, we choose to just end here and stitch to - // the other trace, which allows a side-exit traces to rejoin the - // "main" trace periodically (and also helps protect us against - // pathological behavior where the amount of tier two code explodes - // for a medium-length, branchy code path). This seems to work - // better in practice, but in the future we could be smarter about - // what we do here: - goto done; - } - assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); - RESERVE_RAW(2, "_CHECK_VALIDITY"); - ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); - if (!OPCODE_HAS_NO_SAVE_IP(opcode)) { - RESERVE_RAW(2, "_SET_IP"); - ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)instr, target); + break; } - /* Special case the first instruction, - * so that we can guarantee forward progress */ - if (first && progress_needed) { - assert(first); - if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { - opcode = _PyOpcode_Deopt[opcode]; + case RESUME: + case RESUME_CHECK: + /* Use a special tier 2 version of RESUME_CHECK to allow traces to + * start with RESUME_CHECK */ + ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); + break; + default: + { + const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; + // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) + int nuops = expansion->nuops; + if (nuops == 0) { + DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]); + goto unsupported; } - assert(!OPCODE_HAS_EXIT(opcode)); - assert(!OPCODE_HAS_DEOPT(opcode)); - } + assert(nuops > 0); + uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM + uint32_t orig_target = target; + for (int i = 0; i < nuops; i++) { + oparg = orig_oparg; + target = orig_target; + uint32_t uop = expansion->uops[i].uop; + uint64_t operand = 0; + // Add one to account for the actual opcode/oparg pair: + int offset = expansion->uops[i].offset + 1; + switch (expansion->uops[i].size) { + case OPARG_SIMPLE: + assert(opcode != _JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD); + break; + case OPARG_CACHE_1: + operand = read_u16(&this_instr[offset].cache); + break; + case OPARG_CACHE_2: + operand = read_u32(&this_instr[offset].cache); + break; + case OPARG_CACHE_4: + operand = read_u64(&this_instr[offset].cache); + break; + case OPARG_TOP: // First half of super-instr + assert(orig_oparg <= 255); + oparg = orig_oparg >> 4; + break; + case OPARG_BOTTOM: // Second half of super-instr + assert(orig_oparg <= 255); + oparg = orig_oparg & 0xF; + break; + case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset + oparg = offset; + assert(uop == _SAVE_RETURN_OFFSET); + break; + case OPARG_REPLACED: + uop = _PyUOp_Replacements[uop]; + assert(uop != 0); - if (OPCODE_HAS_EXIT(opcode)) { - // Make space for side exit and final _EXIT_TRACE: - RESERVE_RAW(2, "_EXIT_TRACE"); - max_length--; - } - if (OPCODE_HAS_ERROR(opcode)) { - // Make space for error stub and final _EXIT_TRACE: - RESERVE_RAW(2, "_ERROR_POP_N"); - max_length--; - } - switch (opcode) { - case POP_JUMP_IF_NONE: - case POP_JUMP_IF_NOT_NONE: - case POP_JUMP_IF_FALSE: - case POP_JUMP_IF_TRUE: - { - RESERVE(1); - int counter = instr[1].cache; - int bitcount = _Py_popcount32(counter); - int jump_likely = bitcount > 8; - /* If bitcount is 8 (half the jumps were taken), adjust confidence by 50%. - For values in between, adjust proportionally. */ - if (jump_likely) { - confidence = confidence * bitcount / 16; + uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + if (uop == _TIER2_RESUME_CHECK) { + target = next_inst; + } + else { + int extended_arg = orig_oparg > 255; + uint32_t jump_target = next_inst + orig_oparg + extended_arg; + assert(_Py_GetBaseCodeUnit(old_code, jump_target).op.code == END_FOR); + assert(_Py_GetBaseCodeUnit(old_code, jump_target+1).op.code == POP_ITER); + if (is_for_iter_test[uop]) { + target = jump_target + 1; + } + } + break; + case OPERAND1_1: + assert(trace[trace_length-1].opcode == uop); + operand = read_u16(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + case OPERAND1_2: + assert(trace[trace_length-1].opcode == uop); + operand = read_u32(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + case OPERAND1_4: + assert(trace[trace_length-1].opcode == uop); + operand = read_u64(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + default: + fprintf(stderr, + "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n", + opcode, oparg, nuops, i, + expansion->uops[i].size, + expansion->uops[i].offset); + Py_FatalError("garbled expansion"); } - else { - confidence = confidence * (16 - bitcount) / 16; - } - uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; - DPRINTF(2, "%d: %s(%d): counter=%04x, bitcount=%d, likely=%d, confidence=%d, uopcode=%s\n", - target, _PyOpcode_OpName[opcode], oparg, - counter, bitcount, jump_likely, confidence, _PyUOpName(uopcode)); - if (confidence < CONFIDENCE_CUTOFF) { - DPRINTF(2, "Confidence too low (%d < %d)\n", confidence, CONFIDENCE_CUTOFF); - OPT_STAT_INC(low_confidence); - goto done; - } - _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - _Py_CODEUNIT *target_instr = next_instr + oparg; - if (jump_likely) { - DPRINTF(2, "Jump likely (%04x = %d bits), continue at byte offset %d\n", - instr[1].cache, bitcount, 2 * INSTR_IP(target_instr, code)); - instr = target_instr; - ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(next_instr, code)); - goto top; - } - ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(target_instr, code)); - break; - } + if (uop == _PUSH_FRAME || uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { + PyCodeObject *new_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + PyFunctionObject *new_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - case JUMP_BACKWARD: - case JUMP_BACKWARD_JIT: - ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); - _Py_FALLTHROUGH; - case JUMP_BACKWARD_NO_INTERRUPT: - { - instr += 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] - (int)oparg; - if (jump_seen) { - OPT_STAT_INC(inner_loop); - DPRINTF(2, "JUMP_BACKWARD not to top ends trace\n"); - goto done; - } - jump_seen = true; - goto top; - } - - case JUMP_FORWARD: - { - RESERVE(0); - // This will emit two _SET_IP instructions; leave it to the optimizer - instr += oparg; - break; - } - - case RESUME: - /* Use a special tier 2 version of RESUME_CHECK to allow traces to - * start with RESUME_CHECK */ - ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); - break; - - default: - { - const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; - if (expansion->nuops > 0) { - // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) - int nuops = expansion->nuops; - RESERVE(nuops + 1); /* One extra for exit */ - int16_t last_op = expansion->uops[nuops-1].uop; - if (last_op == _RETURN_VALUE || last_op == _RETURN_GENERATOR || last_op == _YIELD_VALUE) { - // Check for trace stack underflow now: - // We can't bail e.g. in the middle of - // LOAD_CONST + _RETURN_VALUE. - if (trace_stack_depth == 0) { - DPRINTF(2, "Trace stack underflow\n"); - OPT_STAT_INC(trace_stack_underflow); - return 0; + operand = 0; + if (frame->owner < FRAME_OWNED_BY_INTERPRETER) { + // Don't add nested code objects to the dependency. + // It causes endless re-traces. + if (new_func != NULL && !Py_IsNone((PyObject*)new_func) && !(new_code->co_flags & CO_NESTED)) { + operand = (uintptr_t)new_func; + DPRINTF(2, "Adding %p func to op\n", (void *)operand); + _Py_BloomFilter_Add(dependencies, new_func); + } + else if (new_code != NULL && !Py_IsNone((PyObject*)new_code)) { + operand = (uintptr_t)new_code | 1; + DPRINTF(2, "Adding %p code to op\n", (void *)operand); + _Py_BloomFilter_Add(dependencies, new_code); } } - uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM - for (int i = 0; i < nuops; i++) { - oparg = orig_oparg; - uint32_t uop = expansion->uops[i].uop; - uint64_t operand = 0; - // Add one to account for the actual opcode/oparg pair: - int offset = expansion->uops[i].offset + 1; - switch (expansion->uops[i].size) { - case OPARG_SIMPLE: - assert(opcode != JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD); - break; - case OPARG_CACHE_1: - operand = read_u16(&instr[offset].cache); - break; - case OPARG_CACHE_2: - operand = read_u32(&instr[offset].cache); - break; - case OPARG_CACHE_4: - operand = read_u64(&instr[offset].cache); - break; - case OPARG_TOP: // First half of super-instr - oparg = orig_oparg >> 4; - break; - case OPARG_BOTTOM: // Second half of super-instr - oparg = orig_oparg & 0xF; - break; - case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset - oparg = offset; - assert(uop == _SAVE_RETURN_OFFSET); - break; - case OPARG_REPLACED: - uop = _PyUOp_Replacements[uop]; - assert(uop != 0); - uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + (oparg > 255); - if (uop == _TIER2_RESUME_CHECK) { - target = next_inst; - } -#ifdef Py_DEBUG - else { - uint32_t jump_target = next_inst + oparg; - assert(_Py_GetBaseCodeUnit(code, jump_target).op.code == END_FOR); - assert(_Py_GetBaseCodeUnit(code, jump_target+1).op.code == POP_ITER); - } -#endif - break; - case OPERAND1_1: - assert(trace[trace_length-1].opcode == uop); - operand = read_u16(&instr[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - case OPERAND1_2: - assert(trace[trace_length-1].opcode == uop); - operand = read_u32(&instr[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - case OPERAND1_4: - assert(trace[trace_length-1].opcode == uop); - operand = read_u64(&instr[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - default: - fprintf(stderr, - "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n", - opcode, oparg, nuops, i, - expansion->uops[i].size, - expansion->uops[i].offset); - Py_FatalError("garbled expansion"); - } - - if (uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { - TRACE_STACK_POP(); - /* Set the operand to the function or code object returned to, - * to assist optimization passes. (See _PUSH_FRAME below.) - */ - if (func != NULL) { - operand = (uintptr_t)func; - } - else if (code != NULL) { - operand = (uintptr_t)code | 1; - } - else { - operand = 0; - } - ADD_TO_TRACE(uop, oparg, operand, target); - DPRINTF(2, - "Returning to %s (%s:%d) at byte offset %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(instr, code)); - goto top; - } - - if (uop == _PUSH_FRAME) { - assert(i + 1 == nuops); - if (opcode == FOR_ITER_GEN || - opcode == LOAD_ATTR_PROPERTY || - opcode == BINARY_OP_SUBSCR_GETITEM || - opcode == SEND_GEN) - { - DPRINTF(2, "Bailing due to dynamic target\n"); - OPT_STAT_INC(unknown_callee); - return 0; - } - assert(_PyOpcode_Deopt[opcode] == CALL || _PyOpcode_Deopt[opcode] == CALL_KW); - int func_version_offset = - offsetof(_PyCallCache, func_version)/sizeof(_Py_CODEUNIT) - // Add one to account for the actual opcode/oparg pair: - + 1; - uint32_t func_version = read_u32(&instr[func_version_offset].cache); - PyCodeObject *new_code = NULL; - PyFunctionObject *new_func = - _PyFunction_LookupByVersion(func_version, (PyObject **) &new_code); - DPRINTF(2, "Function: version=%#x; new_func=%p, new_code=%p\n", - (int)func_version, new_func, new_code); - if (new_code != NULL) { - if (new_code == code) { - // Recursive call, bail (we could be here forever). - DPRINTF(2, "Bailing on recursive call to %s (%s:%d)\n", - PyUnicode_AsUTF8(new_code->co_qualname), - PyUnicode_AsUTF8(new_code->co_filename), - new_code->co_firstlineno); - OPT_STAT_INC(recursive_call); - ADD_TO_TRACE(uop, oparg, 0, target); - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); - goto done; - } - if (new_code->co_version != func_version) { - // func.__code__ was updated. - // Perhaps it may happen again, so don't bother tracing. - // TODO: Reason about this -- is it better to bail or not? - DPRINTF(2, "Bailing because co_version != func_version\n"); - ADD_TO_TRACE(uop, oparg, 0, target); - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); - goto done; - } - // Increment IP to the return address - instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1; - TRACE_STACK_PUSH(); - _Py_BloomFilter_Add(dependencies, new_code); - /* Set the operand to the callee's function or code object, - * to assist optimization passes. - * We prefer setting it to the function - * but if that's not available but the code is available, - * use the code, setting the low bit so the optimizer knows. - */ - if (new_func != NULL) { - operand = (uintptr_t)new_func; - } - else if (new_code != NULL) { - operand = (uintptr_t)new_code | 1; - } - else { - operand = 0; - } - ADD_TO_TRACE(uop, oparg, operand, target); - code = new_code; - func = new_func; - instr = _PyCode_CODE(code); - DPRINTF(2, - "Continuing in %s (%s:%d) at byte offset %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(instr, code)); - goto top; - } - DPRINTF(2, "Bail, new_code == NULL\n"); - OPT_STAT_INC(unknown_callee); - return 0; - } - - if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) { - assert(i + 1 == nuops); - _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - assert(next_instr->op.code == STORE_FAST); - operand = next_instr->op.arg; - // Skip the STORE_FAST: - instr++; - } - - // All other instructions - ADD_TO_TRACE(uop, oparg, operand, target); - } + ADD_TO_TRACE(uop, oparg, operand, target); + trace[trace_length - 1].operand1 = PyStackRef_IsNone(frame->f_executable) ? 2 : ((int)(frame->stackpointer - _PyFrame_Stackbase(frame))); break; } - DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]); - OPT_UNSUPPORTED_OPCODE(opcode); - goto done; // Break out of loop - } // End default + if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) { + assert(i + 1 == nuops); + _Py_CODEUNIT *next = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + assert(next->op.code == STORE_FAST); + operand = next->op.arg; + } + // All other instructions + ADD_TO_TRACE(uop, oparg, operand, target); + } + break; + } // End default - } // End switch (opcode) + } // End switch (opcode) - instr++; - // Add cache size for opcode - instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - - if (opcode == CALL_LIST_APPEND) { - assert(instr->op.code == POP_TOP); - instr++; + if (needs_guard_ip) { + uint16_t guard_ip = guard_ip_uop[trace[trace_length-1].opcode]; + if (guard_ip == 0) { + DPRINTF(1, "Unknown uop needing guard ip %s\n", _PyOpcode_uop_name[trace[trace_length-1].opcode]); + Py_UNREACHABLE(); } - top: - // Jump here after _PUSH_FRAME or likely branches. - first = false; - } // End for (;;) - -done: - while (trace_stack_depth > 0) { - TRACE_STACK_POP(); + ADD_TO_TRACE(guard_ip, 0, (uintptr_t)next_instr, 0); } - assert(code == initial_code); - // Skip short traces where we can't even translate a single instruction: - if (first) { - OPT_STAT_INC(trace_too_short); - DPRINTF(2, - "No trace for %s (%s:%d) at byte offset %d (no progress)\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code)); + // Loop back to the start + int is_first_instr = _tstate->jit_tracer_state.initial_state.close_loop_instr == next_instr || + _tstate->jit_tracer_state.initial_state.start_instr == next_instr; + if (is_first_instr && _tstate->jit_tracer_state.prev_state.code_curr_size > CODE_SIZE_NO_PROGRESS) { + if (needs_guard_ip) { + ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)next_instr, 0); + } + ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); + goto done; + } + DPRINTF(2, "Trace continuing\n"); + _tstate->jit_tracer_state.prev_state.code_curr_size = trace_length; + _tstate->jit_tracer_state.prev_state.code_max_size = max_length; + return 1; +done: + DPRINTF(2, "Trace done\n"); + _tstate->jit_tracer_state.prev_state.code_curr_size = trace_length; + _tstate->jit_tracer_state.prev_state.code_max_size = max_length; + return 0; +full: + DPRINTF(2, "Trace full\n"); + if (!is_terminator(&_tstate->jit_tracer_state.code_buffer[trace_length-1])) { + // Undo the last few instructions. + trace_length = _tstate->jit_tracer_state.prev_state.code_curr_size; + max_length = _tstate->jit_tracer_state.prev_state.code_max_size; + // We previously reversed one. + max_length += 1; + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + trace[trace_length-1].operand1 = true; // is_control_flow + } + _tstate->jit_tracer_state.prev_state.code_curr_size = trace_length; + _tstate->jit_tracer_state.prev_state.code_max_size = max_length; + return 0; +} + +// Returns 0 for do not enter tracing, 1 on enter tracing. +int +_PyJit_TryInitializeTracing( + PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, + _Py_CODEUNIT *start_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, + _PyExitData *exit, int oparg) +{ + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + // A recursive trace. + // Don't trace into the inner call because it will stomp on the previous trace, causing endless retraces. + if (_tstate->jit_tracer_state.prev_state.code_curr_size > CODE_SIZE_EMPTY) { return 0; } - if (!is_terminator(&trace[trace_length-1])) { - /* Allow space for _EXIT_TRACE */ - max_length += 2; - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + if (oparg > 0xFFFF) { + return 0; } - DPRINTF(1, - "Created a proto-trace for %s (%s:%d) at byte offset %d -- length %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code), - trace_length); - OPT_HIST(trace_length, trace_length_hist); - return trace_length; + if (_tstate->jit_tracer_state.code_buffer == NULL) { + _tstate->jit_tracer_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); + if (_tstate->jit_tracer_state.code_buffer == NULL) { + // Don't error, just go to next instruction. + return 0; + } + } + PyObject *func = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + if (func == NULL) { + return 0; + } + PyCodeObject *code = _PyFrame_GetCode(frame); +#ifdef Py_DEBUG + char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); + int lltrace = 0; + if (python_lltrace != NULL && *python_lltrace >= '0') { + lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that + } + DPRINTF(2, + "Tracing %s (%s:%d) at byte offset %d at chain depth %d\n", + PyUnicode_AsUTF8(code->co_qualname), + PyUnicode_AsUTF8(code->co_filename), + code->co_firstlineno, + 2 * INSTR_IP(close_loop_instr, code), + chain_depth); +#endif + + add_to_trace(_tstate->jit_tracer_state.code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)start_instr, INSTR_IP(start_instr, code)); + add_to_trace(_tstate->jit_tracer_state.code_buffer, 1, _MAKE_WARM, 0, 0, 0); + _tstate->jit_tracer_state.prev_state.code_curr_size = CODE_SIZE_EMPTY; + + _tstate->jit_tracer_state.prev_state.code_max_size = UOP_MAX_TRACE_LENGTH; + _tstate->jit_tracer_state.initial_state.start_instr = start_instr; + _tstate->jit_tracer_state.initial_state.close_loop_instr = close_loop_instr; + _tstate->jit_tracer_state.initial_state.code = (PyCodeObject *)Py_NewRef(code); + _tstate->jit_tracer_state.initial_state.func = (PyFunctionObject *)Py_NewRef(func); + _tstate->jit_tracer_state.initial_state.exit = exit; + _tstate->jit_tracer_state.initial_state.stack_depth = curr_stackdepth; + _tstate->jit_tracer_state.initial_state.chain_depth = chain_depth; + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.dependencies_still_valid = true; + _tstate->jit_tracer_state.prev_state.instr_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); + _tstate->jit_tracer_state.prev_state.instr = curr_instr; + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; + _tstate->jit_tracer_state.prev_state.instr_stacklevel = curr_stackdepth; + _tstate->jit_tracer_state.prev_state.instr_is_super = false; + assert(curr_instr->op.code == JUMP_BACKWARD_JIT || (exit != NULL)); + _tstate->jit_tracer_state.initial_state.jump_backward_instr = curr_instr; + + if (_PyOpcode_Caches[_PyOpcode_Deopt[close_loop_instr->op.code]]) { + close_loop_instr[1].counter = trigger_backoff_counter(); + } + _Py_BloomFilter_Init(&_tstate->jit_tracer_state.prev_state.dependencies); + return 1; } +void +_PyJit_FinalizeTracing(PyThreadState *tstate) +{ + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + Py_CLEAR(_tstate->jit_tracer_state.initial_state.code); + Py_CLEAR(_tstate->jit_tracer_state.initial_state.func); + Py_CLEAR(_tstate->jit_tracer_state.prev_state.instr_code); + _tstate->jit_tracer_state.prev_state.code_curr_size = CODE_SIZE_EMPTY; + _tstate->jit_tracer_state.prev_state.code_max_size = UOP_MAX_TRACE_LENGTH - 1; +} + + #undef RESERVE #undef RESERVE_RAW #undef INSTR_IP @@ -1018,20 +1064,21 @@ count_exits(_PyUOpInstruction *buffer, int length) int exit_count = 0; for (int i = 0; i < length; i++) { int opcode = buffer[i].opcode; - if (opcode == _EXIT_TRACE) { + if (opcode == _EXIT_TRACE || opcode == _DYNAMIC_EXIT) { exit_count++; } } return exit_count; } -static void make_exit(_PyUOpInstruction *inst, int opcode, int target) +static void make_exit(_PyUOpInstruction *inst, int opcode, int target, bool is_control_flow) { inst->opcode = opcode; inst->oparg = 0; inst->operand0 = 0; inst->format = UOP_FORMAT_TARGET; inst->target = target; + inst->operand1 = is_control_flow; #ifdef Py_STATS inst->execution_count = 0; #endif @@ -1075,15 +1122,17 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) exit_op = _HANDLE_PENDING_AND_DEOPT; } int32_t jump_target = target; - if (is_for_iter_test[opcode]) { - /* Target the POP_TOP immediately after the END_FOR, - * leaving only the iterator on the stack. */ - int extended_arg = inst->oparg > 255; - int32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + extended_arg; - jump_target = next_inst + inst->oparg + 1; + if ( + opcode == _GUARD_IP__PUSH_FRAME || + opcode == _GUARD_IP_RETURN_VALUE || + opcode == _GUARD_IP_YIELD_VALUE || + opcode == _GUARD_IP_RETURN_GENERATOR + ) { + exit_op = _DYNAMIC_EXIT; } + bool is_control_flow = (opcode == _GUARD_IS_FALSE_POP || opcode == _GUARD_IS_TRUE_POP || is_for_iter_test[opcode]); if (jump_target != current_jump_target || current_exit_op != exit_op) { - make_exit(&buffer[next_spare], exit_op, jump_target); + make_exit(&buffer[next_spare], exit_op, jump_target, is_control_flow); current_exit_op = exit_op; current_jump_target = jump_target; current_jump = next_spare; @@ -1099,7 +1148,7 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) current_popped = popped; current_error = next_spare; current_error_target = target; - make_exit(&buffer[next_spare], _ERROR_POP_N, 0); + make_exit(&buffer[next_spare], _ERROR_POP_N, 0, false); buffer[next_spare].operand0 = target; next_spare++; } @@ -1157,7 +1206,9 @@ sanity_check(_PyExecutorObject *executor) } bool ended = false; uint32_t i = 0; - CHECK(executor->trace[0].opcode == _START_EXECUTOR || executor->trace[0].opcode == _COLD_EXIT); + CHECK(executor->trace[0].opcode == _START_EXECUTOR || + executor->trace[0].opcode == _COLD_EXIT || + executor->trace[0].opcode == _COLD_DYNAMIC_EXIT); for (; i < executor->code_size; i++) { const _PyUOpInstruction *inst = &executor->trace[i]; uint16_t opcode = inst->opcode; @@ -1189,7 +1240,8 @@ sanity_check(_PyExecutorObject *executor) opcode == _DEOPT || opcode == _HANDLE_PENDING_AND_DEOPT || opcode == _EXIT_TRACE || - opcode == _ERROR_POP_N); + opcode == _ERROR_POP_N || + opcode == _DYNAMIC_EXIT); } } @@ -1202,7 +1254,7 @@ sanity_check(_PyExecutorObject *executor) * and not a NOP. */ static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies) +make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies, int chain_depth) { int exit_count = count_exits(buffer, length); _PyExecutorObject *executor = allocate_executor(exit_count, length); @@ -1212,10 +1264,11 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil /* Initialize exits */ _PyExecutorObject *cold = _PyExecutor_GetColdExecutor(); + _PyExecutorObject *cold_dynamic = _PyExecutor_GetColdDynamicExecutor(); + cold->vm_data.chain_depth = chain_depth; for (int i = 0; i < exit_count; i++) { executor->exits[i].index = i; executor->exits[i].temperature = initial_temperature_backoff_counter(); - executor->exits[i].executor = cold; } int next_exit = exit_count-1; _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length]; @@ -1225,11 +1278,13 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil int opcode = buffer[i].opcode; dest--; *dest = buffer[i]; - assert(opcode != _POP_JUMP_IF_FALSE && opcode != _POP_JUMP_IF_TRUE); - if (opcode == _EXIT_TRACE) { + if (opcode == _EXIT_TRACE || opcode == _DYNAMIC_EXIT) { _PyExitData *exit = &executor->exits[next_exit]; exit->target = buffer[i].target; dest->operand0 = (uint64_t)exit; + exit->executor = opcode == _EXIT_TRACE ? cold : cold_dynamic; + exit->is_dynamic = (char)(opcode == _DYNAMIC_EXIT); + exit->is_control_flow = (char)buffer[i].operand1; next_exit--; } } @@ -1291,38 +1346,32 @@ int effective_trace_length(_PyUOpInstruction *buffer, int length) static int uop_optimize( _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, + PyThreadState *tstate, _PyExecutorObject **exec_ptr, - int curr_stackentries, bool progress_needed) { - _PyBloomFilter dependencies; - _Py_BloomFilter_Init(&dependencies); - PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->jit_uop_buffer == NULL) { - interp->jit_uop_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); - if (interp->jit_uop_buffer == NULL) { - return 0; - } - } - _PyUOpInstruction *buffer = interp->jit_uop_buffer; + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + _PyBloomFilter *dependencies = &_tstate->jit_tracer_state.prev_state.dependencies; + _PyUOpInstruction *buffer = _tstate->jit_tracer_state.code_buffer; OPT_STAT_INC(attempts); char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); bool is_noopt = true; if (env_var == NULL || *env_var == '\0' || *env_var > '0') { is_noopt = false; } - int length = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies, progress_needed); - if (length <= 0) { - // Error or nothing translated - return length; + int curr_stackentries = _tstate->jit_tracer_state.initial_state.stack_depth; + int length = _tstate->jit_tracer_state.prev_state.code_curr_size; + if (length <= CODE_SIZE_NO_PROGRESS) { + return 0; } + assert(length > 0); assert(length < UOP_MAX_TRACE_LENGTH); OPT_STAT_INC(traces_created); if (!is_noopt) { - length = _Py_uop_analyze_and_optimize(frame, buffer, - length, - curr_stackentries, &dependencies); + length = _Py_uop_analyze_and_optimize( + _tstate->jit_tracer_state.initial_state.func, + buffer,length, + curr_stackentries, dependencies); if (length <= 0) { return length; } @@ -1345,14 +1394,14 @@ uop_optimize( OPT_HIST(effective_trace_length(buffer, length), optimized_trace_length_hist); length = prepare_for_execution(buffer, length); assert(length <= UOP_MAX_TRACE_LENGTH); - _PyExecutorObject *executor = make_executor_from_uops(buffer, length, &dependencies); + _PyExecutorObject *executor = make_executor_from_uops( + buffer, length, dependencies, _tstate->jit_tracer_state.initial_state.chain_depth); if (executor == NULL) { return -1; } assert(length <= UOP_MAX_TRACE_LENGTH); // Check executor coldness - PyThreadState *tstate = PyThreadState_Get(); // It's okay if this ends up going negative. if (--tstate->interp->executor_creation_counter == 0) { _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); @@ -1539,6 +1588,35 @@ _PyExecutor_GetColdExecutor(void) return cold; } +_PyExecutorObject * +_PyExecutor_GetColdDynamicExecutor(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->cold_dynamic_executor != NULL) { + assert(interp->cold_dynamic_executor->trace[0].opcode == _COLD_DYNAMIC_EXIT); + return interp->cold_dynamic_executor; + } + _PyExecutorObject *cold = allocate_executor(0, 1); + if (cold == NULL) { + Py_FatalError("Cannot allocate core JIT code"); + } + ((_PyUOpInstruction *)cold->trace)->opcode = _COLD_DYNAMIC_EXIT; +#ifdef _Py_JIT + cold->jit_code = NULL; + cold->jit_size = 0; + // This is initialized to true so we can prevent the executor + // from being immediately detected as cold and invalidated. + cold->vm_data.warm = true; + if (_PyJIT_Compile(cold, cold->trace, 1)) { + Py_DECREF(cold); + Py_FatalError("Cannot allocate core JIT code"); + } +#endif + _Py_SetImmortal((PyObject *)cold); + interp->cold_dynamic_executor = cold; + return cold; +} + void _PyExecutor_ClearExit(_PyExitData *exit) { @@ -1546,7 +1624,12 @@ _PyExecutor_ClearExit(_PyExitData *exit) return; } _PyExecutorObject *old = exit->executor; - exit->executor = _PyExecutor_GetColdExecutor(); + if (exit->is_dynamic) { + exit->executor = _PyExecutor_GetColdDynamicExecutor(); + } + else { + exit->executor = _PyExecutor_GetColdExecutor(); + } Py_DECREF(old); } @@ -1648,6 +1731,18 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is _Py_Executors_InvalidateAll(interp, is_invalidation); } +void +_PyJit_Tracer_InvalidateDependency(PyThreadState *tstate, void *obj) +{ + _PyBloomFilter obj_filter; + _Py_BloomFilter_Init(&obj_filter); + _Py_BloomFilter_Add(&obj_filter, obj); + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if (bloom_filter_may_contain(&_tstate->jit_tracer_state.prev_state.dependencies, &obj_filter)) + { + _tstate->jit_tracer_state.prev_state.dependencies_still_valid = false; + } +} /* Invalidate all executors */ void _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation) @@ -1777,7 +1872,7 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) #ifdef Py_STATS fprintf(out, " %s -- %" PRIu64 "\n", i, opname, inst->execution_count); #else - fprintf(out, " %s\n", i, opname); + fprintf(out, " %s op0=%" PRIu64 "\n", i, opname, inst->operand0); #endif if (inst->opcode == _EXIT_TRACE || inst->opcode == _JUMP_TO_TOP) { break; @@ -1787,6 +1882,8 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) fprintf(out, "]\n\n"); /* Write all the outgoing edges */ + _PyExecutorObject *cold = _PyExecutor_GetColdExecutor(); + _PyExecutorObject *cold_dynamic = _PyExecutor_GetColdDynamicExecutor(); for (uint32_t i = 0; i < executor->code_size; i++) { _PyUOpInstruction const *inst = &executor->trace[i]; uint16_t flags = _PyUop_Flags[inst->opcode]; @@ -1797,10 +1894,10 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) else if (flags & HAS_EXIT_FLAG) { assert(inst->format == UOP_FORMAT_JUMP); _PyUOpInstruction const *exit_inst = &executor->trace[inst->jump_target]; - assert(exit_inst->opcode == _EXIT_TRACE); + assert(exit_inst->opcode == _EXIT_TRACE || exit_inst->opcode == _DYNAMIC_EXIT); exit = (_PyExitData *)exit_inst->operand0; } - if (exit != NULL && exit->executor != NULL) { + if (exit != NULL && exit->executor != cold && exit->executor != cold_dynamic) { fprintf(out, "executor_%p:i%d -> executor_%p:start\n", executor, i, exit->executor); } if (inst->opcode == _EXIT_TRACE || inst->opcode == _JUMP_TO_TOP) { diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index a6add301ccb..8d7b734e17c 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -142,8 +142,10 @@ incorrect_keys(PyObject *obj, uint32_t version) #define STACK_LEVEL() ((int)(stack_pointer - ctx->frame->stack)) #define STACK_SIZE() ((int)(ctx->frame->stack_len)) +#define CURRENT_FRAME_IS_INIT_SHIM() (ctx->frame->code == ((PyCodeObject *)&_Py_InitCleanup)) + #define WITHIN_STACK_BOUNDS() \ - (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE()) + (CURRENT_FRAME_IS_INIT_SHIM() || (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE())) #define GETLOCAL(idx) ((ctx->frame->locals[idx])) @@ -267,7 +269,7 @@ static PyCodeObject * get_current_code_object(JitOptContext *ctx) { - return (PyCodeObject *)ctx->frame->func->func_code; + return (PyCodeObject *)ctx->frame->code; } static PyObject * @@ -298,10 +300,6 @@ optimize_uops( JitOptContext context; JitOptContext *ctx = &context; uint32_t opcode = UINT16_MAX; - int curr_space = 0; - int max_space = 0; - _PyUOpInstruction *first_valid_check_stack = NULL; - _PyUOpInstruction *corresponding_check_stack = NULL; // Make sure that watchers are set up PyInterpreterState *interp = _PyInterpreterState_GET(); @@ -320,13 +318,18 @@ optimize_uops( ctx->frame = frame; _PyUOpInstruction *this_instr = NULL; + JitOptRef *stack_pointer = ctx->frame->stack_pointer; + for (int i = 0; !ctx->done; i++) { assert(i < trace_len); this_instr = &trace[i]; int oparg = this_instr->oparg; opcode = this_instr->opcode; - JitOptRef *stack_pointer = ctx->frame->stack_pointer; + + if (!CURRENT_FRAME_IS_INIT_SHIM()) { + stack_pointer = ctx->frame->stack_pointer; + } #ifdef Py_DEBUG if (get_lltrace() >= 3) { @@ -345,9 +348,11 @@ optimize_uops( Py_UNREACHABLE(); } assert(ctx->frame != NULL); - DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); - ctx->frame->stack_pointer = stack_pointer; - assert(STACK_LEVEL() >= 0); + if (!CURRENT_FRAME_IS_INIT_SHIM()) { + DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); + ctx->frame->stack_pointer = stack_pointer; + assert(STACK_LEVEL() >= 0); + } } if (ctx->out_of_space) { DPRINTF(3, "\n"); @@ -355,27 +360,21 @@ optimize_uops( } if (ctx->contradiction) { // Attempted to push a "bottom" (contradiction) symbol onto the stack. - // This means that the abstract interpreter has hit unreachable code. + // This means that the abstract interpreter has optimized to trace + // to an unreachable estate. // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but hitting - // bottom indicates type instability, so we are probably better off + // bottom usually indicates an optimizer bug, so we are probably better off // retrying later. DPRINTF(3, "\n"); DPRINTF(1, "Hit bottom in abstract interpreter\n"); _Py_uop_abstractcontext_fini(ctx); + OPT_STAT_INC(optimizer_contradiction); return 0; } /* Either reached the end or cannot optimize further, but there * would be no benefit in retrying later */ _Py_uop_abstractcontext_fini(ctx); - if (first_valid_check_stack != NULL) { - assert(first_valid_check_stack->opcode == _CHECK_STACK_SPACE); - assert(max_space > 0); - assert(max_space <= INT_MAX); - assert(max_space <= INT32_MAX); - first_valid_check_stack->opcode = _CHECK_STACK_SPACE_OPERAND; - first_valid_check_stack->operand0 = max_space; - } return trace_len; error: @@ -460,6 +459,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) buffer[pc].opcode = _NOP; } break; + case _EXIT_TRACE: default: { // Cancel out pushes and pops, repeatedly. So: @@ -493,7 +493,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) } /* _PUSH_FRAME doesn't escape or error, but it * does need the IP for the return address */ - bool needs_ip = opcode == _PUSH_FRAME; + bool needs_ip = (opcode == _PUSH_FRAME || opcode == _YIELD_VALUE || opcode == _DYNAMIC_EXIT || opcode == _EXIT_TRACE); if (_PyUop_Flags[opcode] & HAS_ESCAPES_FLAG) { needs_ip = true; may_have_escaped = true; @@ -503,10 +503,14 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) buffer[last_set_ip].opcode = _SET_IP; last_set_ip = -1; } + if (opcode == _EXIT_TRACE) { + return pc + 1; + } break; } case _JUMP_TO_TOP: - case _EXIT_TRACE: + case _DYNAMIC_EXIT: + case _DEOPT: return pc + 1; } } @@ -518,7 +522,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) // > 0 - length of optimized trace int _Py_uop_analyze_and_optimize( - _PyInterpreterFrame *frame, + PyFunctionObject *func, _PyUOpInstruction *buffer, int length, int curr_stacklen, @@ -528,8 +532,8 @@ _Py_uop_analyze_and_optimize( OPT_STAT_INC(optimizer_attempts); length = optimize_uops( - _PyFrame_GetFunction(frame), buffer, - length, curr_stacklen, dependencies); + func, buffer, + length, curr_stacklen, dependencies); if (length == 0) { return length; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index da3d3c96bc1..06fa8a4522a 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -342,7 +342,6 @@ dummy_func(void) { int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { sym_set_type(value, &PyBool_Type); - value = sym_new_truthiness(ctx, value, true); } } @@ -752,8 +751,14 @@ dummy_func(void) { } op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { - new_frame = PyJitRef_NULL; - ctx->done = true; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + PyCodeObject *co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; + } + + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { @@ -764,8 +769,20 @@ dummy_func(void) { } op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame)) { - init_frame = PyJitRef_NULL; - ctx->done = true; + ctx->frame->stack_pointer = stack_pointer - oparg - 2; + _Py_UOpsAbstractFrame *shim = frame_new(ctx, (PyCodeObject *)&_Py_InitCleanup, 0, NULL, 0); + if (shim == NULL) { + break; + } + /* Push self onto stack of shim */ + shim->stack[0] = self; + shim->stack_pointer++; + assert((int)(shim->stack_pointer - shim->stack) == 1); + ctx->frame = shim; + ctx->curr_frame_depth++; + assert((this_instr + 1)->opcode == _PUSH_FRAME); + PyCodeObject *co = get_code_with_logging((this_instr + 1)); + init_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args-1, oparg+1)); } op(_RETURN_VALUE, (retval -- res)) { @@ -773,42 +790,65 @@ dummy_func(void) { JitOptRef temp = PyJitRef_StripReferenceInfo(retval); DEAD(retval); SAVE_STACK(); - PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + int returning_stacklevel = this_instr->operand1; + if (ctx->curr_frame_depth >= 2) { + PyCodeObject *expected_code = ctx->frames[ctx->curr_frame_depth - 2].code; + if (expected_code == returning_code) { + assert((this_instr + 1)->opcode == _GUARD_IP_RETURN_VALUE); + REPLACE_OP((this_instr + 1), _NOP, 0, 0); + } + } + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } stack_pointer = ctx->frame->stack_pointer; - /* Stack space handling */ - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; - RELOAD_STACK(); res = temp; } op(_RETURN_GENERATOR, ( -- res)) { SYNC_SP(); - PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + _Py_BloomFilter_Add(dependencies, returning_code); + int returning_stacklevel = this_instr->operand1; + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } stack_pointer = ctx->frame->stack_pointer; res = sym_new_unknown(ctx); - - /* Stack space handling */ - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; } - op(_YIELD_VALUE, (unused -- value)) { - value = sym_new_unknown(ctx); + op(_YIELD_VALUE, (retval -- value)) { + // Mimics PyStackRef_MakeHeapSafe in the interpreter. + JitOptRef temp = PyJitRef_StripReferenceInfo(retval); + DEAD(retval); + SAVE_STACK(); + ctx->frame->stack_pointer = stack_pointer; + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + _Py_BloomFilter_Add(dependencies, returning_code); + int returning_stacklevel = this_instr->operand1; + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } + stack_pointer = ctx->frame->stack_pointer; + RELOAD_STACK(); + value = temp; } op(_GET_ITER, (iterable -- iter, index_or_null)) { @@ -835,8 +875,6 @@ dummy_func(void) { } op(_CHECK_STACK_SPACE, (unused, unused, unused[oparg] -- unused, unused, unused[oparg])) { - assert(corresponding_check_stack == NULL); - corresponding_check_stack = this_instr; } op (_CHECK_STACK_SPACE_OPERAND, (framesize/2 -- )) { @@ -848,38 +886,29 @@ dummy_func(void) { op(_PUSH_FRAME, (new_frame -- )) { SYNC_SP(); - ctx->frame->stack_pointer = stack_pointer; + if (!CURRENT_FRAME_IS_INIT_SHIM()) { + ctx->frame->stack_pointer = stack_pointer; + } ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); ctx->curr_frame_depth++; stack_pointer = ctx->frame->stack_pointer; uint64_t operand = this_instr->operand0; - if (operand == 0 || (operand & 1)) { - // It's either a code object or NULL + if (operand == 0) { ctx->done = true; break; } - PyFunctionObject *func = (PyFunctionObject *)operand; - PyCodeObject *co = (PyCodeObject *)func->func_code; - assert(PyFunction_Check(func)); - ctx->frame->func = func; - /* Stack space handling */ - int framesize = co->co_framesize; - assert(framesize > 0); - curr_space += framesize; - if (curr_space < 0 || curr_space > INT32_MAX) { - // won't fit in signed 32-bit int - ctx->done = true; - break; + if (!(operand & 1)) { + PyFunctionObject *func = (PyFunctionObject *)operand; + // No need to re-add to dependencies here. Already + // handled by the tracer. + ctx->frame->func = func; } - max_space = curr_space > max_space ? curr_space : max_space; - if (first_valid_check_stack == NULL) { - first_valid_check_stack = corresponding_check_stack; + // Fixed calls don't need IP guards. + if ((this_instr-1)->opcode == _SAVE_RETURN_OFFSET || + (this_instr-1)->opcode == _CREATE_INIT_FRAME) { + assert((this_instr+1)->opcode == _GUARD_IP__PUSH_FRAME); + REPLACE_OP(this_instr+1, _NOP, 0, 0); } - else if (corresponding_check_stack) { - // delete all but the first valid _CHECK_STACK_SPACE - corresponding_check_stack->opcode = _NOP; - } - corresponding_check_stack = NULL; } op(_UNPACK_SEQUENCE, (seq -- values[oparg], top[0])) { @@ -1024,6 +1053,10 @@ dummy_func(void) { ctx->done = true; } + op(_DEOPT, (--)) { + ctx->done = true; + } + op(_REPLACE_WITH_TRUE, (value -- res)) { REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); res = sym_new_const(ctx, Py_True); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b08099d8e2f..01263fe8c7a 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -280,7 +280,6 @@ int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { sym_set_type(value, &PyBool_Type); - value = sym_new_truthiness(ctx, value, true); } stack_pointer[-1] = value; break; @@ -1116,16 +1115,24 @@ JitOptRef temp = PyJitRef_StripReferenceInfo(retval); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + int returning_stacklevel = this_instr->operand1; + if (ctx->curr_frame_depth >= 2) { + PyCodeObject *expected_code = ctx->frames[ctx->curr_frame_depth - 2].code; + if (expected_code == returning_code) { + assert((this_instr + 1)->opcode == _GUARD_IP_RETURN_VALUE); + REPLACE_OP((this_instr + 1), _NOP, 0, 0); + } + } + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } stack_pointer = ctx->frame->stack_pointer; - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; res = temp; stack_pointer[0] = res; stack_pointer += 1; @@ -1167,9 +1174,28 @@ } case _YIELD_VALUE: { + JitOptRef retval; JitOptRef value; - value = sym_new_unknown(ctx); - stack_pointer[-1] = value; + retval = stack_pointer[-1]; + JitOptRef temp = PyJitRef_StripReferenceInfo(retval); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + ctx->frame->stack_pointer = stack_pointer; + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + _Py_BloomFilter_Add(dependencies, returning_code); + int returning_stacklevel = this_instr->operand1; + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } + stack_pointer = ctx->frame->stack_pointer; + value = temp; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -2103,6 +2129,8 @@ break; } + /* _JUMP_BACKWARD_NO_INTERRUPT is not a viable micro-op for tier 2 */ + case _GET_LEN: { JitOptRef obj; JitOptRef len; @@ -2557,8 +2585,6 @@ } case _CHECK_STACK_SPACE: { - assert(corresponding_check_stack == NULL); - corresponding_check_stack = this_instr; break; } @@ -2601,34 +2627,26 @@ new_frame = stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - ctx->frame->stack_pointer = stack_pointer; + if (!CURRENT_FRAME_IS_INIT_SHIM()) { + ctx->frame->stack_pointer = stack_pointer; + } ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); ctx->curr_frame_depth++; stack_pointer = ctx->frame->stack_pointer; uint64_t operand = this_instr->operand0; - if (operand == 0 || (operand & 1)) { + if (operand == 0) { ctx->done = true; break; } - PyFunctionObject *func = (PyFunctionObject *)operand; - PyCodeObject *co = (PyCodeObject *)func->func_code; - assert(PyFunction_Check(func)); - ctx->frame->func = func; - int framesize = co->co_framesize; - assert(framesize > 0); - curr_space += framesize; - if (curr_space < 0 || curr_space > INT32_MAX) { - ctx->done = true; - break; + if (!(operand & 1)) { + PyFunctionObject *func = (PyFunctionObject *)operand; + ctx->frame->func = func; } - max_space = curr_space > max_space ? curr_space : max_space; - if (first_valid_check_stack == NULL) { - first_valid_check_stack = corresponding_check_stack; + if ((this_instr-1)->opcode == _SAVE_RETURN_OFFSET || + (this_instr-1)->opcode == _CREATE_INIT_FRAME) { + assert((this_instr+1)->opcode == _GUARD_IP__PUSH_FRAME); + REPLACE_OP(this_instr+1, _NOP, 0, 0); } - else if (corresponding_check_stack) { - corresponding_check_stack->opcode = _NOP; - } - corresponding_check_stack = NULL; break; } @@ -2761,9 +2779,24 @@ } case _CREATE_INIT_FRAME: { + JitOptRef *args; + JitOptRef self; JitOptRef init_frame; - init_frame = PyJitRef_NULL; - ctx->done = true; + args = &stack_pointer[-oparg]; + self = stack_pointer[-1 - oparg]; + ctx->frame->stack_pointer = stack_pointer - oparg - 2; + _Py_UOpsAbstractFrame *shim = frame_new(ctx, (PyCodeObject *)&_Py_InitCleanup, 0, NULL, 0); + if (shim == NULL) { + break; + } + shim->stack[0] = self; + shim->stack_pointer++; + assert((int)(shim->stack_pointer - shim->stack) == 1); + ctx->frame = shim; + ctx->curr_frame_depth++; + assert((this_instr + 1)->opcode == _PUSH_FRAME); + PyCodeObject *co = get_code_with_logging((this_instr + 1)); + init_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args-1, oparg+1)); stack_pointer[-2 - oparg] = init_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2948,8 +2981,13 @@ case _PY_FRAME_KW: { JitOptRef new_frame; - new_frame = PyJitRef_NULL; - ctx->done = true; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + PyCodeObject *co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; + } + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); stack_pointer[-3 - oparg] = new_frame; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -3005,17 +3043,19 @@ case _RETURN_GENERATOR: { JitOptRef res; - PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + _Py_BloomFilter_Add(dependencies, returning_code); + int returning_stacklevel = this_instr->operand1; + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } stack_pointer = ctx->frame->stack_pointer; res = sym_new_unknown(ctx); - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -3265,6 +3305,10 @@ break; } + case _DYNAMIC_EXIT: { + break; + } + case _CHECK_VALIDITY: { break; } @@ -3399,6 +3443,7 @@ } case _DEOPT: { + ctx->done = true; break; } @@ -3418,3 +3463,23 @@ break; } + case _COLD_DYNAMIC_EXIT: { + break; + } + + case _GUARD_IP__PUSH_FRAME: { + break; + } + + case _GUARD_IP_YIELD_VALUE: { + break; + } + + case _GUARD_IP_RETURN_VALUE: { + break; + } + + case _GUARD_IP_RETURN_GENERATOR: { + break; + } + diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 01cff0b014c..8a71eff465e 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -817,9 +817,14 @@ _Py_uop_frame_new( JitOptRef *args, int arg_len) { - assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); + if (ctx->curr_frame_depth >= MAX_ABSTRACT_FRAME_DEPTH) { + ctx->done = true; + ctx->out_of_space = true; + OPT_STAT_INC(optimizer_frame_overflow); + return NULL; + } _Py_UOpsAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth]; - + frame->code = co; frame->stack_len = co->co_stacksize; frame->locals_len = co->co_nlocalsplus; @@ -901,13 +906,42 @@ _Py_uop_abstractcontext_init(JitOptContext *ctx) } int -_Py_uop_frame_pop(JitOptContext *ctx) +_Py_uop_frame_pop(JitOptContext *ctx, PyCodeObject *co, int curr_stackentries) { _Py_UOpsAbstractFrame *frame = ctx->frame; ctx->n_consumed = frame->locals; + ctx->curr_frame_depth--; - assert(ctx->curr_frame_depth >= 1); - ctx->frame = &ctx->frames[ctx->curr_frame_depth - 1]; + + if (ctx->curr_frame_depth >= 1) { + ctx->frame = &ctx->frames[ctx->curr_frame_depth - 1]; + + // We returned to the correct code. Nothing to do here. + if (co == ctx->frame->code) { + return 0; + } + // Else: the code we recorded doesn't match the code we *think* we're + // returning to. We could trace anything, we can't just return to the + // old frame. We have to restore what the tracer recorded + // as the traced next frame. + // Remove the current frame, and later swap it out with the right one. + else { + ctx->curr_frame_depth--; + } + } + // Else: trace stack underflow. + + // This handles swapping out frames. + assert(curr_stackentries >= 1); + // -1 to stackentries as we push to the stack our return value after this. + _Py_UOpsAbstractFrame *new_frame = _Py_uop_frame_new(ctx, co, curr_stackentries - 1, NULL, 0); + if (new_frame == NULL) { + ctx->done = true; + return 1; + } + + ctx->curr_frame_depth++; + ctx->frame = new_frame; return 0; } diff --git a/Python/pystate.c b/Python/pystate.c index 341c680a403..c12a1418e74 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -552,10 +552,6 @@ init_interpreter(PyInterpreterState *interp, _Py_brc_init_state(interp); #endif -#ifdef _Py_TIER2 - // Ensure the buffer is to be set as NULL. - interp->jit_uop_buffer = NULL; -#endif llist_init(&interp->mem_free_queue.head); llist_init(&interp->asyncio_tasks_head); interp->asyncio_tasks_lock = (PyMutex){0}; @@ -805,10 +801,6 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) #ifdef _Py_TIER2 _Py_ClearExecutorDeletionList(interp); - if (interp->jit_uop_buffer != NULL) { - _PyObject_VirtualFree(interp->jit_uop_buffer, UOP_BUFFER_SIZE); - interp->jit_uop_buffer = NULL; - } #endif _PyAST_Fini(interp); _PyAtExit_Fini(interp); @@ -831,6 +823,14 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) assert(cold->vm_data.warm); _PyExecutor_Free(cold); } + + struct _PyExecutorObject *cold_dynamic = interp->cold_dynamic_executor; + if (cold_dynamic != NULL) { + interp->cold_dynamic_executor = NULL; + assert(cold_dynamic->vm_data.valid); + assert(cold_dynamic->vm_data.warm); + _PyExecutor_Free(cold_dynamic); + } /* We don't clear sysdict and builtins until the end of this function. Because clearing other attributes can execute arbitrary Python code which requires sysdict and builtins. */ @@ -1501,6 +1501,9 @@ init_threadstate(_PyThreadStateImpl *_tstate, _tstate->asyncio_running_loop = NULL; _tstate->asyncio_running_task = NULL; +#ifdef _Py_TIER2 + _tstate->jit_tracer_state.code_buffer = NULL; +#endif tstate->delete_later = NULL; llist_init(&_tstate->mem_free_queue); @@ -1807,6 +1810,14 @@ tstate_delete_common(PyThreadState *tstate, int release_gil) assert(tstate_impl->refcounts.values == NULL); #endif +#if _Py_TIER2 + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if (_tstate->jit_tracer_state.code_buffer != NULL) { + _PyObject_VirtualFree(_tstate->jit_tracer_state.code_buffer, UOP_BUFFER_SIZE); + _tstate->jit_tracer_state.code_buffer = NULL; + } +#endif + HEAD_UNLOCK(runtime); // XXX Unbind in PyThreadState_Clear(), or earlier diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 4621ad250f4..bd4a8cf0d3e 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -359,6 +359,7 @@ Parser/parser.c - soft_keywords - Parser/lexer/lexer.c - type_comment_prefix - Python/ceval.c - _PyEval_BinaryOps - Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS - +Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR - Python/codecs.c - Py_hexdigits - Python/codecs.c - codecs_builtin_error_handlers - Python/codecs.c - ucnhash_capi - diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 9dd7e5dbfba..d39013db4f7 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -34,6 +34,8 @@ class Properties: side_exit: bool pure: bool uses_opcode: bool + needs_guard_ip: bool + unpredictable_jump: bool tier: int | None = None const_oparg: int = -1 needs_prev: bool = False @@ -75,6 +77,8 @@ def from_list(properties: list["Properties"]) -> "Properties": pure=all(p.pure for p in properties), needs_prev=any(p.needs_prev for p in properties), no_save_ip=all(p.no_save_ip for p in properties), + needs_guard_ip=any(p.needs_guard_ip for p in properties), + unpredictable_jump=any(p.unpredictable_jump for p in properties), ) @property @@ -102,6 +106,8 @@ def infallible(self) -> bool: side_exit=False, pure=True, no_save_ip=False, + needs_guard_ip=False, + unpredictable_jump=False, ) @@ -692,6 +698,11 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_Wrap", "PyStackRef_Unwrap", "_PyLong_CheckExactAndCompact", + "_PyExecutor_FromExit", + "_PyJit_TryInitializeTracing", + "_Py_unset_eval_breaker_bit", + "_Py_set_eval_breaker_bit", + "trigger_backoff_counter", ) @@ -882,6 +893,46 @@ def stmt_escapes(stmt: Stmt) -> bool: else: assert False, "Unexpected statement type" +def stmt_has_jump_on_unpredictable_path_body(stmts: list[Stmt] | None, branches_seen: int) -> tuple[bool, int]: + if not stmts: + return False, branches_seen + predict = False + seen = 0 + for st in stmts: + predict_body, seen_body = stmt_has_jump_on_unpredictable_path(st, branches_seen) + predict = predict or predict_body + seen += seen_body + return predict, seen + +def stmt_has_jump_on_unpredictable_path(stmt: Stmt, branches_seen: int) -> tuple[bool, int]: + if isinstance(stmt, BlockStmt): + return stmt_has_jump_on_unpredictable_path_body(stmt.body, branches_seen) + elif isinstance(stmt, SimpleStmt): + for tkn in stmt.contents: + if tkn.text == "JUMPBY": + return True, branches_seen + return False, branches_seen + elif isinstance(stmt, IfStmt): + predict, seen = stmt_has_jump_on_unpredictable_path(stmt.body, branches_seen) + if stmt.else_body: + predict_else, seen_else = stmt_has_jump_on_unpredictable_path(stmt.else_body, branches_seen) + return predict != predict_else, seen + seen_else + 1 + return predict, seen + 1 + elif isinstance(stmt, MacroIfStmt): + predict, seen = stmt_has_jump_on_unpredictable_path_body(stmt.body, branches_seen) + if stmt.else_body: + predict_else, seen_else = stmt_has_jump_on_unpredictable_path_body(stmt.else_body, branches_seen) + return predict != predict_else, seen + seen_else + return predict, seen + elif isinstance(stmt, ForStmt): + unpredictable, branches_seen = stmt_has_jump_on_unpredictable_path(stmt.body, branches_seen) + return unpredictable, branches_seen + 1 + elif isinstance(stmt, WhileStmt): + unpredictable, branches_seen = stmt_has_jump_on_unpredictable_path(stmt.body, branches_seen) + return unpredictable, branches_seen + 1 + else: + assert False, f"Unexpected statement type {stmt}" + def compute_properties(op: parser.CodeDef) -> Properties: escaping_calls = find_escaping_api_calls(op) @@ -909,6 +960,8 @@ def compute_properties(op: parser.CodeDef) -> Properties: escapes = stmt_escapes(op.block) pure = False if isinstance(op, parser.LabelDef) else "pure" in op.annotations no_save_ip = False if isinstance(op, parser.LabelDef) else "no_save_ip" in op.annotations + unpredictable, branches_seen = stmt_has_jump_on_unpredictable_path(op.block, 0) + unpredictable_jump = False if isinstance(op, parser.LabelDef) else (unpredictable and branches_seen > 0) return Properties( escaping_calls=escaping_calls, escapes=escapes, @@ -932,6 +985,11 @@ def compute_properties(op: parser.CodeDef) -> Properties: no_save_ip=no_save_ip, tier=tier_variable(op), needs_prev=variable_used(op, "prev_instr"), + needs_guard_ip=(isinstance(op, parser.InstDef) + and (unpredictable_jump and "replaced" not in op.annotations)) + or variable_used(op, "LOAD_IP") + or variable_used(op, "DISPATCH_INLINED"), + unpredictable_jump=unpredictable_jump, ) def expand(items: list[StackItem], oparg: int) -> list[StackItem]: diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 61e855eb003..0b5f764ec52 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -7,6 +7,7 @@ analysis_error, Label, CodeSection, + Uop, ) from cwriter import CWriter from typing import Callable, TextIO, Iterator, Iterable @@ -107,8 +108,9 @@ class Emitter: labels: dict[str, Label] _replacers: dict[str, ReplacementFunctionType] cannot_escape: bool + jump_prefix: str - def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = False): + def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = False, jump_prefix: str = ""): self._replacers = { "EXIT_IF": self.exit_if, "AT_END_EXIT_IF": self.exit_if_after, @@ -131,6 +133,7 @@ def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = self.out = out self.labels = labels self.cannot_escape = cannot_escape + self.jump_prefix = jump_prefix def dispatch( self, @@ -167,7 +170,7 @@ def deopt_if( family_name = inst.family.name self.emit(f"UPDATE_MISS_STATS({family_name});\n") self.emit(f"assert(_PyOpcode_Deopt[opcode] == ({family_name}));\n") - self.emit(f"JUMP_TO_PREDICTED({family_name});\n") + self.emit(f"JUMP_TO_PREDICTED({self.jump_prefix}{family_name});\n") self.emit("}\n") return not always_true(first_tkn) @@ -198,10 +201,10 @@ def exit_if_after( def goto_error(self, offset: int, storage: Storage) -> str: if offset > 0: - return f"JUMP_TO_LABEL(pop_{offset}_error);" + return f"{self.jump_prefix}JUMP_TO_LABEL(pop_{offset}_error);" if offset < 0: storage.copy().flush(self.out) - return f"JUMP_TO_LABEL(error);" + return f"{self.jump_prefix}JUMP_TO_LABEL(error);" def error_if( self, @@ -421,7 +424,7 @@ def goto_label(self, goto: Token, label: Token, storage: Storage) -> None: elif storage.spilled: raise analysis_error("Cannot jump from spilled label without reloading the stack pointer", goto) self.out.start_line() - self.out.emit("JUMP_TO_LABEL(") + self.out.emit(f"{self.jump_prefix}JUMP_TO_LABEL(") self.out.emit(label) self.out.emit(")") @@ -731,6 +734,10 @@ def cflags(p: Properties) -> str: flags.append("HAS_PURE_FLAG") if p.no_save_ip: flags.append("HAS_NO_SAVE_IP_FLAG") + if p.unpredictable_jump: + flags.append("HAS_UNPREDICTABLE_JUMP_FLAG") + if p.needs_guard_ip: + flags.append("HAS_NEEDS_GUARD_IP_FLAG") if flags: return " | ".join(flags) else: diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index b649b381233..21ae785a0ec 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -56,6 +56,8 @@ "ERROR_NO_POP", "NO_SAVE_IP", "PERIODIC", + "UNPREDICTABLE_JUMP", + "NEEDS_GUARD_IP", ] @@ -201,7 +203,7 @@ def generate_metadata_table(analysis: Analysis, out: CWriter) -> None: out.emit("struct opcode_metadata {\n") out.emit("uint8_t valid_entry;\n") out.emit("uint8_t instr_format;\n") - out.emit("uint16_t flags;\n") + out.emit("uint32_t flags;\n") out.emit("};\n\n") out.emit( f"extern const struct opcode_metadata _PyOpcode_opcode_metadata[{table_size}];\n" diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index 324ef2773ab..36fa1d7fa49 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -31,6 +31,16 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: for target in targets: out.emit(target) out.emit("};\n") + targets = ["&&_unknown_opcode,\n"] * 256 + for name, op in analysis.opmap.items(): + if op < 256: + targets[op] = f"&&record_previous_inst,\n" + out.emit("#if _Py_TIER2\n") + out.emit("static void *opcode_tracing_targets_table[256] = {\n") + for target in targets: + out.emit(target) + out.emit("};\n") + out.emit(f"#endif\n") out.emit("#else /* _Py_TAIL_CALL_INTERP */\n") def function_proto(name: str) -> str: @@ -38,7 +48,9 @@ def function_proto(name: str) -> str: def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: - out.emit("static py_tail_call_funcptr instruction_funcptr_table[256];\n") + out.emit("static py_tail_call_funcptr instruction_funcptr_handler_table[256];\n") + out.emit("\n") + out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256];\n") out.emit("\n") # Emit function prototypes for labels. @@ -60,7 +72,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: out.emit("\n") # Emit the dispatch table. - out.emit("static py_tail_call_funcptr instruction_funcptr_table[256] = {\n") + out.emit("static py_tail_call_funcptr instruction_funcptr_handler_table[256] = {\n") for name in sorted(analysis.instructions.keys()): out.emit(f"[{name}] = _TAIL_CALL_{name},\n") named_values = analysis.opmap.values() @@ -68,6 +80,16 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: if rest not in named_values: out.emit(f"[{rest}] = _TAIL_CALL_UNKNOWN_OPCODE,\n") out.emit("};\n") + + # Emit the tracing dispatch table. + out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {\n") + for name in sorted(analysis.instructions.keys()): + out.emit(f"[{name}] = _TAIL_CALL_record_previous_inst,\n") + named_values = analysis.opmap.values() + for rest in range(256): + if rest not in named_values: + out.emit(f"[{rest}] = _TAIL_CALL_UNKNOWN_OPCODE,\n") + out.emit("};\n") outfile.write("#endif /* _Py_TAIL_CALL_INTERP */\n") arg_parser = argparse.ArgumentParser( diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 1bb5f48658d..ac3e6b94afe 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -63,6 +63,7 @@ class Tier2Emitter(Emitter): def __init__(self, out: CWriter, labels: dict[str, Label]): super().__init__(out, labels) self._replacers["oparg"] = self.oparg + self._replacers["IP_OFFSET_OF"] = self.ip_offset_of def goto_error(self, offset: int, storage: Storage) -> str: # To do: Add jump targets for popping values. @@ -134,10 +135,30 @@ def oparg( self.out.emit_at(uop.name[-1], tkn) return True + def ip_offset_of( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + assert uop.name.startswith("_GUARD_IP") + # LPAREN + next(tkn_iter) + tok = next(tkn_iter) + self.emit(f" OFFSET_OF_{tok.text};\n") + # RPAREN + next(tkn_iter) + # SEMI + next(tkn_iter) + return True -def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: +def write_uop(uop: Uop, emitter: Emitter, stack: Stack, offset_strs: dict[str, tuple[str, str]]) -> Stack: locals: dict[str, Local] = {} try: + if name_offset_pair := offset_strs.get(uop.name): + emitter.emit(f"#define OFFSET_OF_{name_offset_pair[0]} ({name_offset_pair[1]})\n") emitter.out.start_line() if uop.properties.oparg: emitter.emit("oparg = CURRENT_OPARG();\n") @@ -158,6 +179,8 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: idx += 1 _, storage = emitter.emit_tokens(uop, storage, None, False) storage.flush(emitter.out) + if name_offset_pair: + emitter.emit(f"#undef OFFSET_OF_{name_offset_pair[0]}\n") except StackError as ex: raise analysis_error(ex.args[0], uop.body.open) from None return storage.stack @@ -165,6 +188,29 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: SKIPS = ("_EXTENDED_ARG",) +def populate_offset_strs(analysis: Analysis) -> dict[str, tuple[str, str]]: + offset_strs: dict[str, tuple[str, str]] = {} + for name, uop in analysis.uops.items(): + if not f"_GUARD_IP_{name}" in analysis.uops: + continue + tkn_iter = uop.body.tokens() + found = False + offset_str = "" + for token in tkn_iter: + if token.kind == "IDENTIFIER" and token.text == "LOAD_IP": + if found: + raise analysis_error("Cannot have two LOAD_IP in a guarded single uop.", uop.body.open) + offset = [] + while token.kind != "SEMI": + offset.append(token.text) + token = next(tkn_iter) + # 1: to remove the LOAD_IP text + offset_str = "".join(offset[1:]) + found = True + assert offset_str + offset_strs[f"_GUARD_IP_{name}"] = (name, offset_str) + return offset_strs + def generate_tier2( filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool ) -> None: @@ -179,7 +225,9 @@ def generate_tier2( ) out = CWriter(outfile, 2, lines) emitter = Tier2Emitter(out, analysis.labels) + offset_strs = populate_offset_strs(analysis) out.emit("\n") + for name, uop in analysis.uops.items(): if uop.properties.tier == 1: continue @@ -194,13 +242,15 @@ def generate_tier2( out.emit(f"case {uop.name}: {{\n") declare_variables(uop, out) stack = Stack() - stack = write_uop(uop, emitter, stack) + stack = write_uop(uop, emitter, stack, offset_strs) out.start_line() if not uop.properties.always_exits: out.emit("break;\n") out.start_line() out.emit("}") out.emit("\n\n") + + out.emit("\n") outfile.write("#undef TIER_TWO\n") diff --git a/Tools/cases_generator/uop_metadata_generator.py b/Tools/cases_generator/uop_metadata_generator.py index 1cc23837a72..0e0396e5143 100644 --- a/Tools/cases_generator/uop_metadata_generator.py +++ b/Tools/cases_generator/uop_metadata_generator.py @@ -23,13 +23,13 @@ def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: - out.emit("extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1];\n") + out.emit("extern const uint32_t _PyUop_Flags[MAX_UOP_ID+1];\n") out.emit("typedef struct _rep_range { uint8_t start; uint8_t stop; } ReplicationRange;\n") out.emit("extern const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1];\n") out.emit("extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1];\n\n") out.emit("extern int _PyUop_num_popped(int opcode, int oparg);\n\n") out.emit("#ifdef NEED_OPCODE_METADATA\n") - out.emit("const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {\n") + out.emit("const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = {\n") for uop in analysis.uops.values(): if uop.is_viable() and uop.properties.tier != 1: out.emit(f"[{uop.name}] = {cflags(uop.properties)},\n") diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 2f146014a1c..857e926d119 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -55,13 +55,10 @@ do { \ __attribute__((musttail)) return jitted(frame, stack_pointer, tstate); \ } while (0) -#undef GOTO_TIER_ONE -#define GOTO_TIER_ONE(TARGET) \ -do { \ - tstate->current_executor = NULL; \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - return TARGET; \ -} while (0) +#undef GOTO_TIER_ONE_SETUP +#define GOTO_TIER_ONE_SETUP \ + tstate->current_executor = NULL; \ + _PyFrame_SetStackPointer(frame, stack_pointer); #undef LOAD_IP #define LOAD_IP(UNUSED) \ From 209eaff68c3b241c01aece14182cb9ced51526fc Mon Sep 17 00:00:00 2001 From: dr-carlos <77367421+dr-carlos@users.noreply.github.com> Date: Fri, 14 Nov 2025 04:47:17 +1030 Subject: [PATCH 192/313] gh-137969: Fix double evaluation of `ForwardRef`s which rely on globals (#140974) --- Lib/annotationlib.py | 39 +++++++++------- Lib/test/test_annotationlib.py | 45 +++++++++++++++++++ ...-11-04-15-40-35.gh-issue-137969.9VZQVt.rst | 3 ++ 3 files changed, 72 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 2166dbff0ee..33907b1fc2a 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -150,33 +150,42 @@ def evaluate( if globals is None: globals = {} + if type_params is None and owner is not None: + type_params = getattr(owner, "__type_params__", None) + if locals is None: locals = {} if isinstance(owner, type): locals.update(vars(owner)) + elif ( + type_params is not None + or isinstance(self.__cell__, dict) + or self.__extra_names__ + ): + # Create a new locals dict if necessary, + # to avoid mutating the argument. + locals = dict(locals) - if type_params is None and owner is not None: - # "Inject" type parameters into the local namespace - # (unless they are shadowed by assignments *in* the local namespace), - # as a way of emulating annotation scopes when calling `eval()` - type_params = getattr(owner, "__type_params__", None) - - # Type parameters exist in their own scope, which is logically - # between the locals and the globals. We simulate this by adding - # them to the globals. Similar reasoning applies to nonlocals stored in cells. - if type_params is not None or isinstance(self.__cell__, dict): - globals = dict(globals) + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` if type_params is not None: for param in type_params: - globals[param.__name__] = param + locals.setdefault(param.__name__, param) + + # Similar logic can be used for nonlocals, which should not + # override locals. if isinstance(self.__cell__, dict): - for cell_name, cell_value in self.__cell__.items(): + for cell_name, cell in self.__cell__.items(): try: - globals[cell_name] = cell_value.cell_contents + cell_value = cell.cell_contents except ValueError: pass + else: + locals.setdefault(cell_name, cell_value) + if self.__extra_names__: - locals = {**locals, **self.__extra_names__} + locals.update(self.__extra_names__) arg = self.__forward_arg__ if arg.isidentifier() and not keyword.iskeyword(arg): diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 9f3275d5071..8208d0e9c94 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -2149,6 +2149,51 @@ def test_fwdref_invalid_syntax(self): with self.assertRaises(SyntaxError): fr.evaluate() + def test_re_evaluate_generics(self): + global global_alias + + # If we've already run this test before, + # ensure the variable is still undefined + if "global_alias" in globals(): + del global_alias + + class C: + x: global_alias[int] + + # Evaluate the ForwardRef once + evaluated = get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + format=Format.FORWARDREF + ) + + # Now define the global and ensure that the ForwardRef evaluates + global_alias = list + self.assertEqual(evaluated.evaluate(), list[int]) + + def test_fwdref_evaluate_argument_mutation(self): + class C[T]: + nonlocal alias + x: alias[T] + + # Mutable arguments + globals_ = globals() + globals_copy = globals_.copy() + locals_ = locals() + locals_copy = locals_.copy() + + # Evaluate the ForwardRef, ensuring we use __cell__ and type params + get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + globals=globals_, + locals=locals_, + type_params=C.__type_params__, + format=Format.FORWARDREF, + ) + + # Check if the passed in mutable arguments equal the originals + self.assertEqual(globals_, globals_copy) + self.assertEqual(locals_, locals_copy) + + alias = list + def test_fwdref_final_class(self): with self.assertRaises(TypeError): class C(ForwardRef): diff --git a/Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst b/Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst new file mode 100644 index 00000000000..dfa582bdbc8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst @@ -0,0 +1,3 @@ +Fix :meth:`annotationlib.ForwardRef.evaluate` returning +:class:`~annotationlib.ForwardRef` objects which don't update with new +globals. From a486d452c78a7dfcd42561f6c151bf1fef0a756e Mon Sep 17 00:00:00 2001 From: Osama Abdelkader <78818069+osamakader@users.noreply.github.com> Date: Thu, 13 Nov 2025 20:05:28 +0100 Subject: [PATCH 193/313] gh-140601: Add ResourceWarning to iterparse when not closed (GH-140603) When iterparse() opens a file by filename and is not explicitly closed, emit a ResourceWarning to alert developers of the resource leak. Signed-off-by: Osama Abdelkader Co-authored-by: Serhiy Storchaka --- Doc/library/xml.etree.elementtree.rst | 4 ++ Doc/whatsnew/3.15.rst | 6 +++ Lib/test/test_xml_etree.py | 47 +++++++++++++++++++ Lib/xml/etree/ElementTree.py | 12 +++-- ...-10-25-22-55-07.gh-issue-140601.In3MlS.rst | 4 ++ 5 files changed, 69 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 881708a4dd7..cbbc87b4721 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -656,6 +656,10 @@ Functions .. versionchanged:: 3.13 Added the :meth:`!close` method. + .. versionchanged:: next + A :exc:`ResourceWarning` is now emitted if the iterator opened a file + and is not explicitly closed. + .. function:: parse(source, parser=None) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 895616e3049..31594a2e70b 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1244,3 +1244,9 @@ that may require changes to your code. * :meth:`~mmap.mmap.resize` has been removed on platforms that don't support the underlying syscall, instead of raising a :exc:`SystemError`. + +* Resource warning is now emitted for unclosed + :func:`xml.etree.ElementTree.iterparse` iterator if it opened a file. + Use its :meth:`!close` method or the :func:`contextlib.closing` context + manager to close it. + (Contributed by Osama Abdelkader and Serhiy Storchaka in :gh:`140601`.) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 25c084c8b9c..87811199706 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1436,17 +1436,39 @@ def test_nonexistent_file(self): def test_resource_warnings_not_exhausted(self): # Not exhausting the iterator still closes the underlying file (bpo-43292) + # Not closing before del should emit ResourceWarning it = ET.iterparse(SIMPLE_XMLFILE) with warnings_helper.check_no_resource_warning(self): + it.close() del it gc_collect() + it = ET.iterparse(SIMPLE_XMLFILE) + with self.assertWarns(ResourceWarning) as wm: + del it + gc_collect() + # Not 'unclosed file'. + self.assertIn('unclosed iterparse iterator', str(wm.warning)) + self.assertIn(repr(SIMPLE_XMLFILE), str(wm.warning)) + self.assertEqual(wm.filename, __file__) + it = ET.iterparse(SIMPLE_XMLFILE) with warnings_helper.check_no_resource_warning(self): + action, elem = next(it) + it.close() + self.assertEqual((action, elem.tag), ('end', 'element')) + del it, elem + gc_collect() + + it = ET.iterparse(SIMPLE_XMLFILE) + with self.assertWarns(ResourceWarning) as wm: action, elem = next(it) self.assertEqual((action, elem.tag), ('end', 'element')) del it, elem gc_collect() + self.assertIn('unclosed iterparse iterator', str(wm.warning)) + self.assertIn(repr(SIMPLE_XMLFILE), str(wm.warning)) + self.assertEqual(wm.filename, __file__) def test_resource_warnings_failed_iteration(self): self.addCleanup(os_helper.unlink, TESTFN) @@ -1461,16 +1483,41 @@ def test_resource_warnings_failed_iteration(self): next(it) self.assertEqual(str(cm.exception), 'junk after document element: line 1, column 12') + it.close() del cm, it gc_collect() + it = ET.iterparse(TESTFN) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'document')) + with self.assertWarns(ResourceWarning) as wm: + with self.assertRaises(ET.ParseError) as cm: + next(it) + self.assertEqual(str(cm.exception), + 'junk after document element: line 1, column 12') + del cm, it + gc_collect() + self.assertIn('unclosed iterparse iterator', str(wm.warning)) + self.assertIn(repr(TESTFN), str(wm.warning)) + self.assertEqual(wm.filename, __file__) + def test_resource_warnings_exhausted(self): it = ET.iterparse(SIMPLE_XMLFILE) with warnings_helper.check_no_resource_warning(self): list(it) + it.close() del it gc_collect() + it = ET.iterparse(SIMPLE_XMLFILE) + with self.assertWarns(ResourceWarning) as wm: + list(it) + del it + gc_collect() + self.assertIn('unclosed iterparse iterator', str(wm.warning)) + self.assertIn(repr(SIMPLE_XMLFILE), str(wm.warning)) + self.assertEqual(wm.filename, __file__) + def test_close_not_exhausted(self): iterparse = ET.iterparse diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index dafe5b1b8a0..d8c0b1b6216 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -1261,16 +1261,20 @@ def iterator(source): gen = iterator(source) class IterParseIterator(collections.abc.Iterator): __next__ = gen.__next__ + def close(self): + nonlocal close_source if close_source: source.close() + close_source = False gen.close() - def __del__(self): - # TODO: Emit a ResourceWarning if it was not explicitly closed. - # (When the close() method will be supported in all maintained Python versions.) + def __del__(self, _warn=warnings.warn): if close_source: - source.close() + try: + _warn(f"unclosed iterparse iterator {source.name!r}", ResourceWarning, stacklevel=2) + finally: + source.close() it = IterParseIterator() it.root = None diff --git a/Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst b/Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst new file mode 100644 index 00000000000..72666bb8224 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst @@ -0,0 +1,4 @@ +:func:`xml.etree.ElementTree.iterparse` now emits a :exc:`ResourceWarning` +when the iterator is not explicitly closed and was opened with a filename. +This helps developers identify and fix resource leaks. Patch by Osama +Abdelkader. From 4885ecfbda4cc792691e5d488ef6cb09727eb417 Mon Sep 17 00:00:00 2001 From: M Bussonnier Date: Fri, 14 Nov 2025 04:18:54 +0100 Subject: [PATCH 194/313] gh-140790: pdb: Initialize instance variables in Pdb.__init__ (#140791) Initialize lineno, stack, curindex, curframe, currentbp, and _user_requested_quit attributes in `Pdb.__init__``. --- Lib/pdb.py | 10 ++++++++-- .../2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst diff --git a/Lib/pdb.py b/Lib/pdb.py index fdc74198582..b799a113503 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -398,6 +398,12 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, self._current_task = None + self.lineno = None + self.stack = [] + self.curindex = 0 + self.curframe = None + self._user_requested_quit = False + def set_trace(self, frame=None, *, commands=None): Pdb._last_pdb_instance = self if frame is None: @@ -474,7 +480,7 @@ def forget(self): self.lineno = None self.stack = [] self.curindex = 0 - if hasattr(self, 'curframe') and self.curframe: + if self.curframe: self.curframe.f_globals.pop('__pdb_convenience_variables', None) self.curframe = None self.tb_lineno.clear() @@ -1493,7 +1499,7 @@ def checkline(self, filename, lineno, module_globals=None): """ # this method should be callable before starting debugging, so default # to "no globals" if there is no current frame - frame = getattr(self, 'curframe', None) + frame = self.curframe if module_globals is None: module_globals = frame.f_globals if frame else None line = linecache.getline(filename, lineno, module_globals) diff --git a/Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst b/Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst new file mode 100644 index 00000000000..03856f0b9b6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst @@ -0,0 +1 @@ +Initialize all Pdb's instance variables in ``__init__``, remove some hasattr/getattr From a4dd66275b62453bec055d730a8ce7173e519b6d Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 14 Nov 2025 10:38:49 +0100 Subject: [PATCH 195/313] gh-140550: Use a bool for the Py_mod_gil value (GH-141519) This needs a single bit, but was stored as a void* in the module struct. This didn't matter due to packing, but now that there's another bool in the struct, we can save a bit of memory by making md_gil a bool. Variables that changed type are renamed, to detect conflicts. --- Include/internal/pycore_moduleobject.h | 2 +- Lib/test/test_sys.py | 2 +- Objects/moduleobject.c | 15 ++++++++------- Python/import.c | 26 ++++++++++++++------------ 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index 6eef6eaa5df..9a62daf6621 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -30,7 +30,7 @@ typedef struct { PyObject *md_name; bool md_token_is_def; /* if true, `md_token` is the PyModuleDef */ #ifdef Py_GIL_DISABLED - void *md_gil; + bool md_requires_gil; #endif Py_ssize_t md_state_size; traverseproc md_state_traverse; diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 798f58737b1..2f169c1165d 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1725,7 +1725,7 @@ def get_gen(): yield 1 check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit) # module if support.Py_GIL_DISABLED: - md_gil = 'P' + md_gil = '?' else: md_gil = '' check(unittest, size('PPPP?' + md_gil + 'NPPPPP')) diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 9dee03bdb5e..6c1c5f5eb89 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -178,7 +178,7 @@ new_module_notrack(PyTypeObject *mt) m->md_name = NULL; m->md_token_is_def = false; #ifdef Py_GIL_DISABLED - m->md_gil = Py_MOD_GIL_USED; + m->md_requires_gil = true; #endif m->md_state_size = 0; m->md_state_traverse = NULL; @@ -361,7 +361,7 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) m->md_token_is_def = true; module_copy_members_from_deflike(m, module); #ifdef Py_GIL_DISABLED - m->md_gil = Py_MOD_GIL_USED; + m->md_requires_gil = true; #endif return (PyObject*)m; } @@ -380,7 +380,7 @@ module_from_def_and_spec( int has_multiple_interpreters_slot = 0; void *multiple_interpreters = (void *)0; int has_gil_slot = 0; - void *gil_slot = Py_MOD_GIL_USED; + bool requires_gil = true; int has_execution_slots = 0; const char *name; int ret; @@ -474,7 +474,7 @@ module_from_def_and_spec( name); goto error; } - gil_slot = cur_slot->value; + requires_gil = (cur_slot->value != Py_MOD_GIL_NOT_USED); has_gil_slot = 1; break; case Py_mod_abi: @@ -581,9 +581,9 @@ module_from_def_and_spec( mod->md_token = token; } #ifdef Py_GIL_DISABLED - mod->md_gil = gil_slot; + mod->md_requires_gil = requires_gil; #else - (void)gil_slot; + (void)requires_gil; #endif mod->md_exec = m_exec; } else { @@ -664,11 +664,12 @@ PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *slots, PyObject *spec) int PyUnstable_Module_SetGIL(PyObject *module, void *gil) { + bool requires_gil = (gil != Py_MOD_GIL_NOT_USED); if (!PyModule_Check(module)) { PyErr_BadInternalCall(); return -1; } - ((PyModuleObject *)module)->md_gil = gil; + ((PyModuleObject *)module)->md_requires_gil = requires_gil; return 0; } #endif diff --git a/Python/import.c b/Python/import.c index 2afa7c15e6a..b05b40448d0 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1017,9 +1017,10 @@ struct extensions_cache_value { _Py_ext_module_origin origin; #ifdef Py_GIL_DISABLED - /* The module's md_gil slot, for legacy modules that are reinitialized from - m_dict rather than calling their initialization function again. */ - void *md_gil; + /* The module's md_requires_gil member, for legacy modules that are + * reinitialized from m_dict rather than calling their initialization + * function again. */ + bool md_requires_gil; #endif }; @@ -1350,7 +1351,7 @@ static struct extensions_cache_value * _extensions_cache_set(PyObject *path, PyObject *name, PyModuleDef *def, PyModInitFunction m_init, Py_ssize_t m_index, PyObject *m_dict, - _Py_ext_module_origin origin, void *md_gil) + _Py_ext_module_origin origin, bool requires_gil) { struct extensions_cache_value *value = NULL; void *key = NULL; @@ -1405,11 +1406,11 @@ _extensions_cache_set(PyObject *path, PyObject *name, /* m_dict is set by set_cached_m_dict(). */ .origin=origin, #ifdef Py_GIL_DISABLED - .md_gil=md_gil, + .md_requires_gil=requires_gil, #endif }; #ifndef Py_GIL_DISABLED - (void)md_gil; + (void)requires_gil; #endif if (init_cached_m_dict(newvalue, m_dict) < 0) { goto finally; @@ -1547,7 +1548,8 @@ _PyImport_CheckGILForModule(PyObject* module, PyObject *module_name) } if (!PyModule_Check(module) || - ((PyModuleObject *)module)->md_gil == Py_MOD_GIL_USED) { + ((PyModuleObject *)module)->md_requires_gil) + { if (_PyEval_EnableGILPermanent(tstate)) { int warn_result = PyErr_WarnFormat( PyExc_RuntimeWarning, @@ -1725,7 +1727,7 @@ struct singlephase_global_update { Py_ssize_t m_index; PyObject *m_dict; _Py_ext_module_origin origin; - void *md_gil; + bool md_requires_gil; }; static struct extensions_cache_value * @@ -1784,7 +1786,7 @@ update_global_state_for_extension(PyThreadState *tstate, #endif cached = _extensions_cache_set( path, name, def, m_init, singlephase->m_index, m_dict, - singlephase->origin, singlephase->md_gil); + singlephase->origin, singlephase->md_requires_gil); if (cached == NULL) { // XXX Ignore this error? Doing so would effectively // mark the module as not loadable. @@ -1873,7 +1875,7 @@ reload_singlephase_extension(PyThreadState *tstate, if (def->m_base.m_copy != NULL) { // For non-core modules, fetch the GIL slot that was stored by // import_run_extension(). - ((PyModuleObject *)mod)->md_gil = cached->md_gil; + ((PyModuleObject *)mod)->md_requires_gil = cached->md_requires_gil; } #endif /* We can't set mod->md_def if it's missing, @@ -2128,7 +2130,7 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, .m_index=def->m_base.m_index, .origin=info->origin, #ifdef Py_GIL_DISABLED - .md_gil=((PyModuleObject *)mod)->md_gil, + .md_requires_gil=((PyModuleObject *)mod)->md_requires_gil, #endif }; // gh-88216: Extensions and def->m_base.m_copy can be updated @@ -2323,7 +2325,7 @@ _PyImport_FixupBuiltin(PyThreadState *tstate, PyObject *mod, const char *name, .origin=_Py_ext_module_origin_CORE, #ifdef Py_GIL_DISABLED /* Unused when m_dict == NULL. */ - .md_gil=NULL, + .md_requires_gil=false, #endif }; cached = update_global_state_for_extension( From 1e4e59bb3714ba7c6b6297f1a74e231b056f004c Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Fri, 14 Nov 2025 01:43:25 -0800 Subject: [PATCH 196/313] gh-116146: Add C-API to create module from spec and initfunc (GH-139196) Co-authored-by: Kumar Aditya Co-authored-by: Petr Viktorin Co-authored-by: Victor Stinner --- Doc/c-api/import.rst | 21 ++++ Doc/whatsnew/3.15.rst | 4 + Include/cpython/import.h | 7 ++ Lib/test/test_embed.py | 25 ++++ ...-11-08-10-51-50.gh-issue-116146.pCmx6L.rst | 2 + Programs/_testembed.c | 111 ++++++++++++++++++ Python/import.c | 74 ++++++++---- 7 files changed, 223 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 8eabc0406b1..24e673d3d13 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -333,3 +333,24 @@ Importing Modules strings instead of Python :class:`str` objects. .. versionadded:: 3.14 + +.. c:function:: PyObject* PyImport_CreateModuleFromInitfunc(PyObject *spec, PyObject* (*initfunc)(void)) + + This function is a building block that enables embedders to implement + the :py:meth:`~importlib.abc.Loader.create_module` step of custom + static extension importers (e.g. of statically-linked extensions). + + *spec* must be a :class:`~importlib.machinery.ModuleSpec` object. + + *initfunc* must be an :ref:`initialization function `, + the same as for :c:func:`PyImport_AppendInittab`. + + On success, create and return a module object. + This module will not be initialized; call :c:func:`!PyModule_Exec` + to initialize it. + (Custom importers should do this in their + :py:meth:`~importlib.abc.Loader.exec_module` method.) + + On error, return NULL with an exception set. + + .. versionadded:: next diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 31594a2e70b..9393b65ed8e 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1080,6 +1080,10 @@ New features thread state. (Contributed by Victor Stinner in :gh:`139653`.) +* Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating + a module from a *spec* and *initfunc*. + (Contributed by Itamar Oren in :gh:`116146`.) + Changed C APIs -------------- diff --git a/Include/cpython/import.h b/Include/cpython/import.h index 0ce0b1ee6cc..149a20af8b9 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -10,6 +10,13 @@ struct _inittab { PyAPI_DATA(struct _inittab *) PyImport_Inittab; PyAPI_FUNC(int) PyImport_ExtendInittab(struct _inittab *newtab); +// Custom importers may use this API to initialize statically linked +// extension modules directly from a spec and init function, +// without needing to go through inittab +PyAPI_FUNC(PyObject *) PyImport_CreateModuleFromInitfunc( + PyObject *spec, + PyObject *(*initfunc)(void)); + struct _frozen { const char *name; /* ASCII encoded string */ const unsigned char *code; diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 1933f691a78..1078796eae8 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -239,6 +239,31 @@ def test_repeated_init_and_inittab(self): lines = "\n".join(lines) + "\n" self.assertEqual(out, lines) + def test_create_module_from_initfunc(self): + out, err = self.run_embedded_interpreter("test_create_module_from_initfunc") + if support.Py_GIL_DISABLED: + # the test imports a singlephase init extension, so it emits a warning + # under the free-threaded build + expected_runtime_warning = ( + "RuntimeWarning: The global interpreter lock (GIL)" + " has been enabled to load module 'embedded_ext'" + ) + filtered_err_lines = [ + line + for line in err.strip().splitlines() + if expected_runtime_warning not in line + ] + self.assertEqual(filtered_err_lines, []) + else: + self.assertEqual(err, "") + self.assertEqual(out, + "\n" + "my_test_extension.executed='yes'\n" + "my_test_extension.exec_slot_ran='yes'\n" + "\n" + "embedded_ext.executed='yes'\n" + ) + def test_forced_io_encoding(self): # Checks forced configuration of embedded interpreter IO streams env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape") diff --git a/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst b/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst new file mode 100644 index 00000000000..be8043e26dd --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst @@ -0,0 +1,2 @@ +Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating a +module from a *spec* and *initfunc*. Patch by Itamar Oren. diff --git a/Programs/_testembed.c b/Programs/_testembed.c index d3600fecbe2..27224e508bd 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -166,6 +166,8 @@ static PyModuleDef embedded_ext = { static PyObject* PyInit_embedded_ext(void) { + // keep this as a single-phase initialization module; + // see test_create_module_from_initfunc return PyModule_Create(&embedded_ext); } @@ -1894,8 +1896,16 @@ static int test_initconfig_exit(void) } +int +extension_module_exec(PyObject *mod) +{ + return PyModule_AddStringConstant(mod, "exec_slot_ran", "yes"); +} + + static PyModuleDef_Slot extension_slots[] = { {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {Py_mod_exec, extension_module_exec}, {0, NULL} }; @@ -2213,6 +2223,106 @@ static int test_repeated_init_and_inittab(void) return 0; } +static PyObject* +create_module(PyObject* self, PyObject* spec) +{ + PyObject *name = PyObject_GetAttrString(spec, "name"); + if (!name) { + return NULL; + } + if (PyUnicode_EqualToUTF8(name, "my_test_extension")) { + Py_DECREF(name); + return PyImport_CreateModuleFromInitfunc(spec, init_my_test_extension); + } + if (PyUnicode_EqualToUTF8(name, "embedded_ext")) { + Py_DECREF(name); + return PyImport_CreateModuleFromInitfunc(spec, PyInit_embedded_ext); + } + PyErr_Format(PyExc_LookupError, "static module %R not found", name); + Py_DECREF(name); + return NULL; +} + +static PyObject* +exec_module(PyObject* self, PyObject* mod) +{ + if (PyModule_Exec(mod) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyMethodDef create_static_module_methods[] = { + {"create_module", create_module, METH_O, NULL}, + {"exec_module", exec_module, METH_O, NULL}, + {} +}; + +static struct PyModuleDef create_static_module_def = { + PyModuleDef_HEAD_INIT, + .m_name = "create_static_module", + .m_size = 0, + .m_methods = create_static_module_methods, + .m_slots = extension_slots, +}; + +PyMODINIT_FUNC PyInit_create_static_module(void) { + return PyModuleDef_Init(&create_static_module_def); +} + +static int +test_create_module_from_initfunc(void) +{ + wchar_t* argv[] = { + PROGRAM_NAME, + L"-c", + // Multi-phase initialization + L"import my_test_extension;" + L"print(my_test_extension);" + L"print(f'{my_test_extension.executed=}');" + L"print(f'{my_test_extension.exec_slot_ran=}');" + // Single-phase initialization + L"import embedded_ext;" + L"print(embedded_ext);" + L"print(f'{embedded_ext.executed=}');" + }; + PyConfig config; + if (PyImport_AppendInittab("create_static_module", + &PyInit_create_static_module) != 0) { + fprintf(stderr, "PyImport_AppendInittab() failed\n"); + return 1; + } + PyConfig_InitPythonConfig(&config); + config.isolated = 1; + config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv); + init_from_config_clear(&config); + int result = PyRun_SimpleString( + "import sys\n" + "from importlib.util import spec_from_loader\n" + "import create_static_module\n" + "class StaticExtensionImporter:\n" + " _ORIGIN = \"static-extension\"\n" + " @classmethod\n" + " def find_spec(cls, fullname, path, target=None):\n" + " if fullname in {'my_test_extension', 'embedded_ext'}:\n" + " return spec_from_loader(fullname, cls, origin=cls._ORIGIN)\n" + " return None\n" + " @staticmethod\n" + " def create_module(spec):\n" + " return create_static_module.create_module(spec)\n" + " @staticmethod\n" + " def exec_module(module):\n" + " create_static_module.exec_module(module)\n" + " module.executed = 'yes'\n" + "sys.meta_path.append(StaticExtensionImporter)\n" + ); + if (result < 0) { + fprintf(stderr, "PyRun_SimpleString() failed\n"); + return 1; + } + return Py_RunMain(); +} + static void wrap_allocator(PyMemAllocatorEx *allocator); static void unwrap_allocator(PyMemAllocatorEx *allocator); @@ -2396,6 +2506,7 @@ static struct TestCase TestCases[] = { #endif {"test_get_incomplete_frame", test_get_incomplete_frame}, {"test_gilstate_after_finalization", test_gilstate_after_finalization}, + {"test_create_module_from_initfunc", test_create_module_from_initfunc}, {NULL, NULL} }; diff --git a/Python/import.c b/Python/import.c index b05b40448d0..9ab2d3b3552 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2364,8 +2364,23 @@ is_builtin(PyObject *name) return 0; } +static PyModInitFunction +lookup_inittab_initfunc(const struct _Py_ext_module_loader_info* info) +{ + for (struct _inittab *p = INITTAB; p->name != NULL; p++) { + if (_PyUnicode_EqualToASCIIString(info->name, p->name)) { + return (PyModInitFunction)p->initfunc; + } + } + // not found + return NULL; +} + static PyObject* -create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) +create_builtin( + PyThreadState *tstate, PyObject *name, + PyObject *spec, + PyModInitFunction initfunc) { struct _Py_ext_module_loader_info info; if (_Py_ext_module_loader_info_init_for_builtin(&info, name) < 0) { @@ -2396,25 +2411,15 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) _extensions_cache_delete(info.path, info.name); } - struct _inittab *found = NULL; - for (struct _inittab *p = INITTAB; p->name != NULL; p++) { - if (_PyUnicode_EqualToASCIIString(info.name, p->name)) { - found = p; - break; - } - } - if (found == NULL) { - // not found - mod = Py_NewRef(Py_None); - goto finally; - } - - PyModInitFunction p0 = (PyModInitFunction)found->initfunc; + PyModInitFunction p0 = initfunc; if (p0 == NULL) { - /* Cannot re-init internal module ("sys" or "builtins") */ - assert(is_core_module(tstate->interp, info.name, info.path)); - mod = import_add_module(tstate, info.name); - goto finally; + p0 = lookup_inittab_initfunc(&info); + if (p0 == NULL) { + /* Cannot re-init internal module ("sys" or "builtins") */ + assert(is_core_module(tstate->interp, info.name, info.path)); + mod = import_add_module(tstate, info.name); + goto finally; + } } #ifdef Py_GIL_DISABLED @@ -2440,6 +2445,33 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) return mod; } +PyObject* +PyImport_CreateModuleFromInitfunc( + PyObject *spec, PyObject *(*initfunc)(void)) +{ + if (initfunc == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + + PyThreadState *tstate = _PyThreadState_GET(); + + PyObject *name = PyObject_GetAttr(spec, &_Py_ID(name)); + if (name == NULL) { + return NULL; + } + + if (!PyUnicode_Check(name)) { + PyErr_Format(PyExc_TypeError, + "spec name must be string, not %T", name); + Py_DECREF(name); + return NULL; + } + + PyObject *mod = create_builtin(tstate, name, spec, initfunc); + Py_DECREF(name); + return mod; +} /*****************************/ /* the builtin modules table */ @@ -3209,7 +3241,7 @@ bootstrap_imp(PyThreadState *tstate) } // Create the _imp module from its definition. - PyObject *mod = create_builtin(tstate, name, spec); + PyObject *mod = create_builtin(tstate, name, spec, NULL); Py_CLEAR(name); Py_DECREF(spec); if (mod == NULL) { @@ -4369,7 +4401,7 @@ _imp_create_builtin(PyObject *module, PyObject *spec) return NULL; } - PyObject *mod = create_builtin(tstate, name, spec); + PyObject *mod = create_builtin(tstate, name, spec, NULL); Py_DECREF(name); return mod; } From 181a2f4f2e3bed8dc6be5630e9bfb3362194ab3a Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Fri, 14 Nov 2025 11:59:19 +0200 Subject: [PATCH 197/313] gh-139596: Cease caching config.cache & ccache in GH Actions (#141451) --- .github/workflows/build.yml | 5 ----- .github/workflows/reusable-context.yml | 9 --------- .github/workflows/reusable-macos.yml | 3 --- .github/workflows/reusable-san.yml | 3 --- .github/workflows/reusable-ubuntu.yml | 3 --- .github/workflows/reusable-wasi.yml | 6 +----- .gitignore | 1 - 7 files changed, 1 insertion(+), 29 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a0f60c30ac8..8e15400e497 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -205,7 +205,6 @@ jobs: free-threading: true uses: ./.github/workflows/reusable-macos.yml with: - config_hash: ${{ needs.build-context.outputs.config-hash }} free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} @@ -237,7 +236,6 @@ jobs: bolt: true uses: ./.github/workflows/reusable-ubuntu.yml with: - config_hash: ${{ needs.build-context.outputs.config-hash }} bolt-optimizations: ${{ matrix.bolt }} free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} @@ -414,8 +412,6 @@ jobs: needs: build-context if: needs.build-context.outputs.run-tests == 'true' uses: ./.github/workflows/reusable-wasi.yml - with: - config_hash: ${{ needs.build-context.outputs.config-hash }} test-hypothesis: name: "Hypothesis tests on Ubuntu" @@ -600,7 +596,6 @@ jobs: uses: ./.github/workflows/reusable-san.yml with: sanitizer: ${{ matrix.sanitizer }} - config_hash: ${{ needs.build-context.outputs.config-hash }} free-threading: ${{ matrix.free-threading }} cross-build-linux: diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml index d2668ddcac1..66c7cc47de0 100644 --- a/.github/workflows/reusable-context.yml +++ b/.github/workflows/reusable-context.yml @@ -17,9 +17,6 @@ on: # yamllint disable-line rule:truthy # || 'falsy-branch' # }} # - config-hash: - description: Config hash value for use in cache keys - value: ${{ jobs.compute-changes.outputs.config-hash }} # str run-docs: description: Whether to build the docs value: ${{ jobs.compute-changes.outputs.run-docs }} # bool @@ -42,7 +39,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 outputs: - config-hash: ${{ steps.config-hash.outputs.hash }} run-ci-fuzz: ${{ steps.changes.outputs.run-ci-fuzz }} run-docs: ${{ steps.changes.outputs.run-docs }} run-tests: ${{ steps.changes.outputs.run-tests }} @@ -100,8 +96,3 @@ jobs: GITHUB_EVENT_NAME: ${{ github.event_name }} CCF_TARGET_REF: ${{ github.base_ref || github.event.repository.default_branch }} CCF_HEAD_REF: ${{ github.event.pull_request.head.sha || github.sha }} - - - name: Compute hash for config cache key - id: config-hash - run: | - echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index d85c46b96f8..98d557ba1ea 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -3,9 +3,6 @@ name: Reusable macOS on: workflow_call: inputs: - config_hash: - required: true - type: string free-threading: required: false type: boolean diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index 7fe96d1b238..c601d0b7338 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -6,9 +6,6 @@ on: sanitizer: required: true type: string - config_hash: - required: true - type: string free-threading: description: Whether to use free-threaded mode required: false diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 7b93b5f51b0..0c1ebe29ae3 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -3,9 +3,6 @@ name: Reusable Ubuntu on: workflow_call: inputs: - config_hash: - required: true - type: string bolt-optimizations: description: Whether to enable BOLT optimizations required: false diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 8f412288f53..a309ef4e7f4 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -2,10 +2,6 @@ name: Reusable WASI on: workflow_call: - inputs: - config_hash: - required: true - type: string env: FORCE_COLOR: 1 @@ -53,7 +49,7 @@ jobs: - name: "Configure build Python" run: python3 Tools/wasm/wasi configure-build-python -- --config-cache --with-pydebug - name: "Make build Python" - run: python3 Tools/wasm/wasi.py make-build-python + run: python3 Tools/wasm/wasi make-build-python - name: "Configure host" # `--with-pydebug` inferred from configure-build-python run: python3 Tools/wasm/wasi configure-host -- --config-cache diff --git a/.gitignore b/.gitignore index 2bf4925647d..4ea2fd96554 100644 --- a/.gitignore +++ b/.gitignore @@ -135,7 +135,6 @@ Tools/unicode/data/ /config.log /config.status /config.status.lineno -# hendrikmuhs/ccache-action@v1 /.ccache /cross-build/ /jit_stencils*.h From 3bacae55980561cb99095a20a70c45d6174e056d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 14 Nov 2025 11:13:24 +0100 Subject: [PATCH 198/313] gh-131510: Use PyUnstable_Unicode_GET_CACHED_HASH() (GH-141520) Replace code that directly accesses PyASCIIObject.hash with PyUnstable_Unicode_GET_CACHED_HASH(). Remove redundant "assert(PyUnicode_Check(op))" from PyUnstable_Unicode_GET_CACHED_HASH(), _PyASCIIObject_CAST() already implements the check. --- Include/cpython/unicodeobject.h | 1 - Include/internal/pycore_object.h | 3 +-- Objects/dictobject.c | 3 +-- Objects/typeobject.c | 2 +- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 73e3bc44d6c..2853d24c34b 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -301,7 +301,6 @@ static inline Py_ssize_t PyUnicode_GET_LENGTH(PyObject *op) { /* Returns the cached hash, or -1 if not cached yet. */ static inline Py_hash_t PyUnstable_Unicode_GET_CACHED_HASH(PyObject *op) { - assert(PyUnicode_Check(op)); #ifdef Py_GIL_DISABLED return _Py_atomic_load_ssize_relaxed(&_PyASCIIObject_CAST(op)->hash); #else diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 980d6d7764b..fb50acd62da 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -863,8 +863,7 @@ static inline Py_hash_t _PyObject_HashFast(PyObject *op) { if (PyUnicode_CheckExact(op)) { - Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED( - _PyASCIIObject_CAST(op)->hash); + Py_hash_t hash = PyUnstable_Unicode_GET_CACHED_HASH(op); if (hash != -1) { return hash; } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 65eed151c28..14de21f3c67 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -400,8 +400,7 @@ static int _PyObject_InlineValuesConsistencyCheck(PyObject *obj); static inline Py_hash_t unicode_get_hash(PyObject *o) { - assert(PyUnicode_CheckExact(o)); - return FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyASCIIObject_CAST(o)->hash); + return PyUnstable_Unicode_GET_CACHED_HASH(o); } /* Print summary info about the state of the optimized allocator */ diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 58228d62485..61bcc21ce13 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6036,7 +6036,7 @@ static PyObject * update_cache(struct type_cache_entry *entry, PyObject *name, unsigned int version_tag, PyObject *value) { _Py_atomic_store_ptr_relaxed(&entry->value, value); /* borrowed */ - assert(_PyASCIIObject_CAST(name)->hash != -1); + assert(PyUnstable_Unicode_GET_CACHED_HASH(name) != -1); OBJECT_STAT_INC_COND(type_cache_collisions, entry->name != Py_None && entry->name != name); // We're releasing this under the lock for simplicity sake because it's always a // exact unicode object or Py_None so it's safe to do so. From 5ac0b55ebc792936184f8e08697e60d5b3f8b946 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 14 Nov 2025 11:22:18 +0100 Subject: [PATCH 199/313] gh-141376: Remove exceptions from `make smelly` (GH-141392) * Don't ignore initialized data and BSS * Remove exceptions for _init and _fini --- Tools/build/smelly.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/Tools/build/smelly.py b/Tools/build/smelly.py index 9a360412a73..424fa6ad4a1 100755 --- a/Tools/build/smelly.py +++ b/Tools/build/smelly.py @@ -21,8 +21,6 @@ }) IGNORED_EXTENSION = "_ctypes_test" -# Ignore constructor and destructor functions -IGNORED_SYMBOLS = {'_init', '_fini'} def is_local_symbol_type(symtype): @@ -34,19 +32,12 @@ def is_local_symbol_type(symtype): if symtype.islower() and symtype not in "uvw": return True - # Ignore the initialized data section (d and D) and the BSS data - # section. For example, ignore "__bss_start (type: B)" - # and "_edata (type: D)". - if symtype in "bBdD": - return True - return False def get_exported_symbols(library, dynamic=False): print(f"Check that {library} only exports symbols starting with Py or _Py") - # Only look at dynamic symbols args = ['nm', '--no-sort'] if dynamic: args.append('--dynamic') @@ -89,8 +80,6 @@ def get_smelly_symbols(stdout, dynamic=False): if is_local_symbol_type(symtype): local_symbols.append(result) - elif symbol in IGNORED_SYMBOLS: - local_symbols.append(result) else: smelly_symbols.append(result) From ef90261be508b97d682589aac8f00065a9585683 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Fri, 14 Nov 2025 11:20:36 +0000 Subject: [PATCH 200/313] gh-141004: Document `PyOS_InterruptOccurred` (GH-141526) --- Doc/c-api/sys.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 336e3ef9640..ee73c1c8ada 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -123,6 +123,24 @@ Operating System Utilities This is a thin wrapper around either :c:func:`!sigaction` or :c:func:`!signal`. Do not call those functions directly! + +.. c:function:: int PyOS_InterruptOccurred(void) + + Check if a :c:macro:`!SIGINT` signal has been received. + + Returns ``1`` if a :c:macro:`!SIGINT` has occurred and clears the signal flag, + or ``0`` otherwise. + + In most cases, you should prefer :c:func:`PyErr_CheckSignals` over this function. + :c:func:`!PyErr_CheckSignals` invokes the appropriate signal handlers + for all pending signals, allowing Python code to handle the signal properly. + This function only detects :c:macro:`!SIGINT` and does not invoke any Python + signal handlers. + + This function is async-signal-safe and this function cannot fail. + The caller must hold an :term:`attached thread state`. + + .. c:function:: wchar_t* Py_DecodeLocale(const char* arg, size_t *size) .. warning:: From c10fa5be6167b1338ad194f9fe4be4782e025175 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Fri, 14 Nov 2025 09:22:36 -0500 Subject: [PATCH 201/313] gh-131229: Temporarily skip `test_basic_multiple_interpreters_deleted_no_reset` (GH-141552) This is a temporary band-aid to unblock other PRs. Co-authored-by: Kumar Aditya --- Lib/test/test_import/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index fe669bb04df..fd9750eae80 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -3261,6 +3261,7 @@ def test_basic_multiple_interpreters_main_no_reset(self): # * m_copy was copied from interp2 (was from interp1) # * module's global state was updated, not reset + @unittest.skip("gh-131229: This is suddenly very flaky") @no_rerun(reason="rerun not possible; module state is never cleared (see gh-102251)") @requires_subinterpreters def test_basic_multiple_interpreters_deleted_no_reset(self): From 8deaa9393eadf84e6e571be611e0c5a377abf7cd Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 14 Nov 2025 16:49:28 +0200 Subject: [PATCH 202/313] gh-122255: Synchronize warnings in C and Python implementations of the warnings module (GH-122824) In the linecache module and in the Python implementation of the warnings module, a DeprecationWarning is issued when m.__loader__ differs from m.__spec__.loader (like in the C implementation of the warnings module). --- Lib/linecache.py | 63 +++++++++++++++---- Lib/test/test_linecache.py | 32 ++++++++-- Lib/test/test_warnings/__init__.py | 5 +- ...-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst | 4 ++ 4 files changed, 82 insertions(+), 22 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst diff --git a/Lib/linecache.py b/Lib/linecache.py index ef3b2d9136b..b5bf9dbdd3c 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -224,21 +224,58 @@ def lazycache(filename, module_globals): def _make_lazycache_entry(filename, module_globals): if not filename or (filename.startswith('<') and filename.endswith('>')): return None - # Try for a __loader__, if available - if module_globals and '__name__' in module_globals: - spec = module_globals.get('__spec__') - name = getattr(spec, 'name', None) or module_globals['__name__'] - loader = getattr(spec, 'loader', None) - if loader is None: - loader = module_globals.get('__loader__') - get_source = getattr(loader, 'get_source', None) - if name and get_source: - def get_lines(name=name, *args, **kwargs): - return get_source(name, *args, **kwargs) - return (get_lines,) - return None + if module_globals is not None and not isinstance(module_globals, dict): + raise TypeError(f'module_globals must be a dict, not {type(module_globals).__qualname__}') + if not module_globals or '__name__' not in module_globals: + return None + spec = module_globals.get('__spec__') + name = getattr(spec, 'name', None) or module_globals['__name__'] + if name is None: + return None + + loader = _bless_my_loader(module_globals) + if loader is None: + return None + + get_source = getattr(loader, 'get_source', None) + if get_source is None: + return None + + def get_lines(name=name, *args, **kwargs): + return get_source(name, *args, **kwargs) + return (get_lines,) + +def _bless_my_loader(module_globals): + # Similar to _bless_my_loader() in importlib._bootstrap_external, + # but always emits warnings instead of errors. + loader = module_globals.get('__loader__') + if loader is None and '__spec__' not in module_globals: + return None + spec = module_globals.get('__spec__') + + # The __main__ module has __spec__ = None. + if spec is None and module_globals.get('__name__') == '__main__': + return loader + + spec_loader = getattr(spec, 'loader', None) + if spec_loader is None: + import warnings + warnings.warn( + 'Module globals is missing a __spec__.loader', + DeprecationWarning) + return loader + + assert spec_loader is not None + if loader is not None and loader != spec_loader: + import warnings + warnings.warn( + 'Module globals; __loader__ != __spec__.loader', + DeprecationWarning) + return loader + + return spec_loader def _register_code(code, string, name): diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index 02f65338428..fcd94edc611 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -259,22 +259,44 @@ def raise_memoryerror(*args, **kwargs): def test_loader(self): filename = 'scheme://path' - for loader in (None, object(), NoSourceLoader()): + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': None} + self.assertEqual(linecache.getlines(filename, module_globals), []) + + for loader in object(), NoSourceLoader(): linecache.clearcache() module_globals = {'__name__': 'a.b.c', '__loader__': loader} - self.assertEqual(linecache.getlines(filename, module_globals), []) + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(linecache.getlines(filename, module_globals), []) + self.assertEqual(str(w.warning), + 'Module globals is missing a __spec__.loader') linecache.clearcache() module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader()} - self.assertEqual(linecache.getlines(filename, module_globals), - ['source for a.b.c\n']) + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for a.b.c\n']) + self.assertEqual(str(w.warning), + 'Module globals is missing a __spec__.loader') - for spec in (None, object(), ModuleSpec('', FakeLoader())): + for spec in None, object(): linecache.clearcache() module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader(), '__spec__': spec} + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for a.b.c\n']) + self.assertEqual(str(w.warning), + 'Module globals is missing a __spec__.loader') + + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader(), + '__spec__': ModuleSpec('', FakeLoader())} + with self.assertWarns(DeprecationWarning) as w: self.assertEqual(linecache.getlines(filename, module_globals), ['source for a.b.c\n']) + self.assertEqual(str(w.warning), + 'Module globals; __loader__ != __spec__.loader') linecache.clearcache() spec = ModuleSpec('x.y.z', FakeLoader()) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index e6666ddc638..a6af5057cc8 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -727,7 +727,7 @@ def check_module_globals(self, module_globals): def check_module_globals_error(self, module_globals, errmsg, errtype=ValueError): if self.module is py_warnings: - self.check_module_globals(module_globals) + self.check_module_globals_deprecated(module_globals, errmsg) return with self.module.catch_warnings(record=True) as w: self.module.filterwarnings('always') @@ -738,9 +738,6 @@ def check_module_globals_error(self, module_globals, errmsg, errtype=ValueError) self.assertEqual(len(w), 0) def check_module_globals_deprecated(self, module_globals, msg): - if self.module is py_warnings: - self.check_module_globals(module_globals) - return with self.module.catch_warnings(record=True) as w: self.module.filterwarnings('always') self.module.warn_explicit( diff --git a/Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst b/Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst new file mode 100644 index 00000000000..63e71c19f8b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst @@ -0,0 +1,4 @@ +In the :mod:`linecache` module and in the Python implementation of the +:mod:`warnings` module, a ``DeprecationWarning`` is issued when +``mod.__loader__`` differs from ``mod.__spec__.loader`` (like in the C +implementation of the :mod:`!warnings` module). From 49e74210cb652d8bd538a4cc887f507396cfc893 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 14 Nov 2025 15:50:03 +0100 Subject: [PATCH 203/313] gh-139344: Remove pending removal notice for undeprecated importlib.resources API (GH-141507) --- Doc/deprecations/pending-removal-in-3.13.rst | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/Doc/deprecations/pending-removal-in-3.13.rst b/Doc/deprecations/pending-removal-in-3.13.rst index 2fd2f12cc6a..d5b8c80e8f9 100644 --- a/Doc/deprecations/pending-removal-in-3.13.rst +++ b/Doc/deprecations/pending-removal-in-3.13.rst @@ -38,15 +38,3 @@ APIs: * :meth:`!unittest.TestProgram.usageExit` (:gh:`67048`) * :class:`!webbrowser.MacOSX` (:gh:`86421`) * :class:`classmethod` descriptor chaining (:gh:`89519`) -* :mod:`importlib.resources` deprecated methods: - - * ``contents()`` - * ``is_resource()`` - * ``open_binary()`` - * ``open_text()`` - * ``path()`` - * ``read_binary()`` - * ``read_text()`` - - Use :func:`importlib.resources.files` instead. Refer to `importlib-resources: Migrating from Legacy - `_ (:gh:`106531`) From 10bec7c1eb3ee27f490a067426eef452b15f78f9 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Fri, 14 Nov 2025 19:52:01 +0500 Subject: [PATCH 204/313] GH-141312: Allow only integers to longrangeiter_setstate state (GH-141317) This fixes an assertion error when the new computed start is not an integer. --- Lib/test/test_range.py | 10 ++++++++++ .../2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst | 2 ++ Objects/rangeobject.c | 5 +++++ 3 files changed, 17 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index 3870b153688..2c9c290e890 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -470,6 +470,16 @@ def test_iterator_setstate(self): it.__setstate__(2**64 - 7) self.assertEqual(list(it), [12, 10]) + def test_iterator_invalid_setstate(self): + for invalid_value in (1.0, ""): + ranges = (('rangeiter', range(10, 100, 2)), + ('longrangeiter', range(10, 2**65, 2))) + for rng_name, rng in ranges: + with self.subTest(invalid_value=invalid_value, range=rng_name): + it = iter(rng) + with self.assertRaises(TypeError): + it.__setstate__(invalid_value) + def test_odd_bug(self): # This used to raise a "SystemError: NULL result without error" # because the range validation step was eating the exception diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst new file mode 100644 index 00000000000..fdb136cef3f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst @@ -0,0 +1,2 @@ +Fix the assertion failure in the ``__setstate__`` method of the range iterator +when a non-integer argument is passed. Patch by Sergey Miryanov. diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index f8cdfe68a64..e93346fb277 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -1042,6 +1042,11 @@ longrangeiter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) static PyObject * longrangeiter_setstate(PyObject *op, PyObject *state) { + if (!PyLong_CheckExact(state)) { + PyErr_Format(PyExc_TypeError, "state must be an int, not %T", state); + return NULL; + } + longrangeiterobject *r = (longrangeiterobject*)op; PyObject *zero = _PyLong_GetZero(); // borrowed reference int cmp; From fa245df4a0848c15cf8d907c10fc92819994b866 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Fri, 14 Nov 2025 19:55:04 +0500 Subject: [PATCH 205/313] GH-141509: Fix warning about remaining subinterpreters (GH-141528) Co-authored-by: Peter Bierma --- Lib/test/test_interpreters/test_api.py | 2 +- .../2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst | 3 +++ Python/pylifecycle.c | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 9a5ee03e472..fd9e46bf335 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -432,7 +432,7 @@ def test_cleanup_in_repl(self): exit()""" stdout, stderr = repl.communicate(script) self.assertIsNone(stderr) - self.assertIn(b"remaining subinterpreters", stdout) + self.assertIn(b"Interpreter.close()", stdout) self.assertNotIn(b"Traceback", stdout) @support.requires_subprocess() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst new file mode 100644 index 00000000000..a51aa495228 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst @@ -0,0 +1,3 @@ +Suggest using :meth:`concurrent.interpreters.Interpreter.close` instead of the +private ``_interpreters.destroy`` function when warning about remaining subinterpreters. +Patch by Sergey Miryanov. diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 805805ef188..67368b5ce07 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2643,7 +2643,7 @@ finalize_subinterpreters(void) (void)PyErr_WarnEx( PyExc_RuntimeWarning, "remaining subinterpreters; " - "destroy them with _interpreters.destroy()", + "close them with Interpreter.close()", 0); /* Swap out the current tstate, which we know must belong From a415a1812c4d7798131d077c8776503bb3e1844f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 14 Nov 2025 15:56:37 +0100 Subject: [PATCH 206/313] gh-139653: Remove assertions in _Py_InitializeRecursionLimits() (#141551) These checks were invalid and failed randomly on FreeBSD and Alpine Linux. --- Python/ceval.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index b76c9ec2811..31b81a37464 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -523,13 +523,6 @@ _Py_InitializeRecursionLimits(PyThreadState *tstate) _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; ts->c_stack_init_base = base; ts->c_stack_init_top = top; - - // Test the stack pointer -#if !defined(NDEBUG) && !defined(__wasi__) - uintptr_t here_addr = _Py_get_machine_stack_pointer(); - assert(ts->c_stack_soft_limit < here_addr); - assert(here_addr < ts->c_stack_top); -#endif } From eab7385858025df9fcb0131f71ec4a46d44e3ae9 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 14 Nov 2025 16:05:42 +0100 Subject: [PATCH 207/313] gh-116146: Avoid empty braces in _testembed.c (GH-141556) --- Programs/_testembed.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 27224e508bd..d0d7d5f03fb 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -2255,7 +2255,7 @@ exec_module(PyObject* self, PyObject* mod) static PyMethodDef create_static_module_methods[] = { {"create_module", create_module, METH_O, NULL}, {"exec_module", exec_module, METH_O, NULL}, - {} + {NULL} }; static struct PyModuleDef create_static_module_def = { From b101e9d36b1aed2bb4bca8aec3e1cc1d1df4f79e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 14 Nov 2025 15:23:01 +0000 Subject: [PATCH 208/313] Add PyManager troubleshooting steps for direct launch of script files (GH-141530) --- Doc/using/windows.rst | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index e6619b73bd2..ee182519199 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -4,6 +4,8 @@ .. _Microsoft Store app: https://apps.microsoft.com/detail/9NQ7512CXL7T +.. _legacy launcher: https://www.python.org/ftp/python/3.14.0/win32/launcher.msi + .. _using-on-windows: ************************* @@ -543,12 +545,9 @@ configuration option. The behaviour of shebangs in the Python install manager is subtly different from the previous ``py.exe`` launcher, and the old configuration options no longer apply. If you are specifically reliant on the old behaviour or - configuration, we recommend keeping the legacy launcher. It may be - `downloaded independently `_ - and installed on its own. The legacy launcher's ``py`` command will override - PyManager's one, and you will need to use ``pymanager`` commands for - installing and uninstalling. - + configuration, we recommend installing the `legacy launcher`_. The legacy + launcher's ``py`` command will override PyManager's one by default, and you + will need to use ``pymanager`` commands for installing and uninstalling. .. _Add-AppxPackage: https://learn.microsoft.com/powershell/module/appx/add-appxpackage @@ -859,6 +858,17 @@ default). These scripts are separated for each runtime, and so you may need to add multiple paths. + * - Typing ``script-name.py`` in the terminal opens in a new window. + - This is a known limitation of the operating system. Either specify ``py`` + before the script name, create a batch file containing ``@py "%~dpn0.py" %*`` + with the same name as the script, or install the `legacy launcher`_ + and select it as the association for scripts. + + * - Drag-dropping files onto a script doesn't work + - This is a known limitation of the operating system. It is supported with + the `legacy launcher`_, or with the Python install manager when installed + from the MSI. + .. _windows-embeddable: From da7f4e4b22020cfc6c5b5918756e454ef281848d Mon Sep 17 00:00:00 2001 From: Locked-chess-official <13140752715@163.com> Date: Fri, 14 Nov 2025 23:52:14 +0800 Subject: [PATCH 209/313] gh-141488: Add `Py_` prefix to Include/datetime.h macros (#141493) --- Include/datetime.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Include/datetime.h b/Include/datetime.h index b78cc0e8e2e..ed36e6e48c8 100644 --- a/Include/datetime.h +++ b/Include/datetime.h @@ -1,8 +1,8 @@ /* datetime.h */ #ifndef Py_LIMITED_API -#ifndef DATETIME_H -#define DATETIME_H +#ifndef Py_DATETIME_H +#define Py_DATETIME_H #ifdef __cplusplus extern "C" { #endif @@ -263,5 +263,5 @@ static PyDateTime_CAPI *PyDateTimeAPI = NULL; #ifdef __cplusplus } #endif -#endif +#endif /* !Py_DATETIME_H */ #endif /* !Py_LIMITED_API */ From f26ed455d5582a7d66618acf2a93bc4b22a84b47 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Fri, 14 Nov 2025 23:17:59 +0530 Subject: [PATCH 210/313] gh-114203: skip locking if object is already locked by two-mutex critical section (#141476) --- ...-11-14-16-25-15.gh-issue-114203.n3tlQO.rst | 1 + .../test_critical_sections.c | 101 ++++++++++++++++++ Python/critical_section.c | 23 +++- 3 files changed, 120 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst new file mode 100644 index 00000000000..883f9333cae --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst @@ -0,0 +1 @@ +Skip locking if object is already locked by two-mutex critical section. diff --git a/Modules/_testinternalcapi/test_critical_sections.c b/Modules/_testinternalcapi/test_critical_sections.c index e0ba37abcdd..e3b2fe716d4 100644 --- a/Modules/_testinternalcapi/test_critical_sections.c +++ b/Modules/_testinternalcapi/test_critical_sections.c @@ -284,10 +284,111 @@ test_critical_sections_gc(PyObject *self, PyObject *Py_UNUSED(args)) #endif +#ifdef Py_GIL_DISABLED + +static PyObject * +test_critical_section1_reacquisition(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyObject *a = PyDict_New(); + assert(a != NULL); + + PyCriticalSection cs1, cs2; + // First acquisition of critical section on object locks it + PyCriticalSection_Begin(&cs1, a); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert(_PyThreadState_GET()->critical_section == (uintptr_t)&cs1); + // Attempting to re-acquire critical section on same object which + // is already locked by top-most critical section is a no-op. + PyCriticalSection_Begin(&cs2, a); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert(_PyThreadState_GET()->critical_section == (uintptr_t)&cs1); + // Releasing second critical section is a no-op. + PyCriticalSection_End(&cs2); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert(_PyThreadState_GET()->critical_section == (uintptr_t)&cs1); + // Releasing first critical section unlocks the object + PyCriticalSection_End(&cs1); + assert(!PyMutex_IsLocked(&a->ob_mutex)); + + Py_DECREF(a); + Py_RETURN_NONE; +} + +static PyObject * +test_critical_section2_reacquisition(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyObject *a = PyDict_New(); + assert(a != NULL); + PyObject *b = PyDict_New(); + assert(b != NULL); + + PyCriticalSection2 cs; + // First acquisition of critical section on objects locks them + PyCriticalSection2_Begin(&cs, a, b); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + + // Attempting to re-acquire critical section on either of two + // objects already locked by top-most critical section is a no-op. + + // Check re-acquiring on first object + PyCriticalSection a_cs; + PyCriticalSection_Begin(&a_cs, a); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + // Releasing critical section on either object is a no-op. + PyCriticalSection_End(&a_cs); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + + // Check re-acquiring on second object + PyCriticalSection b_cs; + PyCriticalSection_Begin(&b_cs, b); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + // Releasing critical section on either object is a no-op. + PyCriticalSection_End(&b_cs); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + + // Releasing critical section on both objects unlocks them + PyCriticalSection2_End(&cs); + assert(!PyMutex_IsLocked(&a->ob_mutex)); + assert(!PyMutex_IsLocked(&b->ob_mutex)); + + Py_DECREF(a); + Py_DECREF(b); + Py_RETURN_NONE; +} + +#endif // Py_GIL_DISABLED + static PyMethodDef test_methods[] = { {"test_critical_sections", test_critical_sections, METH_NOARGS}, {"test_critical_sections_nest", test_critical_sections_nest, METH_NOARGS}, {"test_critical_sections_suspend", test_critical_sections_suspend, METH_NOARGS}, +#ifdef Py_GIL_DISABLED + {"test_critical_section1_reacquisition", test_critical_section1_reacquisition, METH_NOARGS}, + {"test_critical_section2_reacquisition", test_critical_section2_reacquisition, METH_NOARGS}, +#endif #ifdef Py_CAN_START_THREADS {"test_critical_sections_threads", test_critical_sections_threads, METH_NOARGS}, {"test_critical_sections_gc", test_critical_sections_gc, METH_NOARGS}, diff --git a/Python/critical_section.c b/Python/critical_section.c index e628ba2f6d1..218b580e951 100644 --- a/Python/critical_section.c +++ b/Python/critical_section.c @@ -24,11 +24,24 @@ _PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m) // As an optimisation for locking the same object recursively, skip // locking if the mutex is currently locked by the top-most critical // section. - if (tstate->critical_section && - untag_critical_section(tstate->critical_section)->_cs_mutex == m) { - c->_cs_mutex = NULL; - c->_cs_prev = 0; - return; + // If the top-most critical section is a two-mutex critical section, + // then locking is skipped if either mutex is m. + if (tstate->critical_section) { + PyCriticalSection *prev = untag_critical_section(tstate->critical_section); + if (prev->_cs_mutex == m) { + c->_cs_mutex = NULL; + c->_cs_prev = 0; + return; + } + if (tstate->critical_section & _Py_CRITICAL_SECTION_TWO_MUTEXES) { + PyCriticalSection2 *prev2 = (PyCriticalSection2 *) + untag_critical_section(tstate->critical_section); + if (prev2->_cs_mutex2 == m) { + c->_cs_mutex = NULL; + c->_cs_prev = 0; + return; + } + } } c->_cs_mutex = NULL; c->_cs_prev = (uintptr_t)tstate->critical_section; From 1281be1caf9357ee2a68f7370a88b5cff0110e15 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Sat, 15 Nov 2025 00:38:39 +0300 Subject: [PATCH 211/313] gh-141367: Use CALL_LIST_APPEND instruction only for lists, not for list subclasses (GH-141398) Co-authored-by: Ken Jin --- Include/internal/pycore_code.h | 4 +-- Lib/test/test_opcache.py | 27 +++++++++++++++++++ ...-11-11-13-40-45.gh-issue-141367.I5KY7F.rst | 2 ++ Python/bytecodes.c | 3 +-- Python/executor_cases.c.h | 4 --- Python/generated_cases.c.h | 7 +---- Python/specialize.c | 17 +++++++----- 7 files changed, 44 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 9748e036bf2..cb9c0aa27a1 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -311,8 +311,8 @@ PyAPI_FUNC(void) _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins _Py_CODEUNIT *instr, PyObject *name); PyAPI_FUNC(void) _Py_Specialize_StoreSubscr(_PyStackRef container, _PyStackRef sub, _Py_CODEUNIT *instr); -PyAPI_FUNC(void) _Py_Specialize_Call(_PyStackRef callable, _Py_CODEUNIT *instr, - int nargs); +PyAPI_FUNC(void) _Py_Specialize_Call(_PyStackRef callable, _PyStackRef self_or_null, + _Py_CODEUNIT *instr, int nargs); PyAPI_FUNC(void) _Py_Specialize_CallKw(_PyStackRef callable, _Py_CODEUNIT *instr, int nargs); PyAPI_FUNC(void) _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index f23f8c053e8..c7eea75117d 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1872,6 +1872,33 @@ def for_iter_generator(): self.assert_specialized(for_iter_generator, "FOR_ITER_GEN") self.assert_no_opcode(for_iter_generator, "FOR_ITER") + @cpython_only + @requires_specialization_ft + def test_call_list_append(self): + # gh-141367: only exact lists should use + # CALL_LIST_APPEND instruction after specialization. + + r = range(_testinternalcapi.SPECIALIZATION_THRESHOLD) + + def list_append(l): + for _ in r: + l.append(1) + + list_append([]) + self.assert_specialized(list_append, "CALL_LIST_APPEND") + self.assert_no_opcode(list_append, "CALL_METHOD_DESCRIPTOR_O") + self.assert_no_opcode(list_append, "CALL") + + def my_list_append(l): + for _ in r: + l.append(1) + + class MyList(list): pass + my_list_append(MyList()) + self.assert_specialized(my_list_append, "CALL_METHOD_DESCRIPTOR_O") + self.assert_no_opcode(my_list_append, "CALL_LIST_APPEND") + self.assert_no_opcode(my_list_append, "CALL") + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst new file mode 100644 index 00000000000..cb830fcd9e1 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst @@ -0,0 +1,2 @@ +Specialize ``CALL_LIST_APPEND`` instruction only for lists, not for list +subclasses, to avoid unnecessary deopt. Patch by Mikhail Efimov. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 2c798855a71..8a7b784bb9e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3689,7 +3689,7 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL); @@ -4395,7 +4395,6 @@ dummy_func( assert(oparg == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - DEOPT_IF(!PyList_CheckExact(self_o)); DEOPT_IF(!LOCK_OBJECT(self_o)); STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 7ba2e9d0d92..6796abf84ac 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -6037,10 +6037,6 @@ callable = stack_pointer[-3]; assert(oparg == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - if (!PyList_CheckExact(self_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } if (!LOCK_OBJECT(self_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a984da6dc91..01f65d9dd37 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1533,7 +1533,7 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } @@ -3470,11 +3470,6 @@ self = nos; assert(oparg == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - if (!PyList_CheckExact(self_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } if (!LOCK_OBJECT(self_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); diff --git a/Python/specialize.c b/Python/specialize.c index 2193596a331..19433bc7a74 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1602,8 +1602,8 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) } static int -specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, - int nargs) +specialize_method_descriptor(PyMethodDescrObject *descr, PyObject *self_or_null, + _Py_CODEUNIT *instr, int nargs) { switch (descr->d_method->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | @@ -1627,8 +1627,11 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, bool pop = (next.op.code == POP_TOP); int oparg = instr->op.arg; if ((PyObject *)descr == list_append && oparg == 1 && pop) { - specialize(instr, CALL_LIST_APPEND); - return 0; + assert(self_or_null != NULL); + if (PyList_CheckExact(self_or_null)) { + specialize(instr, CALL_LIST_APPEND); + return 0; + } } specialize(instr, CALL_METHOD_DESCRIPTOR_O); return 0; @@ -1766,7 +1769,7 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) } Py_NO_INLINE void -_Py_Specialize_Call(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) +_Py_Specialize_Call(_PyStackRef callable_st, _PyStackRef self_or_null_st, _Py_CODEUNIT *instr, int nargs) { PyObject *callable = PyStackRef_AsPyObjectBorrow(callable_st); @@ -1784,7 +1787,9 @@ _Py_Specialize_Call(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) fail = specialize_class_call(callable, instr, nargs); } else if (Py_IS_TYPE(callable, &PyMethodDescr_Type)) { - fail = specialize_method_descriptor((PyMethodDescrObject *)callable, instr, nargs); + PyObject *self_or_null = PyStackRef_AsPyObjectBorrow(self_or_null_st); + fail = specialize_method_descriptor((PyMethodDescrObject *)callable, + self_or_null, instr, nargs); } else if (PyMethod_Check(callable)) { PyObject *func = ((PyMethodObject *)callable)->im_func; From f0a8bc737ab2f04d4196eee154cb1e17e26ad585 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 14 Nov 2025 17:25:45 -0600 Subject: [PATCH 212/313] gh-140938: Raise ValueError for infinite inputs to stdev/pstdev (GH-141531) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Raise ValueError for infinite inputs to stdev/pstdev --- Co-authored-by: Gregory P. Smith <68491+gpshead@users.noreply.github.com> Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/statistics.py | 18 ++++++++++++++---- Lib/test/test_statistics.py | 9 ++++++++- ...5-11-13-14-51-30.gh-issue-140938.kXsHHv.rst | 2 ++ 3 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst diff --git a/Lib/statistics.py b/Lib/statistics.py index 3d805cb0739..26cf925529e 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -619,9 +619,14 @@ def stdev(data, xbar=None): if n < 2: raise StatisticsError('stdev requires at least two data points') mss = ss / (n - 1) + try: + mss_numerator = mss.numerator + mss_denominator = mss.denominator + except AttributeError: + raise ValueError('inf or nan encountered in data') if issubclass(T, Decimal): - return _decimal_sqrt_of_frac(mss.numerator, mss.denominator) - return _float_sqrt_of_frac(mss.numerator, mss.denominator) + return _decimal_sqrt_of_frac(mss_numerator, mss_denominator) + return _float_sqrt_of_frac(mss_numerator, mss_denominator) def pstdev(data, mu=None): @@ -637,9 +642,14 @@ def pstdev(data, mu=None): if n < 1: raise StatisticsError('pstdev requires at least one data point') mss = ss / n + try: + mss_numerator = mss.numerator + mss_denominator = mss.denominator + except AttributeError: + raise ValueError('inf or nan encountered in data') if issubclass(T, Decimal): - return _decimal_sqrt_of_frac(mss.numerator, mss.denominator) - return _float_sqrt_of_frac(mss.numerator, mss.denominator) + return _decimal_sqrt_of_frac(mss_numerator, mss_denominator) + return _float_sqrt_of_frac(mss_numerator, mss_denominator) ## Statistics for relations between two inputs ############################# diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 8250b0aef09..677a87b51b9 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2005,7 +2005,6 @@ def test_iter_list_same(self): expected = self.func(data) self.assertEqual(self.func(iter(data)), expected) - class TestPVariance(VarianceStdevMixin, NumericTestCase, UnivariateTypeMixin): # Tests for population variance. def setUp(self): @@ -2113,6 +2112,14 @@ def test_center_not_at_mean(self): self.assertEqual(self.func(data), 2.5) self.assertEqual(self.func(data, mu=0.5), 6.5) + def test_gh_140938(self): + # Inputs with inf/nan should raise a ValueError + with self.assertRaises(ValueError): + self.func([1.0, math.inf]) + with self.assertRaises(ValueError): + self.func([1.0, math.nan]) + + class TestSqrtHelpers(unittest.TestCase): def test_integer_sqrt_of_frac_rto(self): diff --git a/Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst b/Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst new file mode 100644 index 00000000000..bd3044002a2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst @@ -0,0 +1,2 @@ +The :func:`statistics.stdev` and :func:`statistics.pstdev` functions now raise a +:exc:`ValueError` when the input contains an infinity or a NaN. From 453d886f8592d2f4346d5621b1e4ff31c24338d5 Mon Sep 17 00:00:00 2001 From: Guo Ci Date: Fri, 14 Nov 2025 19:13:37 -0500 Subject: [PATCH 213/313] GH-90344: replace single-call `io.IncrementalNewlineDecoder` usage with non-incremental newline decoders (GH-30276) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Brett Cannon --- Lib/doctest.py | 9 ++------- Lib/importlib/_bootstrap_external.py | 3 +-- Lib/test/test_importlib/test_abc.py | 2 +- .../2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst | 1 + 4 files changed, 5 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst diff --git a/Lib/doctest.py b/Lib/doctest.py index 92a2ab4f7e6..ad8fb900f69 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -104,7 +104,7 @@ def _test(): import traceback import types import unittest -from io import StringIO, IncrementalNewlineDecoder +from io import StringIO, TextIOWrapper, BytesIO from collections import namedtuple import _colorize # Used in doctests from _colorize import ANSIColors, can_colorize @@ -237,10 +237,6 @@ def _normalize_module(module, depth=2): else: raise TypeError("Expected a module, string, or None") -def _newline_convert(data): - # The IO module provides a handy decoder for universal newline conversion - return IncrementalNewlineDecoder(None, True).decode(data, True) - def _load_testfile(filename, package, module_relative, encoding): if module_relative: package = _normalize_module(package, 3) @@ -252,10 +248,9 @@ def _load_testfile(filename, package, module_relative, encoding): pass if hasattr(loader, 'get_data'): file_contents = loader.get_data(filename) - file_contents = file_contents.decode(encoding) # get_data() opens files as 'rb', so one must do the equivalent # conversion as universal newlines would do. - return _newline_convert(file_contents), filename + return TextIOWrapper(BytesIO(file_contents), encoding=encoding, newline=None).read(), filename with open(filename, encoding=encoding) as f: return f.read(), filename diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 4ab0e79ea6e..192c0261408 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -552,8 +552,7 @@ def decode_source(source_bytes): import tokenize # To avoid bootstrap issues. source_bytes_readline = _io.BytesIO(source_bytes).readline encoding = tokenize.detect_encoding(source_bytes_readline) - newline_decoder = _io.IncrementalNewlineDecoder(None, True) - return newline_decoder.decode(source_bytes.decode(encoding[0])) + return _io.TextIOWrapper(_io.BytesIO(source_bytes), encoding=encoding[0], newline=None).read() # Module specifications ####################################################### diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index dd943210ffc..bd1540ce403 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -904,7 +904,7 @@ def test_universal_newlines(self): mock = self.SourceOnlyLoaderMock('mod.file') source = "x = 42\r\ny = -13\r\n" mock.source = source.encode('utf-8') - expect = io.IncrementalNewlineDecoder(None, True).decode(source) + expect = io.StringIO(source, newline=None).getvalue() self.assertEqual(mock.get_source(name), expect) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst new file mode 100644 index 00000000000..b1d05354f65 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst @@ -0,0 +1 @@ +Replace :class:`io.IncrementalNewlineDecoder` with non incremental newline decoders in codebase where :meth:`!io.IncrementalNewlineDecoder.decode` was being called once. From 53d65c840e038ce9a5782fbd3da963c7aba90570 Mon Sep 17 00:00:00 2001 From: Takuya UESHIN Date: Fri, 14 Nov 2025 16:59:51 -0800 Subject: [PATCH 214/313] gh-136442: Fix unittest to return exit code 5 when setUpClass raises an exception (#136487) --- Lib/test/test_unittest/test_program.py | 20 +++++++++++++++++++ Lib/unittest/main.py | 10 +++++----- ...-07-09-21-45-51.gh-issue-136442.jlbklP.rst | 1 + 3 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py index 6092ed292d8..8ed92373e5e 100644 --- a/Lib/test/test_unittest/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -75,6 +75,14 @@ def testUnexpectedSuccess(self): class Empty(unittest.TestCase): pass + class SetUpClassFailure(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + raise Exception + def testPass(self): + pass + class TestLoader(unittest.TestLoader): """Test loader that returns a suite containing the supplied testcase.""" @@ -191,6 +199,18 @@ def test_ExitEmptySuite(self): out = stream.getvalue() self.assertIn('\nNO TESTS RAN\n', out) + def test_ExitSetUpClassFailureSuite(self): + stream = BufferedWriter() + with self.assertRaises(SystemExit) as cm: + unittest.main( + argv=["setup_class_failure"], + testRunner=unittest.TextTestRunner(stream=stream), + testLoader=self.TestLoader(self.SetUpClassFailure)) + self.assertEqual(cm.exception.code, 1) + out = stream.getvalue() + self.assertIn("ERROR: setUpClass", out) + self.assertIn("SetUpClassFailure", out) + class InitialisableProgram(unittest.TestProgram): exit = False diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index 6fd949581f3..be99d93c78c 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -269,12 +269,12 @@ def runTests(self): testRunner = self.testRunner self.result = testRunner.run(self.test) if self.exit: - if self.result.testsRun == 0 and len(self.result.skipped) == 0: - sys.exit(_NO_TESTS_EXITCODE) - elif self.result.wasSuccessful(): - sys.exit(0) - else: + if not self.result.wasSuccessful(): sys.exit(1) + elif self.result.testsRun == 0 and len(self.result.skipped) == 0: + sys.exit(_NO_TESTS_EXITCODE) + else: + sys.exit(0) main = TestProgram diff --git a/Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst b/Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst new file mode 100644 index 00000000000..f87fb1113ca --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst @@ -0,0 +1 @@ +Use exitcode ``1`` instead of ``5`` if :func:`unittest.TestCase.setUpClass` raises an exception From 4ceb077c5cea30fef734f4c4e92c18d978be6c38 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 15 Nov 2025 02:23:54 +0000 Subject: [PATCH 215/313] gh-141579: Fix perf_jit backend in sys.activate_stack_trampoline() (#141580) --- Lib/test/test_perf_profiler.py | 18 ++++++++++++++++++ ...5-11-15-01-21-00.gh-issue-141579.aB7cD9.rst | 2 ++ Python/sysmodule.c | 16 ++++++++-------- 3 files changed, 28 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index 13424991639..e6852c93e69 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -238,6 +238,24 @@ def test_sys_api_get_status(self): """ assert_python_ok("-c", code, PYTHON_JIT="0") + def test_sys_api_perf_jit_backend(self): + code = """if 1: + import sys + sys.activate_stack_trampoline("perf_jit") + assert sys.is_stack_trampoline_active() is True + sys.deactivate_stack_trampoline() + assert sys.is_stack_trampoline_active() is False + """ + assert_python_ok("-c", code, PYTHON_JIT="0") + + def test_sys_api_with_existing_perf_jit_trampoline(self): + code = """if 1: + import sys + sys.activate_stack_trampoline("perf_jit") + sys.activate_stack_trampoline("perf_jit") + """ + assert_python_ok("-c", code, PYTHON_JIT="0") + def is_unwinding_reliable_with_frame_pointers(): cflags = sysconfig.get_config_var("PY_CORE_CFLAGS") diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst new file mode 100644 index 00000000000..8ab9979c399 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst @@ -0,0 +1,2 @@ +Fix :func:`sys.activate_stack_trampoline` to properly support the +``perf_jit`` backend. Patch by Pablo Galindo. diff --git a/Python/sysmodule.c b/Python/sysmodule.c index a611844f76e..b4b441bf4d9 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2380,14 +2380,14 @@ sys_activate_stack_trampoline_impl(PyObject *module, const char *backend) return NULL; } } - else if (strcmp(backend, "perf_jit") == 0) { - _PyPerf_Callbacks cur_cb; - _PyPerfTrampoline_GetCallbacks(&cur_cb); - if (cur_cb.write_state != _Py_perfmap_jit_callbacks.write_state) { - if (_PyPerfTrampoline_SetCallbacks(&_Py_perfmap_jit_callbacks) < 0 ) { - PyErr_SetString(PyExc_ValueError, "can't activate perf jit trampoline"); - return NULL; - } + } + else if (strcmp(backend, "perf_jit") == 0) { + _PyPerf_Callbacks cur_cb; + _PyPerfTrampoline_GetCallbacks(&cur_cb); + if (cur_cb.write_state != _Py_perfmap_jit_callbacks.write_state) { + if (_PyPerfTrampoline_SetCallbacks(&_Py_perfmap_jit_callbacks) < 0 ) { + PyErr_SetString(PyExc_ValueError, "can't activate perf jit trampoline"); + return NULL; } } } From ed81baf81f144e14510c492b71cf860472b0a0b7 Mon Sep 17 00:00:00 2001 From: Yongzi Li <204532581+Yzi-Li@users.noreply.github.com> Date: Sun, 16 Nov 2025 00:14:23 +0800 Subject: [PATCH 216/313] gh-140458: `xmlrpc.client` raises Fault, does not returns it. (GH-140759) --- Doc/library/xmlrpc.client.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index a21c7d3e4e3..e4912629aac 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -179,9 +179,9 @@ ServerProxy Objects A :class:`ServerProxy` instance has a method corresponding to each remote procedure call accepted by the XML-RPC server. Calling the method performs an RPC, dispatched by both name and argument signature (e.g. the same method name -can be overloaded with multiple argument signatures). The RPC finishes by -returning a value, which may be either returned data in a conformant type or a -:class:`Fault` or :class:`ProtocolError` object indicating an error. +can be overloaded with multiple argument signatures). The RPC finishes either +by returning data in a conformant type or by raising a :class:`Fault` or +:class:`ProtocolError` exception indicating an error. Servers that support the XML introspection API support some common methods grouped under the reserved :attr:`~ServerProxy.system` attribute: From 85f3009d7504ddcc01de715c494067e89c16303c Mon Sep 17 00:00:00 2001 From: Shamil Date: Sat, 15 Nov 2025 20:46:54 +0300 Subject: [PATCH 217/313] gh-141553: Fix incorrect function signatures in `_testmultiphase` (#141554) --- Modules/_testmultiphase.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index 220fa888e49..cd2d7b65598 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -1061,7 +1061,7 @@ PyModInit__test_from_modexport_exception(void) } static PyObject * -modexport_create_string(PyObject *spec, PyObject *def) +modexport_create_string(PyObject *spec, PyModuleDef *def) { assert(def == NULL); return PyUnicode_FromString("is this \xf0\x9f\xa6\x8b... a module?"); @@ -1138,8 +1138,9 @@ modexport_get_empty_slots(PyObject *mod, PyObject *arg) } static void -modexport_smoke_free(PyObject *mod) +modexport_smoke_free(void *op) { + PyObject *mod = (PyObject *)op; int *state = PyModule_GetState(mod); if (!state) { PyErr_FormatUnraisable("Exception ignored in module %R free", mod); From ed73c909f278a1eb558b120ef8ed2c0f8528bf58 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Sun, 16 Nov 2025 04:19:41 +0800 Subject: [PATCH 218/313] gh-139109: JIT _EXIT_TRACE to ENTER_EXECUTOR rather than _DEOPT (GH-141573) --- Include/internal/pycore_optimizer.h | 2 +- Lib/test/test_capi/test_opt.py | 36 +++++++++++++++++++++++++++++ Python/bytecodes.c | 4 ++-- Python/generated_cases.c.h | 4 ++-- Python/optimizer.c | 6 ++--- 5 files changed, 44 insertions(+), 8 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 653285a2c6b..0307a174e77 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -362,7 +362,7 @@ PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); #endif -int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, bool stop_tracing); +int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int stop_tracing_opcode); int _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index f06c6cbda29..25372fee58e 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -40,6 +40,17 @@ def get_first_executor(func): pass return None +def get_all_executors(func): + code = func.__code__ + co_code = code.co_code + executors = [] + for i in range(0, len(co_code), 2): + try: + executors.append(_opcode.get_executor(code, i)) + except ValueError: + pass + return executors + def iter_opnames(ex): for item in ex: @@ -2629,6 +2640,31 @@ def gen(): next(g) """ % _testinternalcapi.SPECIALIZATION_THRESHOLD)) + def test_executor_side_exits_create_another_executor(self): + def f(): + for x in range(TIER2_THRESHOLD + 3): + for y in range(TIER2_THRESHOLD + 3): + z = x + y + + f() + all_executors = get_all_executors(f) + # Inner loop warms up first. + # Outer loop warms up later, linking to the inner one. + # Therefore, we have at least two executors. + self.assertGreaterEqual(len(all_executors), 2) + for executor in all_executors: + opnames = list(get_opnames(executor)) + # Assert all executors first terminator ends in + # _EXIT_TRACE or _JUMP_TO_TOP, not _DEOPT + for idx, op in enumerate(opnames): + if op == "_EXIT_TRACE" or op == "_JUMP_TO_TOP": + break + elif op == "_DEOPT": + self.fail(f"_DEOPT encountered first at executor" + f" {executor} at offset {idx} rather" + f" than expected _EXIT_TRACE") + + def global_identity(x): return x diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8a7b784bb9e..565eaa7a599 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5643,7 +5643,7 @@ dummy_func( bool stop_tracing = (opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); - int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); if (full) { LEAVE_TRACING(); int err = stop_tracing_and_jit(tstate, frame); @@ -5683,7 +5683,7 @@ dummy_func( #if _Py_TIER2 assert(IS_JIT_TRACING()); int opcode = next_instr->op.code; - _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, true); + _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, _EXIT_TRACE); LEAVE_TRACING(); int err = stop_tracing_and_jit(tstate, frame); ERROR_IF(err < 0); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 01f65d9dd37..0d4678df68c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -12263,7 +12263,7 @@ JUMP_TO_LABEL(error); opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); _PyFrame_SetStackPointer(frame, stack_pointer); - int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); stack_pointer = _PyFrame_GetStackPointer(frame); if (full) { LEAVE_TRACING(); @@ -12309,7 +12309,7 @@ JUMP_TO_LABEL(error); assert(IS_JIT_TRACING()); int opcode = next_instr->op.code; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, true); + _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, _EXIT_TRACE); stack_pointer = _PyFrame_GetStackPointer(frame); LEAVE_TRACING(); _PyFrame_SetStackPointer(frame, stack_pointer); diff --git a/Python/optimizer.c b/Python/optimizer.c index 65007a256d0..9db894f0bf0 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -574,7 +574,7 @@ _PyJit_translate_single_bytecode_to_trace( PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, - bool stop_tracing) + int stop_tracing_opcode) { #ifdef Py_DEBUG @@ -637,8 +637,8 @@ _PyJit_translate_single_bytecode_to_trace( goto full; } - if (stop_tracing) { - ADD_TO_TRACE(_DEOPT, 0, 0, target); + if (stop_tracing_opcode != 0) { + ADD_TO_TRACE(stop_tracing_opcode, 0, 0, target); goto done; } From e33afa7ddbca3fca38f4ec4369b620c37cb092e2 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sun, 16 Nov 2025 18:50:54 +0000 Subject: [PATCH 219/313] gh-141004: Document the `PyPickleBuffer_*` C API (GH-141630) Co-authored-by: Peter Bierma --- Doc/c-api/concrete.rst | 1 + Doc/c-api/picklebuffer.rst | 59 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 Doc/c-api/picklebuffer.rst diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst index a5c5a53236c..1746fe95eaa 100644 --- a/Doc/c-api/concrete.rst +++ b/Doc/c-api/concrete.rst @@ -109,6 +109,7 @@ Other Objects descriptor.rst slice.rst memoryview.rst + picklebuffer.rst weakref.rst capsule.rst frame.rst diff --git a/Doc/c-api/picklebuffer.rst b/Doc/c-api/picklebuffer.rst new file mode 100644 index 00000000000..9e2d92341b0 --- /dev/null +++ b/Doc/c-api/picklebuffer.rst @@ -0,0 +1,59 @@ +.. highlight:: c + +.. _picklebuffer-objects: + +.. index:: + pair: object; PickleBuffer + +Pickle buffer objects +--------------------- + +.. versionadded:: 3.8 + +A :class:`pickle.PickleBuffer` object wraps a :ref:`buffer-providing object +` for out-of-band data transfer with the :mod:`pickle` module. + + +.. c:var:: PyTypeObject PyPickleBuffer_Type + + This instance of :c:type:`PyTypeObject` represents the Python pickle buffer type. + This is the same object as :class:`pickle.PickleBuffer` in the Python layer. + + +.. c:function:: int PyPickleBuffer_Check(PyObject *op) + + Return true if *op* is a pickle buffer instance. + This function always succeeds. + + +.. c:function:: PyObject *PyPickleBuffer_FromObject(PyObject *obj) + + Create a pickle buffer from the object *obj*. + + This function will fail if *obj* doesn't support the :ref:`buffer protocol `. + + On success, return a new pickle buffer instance. + On failure, set an exception and return ``NULL``. + + Analogous to calling :class:`pickle.PickleBuffer` with *obj* in Python. + + +.. c:function:: const Py_buffer *PyPickleBuffer_GetBuffer(PyObject *picklebuf) + + Get a pointer to the underlying :c:type:`Py_buffer` that the pickle buffer wraps. + + The returned pointer is valid as long as *picklebuf* is alive and has not been + released. The caller must not modify or free the returned :c:type:`Py_buffer`. + If the pickle buffer has been released, raise :exc:`ValueError`. + + On success, return a pointer to the buffer view. + On failure, set an exception and return ``NULL``. + + +.. c:function:: int PyPickleBuffer_Release(PyObject *picklebuf) + + Release the underlying buffer held by the pickle buffer. + + Return ``0`` on success. On failure, set an exception and return ``-1``. + + Analogous to calling :meth:`pickle.PickleBuffer.release` in Python. From 5348c200f5b26d6dd21d900b2b4cb684150d4b01 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sun, 16 Nov 2025 10:58:28 -0800 Subject: [PATCH 220/313] gh-125115 : Refactor the pdb parsing issue so positional arguments can pass through (#140933) --- Lib/pdb.py | 86 ++++++++++--------- Lib/test/test_pdb.py | 5 +- ...-11-03-05-38-31.gh-issue-125115.jGS8MN.rst | 1 + 3 files changed, 52 insertions(+), 40 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst diff --git a/Lib/pdb.py b/Lib/pdb.py index b799a113503..76bb28d7396 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -3548,7 +3548,15 @@ def exit_with_permission_help_text(): sys.exit(1) -def main(): +def parse_args(): + # We want pdb to be as intuitive as possible to users, so we need to do some + # heuristic parsing to deal with ambiguity. + # For example: + # "python -m pdb -m foo -p 1" should pass "-p 1" to "foo". + # "python -m pdb foo.py -m bar" should pass "-m bar" to "foo.py". + # "python -m pdb -m foo -m bar" should pass "-m bar" to "foo". + # This require some customized parsing logic to find the actual debug target. + import argparse parser = argparse.ArgumentParser( @@ -3559,28 +3567,48 @@ def main(): color=True, ) - # We need to maunally get the script from args, because the first positional - # arguments could be either the script we need to debug, or the argument - # to the -m module + # Get all the commands out first. For backwards compatibility, we allow + # -c commands to be after the target. parser.add_argument('-c', '--command', action='append', default=[], metavar='command', dest='commands', help='pdb commands to execute as if given in a .pdbrc file') - parser.add_argument('-m', metavar='module', dest='module') - parser.add_argument('-p', '--pid', type=int, help="attach to the specified PID", default=None) - - if len(sys.argv) == 1: - # If no arguments were given (python -m pdb), print the whole help message. - # Without this check, argparse would only complain about missing required arguments. - parser.print_help() - sys.exit(2) opts, args = parser.parse_known_args() - if opts.pid: - # If attaching to a remote pid, unrecognized arguments are not allowed. - # This will raise an error if there are extra unrecognized arguments. - opts = parser.parse_args() - if opts.module: - parser.error("argument -m: not allowed with argument --pid") + if not args: + # If no arguments were given (python -m pdb), print the whole help message. + # Without this check, argparse would only complain about missing required arguments. + # We need to add the arguments definitions here to get a proper help message. + parser.add_argument('-m', metavar='module', dest='module') + parser.add_argument('-p', '--pid', type=int, help="attach to the specified PID", default=None) + parser.print_help() + sys.exit(2) + elif args[0] == '-p' or args[0] == '--pid': + # Attach to a pid + parser.add_argument('-p', '--pid', type=int, help="attach to the specified PID", default=None) + opts, args = parser.parse_known_args() + if args: + # For --pid, any extra arguments are invalid. + parser.error(f"unrecognized arguments: {' '.join(args)}") + elif args[0] == '-m': + # Debug a module, we only need the first -m module argument. + # The rest is passed to the module itself. + parser.add_argument('-m', metavar='module', dest='module') + opt_module = parser.parse_args(args[:2]) + opts.module = opt_module.module + args = args[2:] + elif args[0].startswith('-'): + # Invalid argument before the script name. + invalid_args = list(itertools.takewhile(lambda a: a.startswith('-'), args)) + parser.error(f"unrecognized arguments: {' '.join(invalid_args)}") + + # Otherwise it's debugging a script and we already parsed all -c commands. + + return opts, args + +def main(): + opts, args = parse_args() + + if getattr(opts, 'pid', None) is not None: try: attach(opts.pid, opts.commands) except RuntimeError: @@ -3592,30 +3620,10 @@ def main(): except PermissionError: exit_with_permission_help_text() return - elif opts.module: - # If a module is being debugged, we consider the arguments after "-m module" to - # be potential arguments to the module itself. We need to parse the arguments - # before "-m" to check if there is any invalid argument. - # e.g. "python -m pdb -m foo --spam" means passing "--spam" to "foo" - # "python -m pdb --spam -m foo" means passing "--spam" to "pdb" and is invalid - idx = sys.argv.index('-m') - args_to_pdb = sys.argv[1:idx] - # This will raise an error if there are invalid arguments - parser.parse_args(args_to_pdb) - else: - # If a script is being debugged, then pdb expects the script name as the first argument. - # Anything before the script is considered an argument to pdb itself, which would - # be invalid because it's not parsed by argparse. - invalid_args = list(itertools.takewhile(lambda a: a.startswith('-'), args)) - if invalid_args: - parser.error(f"unrecognized arguments: {' '.join(invalid_args)}") - - if opts.module: + elif getattr(opts, 'module', None) is not None: file = opts.module target = _ModuleTarget(file) else: - if not args: - parser.error("no module or script to run") file = args.pop(0) if file.endswith('.pyz'): target = _ZipTarget(file) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 9a7d8550035..2ca689e0adf 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -3974,7 +3974,10 @@ def test_run_module_with_args(self): commands = """ continue """ - self._run_pdb(["calendar", "-m"], commands, expected_returncode=2) + self._run_pdb(["calendar", "-m"], commands, expected_returncode=1) + + _, stderr = self._run_pdb(["-m", "calendar", "-p", "1"], commands) + self.assertIn("unrecognized arguments: -p", stderr) stdout, _ = self._run_pdb(["-m", "calendar", "1"], commands) self.assertIn("December", stdout) diff --git a/Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst b/Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst new file mode 100644 index 00000000000..d36debec3ed --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst @@ -0,0 +1 @@ +Refactor the :mod:`pdb` parsing issue so positional arguments can pass through intuitively. From be699d6c7c8793d3eb464f2e5d3f10262fe3bc37 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 16 Nov 2025 14:25:50 -0500 Subject: [PATCH 221/313] gh-141004: Document missing `PyCFunction*` and `PyCMethod*` APIs (GH-141253) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/c-api/structures.rst | 93 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 58dd915e04f..414dfdc84e6 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -447,6 +447,25 @@ definition with the same method name. slot. This is helpful because calls to PyCFunctions are optimized more than wrapper object calls. + +.. c:var:: PyTypeObject PyCMethod_Type + + The type object corresponding to Python C method objects. This is + available as :class:`types.BuiltinMethodType` in the Python layer. + + +.. c:function:: int PyCMethod_Check(PyObject *op) + + Return true if *op* is an instance of the :c:type:`PyCMethod_Type` type + or a subtype of it. This function always succeeds. + + +.. c:function:: int PyCMethod_CheckExact(PyObject *op) + + This is the same as :c:func:`PyCMethod_Check`, but does not account for + subtypes. + + .. c:function:: PyObject * PyCMethod_New(PyMethodDef *ml, PyObject *self, PyObject *module, PyTypeObject *cls) Turn *ml* into a Python :term:`callable` object. @@ -472,6 +491,24 @@ definition with the same method name. .. versionadded:: 3.9 +.. c:var:: PyTypeObject PyCFunction_Type + + The type object corresponding to Python C function objects. This is + available as :class:`types.BuiltinFunctionType` in the Python layer. + + +.. c:function:: int PyCFunction_Check(PyObject *op) + + Return true if *op* is an instance of the :c:type:`PyCFunction_Type` type + or a subtype of it. This function always succeeds. + + +.. c:function:: int PyCFunction_CheckExact(PyObject *op) + + This is the same as :c:func:`PyCFunction_Check`, but does not account for + subtypes. + + .. c:function:: PyObject * PyCFunction_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module) Equivalent to ``PyCMethod_New(ml, self, module, NULL)``. @@ -482,6 +519,62 @@ definition with the same method name. Equivalent to ``PyCMethod_New(ml, self, NULL, NULL)``. +.. c:function:: int PyCFunction_GetFlags(PyObject *func) + + Get the function's flags on *func* as they were passed to + :c:member:`~PyMethodDef.ml_flags`. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns the function's flags on success, and ``-1`` with an + exception set on failure. + + +.. c:function:: int PyCFunction_GET_FLAGS(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetFlags`, but without error + or type checking. + + +.. c:function:: PyCFunction PyCFunction_GetFunction(PyObject *func) + + Get the function pointer on *func* as it was passed to + :c:member:`~PyMethodDef.ml_meth`. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns the function pointer on success, and ``NULL`` with an + exception set on failure. + + +.. c:function:: int PyCFunction_GET_FUNCTION(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetFunction`, but without error + or type checking. + + +.. c:function:: PyObject *PyCFunction_GetSelf(PyObject *func) + + Get the "self" object on *func*. This is the object that would be passed + to the first argument of a :c:type:`PyCFunction`. For C function objects + created through a :c:type:`PyMethodDef` on a :c:type:`PyModuleDef`, this + is the resulting module object. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns a :term:`borrowed reference` to the "self" object + on success, and ``NULL`` with an exception set on failure. + + +.. c:function:: PyObject *PyCFunction_GET_SELF(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetSelf`, but without error or + type checking. + + Accessing attributes of extension types --------------------------------------- From 8be3b2f479431f670f2e81e41b52e698c0806289 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sun, 16 Nov 2025 13:57:07 -0800 Subject: [PATCH 222/313] gh-136057: Allow step and next to step over for loops (#136160) --- Lib/bdb.py | 22 ++++++++++--- Lib/test/test_pdb.py | 31 +++++++++++++++++++ ...-07-01-04-57-57.gh-issue-136057.4-t596.rst | 1 + 3 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst diff --git a/Lib/bdb.py b/Lib/bdb.py index efc3e0a235a..50cf2b3f5b3 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -199,6 +199,8 @@ def __init__(self, skip=None, backend='settrace'): self.frame_returning = None self.trace_opcodes = False self.enterframe = None + self.cmdframe = None + self.cmdlineno = None self.code_linenos = weakref.WeakKeyDictionary() self.backend = backend if backend == 'monitoring': @@ -297,7 +299,12 @@ def dispatch_line(self, frame): self.user_line(). Raise BdbQuit if self.quitting is set. Return self.trace_dispatch to continue tracing in this scope. """ - if self.stop_here(frame) or self.break_here(frame): + # GH-136057 + # For line events, we don't want to stop at the same line where + # the latest next/step command was issued. + if (self.stop_here(frame) or self.break_here(frame)) and not ( + self.cmdframe == frame and self.cmdlineno == frame.f_lineno + ): self.user_line(frame) self.restart_events() if self.quitting: raise BdbQuit @@ -526,7 +533,8 @@ def _set_trace_opcodes(self, trace_opcodes): if self.monitoring_tracer: self.monitoring_tracer.update_local_events() - def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False): + def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False, + cmdframe=None, cmdlineno=None): """Set the attributes for stopping. If stoplineno is greater than or equal to 0, then stop at line @@ -539,6 +547,10 @@ def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False): # stoplineno >= 0 means: stop at line >= the stoplineno # stoplineno -1 means: don't stop at all self.stoplineno = stoplineno + # cmdframe/cmdlineno is the frame/line number when the user issued + # step/next commands. + self.cmdframe = cmdframe + self.cmdlineno = cmdlineno self._set_trace_opcodes(opcode) def _set_caller_tracefunc(self, current_frame): @@ -564,7 +576,9 @@ def set_until(self, frame, lineno=None): def set_step(self): """Stop after one line of code.""" - self._set_stopinfo(None, None) + # set_step() could be called from signal handler so enterframe might be None + self._set_stopinfo(None, None, cmdframe=self.enterframe, + cmdlineno=getattr(self.enterframe, 'f_lineno', None)) def set_stepinstr(self): """Stop before the next instruction.""" @@ -572,7 +586,7 @@ def set_stepinstr(self): def set_next(self, frame): """Stop on the next line in or below the given frame.""" - self._set_stopinfo(frame, None) + self._set_stopinfo(frame, None, cmdframe=frame, cmdlineno=frame.f_lineno) def set_return(self, frame): """Stop when returning from the given frame.""" diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 2ca689e0adf..9d89008756a 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -3232,6 +3232,37 @@ def test_pdb_issue_gh_127321(): """ +def test_pdb_issue_gh_136057(): + """See GH-136057 + "step" and "next" commands should be able to get over list comprehensions + >>> def test_function(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... lst = [i for i in range(10)] + ... for i in lst: pass + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'next', + ... 'next', + ... 'step', + ... 'continue', + ... ]): + ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) next + > (3)test_function() + -> lst = [i for i in range(10)] + (Pdb) next + > (4)test_function() + -> for i in lst: pass + (Pdb) step + --Return-- + > (4)test_function()->None + -> for i in lst: pass + (Pdb) continue + """ + + def test_pdb_issue_gh_80731(): """See GH-80731 diff --git a/Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst b/Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst new file mode 100644 index 00000000000..e237a0e98cc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst @@ -0,0 +1 @@ +Fixed the bug in :mod:`pdb` and :mod:`bdb` where ``next`` and ``step`` can't go over the line if a loop exists in the line. From 7800b78067162fc9d7cb6926f703fe14dee1702a Mon Sep 17 00:00:00 2001 From: SubbaraoGarlapati <53627478+SubbaraoGarlapati@users.noreply.github.com> Date: Mon, 17 Nov 2025 06:23:12 -0500 Subject: [PATCH 223/313] fix memory order of `_Py_atomic_store_uint_release` (#141562) --- Include/cpython/pyatomic_std.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 69a8b9e615e..7176f667a40 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -948,14 +948,6 @@ _Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) memory_order_relaxed); } -static inline void -_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) -{ - _Py_USING_STD; - atomic_store_explicit((_Atomic(unsigned int)*)obj, value, - memory_order_relaxed); -} - static inline void _Py_atomic_store_long_relaxed(long *obj, long value) { @@ -1031,6 +1023,14 @@ _Py_atomic_store_int_release(int *obj, int value) memory_order_release); } +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned int)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { From 31ea3f3c76b33e8e3cc098721266fe17f459e75d Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Mon, 17 Nov 2025 11:32:00 +0000 Subject: [PATCH 224/313] gh-141018: Update `.exe`, `.dll`, `.rtf` and `.jpg` mime types in `mimetypes` (#141023) --- Lib/mimetypes.py | 8 +++---- Lib/test/test_mimetypes.py | 24 ++++++++----------- ...-11-04-20-08-41.gh-issue-141018.d_oyOI.rst | 2 ++ 3 files changed, 15 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 42477713c78..07ac079186f 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -489,8 +489,6 @@ def _default_mime_types(): '.cjs' : 'application/node', '.bin' : 'application/octet-stream', '.a' : 'application/octet-stream', - '.dll' : 'application/octet-stream', - '.exe' : 'application/octet-stream', '.o' : 'application/octet-stream', '.obj' : 'application/octet-stream', '.so' : 'application/octet-stream', @@ -501,12 +499,15 @@ def _default_mime_types(): '.p7c' : 'application/pkcs7-mime', '.ps' : 'application/postscript', '.eps' : 'application/postscript', + '.rtf' : 'application/rtf', '.texi' : 'application/texinfo', '.texinfo': 'application/texinfo', '.toml' : 'application/toml', '.trig' : 'application/trig', '.m3u' : 'application/vnd.apple.mpegurl', '.m3u8' : 'application/vnd.apple.mpegurl', + '.dll' : 'application/vnd.microsoft.portable-executable', + '.exe' : 'application/vnd.microsoft.portable-executable', '.xls' : 'application/vnd.ms-excel', '.xlb' : 'application/vnd.ms-excel', '.eot' : 'application/vnd.ms-fontobject', @@ -649,7 +650,6 @@ def _default_mime_types(): '.pl' : 'text/plain', '.srt' : 'text/plain', '.rtx' : 'text/richtext', - '.rtf' : 'text/rtf', '.tsv' : 'text/tab-separated-values', '.vtt' : 'text/vtt', '.py' : 'text/x-python', @@ -682,11 +682,9 @@ def _default_mime_types(): # Please sort these too common_types = _common_types_default = { - '.rtf' : 'application/rtf', '.apk' : 'application/vnd.android.package-archive', '.midi': 'audio/midi', '.mid' : 'audio/midi', - '.jpg' : 'image/jpg', '.pict': 'image/pict', '.pct' : 'image/pict', '.pic' : 'image/pict', diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index 73414498359..0f29640bc1c 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -112,13 +112,12 @@ def test_non_standard_types(self): eq = self.assertEqual # First try strict eq(self.db.guess_file_type('foo.xul', strict=True), (None, None)) - eq(self.db.guess_extension('image/jpg', strict=True), None) # And then non-strict eq(self.db.guess_file_type('foo.xul', strict=False), ('text/xul', None)) eq(self.db.guess_file_type('foo.XUL', strict=False), ('text/xul', None)) eq(self.db.guess_file_type('foo.invalid', strict=False), (None, None)) - eq(self.db.guess_extension('image/jpg', strict=False), '.jpg') - eq(self.db.guess_extension('image/JPG', strict=False), '.jpg') + eq(self.db.guess_extension('image/jpeg', strict=False), '.jpg') + eq(self.db.guess_extension('image/JPEG', strict=False), '.jpg') def test_filename_with_url_delimiters(self): # bpo-38449: URL delimiters cases should be handled also. @@ -179,8 +178,8 @@ def test_guess_all_types(self): self.assertTrue(set(all) >= {'.bat', '.c', '.h', '.ksh', '.pl', '.txt'}) self.assertEqual(len(set(all)), len(all)) # no duplicates # And now non-strict - all = self.db.guess_all_extensions('image/jpg', strict=False) - self.assertEqual(all, ['.jpg']) + all = self.db.guess_all_extensions('image/jpeg', strict=False) + self.assertEqual(all, ['.jpg', '.jpe', '.jpeg']) # And now for no hits all = self.db.guess_all_extensions('image/jpg', strict=True) self.assertEqual(all, []) @@ -231,6 +230,7 @@ def check_extensions(): ("application/ogg", ".ogx"), ("application/pdf", ".pdf"), ("application/postscript", ".ps"), + ("application/rtf", ".rtf"), ("application/texinfo", ".texi"), ("application/toml", ".toml"), ("application/vnd.apple.mpegurl", ".m3u"), @@ -281,7 +281,6 @@ def check_extensions(): ("model/stl", ".stl"), ("text/html", ".html"), ("text/plain", ".txt"), - ("text/rtf", ".rtf"), ("text/x-rst", ".rst"), ("video/matroska", ".mkv"), ("video/matroska-3d", ".mk3d"), @@ -372,9 +371,7 @@ def test_keywords_args_api(self): self.assertEqual(self.db.guess_type( url="scheme:foo.html", strict=True), ("text/html", None)) self.assertEqual(self.db.guess_all_extensions( - type='image/jpg', strict=True), []) - self.assertEqual(self.db.guess_extension( - type='image/jpg', strict=False), '.jpg') + type='image/jpeg', strict=True), ['.jpg', '.jpe', '.jpeg']) def test_added_types_are_used(self): mimetypes.add_type('testing/default-type', '') @@ -452,15 +449,15 @@ def test_parse_args(self): args, help_text = mimetypes._parse_args("--invalid") self.assertTrue(help_text.startswith("usage: ")) - args, _ = mimetypes._parse_args(shlex.split("-l -e image/jpg")) + args, _ = mimetypes._parse_args(shlex.split("-l -e image/jpeg")) self.assertTrue(args.extension) self.assertTrue(args.lenient) - self.assertEqual(args.type, ["image/jpg"]) + self.assertEqual(args.type, ["image/jpeg"]) - args, _ = mimetypes._parse_args(shlex.split("-e image/jpg")) + args, _ = mimetypes._parse_args(shlex.split("-e image/jpeg")) self.assertTrue(args.extension) self.assertFalse(args.lenient) - self.assertEqual(args.type, ["image/jpg"]) + self.assertEqual(args.type, ["image/jpeg"]) args, _ = mimetypes._parse_args(shlex.split("-l foo.webp")) self.assertFalse(args.extension) @@ -491,7 +488,6 @@ def test_multiple_inputs_error(self): def test_invocation(self): for command, expected in [ - ("-l -e image/jpg", ".jpg"), ("-e image/jpeg", ".jpg"), ("-l foo.webp", "type: image/webp encoding: None"), ]: diff --git a/Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst b/Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst new file mode 100644 index 00000000000..e776515a9fb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst @@ -0,0 +1,2 @@ +:mod:`mimetypes`: Update ``.exe``, ``.dll``, ``.rtf`` and (when +``strict=False``) ``.jpg`` to their correct IANA mime type. From df8091d516f874bd8222569794229ea77fb3a0a3 Mon Sep 17 00:00:00 2001 From: Tamzin Hadasa Kelly Date: Mon, 17 Nov 2025 18:35:01 +0700 Subject: [PATCH 225/313] gh-141650: Fix typo in `xml.sax.saxutils.unescape` documentation (#141652) --- Doc/library/xml.sax.utils.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/xml.sax.utils.rst b/Doc/library/xml.sax.utils.rst index 5ee11d58c3d..7731f03d875 100644 --- a/Doc/library/xml.sax.utils.rst +++ b/Doc/library/xml.sax.utils.rst @@ -37,7 +37,7 @@ or as base classes. You can unescape other strings of data by passing a dictionary as the optional *entities* parameter. The keys and values must all be strings; each key will be - replaced with its corresponding value. ``'&'``, ``'<'``, and ``'>'`` + replaced with its corresponding value. ``'&'``, ``'<'``, and ``'>'`` are always unescaped, even if *entities* is provided. From d527d3bf8beb9cd26c179f2c0111d635cdaa9cd3 Mon Sep 17 00:00:00 2001 From: dereckduran <67027239+dereckduran@users.noreply.github.com> Date: Mon, 17 Nov 2025 03:44:44 -0800 Subject: [PATCH 226/313] gh-62480: De-personalize "Coping with mutable arguments" section in `unittest.mock` examples (#141323) --- Doc/library/unittest.mock-examples.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index 6af4298d44f..61c75b5a03b 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -863,9 +863,9 @@ Here's one solution that uses the :attr:`~Mock.side_effect` functionality. If you provide a ``side_effect`` function for a mock then ``side_effect`` will be called with the same args as the mock. This gives us an opportunity to copy the arguments and store them for later assertions. In this -example I'm using *another* mock to store the arguments so that I can use the +example we're using *another* mock to store the arguments so that we can use the mock methods for doing the assertion. Again a helper function sets this up for -me. :: +us. :: >>> from copy import deepcopy >>> from unittest.mock import Mock, patch, DEFAULT From 20b64bdf23b88e44f72bc49f8bc783ae8ca21511 Mon Sep 17 00:00:00 2001 From: Thomas Ballard Date: Mon, 17 Nov 2025 06:47:28 -0500 Subject: [PATCH 227/313] Docs: Fix typo in socketserver documentation (#140956) --- Doc/library/socketserver.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 7bc2f7afbbb..491b8769f44 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -546,7 +546,7 @@ The difference is that the ``readline()`` call in the second handler will call first handler had to use a ``recv()`` loop to accumulate data until a newline itself. If it had just used a single ``recv()`` without the loop it would just have returned what has been received so far from the client. -TCP is stream based: data arrives in the order it was sent, but there no +TCP is stream based: data arrives in the order it was sent, but there is no correlation between client ``send()`` or ``sendall()`` calls and the number of ``recv()`` calls on the server required to receive it. From 994ab5c922b179ab1884f05b3440c24db9e9733d Mon Sep 17 00:00:00 2001 From: yihong Date: Mon, 17 Nov 2025 20:43:14 +0800 Subject: [PATCH 228/313] gh-140729: Add __mp_main__ as a duplicate for __main__ for pickle to work (#140735) --- Lib/profiling/sampling/_sync_coordinator.py | 11 +++- .../test_profiling/test_sampling_profiler.py | 52 ++++++++++++++++++- ...-10-29-11-31-59.gh-issue-140729.t9JsNt.rst | 2 + 3 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst diff --git a/Lib/profiling/sampling/_sync_coordinator.py b/Lib/profiling/sampling/_sync_coordinator.py index adb040e89cc..be63dbe3e90 100644 --- a/Lib/profiling/sampling/_sync_coordinator.py +++ b/Lib/profiling/sampling/_sync_coordinator.py @@ -10,6 +10,7 @@ import socket import runpy import time +import types from typing import List, NoReturn @@ -175,15 +176,21 @@ def _execute_script(script_path: str, script_args: List[str], cwd: str) -> None: try: with open(script_path, 'rb') as f: source_code = f.read() + except FileNotFoundError as e: raise TargetError(f"Script file not found: {script_path}") from e except PermissionError as e: raise TargetError(f"Permission denied reading script: {script_path}") from e try: - # Compile and execute the script + main_module = types.ModuleType("__main__") + main_module.__file__ = script_path + main_module.__builtins__ = __builtins__ + # gh-140729: Create a __mp_main__ module to allow pickling + sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module + code = compile(source_code, script_path, 'exec', module='__main__') - exec(code, {'__name__': '__main__', '__file__': script_path}) + exec(code, main_module.__dict__) except SyntaxError as e: raise TargetError(f"Syntax error in script {script_path}: {e}") from e except SystemExit: diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index 5b924cb2453..0ba6799a1ce 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -22,7 +22,13 @@ from profiling.sampling.gecko_collector import GeckoCollector from test.support.os_helper import unlink -from test.support import force_not_colorized_test_class, SHORT_TIMEOUT +from test.support import ( + force_not_colorized_test_class, + SHORT_TIMEOUT, + script_helper, + os_helper, + SuppressCrashReport, +) from test.support.socket_helper import find_unused_port from test.support import requires_subprocess, is_emscripten from test.support import captured_stdout, captured_stderr @@ -3009,5 +3015,49 @@ def test_parse_mode_function(self): profiling.sampling.sample._parse_mode("invalid") +@requires_subprocess() +@skip_if_not_supported +class TestProcessPoolExecutorSupport(unittest.TestCase): + """ + Test that ProcessPoolExecutor works correctly with profiling.sampling. + """ + + def test_process_pool_executor_pickle(self): + # gh-140729: test use ProcessPoolExecutor.map() can sampling + test_script = ''' +import concurrent.futures + +def worker(x): + return x * 2 + +if __name__ == "__main__": + with concurrent.futures.ProcessPoolExecutor() as executor: + results = list(executor.map(worker, [1, 2, 3])) + print(f"Results: {results}") +''' + with os_helper.temp_dir() as temp_dir: + script = script_helper.make_script( + temp_dir, 'test_process_pool_executor_pickle', test_script + ) + with SuppressCrashReport(): + with script_helper.spawn_python( + "-m", "profiling.sampling.sample", + "-d", "5", + "-i", "100000", + script, + stderr=subprocess.PIPE, + text=True + ) as proc: + proc.wait(timeout=SHORT_TIMEOUT) + stdout = proc.stdout.read() + stderr = proc.stderr.read() + + if "PermissionError" in stderr: + self.skipTest("Insufficient permissions for remote profiling") + + self.assertIn("Results: [2, 4, 6]", stdout) + self.assertNotIn("Can't pickle", stderr) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst new file mode 100644 index 00000000000..6725547667f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst @@ -0,0 +1,2 @@ +Fix pickling error in the sampling profiler when using ``concurrent.futures.ProcessPoolExecutor`` +script can not be properly pickled and executed in worker processes. From 89a914c58db1661cb9da4f3b9e52c20bb4b02287 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 17 Nov 2025 12:46:26 +0000 Subject: [PATCH 229/313] gh-135953: Add GIL contention markers to sampling profiler Gecko format (#139485) This commit enhances the Gecko format reporter in the sampling profiler to include markers for GIL acquisition events. --- Include/cpython/pystate.h | 3 + Include/internal/pycore_debug_offsets.h | 4 + Lib/profiling/sampling/collector.py | 31 +-- Lib/profiling/sampling/gecko_collector.py | 237 ++++++++++++++++-- Lib/profiling/sampling/sample.py | 37 ++- Lib/test/test_external_inspection.py | 154 +++++++++++- .../test_profiling/test_sampling_profiler.py | 116 ++++++++- Modules/_remote_debugging_module.c | 123 +++++++-- Python/ceval_gil.c | 4 + 9 files changed, 627 insertions(+), 82 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index c53abe43ebe..1e1e46ea4c0 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -113,6 +113,9 @@ struct _ts { /* Currently holds the GIL. Must be its own field to avoid data races */ int holds_gil; + /* Currently requesting the GIL */ + int gil_requested; + int _whence; /* Thread state (_Py_THREAD_ATTACHED, _Py_THREAD_DETACHED, _Py_THREAD_SUSPENDED). diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index 8e7cd16acff..f6d50bf5df7 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -106,6 +106,8 @@ typedef struct _Py_DebugOffsets { uint64_t native_thread_id; uint64_t datastack_chunk; uint64_t status; + uint64_t holds_gil; + uint64_t gil_requested; } thread_state; // InterpreterFrame offset; @@ -273,6 +275,8 @@ typedef struct _Py_DebugOffsets { .native_thread_id = offsetof(PyThreadState, native_thread_id), \ .datastack_chunk = offsetof(PyThreadState, datastack_chunk), \ .status = offsetof(PyThreadState, _status), \ + .holds_gil = offsetof(PyThreadState, holds_gil), \ + .gil_requested = offsetof(PyThreadState, gil_requested), \ }, \ .interpreter_frame = { \ .size = sizeof(_PyInterpreterFrame), \ diff --git a/Lib/profiling/sampling/collector.py b/Lib/profiling/sampling/collector.py index b7a033ac0a6..3c2325ef772 100644 --- a/Lib/profiling/sampling/collector.py +++ b/Lib/profiling/sampling/collector.py @@ -1,17 +1,14 @@ from abc import ABC, abstractmethod -# Enums are slow -THREAD_STATE_RUNNING = 0 -THREAD_STATE_IDLE = 1 -THREAD_STATE_GIL_WAIT = 2 -THREAD_STATE_UNKNOWN = 3 - -STATUS = { - THREAD_STATE_RUNNING: "running", - THREAD_STATE_IDLE: "idle", - THREAD_STATE_GIL_WAIT: "gil_wait", - THREAD_STATE_UNKNOWN: "unknown", -} +# Thread status flags +try: + from _remote_debugging import THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU, THREAD_STATUS_UNKNOWN, THREAD_STATUS_GIL_REQUESTED +except ImportError: + # Fallback for tests or when module is not available + THREAD_STATUS_HAS_GIL = (1 << 0) + THREAD_STATUS_ON_CPU = (1 << 1) + THREAD_STATUS_UNKNOWN = (1 << 2) + THREAD_STATUS_GIL_REQUESTED = (1 << 3) class Collector(ABC): @abstractmethod @@ -26,8 +23,14 @@ def _iter_all_frames(self, stack_frames, skip_idle=False): """Iterate over all frame stacks from all interpreters and threads.""" for interpreter_info in stack_frames: for thread_info in interpreter_info.threads: - if skip_idle and thread_info.status != THREAD_STATE_RUNNING: - continue + # skip_idle now means: skip if thread is not actively running + # A thread is "active" if it has the GIL OR is on CPU + if skip_idle: + status_flags = thread_info.status + has_gil = bool(status_flags & THREAD_STATUS_HAS_GIL) + on_cpu = bool(status_flags & THREAD_STATUS_ON_CPU) + if not (has_gil or on_cpu): + continue frames = thread_info.frame_info if frames: yield frames, thread_info.thread_id diff --git a/Lib/profiling/sampling/gecko_collector.py b/Lib/profiling/sampling/gecko_collector.py index 548acbf24b7..6c6700f1130 100644 --- a/Lib/profiling/sampling/gecko_collector.py +++ b/Lib/profiling/sampling/gecko_collector.py @@ -1,9 +1,20 @@ +import itertools import json import os import platform +import sys +import threading import time -from .collector import Collector, THREAD_STATE_RUNNING +from .collector import Collector +try: + from _remote_debugging import THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU, THREAD_STATUS_UNKNOWN, THREAD_STATUS_GIL_REQUESTED +except ImportError: + # Fallback if module not available (shouldn't happen in normal use) + THREAD_STATUS_HAS_GIL = (1 << 0) + THREAD_STATUS_ON_CPU = (1 << 1) + THREAD_STATUS_UNKNOWN = (1 << 2) + THREAD_STATUS_GIL_REQUESTED = (1 << 3) # Categories matching Firefox Profiler expectations @@ -11,14 +22,20 @@ {"name": "Other", "color": "grey", "subcategories": ["Other"]}, {"name": "Python", "color": "yellow", "subcategories": ["Other"]}, {"name": "Native", "color": "blue", "subcategories": ["Other"]}, - {"name": "Idle", "color": "transparent", "subcategories": ["Other"]}, + {"name": "GC", "color": "orange", "subcategories": ["Other"]}, + {"name": "GIL", "color": "green", "subcategories": ["Other"]}, + {"name": "CPU", "color": "purple", "subcategories": ["Other"]}, + {"name": "Code Type", "color": "red", "subcategories": ["Other"]}, ] # Category indices CATEGORY_OTHER = 0 CATEGORY_PYTHON = 1 CATEGORY_NATIVE = 2 -CATEGORY_IDLE = 3 +CATEGORY_GC = 3 +CATEGORY_GIL = 4 +CATEGORY_CPU = 5 +CATEGORY_CODE_TYPE = 6 # Subcategory indices DEFAULT_SUBCATEGORY = 0 @@ -58,6 +75,56 @@ def __init__(self, *, skip_idle=False): self.last_sample_time = 0 self.interval = 1.0 # Will be calculated from actual sampling + # State tracking for interval markers (tid -> start_time) + self.has_gil_start = {} # Thread has the GIL + self.no_gil_start = {} # Thread doesn't have the GIL + self.on_cpu_start = {} # Thread is running on CPU + self.off_cpu_start = {} # Thread is off CPU + self.python_code_start = {} # Thread running Python code (has GIL) + self.native_code_start = {} # Thread running native code (on CPU without GIL) + self.gil_wait_start = {} # Thread waiting for GIL + + # GC event tracking: track GC start time per thread + self.gc_start_per_thread = {} # tid -> start_time + + # Track which threads have been initialized for state tracking + self.initialized_threads = set() + + def _track_state_transition(self, tid, condition, active_dict, inactive_dict, + active_name, inactive_name, category, current_time): + """Track binary state transitions and emit markers. + + Args: + tid: Thread ID + condition: Whether the active state is true + active_dict: Dict tracking start time of active state + inactive_dict: Dict tracking start time of inactive state + active_name: Name for active state marker + inactive_name: Name for inactive state marker + category: Gecko category for the markers + current_time: Current timestamp + """ + # On first observation of a thread, just record the current state + # without creating a marker (we don't know what the previous state was) + if tid not in self.initialized_threads: + if condition: + active_dict[tid] = current_time + else: + inactive_dict[tid] = current_time + return + + # For already-initialized threads, track transitions + if condition: + active_dict.setdefault(tid, current_time) + if tid in inactive_dict: + self._add_marker(tid, inactive_name, inactive_dict.pop(tid), + current_time, category) + else: + inactive_dict.setdefault(tid, current_time) + if tid in active_dict: + self._add_marker(tid, active_name, active_dict.pop(tid), + current_time, category) + def collect(self, stack_frames): """Collect a sample from stack frames.""" current_time = (time.time() * 1000) - self.start_time @@ -69,19 +136,12 @@ def collect(self, stack_frames): ) / self.sample_count self.last_sample_time = current_time + # Process threads and track GC per thread for interpreter_info in stack_frames: for thread_info in interpreter_info.threads: - if ( - self.skip_idle - and thread_info.status != THREAD_STATE_RUNNING - ): - continue - frames = thread_info.frame_info - if not frames: - continue - tid = thread_info.thread_id + gc_collecting = thread_info.gc_collecting # Initialize thread if needed if tid not in self.threads: @@ -89,6 +149,80 @@ def collect(self, stack_frames): thread_data = self.threads[tid] + # Decode status flags + status_flags = thread_info.status + has_gil = bool(status_flags & THREAD_STATUS_HAS_GIL) + on_cpu = bool(status_flags & THREAD_STATUS_ON_CPU) + gil_requested = bool(status_flags & THREAD_STATUS_GIL_REQUESTED) + + # Track GIL possession (Has GIL / No GIL) + self._track_state_transition( + tid, has_gil, self.has_gil_start, self.no_gil_start, + "Has GIL", "No GIL", CATEGORY_GIL, current_time + ) + + # Track CPU state (On CPU / Off CPU) + self._track_state_transition( + tid, on_cpu, self.on_cpu_start, self.off_cpu_start, + "On CPU", "Off CPU", CATEGORY_CPU, current_time + ) + + # Track code type (Python Code / Native Code) + # This is tri-state: Python (has_gil), Native (on_cpu without gil), or Neither + if has_gil: + self._track_state_transition( + tid, True, self.python_code_start, self.native_code_start, + "Python Code", "Native Code", CATEGORY_CODE_TYPE, current_time + ) + elif on_cpu: + self._track_state_transition( + tid, True, self.native_code_start, self.python_code_start, + "Native Code", "Python Code", CATEGORY_CODE_TYPE, current_time + ) + else: + # Thread is idle (neither has GIL nor on CPU) - close any open code markers + # This handles the third state that _track_state_transition doesn't cover + if tid in self.initialized_threads: + if tid in self.python_code_start: + self._add_marker(tid, "Python Code", self.python_code_start.pop(tid), + current_time, CATEGORY_CODE_TYPE) + if tid in self.native_code_start: + self._add_marker(tid, "Native Code", self.native_code_start.pop(tid), + current_time, CATEGORY_CODE_TYPE) + + # Track "Waiting for GIL" intervals (one-sided tracking) + if gil_requested: + self.gil_wait_start.setdefault(tid, current_time) + elif tid in self.gil_wait_start: + self._add_marker(tid, "Waiting for GIL", self.gil_wait_start.pop(tid), + current_time, CATEGORY_GIL) + + # Track GC events - attribute to all threads that hold the GIL during GC + # (GC is interpreter-wide but runs on whichever thread(s) have the GIL) + # If GIL switches during GC, multiple threads will get GC markers + if gc_collecting and has_gil: + # Start GC marker if not already started for this thread + if tid not in self.gc_start_per_thread: + self.gc_start_per_thread[tid] = current_time + elif tid in self.gc_start_per_thread: + # End GC marker if it was running for this thread + # (either GC finished or thread lost GIL) + self._add_marker(tid, "GC Collecting", self.gc_start_per_thread.pop(tid), + current_time, CATEGORY_GC) + + # Mark thread as initialized after processing all state transitions + self.initialized_threads.add(tid) + + # Categorize: idle if neither has GIL nor on CPU + is_idle = not has_gil and not on_cpu + + # Skip idle threads if skip_idle is enabled + if self.skip_idle and is_idle: + continue + + if not frames: + continue + # Process the stack stack_index = self._process_stack(thread_data, frames) @@ -102,7 +236,6 @@ def collect(self, stack_frames): def _create_thread(self, tid): """Create a new thread structure with processed profile format.""" - import threading # Determine if this is the main thread try: @@ -181,7 +314,7 @@ def _create_thread(self, tid): "functionSize": [], "length": 0, }, - # Markers - processed format + # Markers - processed format (arrays) "markers": { "data": [], "name": [], @@ -215,6 +348,27 @@ def _intern_string(self, s): self.global_string_map[s] = idx return idx + def _add_marker(self, tid, name, start_time, end_time, category): + """Add an interval marker for a specific thread.""" + if tid not in self.threads: + return + + thread_data = self.threads[tid] + duration = end_time - start_time + + name_idx = self._intern_string(name) + markers = thread_data["markers"] + markers["name"].append(name_idx) + markers["startTime"].append(start_time) + markers["endTime"].append(end_time) + markers["phase"].append(1) # 1 = interval marker + markers["category"].append(category) + markers["data"].append({ + "type": name.replace(" ", ""), + "duration": duration, + "tid": tid + }) + def _process_stack(self, thread_data, frames): """Process a stack and return the stack index.""" if not frames: @@ -383,15 +537,63 @@ def _get_or_create_frame(self, thread_data, func_idx, lineno): frame_cache[frame_key] = frame_idx return frame_idx + def _finalize_markers(self): + """Close any open markers at the end of profiling.""" + end_time = self.last_sample_time + + # Close all open markers for each thread using a generic approach + marker_states = [ + (self.has_gil_start, "Has GIL", CATEGORY_GIL), + (self.no_gil_start, "No GIL", CATEGORY_GIL), + (self.on_cpu_start, "On CPU", CATEGORY_CPU), + (self.off_cpu_start, "Off CPU", CATEGORY_CPU), + (self.python_code_start, "Python Code", CATEGORY_CODE_TYPE), + (self.native_code_start, "Native Code", CATEGORY_CODE_TYPE), + (self.gil_wait_start, "Waiting for GIL", CATEGORY_GIL), + (self.gc_start_per_thread, "GC Collecting", CATEGORY_GC), + ] + + for state_dict, marker_name, category in marker_states: + for tid in list(state_dict.keys()): + self._add_marker(tid, marker_name, state_dict[tid], end_time, category) + del state_dict[tid] + def export(self, filename): """Export the profile to a Gecko JSON file.""" + if self.sample_count > 0 and self.last_sample_time > 0: self.interval = self.last_sample_time / self.sample_count - profile = self._build_profile() + # Spinner for progress indication + spinner = itertools.cycle(['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']) + stop_spinner = threading.Event() - with open(filename, "w") as f: - json.dump(profile, f, separators=(",", ":")) + def spin(): + message = 'Building Gecko profile...' + while not stop_spinner.is_set(): + sys.stderr.write(f'\r{next(spinner)} {message}') + sys.stderr.flush() + time.sleep(0.1) + # Clear the spinner line + sys.stderr.write('\r' + ' ' * (len(message) + 3) + '\r') + sys.stderr.flush() + + spinner_thread = threading.Thread(target=spin, daemon=True) + spinner_thread.start() + + try: + # Finalize any open markers before building profile + self._finalize_markers() + + profile = self._build_profile() + + with open(filename, "w") as f: + json.dump(profile, f, separators=(",", ":")) + finally: + stop_spinner.set() + spinner_thread.join(timeout=1.0) + # Small delay to ensure the clear happens + time.sleep(0.01) print(f"Gecko profile written to {filename}") print( @@ -416,6 +618,7 @@ def _build_profile(self): frame_table["length"] = len(frame_table["func"]) func_table["length"] = len(func_table["name"]) resource_table["length"] = len(resource_table["name"]) + thread_data["markers"]["length"] = len(thread_data["markers"]["name"]) # Clean up internal caches del thread_data["_stackCache"] diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 7a0f739a542..5ca68911d8a 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -21,6 +21,7 @@ PROFILING_MODE_WALL = 0 PROFILING_MODE_CPU = 1 PROFILING_MODE_GIL = 2 +PROFILING_MODE_ALL = 3 # Combines GIL + CPU checks def _parse_mode(mode_string): @@ -136,18 +137,20 @@ def _run_with_sync(original_cmd): class SampleProfiler: - def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL): + def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, skip_non_matching_threads=True): self.pid = pid self.sample_interval_usec = sample_interval_usec self.all_threads = all_threads if _FREE_THREADED_BUILD: self.unwinder = _remote_debugging.RemoteUnwinder( - self.pid, all_threads=self.all_threads, mode=mode + self.pid, all_threads=self.all_threads, mode=mode, + skip_non_matching_threads=skip_non_matching_threads ) else: only_active_threads = bool(self.all_threads) self.unwinder = _remote_debugging.RemoteUnwinder( - self.pid, only_active_thread=only_active_threads, mode=mode + self.pid, only_active_thread=only_active_threads, mode=mode, + skip_non_matching_threads=skip_non_matching_threads ) # Track sample intervals and total sample count self.sample_intervals = deque(maxlen=100) @@ -614,14 +617,21 @@ def sample( realtime_stats=False, mode=PROFILING_MODE_WALL, ): + # PROFILING_MODE_ALL implies no skipping at all + if mode == PROFILING_MODE_ALL: + skip_non_matching_threads = False + skip_idle = False + else: + # Determine skip settings based on output format and mode + skip_non_matching_threads = output_format != "gecko" + skip_idle = mode != PROFILING_MODE_WALL + profiler = SampleProfiler( - pid, sample_interval_usec, all_threads=all_threads, mode=mode + pid, sample_interval_usec, all_threads=all_threads, mode=mode, + skip_non_matching_threads=skip_non_matching_threads ) profiler.realtime_stats = realtime_stats - # Determine skip_idle for collector compatibility - skip_idle = mode != PROFILING_MODE_WALL - collector = None match output_format: case "pstats": @@ -633,7 +643,8 @@ def sample( collector = FlamegraphCollector(skip_idle=skip_idle) filename = filename or f"flamegraph.{pid}.html" case "gecko": - collector = GeckoCollector(skip_idle=skip_idle) + # Gecko format never skips idle threads to show full thread states + collector = GeckoCollector(skip_idle=False) filename = filename or f"gecko.{pid}.json" case _: raise ValueError(f"Invalid output format: {output_format}") @@ -882,6 +893,10 @@ def main(): if args.format in ("collapsed", "gecko"): _validate_collapsed_format_args(args, parser) + # Validate that --mode is not used with --gecko + if args.format == "gecko" and args.mode != "wall": + parser.error("--mode option is incompatible with --gecko format. Gecko format automatically uses ALL mode (GIL + CPU analysis).") + sort_value = args.sort if args.sort is not None else 2 if args.module is not None and not args.module: @@ -900,7 +915,11 @@ def main(): elif target_count > 1: parser.error("only one target type can be specified: -p/--pid, -m/--module, or script") - mode = _parse_mode(args.mode) + # Use PROFILING_MODE_ALL for gecko format, otherwise parse user's choice + if args.format == "gecko": + mode = PROFILING_MODE_ALL + else: + mode = _parse_mode(args.mode) if args.pid: sample( diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 01720457e61..60e5000cd72 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -23,6 +23,12 @@ PROFILING_MODE_WALL = 0 PROFILING_MODE_CPU = 1 PROFILING_MODE_GIL = 2 +PROFILING_MODE_ALL = 3 + +# Thread status flags +THREAD_STATUS_HAS_GIL = (1 << 0) +THREAD_STATUS_ON_CPU = (1 << 1) +THREAD_STATUS_UNKNOWN = (1 << 2) try: from concurrent import interpreters @@ -1763,11 +1769,14 @@ def busy(): for thread_info in interpreter_info.threads: statuses[thread_info.thread_id] = thread_info.status - # Check if sleeper thread is idle and busy thread is running + # Check if sleeper thread is off CPU and busy thread is on CPU + # In the new flags system: + # - sleeper should NOT have ON_CPU flag (off CPU) + # - busy should have ON_CPU flag if (sleeper_tid in statuses and busy_tid in statuses and - statuses[sleeper_tid] == 1 and - statuses[busy_tid] == 0): + not (statuses[sleeper_tid] & THREAD_STATUS_ON_CPU) and + (statuses[busy_tid] & THREAD_STATUS_ON_CPU)): break time.sleep(0.5) # Give a bit of time to let threads settle except PermissionError: @@ -1779,8 +1788,8 @@ def busy(): self.assertIsNotNone(busy_tid, "Busy thread id not received") self.assertIn(sleeper_tid, statuses, "Sleeper tid not found in sampled threads") self.assertIn(busy_tid, statuses, "Busy tid not found in sampled threads") - self.assertEqual(statuses[sleeper_tid], 1, "Sleeper thread should be idle (1)") - self.assertEqual(statuses[busy_tid], 0, "Busy thread should be running (0)") + self.assertFalse(statuses[sleeper_tid] & THREAD_STATUS_ON_CPU, "Sleeper thread should be off CPU") + self.assertTrue(statuses[busy_tid] & THREAD_STATUS_ON_CPU, "Busy thread should be on CPU") finally: if client_socket is not None: @@ -1875,11 +1884,14 @@ def busy(): for thread_info in interpreter_info.threads: statuses[thread_info.thread_id] = thread_info.status - # Check if sleeper thread is idle (status 2 for GIL mode) and busy thread is running + # Check if sleeper thread doesn't have GIL and busy thread has GIL + # In the new flags system: + # - sleeper should NOT have HAS_GIL flag (waiting for GIL) + # - busy should have HAS_GIL flag if (sleeper_tid in statuses and busy_tid in statuses and - statuses[sleeper_tid] == 2 and - statuses[busy_tid] == 0): + not (statuses[sleeper_tid] & THREAD_STATUS_HAS_GIL) and + (statuses[busy_tid] & THREAD_STATUS_HAS_GIL)): break time.sleep(0.5) # Give a bit of time to let threads settle except PermissionError: @@ -1891,8 +1903,8 @@ def busy(): self.assertIsNotNone(busy_tid, "Busy thread id not received") self.assertIn(sleeper_tid, statuses, "Sleeper tid not found in sampled threads") self.assertIn(busy_tid, statuses, "Busy tid not found in sampled threads") - self.assertEqual(statuses[sleeper_tid], 2, "Sleeper thread should be idle (1)") - self.assertEqual(statuses[busy_tid], 0, "Busy thread should be running (0)") + self.assertFalse(statuses[sleeper_tid] & THREAD_STATUS_HAS_GIL, "Sleeper thread should not have GIL") + self.assertTrue(statuses[busy_tid] & THREAD_STATUS_HAS_GIL, "Busy thread should have GIL") finally: if client_socket is not None: @@ -1900,6 +1912,128 @@ def busy(): p.terminate() p.wait(timeout=SHORT_TIMEOUT) + @unittest.skipIf( + sys.platform not in ("linux", "darwin", "win32"), + "Test only runs on supported platforms (Linux, macOS, or Windows)", + ) + @unittest.skipIf(sys.platform == "android", "Android raises Linux-specific exception") + def test_thread_status_all_mode_detection(self): + port = find_unused_port() + script = textwrap.dedent( + f"""\ + import socket + import threading + import time + import sys + + def sleeper_thread(): + conn = socket.create_connection(("localhost", {port})) + conn.sendall(b"sleeper:" + str(threading.get_native_id()).encode()) + while True: + time.sleep(1) + + def busy_thread(): + conn = socket.create_connection(("localhost", {port})) + conn.sendall(b"busy:" + str(threading.get_native_id()).encode()) + while True: + sum(range(100000)) + + t1 = threading.Thread(target=sleeper_thread) + t2 = threading.Thread(target=busy_thread) + t1.start() + t2.start() + t1.join() + t2.join() + """ + ) + + with os_helper.temp_dir() as tmp_dir: + script_file = make_script(tmp_dir, "script", script) + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.bind(("localhost", port)) + server_socket.listen(2) + server_socket.settimeout(SHORT_TIMEOUT) + + p = subprocess.Popen( + [sys.executable, script_file], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + client_sockets = [] + try: + sleeper_tid = None + busy_tid = None + + # Receive thread IDs from the child process + for _ in range(2): + client_socket, _ = server_socket.accept() + client_sockets.append(client_socket) + line = client_socket.recv(1024) + if line: + if line.startswith(b"sleeper:"): + try: + sleeper_tid = int(line.split(b":")[-1]) + except Exception: + pass + elif line.startswith(b"busy:"): + try: + busy_tid = int(line.split(b":")[-1]) + except Exception: + pass + + server_socket.close() + + attempts = 10 + statuses = {} + try: + unwinder = RemoteUnwinder(p.pid, all_threads=True, mode=PROFILING_MODE_ALL, + skip_non_matching_threads=False) + for _ in range(attempts): + traces = unwinder.get_stack_trace() + # Find threads and their statuses + statuses = {} + for interpreter_info in traces: + for thread_info in interpreter_info.threads: + statuses[thread_info.thread_id] = thread_info.status + + # Check ALL mode provides both GIL and CPU info + # - sleeper should NOT have ON_CPU and NOT have HAS_GIL + # - busy should have ON_CPU and have HAS_GIL + if (sleeper_tid in statuses and + busy_tid in statuses and + not (statuses[sleeper_tid] & THREAD_STATUS_ON_CPU) and + not (statuses[sleeper_tid] & THREAD_STATUS_HAS_GIL) and + (statuses[busy_tid] & THREAD_STATUS_ON_CPU) and + (statuses[busy_tid] & THREAD_STATUS_HAS_GIL)): + break + time.sleep(0.5) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + + self.assertIsNotNone(sleeper_tid, "Sleeper thread id not received") + self.assertIsNotNone(busy_tid, "Busy thread id not received") + self.assertIn(sleeper_tid, statuses, "Sleeper tid not found in sampled threads") + self.assertIn(busy_tid, statuses, "Busy tid not found in sampled threads") + + # Sleeper thread: off CPU, no GIL + self.assertFalse(statuses[sleeper_tid] & THREAD_STATUS_ON_CPU, "Sleeper should be off CPU") + self.assertFalse(statuses[sleeper_tid] & THREAD_STATUS_HAS_GIL, "Sleeper should not have GIL") + + # Busy thread: on CPU, has GIL + self.assertTrue(statuses[busy_tid] & THREAD_STATUS_ON_CPU, "Busy should be on CPU") + self.assertTrue(statuses[busy_tid] & THREAD_STATUS_HAS_GIL, "Busy should have GIL") + + finally: + for client_socket in client_sockets: + client_socket.close() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + p.stdout.close() + p.stderr.close() if __name__ == "__main__": diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index 0ba6799a1ce..ae9bf3ef2e5 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -63,12 +63,14 @@ def __repr__(self): class MockThreadInfo: """Mock ThreadInfo for testing since the real one isn't accessible.""" - def __init__(self, thread_id, frame_info): + def __init__(self, thread_id, frame_info, status=0, gc_collecting=False): # Default to THREAD_STATE_RUNNING (0) self.thread_id = thread_id self.frame_info = frame_info + self.status = status + self.gc_collecting = gc_collecting def __repr__(self): - return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info})" + return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info}, status={self.status}, gc_collecting={self.gc_collecting})" class MockInterpreterInfo: @@ -674,6 +676,97 @@ def test_gecko_collector_export(self): self.assertIn("func2", string_array) self.assertIn("other_func", string_array) + def test_gecko_collector_markers(self): + """Test Gecko profile markers for GIL and CPU state tracking.""" + try: + from _remote_debugging import THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU, THREAD_STATUS_GIL_REQUESTED + except ImportError: + THREAD_STATUS_HAS_GIL = (1 << 0) + THREAD_STATUS_ON_CPU = (1 << 1) + THREAD_STATUS_GIL_REQUESTED = (1 << 3) + + collector = GeckoCollector() + + # Status combinations for different thread states + HAS_GIL_ON_CPU = THREAD_STATUS_HAS_GIL | THREAD_STATUS_ON_CPU # Running Python code + NO_GIL_ON_CPU = THREAD_STATUS_ON_CPU # Running native code + WAITING_FOR_GIL = THREAD_STATUS_GIL_REQUESTED # Waiting for GIL + + # Simulate thread state transitions + collector.collect([ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [("test.py", 10, "python_func")], status=HAS_GIL_ON_CPU) + ]) + ]) + + collector.collect([ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [("test.py", 15, "wait_func")], status=WAITING_FOR_GIL) + ]) + ]) + + collector.collect([ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [("test.py", 20, "python_func2")], status=HAS_GIL_ON_CPU) + ]) + ]) + + collector.collect([ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [("native.c", 100, "native_func")], status=NO_GIL_ON_CPU) + ]) + ]) + + profile_data = collector._build_profile() + + # Verify we have threads with markers + self.assertIn("threads", profile_data) + self.assertEqual(len(profile_data["threads"]), 1) + thread_data = profile_data["threads"][0] + + # Check markers exist + self.assertIn("markers", thread_data) + markers = thread_data["markers"] + + # Should have marker arrays + self.assertIn("name", markers) + self.assertIn("startTime", markers) + self.assertIn("endTime", markers) + self.assertIn("category", markers) + self.assertGreater(markers["length"], 0, "Should have generated markers") + + # Get marker names from string table + string_array = profile_data["shared"]["stringArray"] + marker_names = [string_array[idx] for idx in markers["name"]] + + # Verify we have different marker types + marker_name_set = set(marker_names) + + # Should have "Has GIL" markers (when thread had GIL) + self.assertIn("Has GIL", marker_name_set, "Should have 'Has GIL' markers") + + # Should have "No GIL" markers (when thread didn't have GIL) + self.assertIn("No GIL", marker_name_set, "Should have 'No GIL' markers") + + # Should have "On CPU" markers (when thread was on CPU) + self.assertIn("On CPU", marker_name_set, "Should have 'On CPU' markers") + + # Should have "Waiting for GIL" markers (when thread was waiting) + self.assertIn("Waiting for GIL", marker_name_set, "Should have 'Waiting for GIL' markers") + + # Verify marker structure + for i in range(markers["length"]): + # All markers should be interval markers (phase = 1) + self.assertEqual(markers["phase"][i], 1, f"Marker {i} should be interval marker") + + # All markers should have valid time range + start_time = markers["startTime"][i] + end_time = markers["endTime"][i] + self.assertLessEqual(start_time, end_time, f"Marker {i} should have valid time range") + + # All markers should have valid category + self.assertGreaterEqual(markers["category"][i], 0, f"Marker {i} should have valid category") + def test_pstats_collector_export(self): collector = PstatsCollector( sample_interval_usec=1000000 @@ -2625,19 +2718,30 @@ def test_mode_validation(self): def test_frames_filtered_with_skip_idle(self): """Test that frames are actually filtered when skip_idle=True.""" + # Import thread status flags + try: + from _remote_debugging import THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU + except ImportError: + THREAD_STATUS_HAS_GIL = (1 << 0) + THREAD_STATUS_ON_CPU = (1 << 1) + # Create mock frames with different thread statuses class MockThreadInfoWithStatus: def __init__(self, thread_id, frame_info, status): self.thread_id = thread_id self.frame_info = frame_info self.status = status + self.gc_collecting = False + + # Create test data: active thread (HAS_GIL | ON_CPU), idle thread (neither), and another active thread + ACTIVE_STATUS = THREAD_STATUS_HAS_GIL | THREAD_STATUS_ON_CPU # Has GIL and on CPU + IDLE_STATUS = 0 # Neither has GIL nor on CPU - # Create test data: running thread, idle thread, and another running thread test_frames = [ MockInterpreterInfo(0, [ - MockThreadInfoWithStatus(1, [MockFrameInfo("active1.py", 10, "active_func1")], 0), # RUNNING - MockThreadInfoWithStatus(2, [MockFrameInfo("idle.py", 20, "idle_func")], 1), # IDLE - MockThreadInfoWithStatus(3, [MockFrameInfo("active2.py", 30, "active_func2")], 0), # RUNNING + MockThreadInfoWithStatus(1, [MockFrameInfo("active1.py", 10, "active_func1")], ACTIVE_STATUS), + MockThreadInfoWithStatus(2, [MockFrameInfo("idle.py", 20, "idle_func")], IDLE_STATUS), + MockThreadInfoWithStatus(3, [MockFrameInfo("active2.py", 30, "active_func2")], ACTIVE_STATUS), ]) ] diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index c6ced39c70c..d190b3c9faf 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -11,6 +11,7 @@ * HEADERS AND INCLUDES * ============================================================================ */ +#include #include #include #include @@ -81,6 +82,8 @@ typedef enum _WIN32_THREADSTATE { #define SIZEOF_TYPE_OBJ sizeof(PyTypeObject) #define SIZEOF_UNICODE_OBJ sizeof(PyUnicodeObject) #define SIZEOF_LONG_OBJ sizeof(PyLongObject) +#define SIZEOF_GC_RUNTIME_STATE sizeof(struct _gc_runtime_state) +#define SIZEOF_INTERPRETER_STATE sizeof(PyInterpreterState) // Calculate the minimum buffer size needed to read interpreter state fields // We need to read code_object_generation and potentially tlbc_generation @@ -178,8 +181,9 @@ static PyStructSequence_Desc CoroInfo_desc = { // ThreadInfo structseq type - replaces 2-tuple (thread_id, frame_info) static PyStructSequence_Field ThreadInfo_fields[] = { {"thread_id", "Thread ID"}, - {"status", "Thread status"}, + {"status", "Thread status (flags: HAS_GIL, ON_CPU, UNKNOWN or legacy enum)"}, {"frame_info", "Frame information"}, + {"gc_collecting", "Whether GC is collecting (interpreter-level)"}, {NULL} }; @@ -187,7 +191,7 @@ static PyStructSequence_Desc ThreadInfo_desc = { "_remote_debugging.ThreadInfo", "Information about a thread", ThreadInfo_fields, - 2 + 3 }; // InterpreterInfo structseq type - replaces 2-tuple (interpreter_id, thread_list) @@ -247,9 +251,16 @@ enum _ThreadState { enum _ProfilingMode { PROFILING_MODE_WALL = 0, PROFILING_MODE_CPU = 1, - PROFILING_MODE_GIL = 2 + PROFILING_MODE_GIL = 2, + PROFILING_MODE_ALL = 3 // Combines GIL + CPU checks }; +// Thread status flags (can be combined) +#define THREAD_STATUS_HAS_GIL (1 << 0) // Thread has the GIL +#define THREAD_STATUS_ON_CPU (1 << 1) // Thread is running on CPU +#define THREAD_STATUS_UNKNOWN (1 << 2) // Status could not be determined +#define THREAD_STATUS_GIL_REQUESTED (1 << 3) // Thread is waiting for the GIL + typedef struct { PyObject_HEAD proc_handle_t handle; @@ -2650,34 +2661,70 @@ unwind_stack_for_thread( long tid = GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.native_thread_id); - // Calculate thread status based on mode - int status = THREAD_STATE_UNKNOWN; - if (unwinder->mode == PROFILING_MODE_CPU) { - long pthread_id = GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.thread_id); - status = get_thread_status(unwinder, tid, pthread_id); - if (status == -1) { - PyErr_Print(); - PyErr_SetString(PyExc_RuntimeError, "Failed to get thread status"); - goto error; - } - } else if (unwinder->mode == PROFILING_MODE_GIL) { + // Read GC collecting state from the interpreter (before any skip checks) + uintptr_t interp_addr = GET_MEMBER(uintptr_t, ts, unwinder->debug_offsets.thread_state.interp); + + // Read the GC runtime state from the interpreter state + uintptr_t gc_addr = interp_addr + unwinder->debug_offsets.interpreter_state.gc; + char gc_state[SIZEOF_GC_RUNTIME_STATE]; + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, gc_addr, unwinder->debug_offsets.gc.size, gc_state) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read GC state"); + goto error; + } + + int gc_collecting = GET_MEMBER(int, gc_state, unwinder->debug_offsets.gc.collecting); + + // Calculate thread status using flags (always) + int status_flags = 0; + + // Check GIL status + int has_gil = 0; + int gil_requested = 0; #ifdef Py_GIL_DISABLED - // All threads are considered running in free threading builds if they have a thread state attached - int active = GET_MEMBER(_thread_status, ts, unwinder->debug_offsets.thread_state.status).active; - status = active ? THREAD_STATE_RUNNING : THREAD_STATE_GIL_WAIT; + int active = GET_MEMBER(_thread_status, ts, unwinder->debug_offsets.thread_state.status).active; + has_gil = active; #else - status = (*current_tstate == gil_holder_tstate) ? THREAD_STATE_RUNNING : THREAD_STATE_GIL_WAIT; + // Read holds_gil directly from thread state + has_gil = GET_MEMBER(int, ts, unwinder->debug_offsets.thread_state.holds_gil); + + // Check if thread is actively requesting the GIL + if (unwinder->debug_offsets.thread_state.gil_requested != 0) { + gil_requested = GET_MEMBER(int, ts, unwinder->debug_offsets.thread_state.gil_requested); + } + + // Set GIL_REQUESTED flag if thread is waiting + if (!has_gil && gil_requested) { + status_flags |= THREAD_STATUS_GIL_REQUESTED; + } #endif - } else { - // PROFILING_MODE_WALL - all threads are considered running - status = THREAD_STATE_RUNNING; + if (has_gil) { + status_flags |= THREAD_STATUS_HAS_GIL; + } + + // Assert that we never have both HAS_GIL and GIL_REQUESTED set at the same time + // This would indicate a race condition in the GIL state tracking + assert(!(has_gil && gil_requested)); + + // Check CPU status + long pthread_id = GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.thread_id); + int cpu_status = get_thread_status(unwinder, tid, pthread_id); + if (cpu_status == -1) { + status_flags |= THREAD_STATUS_UNKNOWN; + } else if (cpu_status == THREAD_STATE_RUNNING) { + status_flags |= THREAD_STATUS_ON_CPU; } // Check if we should skip this thread based on mode int should_skip = 0; - if (unwinder->skip_non_matching_threads && status != THREAD_STATE_RUNNING && - (unwinder->mode == PROFILING_MODE_CPU || unwinder->mode == PROFILING_MODE_GIL)) { - should_skip = 1; + if (unwinder->skip_non_matching_threads) { + if (unwinder->mode == PROFILING_MODE_CPU) { + // Skip if not on CPU + should_skip = !(status_flags & THREAD_STATUS_ON_CPU); + } else if (unwinder->mode == PROFILING_MODE_GIL) { + // Skip if doesn't have GIL + should_skip = !(status_flags & THREAD_STATUS_HAS_GIL); + } + // PROFILING_MODE_WALL and PROFILING_MODE_ALL never skip } if (should_skip) { @@ -2719,16 +2766,25 @@ unwind_stack_for_thread( goto error; } - PyObject *py_status = PyLong_FromLong(status); + // Always use status_flags + PyObject *py_status = PyLong_FromLong(status_flags); if (py_status == NULL) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create thread status"); goto error; } - PyErr_Print(); + PyObject *py_gc_collecting = PyBool_FromLong(gc_collecting); + if (py_gc_collecting == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create gc_collecting"); + Py_DECREF(py_status); + goto error; + } + + // py_status contains status flags (bitfield) PyStructSequence_SetItem(result, 0, thread_id); PyStructSequence_SetItem(result, 1, py_status); // Steals reference PyStructSequence_SetItem(result, 2, frame_info); // Steals reference + PyStructSequence_SetItem(result, 3, py_gc_collecting); // Steals reference cleanup_stack_chunks(&chunks); return result; @@ -3401,6 +3457,21 @@ _remote_debugging_exec(PyObject *m) if (rc < 0) { return -1; } + + // Add thread status flag constants + if (PyModule_AddIntConstant(m, "THREAD_STATUS_HAS_GIL", THREAD_STATUS_HAS_GIL) < 0) { + return -1; + } + if (PyModule_AddIntConstant(m, "THREAD_STATUS_ON_CPU", THREAD_STATUS_ON_CPU) < 0) { + return -1; + } + if (PyModule_AddIntConstant(m, "THREAD_STATUS_UNKNOWN", THREAD_STATUS_UNKNOWN) < 0) { + return -1; + } + if (PyModule_AddIntConstant(m, "THREAD_STATUS_GIL_REQUESTED", THREAD_STATUS_GIL_REQUESTED) < 0) { + return -1; + } + if (RemoteDebugging_InitState(st) < 0) { return -1; } diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 9b6506ac332..f6ada3892f8 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -207,6 +207,7 @@ drop_gil_impl(PyThreadState *tstate, struct _gil_runtime_state *gil) _Py_atomic_store_int_relaxed(&gil->locked, 0); if (tstate != NULL) { tstate->holds_gil = 0; + tstate->gil_requested = 0; } COND_SIGNAL(gil->cond); MUTEX_UNLOCK(gil->mutex); @@ -320,6 +321,8 @@ take_gil(PyThreadState *tstate) MUTEX_LOCK(gil->mutex); + tstate->gil_requested = 1; + int drop_requested = 0; while (_Py_atomic_load_int_relaxed(&gil->locked)) { unsigned long saved_switchnum = gil->switch_number; @@ -407,6 +410,7 @@ take_gil(PyThreadState *tstate) } assert(_PyThreadState_CheckConsistency(tstate)); + tstate->gil_requested = 0; tstate->holds_gil = 1; _Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT); update_eval_breaker_for_thread(interp, tstate); From 336366fd7ca61858572fdb78e2bd79014b215f19 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Mon, 17 Nov 2025 05:39:00 -0800 Subject: [PATCH 230/313] GH-140643: Add `` and `` frames to the sampling profiler (#141108) - Introduce a new field in the GC state to store the frame that initiated garbage collection. - Update RemoteUnwinder to include options for including "" and "" frames in the stack trace. - Modify the sampling profiler to accept parameters for controlling the inclusion of native and GC frames. - Enhance the stack collector to properly format and append these frames during profiling. - Add tests to verify the correct behavior of the profiler with respect to native and GC frames, including options to exclude them. Co-authored-by: Pablo Galindo Salgado --- Doc/library/profile.rst | 12 +- Include/internal/pycore_debug_offsets.h | 2 + .../pycore_global_objects_fini_generated.h | 4 + Include/internal/pycore_global_strings.h | 4 + Include/internal/pycore_interp_structs.h | 3 + Include/internal/pycore_interpframe_structs.h | 1 - .../internal/pycore_runtime_init_generated.h | 4 + .../internal/pycore_unicodeobject_generated.h | 16 ++ Lib/profiling/sampling/flamegraph.js | 28 ++- Lib/profiling/sampling/sample.py | 26 ++- Lib/profiling/sampling/stack_collector.py | 18 +- Lib/test/test_external_inspection.py | 2 + .../test_profiling/test_sampling_profiler.py | 208 +++++++++++++++++- ...-11-05-19-50-37.gh-issue-140643.QCEOqG.rst | 3 + Modules/_remote_debugging_module.c | 170 +++++++++----- Modules/clinic/_remote_debugging_module.c.h | 46 +++- Python/gc.c | 2 + Python/gc_free_threading.c | 2 + 18 files changed, 465 insertions(+), 86 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index faf8079db3d..5bf36b13c6d 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -265,6 +265,14 @@ Profile with real-time sampling statistics:: Sample all threads in the process instead of just the main thread +.. option:: --native + + Include artificial ```` frames to denote calls to non-Python code. + +.. option:: --no-gc + + Don't include artificial ```` frames to denote active garbage collection. + .. option:: --realtime-stats Print real-time sampling statistics during profiling @@ -349,7 +357,7 @@ This section documents the programmatic interface for the :mod:`!profiling.sampl For command-line usage, see :ref:`sampling-profiler-cli`. For conceptual information about statistical profiling, see :ref:`statistical-profiling` -.. function:: sample(pid, *, sort=2, sample_interval_usec=100, duration_sec=10, filename=None, all_threads=False, limit=None, show_summary=True, output_format="pstats", realtime_stats=False) +.. function:: sample(pid, *, sort=2, sample_interval_usec=100, duration_sec=10, filename=None, all_threads=False, limit=None, show_summary=True, output_format="pstats", realtime_stats=False, native=False, gc=True) Sample a Python process and generate profiling data. @@ -367,6 +375,8 @@ about statistical profiling, see :ref:`statistical-profiling` :param bool show_summary: Whether to show summary statistics (default: True) :param str output_format: Output format - 'pstats' or 'collapsed' (default: 'pstats') :param bool realtime_stats: Whether to display real-time statistics (default: False) + :param bool native: Whether to include ```` frames (default: False) + :param bool gc: Whether to include ```` frames (default: True) :raises ValueError: If output_format is not 'pstats' or 'collapsed' diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index f6d50bf5df7..0f17bf17f82 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -212,6 +212,7 @@ typedef struct _Py_DebugOffsets { struct _gc { uint64_t size; uint64_t collecting; + uint64_t frame; } gc; // Generator object offset; @@ -355,6 +356,7 @@ typedef struct _Py_DebugOffsets { .gc = { \ .size = sizeof(struct _gc_runtime_state), \ .collecting = offsetof(struct _gc_runtime_state, collecting), \ + .frame = offsetof(struct _gc_runtime_state, frame), \ }, \ .gen_object = { \ .size = sizeof(PyGenObject), \ diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 92ded14891a..ecef4364cc3 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1326,10 +1326,12 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot_locals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(empty)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(format)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(gc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(generic_base)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(json_decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(kwdefaults)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(list_err)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(native)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(str_replace_inf)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(type_params)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(utf_8)); @@ -1763,6 +1765,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fullerror)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(func)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(future)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(gc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(generation)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get_debug)); @@ -1906,6 +1909,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(name_from)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespace_separator)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespaces)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(native)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ndigits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(nested)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(new_file_name)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index cd21b0847b7..4dd73291df4 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -46,10 +46,12 @@ struct _Py_global_strings { STRUCT_FOR_STR(dot_locals, ".") STRUCT_FOR_STR(empty, "") STRUCT_FOR_STR(format, ".format") + STRUCT_FOR_STR(gc, "") STRUCT_FOR_STR(generic_base, ".generic_base") STRUCT_FOR_STR(json_decoder, "json.decoder") STRUCT_FOR_STR(kwdefaults, ".kwdefaults") STRUCT_FOR_STR(list_err, "list index out of range") + STRUCT_FOR_STR(native, "") STRUCT_FOR_STR(str_replace_inf, "1e309") STRUCT_FOR_STR(type_params, ".type_params") STRUCT_FOR_STR(utf_8, "utf-8") @@ -486,6 +488,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(fullerror) STRUCT_FOR_ID(func) STRUCT_FOR_ID(future) + STRUCT_FOR_ID(gc) STRUCT_FOR_ID(generation) STRUCT_FOR_ID(get) STRUCT_FOR_ID(get_debug) @@ -629,6 +632,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(name_from) STRUCT_FOR_ID(namespace_separator) STRUCT_FOR_ID(namespaces) + STRUCT_FOR_ID(native) STRUCT_FOR_ID(ndigits) STRUCT_FOR_ID(nested) STRUCT_FOR_ID(new_file_name) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 9e4504479cd..f861d3abd96 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -212,6 +212,9 @@ struct _gc_runtime_state { struct gc_generation_stats generation_stats[NUM_GENERATIONS]; /* true if we are currently running the collector */ int collecting; + // The frame that started the current collection. It might be NULL even when + // collecting (if no Python frame is running): + _PyInterpreterFrame *frame; /* list of uncollectable objects */ PyObject *garbage; /* a list of callbacks to be invoked when collection is performed */ diff --git a/Include/internal/pycore_interpframe_structs.h b/Include/internal/pycore_interpframe_structs.h index 835b8e58194..38510685f40 100644 --- a/Include/internal/pycore_interpframe_structs.h +++ b/Include/internal/pycore_interpframe_structs.h @@ -24,7 +24,6 @@ enum _frameowner { FRAME_OWNED_BY_GENERATOR = 1, FRAME_OWNED_BY_FRAME_OBJECT = 2, FRAME_OWNED_BY_INTERPRETER = 3, - FRAME_OWNED_BY_CSTACK = 4, }; struct _PyInterpreterFrame { diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 50d82d0a365..08f8d0e59d1 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1321,10 +1321,12 @@ extern "C" { INIT_STR(dot_locals, "."), \ INIT_STR(empty, ""), \ INIT_STR(format, ".format"), \ + INIT_STR(gc, ""), \ INIT_STR(generic_base, ".generic_base"), \ INIT_STR(json_decoder, "json.decoder"), \ INIT_STR(kwdefaults, ".kwdefaults"), \ INIT_STR(list_err, "list index out of range"), \ + INIT_STR(native, ""), \ INIT_STR(str_replace_inf, "1e309"), \ INIT_STR(type_params, ".type_params"), \ INIT_STR(utf_8, "utf-8"), \ @@ -1761,6 +1763,7 @@ extern "C" { INIT_ID(fullerror), \ INIT_ID(func), \ INIT_ID(future), \ + INIT_ID(gc), \ INIT_ID(generation), \ INIT_ID(get), \ INIT_ID(get_debug), \ @@ -1904,6 +1907,7 @@ extern "C" { INIT_ID(name_from), \ INIT_ID(namespace_separator), \ INIT_ID(namespaces), \ + INIT_ID(native), \ INIT_ID(ndigits), \ INIT_ID(nested), \ INIT_ID(new_file_name), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index b4d920154b6..b1e57126b92 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1732,6 +1732,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(gc); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(generation); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2304,6 +2308,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(native); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ndigits); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3236,6 +3244,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(gc); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_STR(anon_null); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3260,6 +3272,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(native); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_STR(anon_setcomp); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Lib/profiling/sampling/flamegraph.js b/Lib/profiling/sampling/flamegraph.js index 95ad7ca6184..670ca22d442 100644 --- a/Lib/profiling/sampling/flamegraph.js +++ b/Lib/profiling/sampling/flamegraph.js @@ -151,17 +151,22 @@ function createPythonTooltip(data) { const funcname = resolveString(d.data.funcname) || resolveString(d.data.name); const filename = resolveString(d.data.filename) || ""; + // Don't show file location for special frames like and + const isSpecialFrame = filename === "~"; + const fileLocationHTML = isSpecialFrame ? "" : ` +

+ ${filename}${d.data.lineno ? ":" + d.data.lineno : ""} +
`; + const tooltipHTML = `
${funcname}
-
- ${filename}${d.data.lineno ? ":" + d.data.lineno : ""} -
+ ${fileLocationHTML}
Execution Time: @@ -474,14 +479,23 @@ function populateStats(data) { if (i < hotSpots.length && hotSpots[i]) { const hotspot = hotSpots[i]; const filename = hotspot.filename || 'unknown'; - const basename = filename !== 'unknown' ? filename.split('/').pop() : 'unknown'; const lineno = hotspot.lineno ?? '?'; let funcDisplay = hotspot.funcname || 'unknown'; if (funcDisplay.length > 35) { funcDisplay = funcDisplay.substring(0, 32) + '...'; } - document.getElementById(`hotspot-file-${num}`).textContent = `${basename}:${lineno}`; + // Don't show file:line for special frames like and + const isSpecialFrame = filename === '~' && (lineno === 0 || lineno === '?'); + let fileDisplay; + if (isSpecialFrame) { + fileDisplay = '--'; + } else { + const basename = filename !== 'unknown' ? filename.split('/').pop() : 'unknown'; + fileDisplay = `${basename}:${lineno}`; + } + + document.getElementById(`hotspot-file-${num}`).textContent = fileDisplay; document.getElementById(`hotspot-func-${num}`).textContent = funcDisplay; document.getElementById(`hotspot-detail-${num}`).textContent = `${hotspot.directPercent.toFixed(1)}% samples (${hotspot.directSamples.toLocaleString()})`; } else { diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 5ca68911d8a..713931a639d 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -137,19 +137,19 @@ def _run_with_sync(original_cmd): class SampleProfiler: - def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, skip_non_matching_threads=True): + def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, native=False, gc=True, skip_non_matching_threads=True): self.pid = pid self.sample_interval_usec = sample_interval_usec self.all_threads = all_threads if _FREE_THREADED_BUILD: self.unwinder = _remote_debugging.RemoteUnwinder( - self.pid, all_threads=self.all_threads, mode=mode, + self.pid, all_threads=self.all_threads, mode=mode, native=native, gc=gc, skip_non_matching_threads=skip_non_matching_threads ) else: only_active_threads = bool(self.all_threads) self.unwinder = _remote_debugging.RemoteUnwinder( - self.pid, only_active_thread=only_active_threads, mode=mode, + self.pid, only_active_thread=only_active_threads, mode=mode, native=native, gc=gc, skip_non_matching_threads=skip_non_matching_threads ) # Track sample intervals and total sample count @@ -616,6 +616,8 @@ def sample( output_format="pstats", realtime_stats=False, mode=PROFILING_MODE_WALL, + native=False, + gc=True, ): # PROFILING_MODE_ALL implies no skipping at all if mode == PROFILING_MODE_ALL: @@ -627,7 +629,7 @@ def sample( skip_idle = mode != PROFILING_MODE_WALL profiler = SampleProfiler( - pid, sample_interval_usec, all_threads=all_threads, mode=mode, + pid, sample_interval_usec, all_threads=all_threads, mode=mode, native=native, gc=gc, skip_non_matching_threads=skip_non_matching_threads ) profiler.realtime_stats = realtime_stats @@ -717,6 +719,8 @@ def wait_for_process_and_sample(pid, sort_value, args): output_format=args.format, realtime_stats=args.realtime_stats, mode=mode, + native=args.native, + gc=args.gc, ) @@ -767,9 +771,19 @@ def main(): sampling_group.add_argument( "--realtime-stats", action="store_true", - default=False, help="Print real-time sampling statistics (Hz, mean, min, max, stdev) during profiling", ) + sampling_group.add_argument( + "--native", + action="store_true", + help="Include artificial \"\" frames to denote calls to non-Python code.", + ) + sampling_group.add_argument( + "--no-gc", + action="store_false", + dest="gc", + help="Don't include artificial \"\" frames to denote active garbage collection.", + ) # Mode options mode_group = parser.add_argument_group("Mode options") @@ -934,6 +948,8 @@ def main(): output_format=args.format, realtime_stats=args.realtime_stats, mode=mode, + native=args.native, + gc=args.gc, ) elif args.module or args.args: if args.module: diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index bc38151e067..1436811976a 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -36,10 +36,16 @@ def process_frames(self, frames, thread_id): def export(self, filename): lines = [] for (call_tree, thread_id), count in self.stack_counter.items(): - stack_str = ";".join( - f"{os.path.basename(f[0])}:{f[2]}:{f[1]}" for f in call_tree - ) - lines.append((f"tid:{thread_id};{stack_str}", count)) + parts = [f"tid:{thread_id}"] + for file, line, func in call_tree: + # This is what pstats does for "special" frames: + if file == "~" and line == 0: + part = func + else: + part = f"{os.path.basename(file)}:{func}:{line}" + parts.append(part) + stack_str = ";".join(parts) + lines.append((stack_str, count)) lines.sort(key=lambda x: (-x[1], x[0])) @@ -98,6 +104,10 @@ def export(self, filename): def _format_function_name(func): filename, lineno, funcname = func + # Special frames like and should not show file:line + if filename == "~" and lineno == 0: + return funcname + if len(filename) > 50: parts = filename.split("/") if len(parts) > 2: diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 60e5000cd72..7decd8f32d5 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -159,6 +159,8 @@ def foo(): FrameInfo([script_name, 12, "baz"]), FrameInfo([script_name, 9, "bar"]), FrameInfo([threading.__file__, ANY, "Thread.run"]), + FrameInfo([threading.__file__, ANY, "Thread._bootstrap_inner"]), + FrameInfo([threading.__file__, ANY, "Thread._bootstrap"]), ] # Is possible that there are more threads, so we check that the # expected stack traces are in the result (looking at you Windows!) diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index ae9bf3ef2e5..a24dbb55cd7 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -2025,7 +2025,6 @@ def test_sample_target_script(self): # Should see some of our test functions self.assertIn("slow_fibonacci", output) - def test_sample_target_module(self): tempdir = tempfile.TemporaryDirectory(delete=False) self.addCleanup(lambda x: shutil.rmtree(x), tempdir.name) @@ -2264,7 +2263,9 @@ def test_cli_module_argument_parsing(self): show_summary=True, output_format="pstats", realtime_stats=False, - mode=0 + mode=0, + native=False, + gc=True, ) @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") @@ -2292,7 +2293,9 @@ def test_cli_module_with_arguments(self): show_summary=True, output_format="pstats", realtime_stats=False, - mode=0 + mode=0, + native=False, + gc=True, ) @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") @@ -2320,7 +2323,9 @@ def test_cli_script_argument_parsing(self): show_summary=True, output_format="pstats", realtime_stats=False, - mode=0 + mode=0, + native=False, + gc=True, ) @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") @@ -2420,7 +2425,9 @@ def test_cli_module_with_profiler_options(self): show_summary=True, output_format="pstats", realtime_stats=False, - mode=0 + mode=0, + native=False, + gc=True, ) @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") @@ -2454,7 +2461,9 @@ def test_cli_script_with_profiler_options(self): show_summary=True, output_format="collapsed", realtime_stats=False, - mode=0 + mode=0, + native=False, + gc=True, ) def test_cli_empty_module_name(self): @@ -2666,7 +2675,9 @@ def test_argument_parsing_basic(self): show_summary=True, output_format="pstats", realtime_stats=False, - mode=0 + mode=0, + native=False, + gc=True, ) def test_sort_options(self): @@ -3121,6 +3132,187 @@ def test_parse_mode_function(self): @requires_subprocess() @skip_if_not_supported +class TestGCFrameTracking(unittest.TestCase): + """Tests for GC frame tracking in the sampling profiler.""" + + @classmethod + def setUpClass(cls): + """Create a static test script with GC frames and CPU-intensive work.""" + cls.gc_test_script = ''' +import gc + +class ExpensiveGarbage: + """Class that triggers GC with expensive finalizer (callback).""" + def __init__(self): + self.cycle = self + + def __del__(self): + # CPU-intensive work in the finalizer callback + result = 0 + for i in range(100000): + result += i * i + if i % 1000 == 0: + result = result % 1000000 + +def main_loop(): + """Main loop that triggers GC with expensive callback.""" + while True: + ExpensiveGarbage() + gc.collect() + +if __name__ == "__main__": + main_loop() +''' + + def test_gc_frames_enabled(self): + """Test that GC frames appear when gc tracking is enabled.""" + with ( + test_subprocess(self.gc_test_script) as subproc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + sample_interval_usec=5000, + show_summary=False, + native=False, + gc=True, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + output = captured_output.getvalue() + + # Should capture samples + self.assertIn("Captured", output) + self.assertIn("samples", output) + + # GC frames should be present + self.assertIn("", output) + + def test_gc_frames_disabled(self): + """Test that GC frames do not appear when gc tracking is disabled.""" + with ( + test_subprocess(self.gc_test_script) as subproc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + sample_interval_usec=5000, + show_summary=False, + native=False, + gc=False, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + output = captured_output.getvalue() + + # Should capture samples + self.assertIn("Captured", output) + self.assertIn("samples", output) + + # GC frames should NOT be present + self.assertNotIn("", output) + + +@requires_subprocess() +@skip_if_not_supported +class TestNativeFrameTracking(unittest.TestCase): + """Tests for native frame tracking in the sampling profiler.""" + + @classmethod + def setUpClass(cls): + """Create a static test script with native frames and CPU-intensive work.""" + cls.native_test_script = ''' +import operator + +def main_loop(): + while True: + # Native code in the middle of the stack: + operator.call(inner) + +def inner(): + # Python code at the top of the stack: + for _ in range(1_000_0000): + pass + +if __name__ == "__main__": + main_loop() +''' + + def test_native_frames_enabled(self): + """Test that native frames appear when native tracking is enabled.""" + collapsed_file = tempfile.NamedTemporaryFile( + suffix=".txt", delete=False + ) + self.addCleanup(close_and_unlink, collapsed_file) + + with ( + test_subprocess(self.native_test_script) as subproc, + ): + # Suppress profiler output when testing file export + with ( + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + filename=collapsed_file.name, + output_format="collapsed", + sample_interval_usec=1000, + native=True, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + # Verify file was created and contains valid data + self.assertTrue(os.path.exists(collapsed_file.name)) + self.assertGreater(os.path.getsize(collapsed_file.name), 0) + + # Check file format + with open(collapsed_file.name, "r") as f: + content = f.read() + + lines = content.strip().split("\n") + self.assertGreater(len(lines), 0) + + stacks = [line.rsplit(" ", 1)[0] for line in lines] + + # Most samples should have native code in the middle of the stack: + self.assertTrue(any(";;" in stack for stack in stacks)) + + # No samples should have native code at the top of the stack: + self.assertFalse(any(stack.endswith(";") for stack in stacks)) + + def test_native_frames_disabled(self): + """Test that native frames do not appear when native tracking is disabled.""" + with ( + test_subprocess(self.native_test_script) as subproc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + sample_interval_usec=5000, + show_summary=False, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + output = captured_output.getvalue() + # Native frames should NOT be present: + self.assertNotIn("", output) + + class TestProcessPoolExecutorSupport(unittest.TestCase): """ Test that ProcessPoolExecutor works correctly with profiling.sampling. @@ -3161,7 +3353,5 @@ def worker(x): self.assertIn("Results: [2, 4, 6]", stdout) self.assertNotIn("Can't pickle", stderr) - - if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst new file mode 100644 index 00000000000..e1202dd1a17 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst @@ -0,0 +1,3 @@ +Add support for ```` and ```` frames to +:mod:`!profiling.sampling` output to denote active garbage collection and +calls to native code. diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index d190b3c9faf..51b3c6bac02 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -26,8 +26,9 @@ #include "Python.h" #include // _Py_DebugOffsets #include // FRAME_SUSPENDED_YIELD_FROM -#include // FRAME_OWNED_BY_CSTACK +#include // FRAME_OWNED_BY_INTERPRETER #include // struct llist_node +#include // _PyLong_GetZero #include // Py_TAG_BITS #include "../Python/remote_debug.h" @@ -92,14 +93,16 @@ typedef enum _WIN32_THREADSTATE { #endif #ifdef Py_GIL_DISABLED -#define INTERP_STATE_MIN_SIZE MAX(MAX(MAX(offsetof(PyInterpreterState, _code_object_generation) + sizeof(uint64_t), \ - offsetof(PyInterpreterState, tlbc_indices.tlbc_generation) + sizeof(uint32_t)), \ - offsetof(PyInterpreterState, threads.head) + sizeof(void*)), \ - offsetof(PyInterpreterState, _gil.last_holder) + sizeof(PyThreadState*)) +#define INTERP_STATE_MIN_SIZE MAX(MAX(MAX(MAX(offsetof(PyInterpreterState, _code_object_generation) + sizeof(uint64_t), \ + offsetof(PyInterpreterState, tlbc_indices.tlbc_generation) + sizeof(uint32_t)), \ + offsetof(PyInterpreterState, threads.head) + sizeof(void*)), \ + offsetof(PyInterpreterState, _gil.last_holder) + sizeof(PyThreadState*)), \ + offsetof(PyInterpreterState, gc.frame) + sizeof(_PyInterpreterFrame *)) #else -#define INTERP_STATE_MIN_SIZE MAX(MAX(offsetof(PyInterpreterState, _code_object_generation) + sizeof(uint64_t), \ - offsetof(PyInterpreterState, threads.head) + sizeof(void*)), \ - offsetof(PyInterpreterState, _gil.last_holder) + sizeof(PyThreadState*)) +#define INTERP_STATE_MIN_SIZE MAX(MAX(MAX(offsetof(PyInterpreterState, _code_object_generation) + sizeof(uint64_t), \ + offsetof(PyInterpreterState, threads.head) + sizeof(void*)), \ + offsetof(PyInterpreterState, _gil.last_holder) + sizeof(PyThreadState*)), \ + offsetof(PyInterpreterState, gc.frame) + sizeof(_PyInterpreterFrame *)) #endif #define INTERP_STATE_BUFFER_SIZE MAX(INTERP_STATE_MIN_SIZE, 256) @@ -276,6 +279,8 @@ typedef struct { int only_active_thread; int mode; // Use enum _ProfilingMode values int skip_non_matching_threads; // New option to skip threads that don't match mode + int native; + int gc; RemoteDebuggingState *cached_state; // Cached module state #ifdef Py_GIL_DISABLED // TLBC cache invalidation tracking @@ -1812,6 +1817,25 @@ parse_linetable(const uintptr_t addrq, const char* linetable, int firstlineno, L * CODE OBJECT AND FRAME PARSING FUNCTIONS * ============================================================================ */ +static PyObject * +make_frame_info(RemoteUnwinderObject *unwinder, PyObject *file, PyObject *line, + PyObject *func) +{ + RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); + PyObject *info = PyStructSequence_New(state->FrameInfo_Type); + if (info == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create FrameInfo"); + return NULL; + } + Py_INCREF(file); + Py_INCREF(line); + Py_INCREF(func); + PyStructSequence_SetItem(info, 0, file); + PyStructSequence_SetItem(info, 1, line); + PyStructSequence_SetItem(info, 2, func); + return info; +} + static int parse_code_object(RemoteUnwinderObject *unwinder, PyObject **result, @@ -1825,8 +1849,6 @@ parse_code_object(RemoteUnwinderObject *unwinder, PyObject *func = NULL; PyObject *file = NULL; PyObject *linetable = NULL; - PyObject *lineno = NULL; - PyObject *tuple = NULL; #ifdef Py_GIL_DISABLED // In free threading builds, code object addresses might have the low bit set @@ -1948,25 +1970,18 @@ parse_code_object(RemoteUnwinderObject *unwinder, info.lineno = -1; } - lineno = PyLong_FromLong(info.lineno); + PyObject *lineno = PyLong_FromLong(info.lineno); if (!lineno) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create line number object"); goto error; } - RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); - tuple = PyStructSequence_New(state->FrameInfo_Type); + PyObject *tuple = make_frame_info(unwinder, meta->file_name, lineno, meta->func_name); + Py_DECREF(lineno); if (!tuple) { - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create FrameInfo for code object"); goto error; } - Py_INCREF(meta->func_name); - Py_INCREF(meta->file_name); - PyStructSequence_SetItem(tuple, 0, meta->file_name); - PyStructSequence_SetItem(tuple, 1, lineno); - PyStructSequence_SetItem(tuple, 2, meta->func_name); - *result = tuple; return 0; @@ -1974,8 +1989,6 @@ parse_code_object(RemoteUnwinderObject *unwinder, Py_XDECREF(func); Py_XDECREF(file); Py_XDECREF(linetable); - Py_XDECREF(lineno); - Py_XDECREF(tuple); return -1; } @@ -2116,6 +2129,7 @@ parse_frame_from_chunks( PyObject **result, uintptr_t address, uintptr_t *previous_frame, + uintptr_t *stackpointer, StackChunkList *chunks ) { void *frame_ptr = find_frame_in_chunks(chunks, address); @@ -2126,6 +2140,7 @@ parse_frame_from_chunks( char *frame = (char *)frame_ptr; *previous_frame = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.previous); + *stackpointer = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.stackpointer); uintptr_t code_object = GET_MEMBER_NO_TAG(uintptr_t, frame_ptr, unwinder->debug_offsets.interpreter_frame.executable); int frame_valid = is_frame_valid(unwinder, (uintptr_t)frame, code_object); if (frame_valid != 1) { @@ -2238,8 +2253,7 @@ is_frame_valid( void* frame = (void*)frame_addr; - if (GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner) == FRAME_OWNED_BY_CSTACK || - GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner) == FRAME_OWNED_BY_INTERPRETER) { + if (GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner) == FRAME_OWNED_BY_INTERPRETER) { return 0; // C frame } @@ -2458,8 +2472,9 @@ process_frame_chain( RemoteUnwinderObject *unwinder, uintptr_t initial_frame_addr, StackChunkList *chunks, - PyObject *frame_info -) { + PyObject *frame_info, + uintptr_t gc_frame) +{ uintptr_t frame_addr = initial_frame_addr; uintptr_t prev_frame_addr = 0; const size_t MAX_FRAMES = 1024; @@ -2468,6 +2483,7 @@ process_frame_chain( while ((void*)frame_addr != NULL) { PyObject *frame = NULL; uintptr_t next_frame_addr = 0; + uintptr_t stackpointer = 0; if (++frame_count > MAX_FRAMES) { PyErr_SetString(PyExc_RuntimeError, "Too many stack frames (possible infinite loop)"); @@ -2476,7 +2492,7 @@ process_frame_chain( } // Try chunks first, fallback to direct memory read - if (parse_frame_from_chunks(unwinder, &frame, frame_addr, &next_frame_addr, chunks) < 0) { + if (parse_frame_from_chunks(unwinder, &frame, frame_addr, &next_frame_addr, &stackpointer, chunks) < 0) { PyErr_Clear(); uintptr_t address_of_code_object = 0; if (parse_frame_object(unwinder, &frame, frame_addr, &address_of_code_object ,&next_frame_addr) < 0) { @@ -2484,26 +2500,63 @@ process_frame_chain( return -1; } } - - if (!frame) { - break; - } - - if (prev_frame_addr && frame_addr != prev_frame_addr) { - PyErr_Format(PyExc_RuntimeError, - "Broken frame chain: expected frame at 0x%lx, got 0x%lx", - prev_frame_addr, frame_addr); - Py_DECREF(frame); - set_exception_cause(unwinder, PyExc_RuntimeError, "Frame chain consistency check failed"); + if (frame == NULL && PyList_GET_SIZE(frame_info) == 0) { + // If the first frame is missing, the chain is broken: + const char *e = "Failed to parse initial frame in chain"; + PyErr_SetString(PyExc_RuntimeError, e); return -1; } - - if (PyList_Append(frame_info, frame) == -1) { - Py_DECREF(frame); - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append frame to frame info list"); - return -1; + PyObject *extra_frame = NULL; + // This frame kicked off the current GC collection: + if (unwinder->gc && frame_addr == gc_frame) { + _Py_DECLARE_STR(gc, ""); + extra_frame = &_Py_STR(gc); + } + // Otherwise, check for native frames to insert: + else if (unwinder->native && + // We've reached an interpreter trampoline frame: + frame == NULL && + // Bottommost frame is always native, so skip that one: + next_frame_addr && + // Only suppress native frames if GC tracking is enabled and the next frame will be a GC frame: + !(unwinder->gc && next_frame_addr == gc_frame)) + { + _Py_DECLARE_STR(native, ""); + extra_frame = &_Py_STR(native); + } + if (extra_frame) { + // Use "~" as file and 0 as line, since that's what pstats uses: + PyObject *extra_frame_info = make_frame_info( + unwinder, _Py_LATIN1_CHR('~'), _PyLong_GetZero(), extra_frame); + if (extra_frame_info == NULL) { + return -1; + } + int error = PyList_Append(frame_info, extra_frame_info); + Py_DECREF(extra_frame_info); + if (error) { + const char *e = "Failed to append extra frame to frame info list"; + set_exception_cause(unwinder, PyExc_RuntimeError, e); + return -1; + } + } + if (frame) { + if (prev_frame_addr && frame_addr != prev_frame_addr) { + const char *f = "Broken frame chain: expected frame at 0x%lx, got 0x%lx"; + PyErr_Format(PyExc_RuntimeError, f, prev_frame_addr, frame_addr); + Py_DECREF(frame); + const char *e = "Frame chain consistency check failed"; + set_exception_cause(unwinder, PyExc_RuntimeError, e); + return -1; + } + + if (PyList_Append(frame_info, frame) == -1) { + Py_DECREF(frame); + const char *e = "Failed to append frame to frame info list"; + set_exception_cause(unwinder, PyExc_RuntimeError, e); + return -1; + } + Py_DECREF(frame); } - Py_DECREF(frame); prev_frame_addr = next_frame_addr; frame_addr = next_frame_addr; @@ -2644,7 +2697,8 @@ static PyObject* unwind_stack_for_thread( RemoteUnwinderObject *unwinder, uintptr_t *current_tstate, - uintptr_t gil_holder_tstate + uintptr_t gil_holder_tstate, + uintptr_t gc_frame ) { PyObject *frame_info = NULL; PyObject *thread_id = NULL; @@ -2746,7 +2800,7 @@ unwind_stack_for_thread( goto error; } - if (process_frame_chain(unwinder, frame_addr, &chunks, frame_info) < 0) { + if (process_frame_chain(unwinder, frame_addr, &chunks, frame_info, gc_frame) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to process frame chain"); goto error; } @@ -2818,6 +2872,8 @@ _remote_debugging.RemoteUnwinder.__init__ mode: int = 0 debug: bool = False skip_non_matching_threads: bool = True + native: bool = False + gc: bool = False Initialize a new RemoteUnwinder object for debugging a remote Python process. @@ -2832,6 +2888,10 @@ Initialize a new RemoteUnwinder object for debugging a remote Python process. lead to the exception. skip_non_matching_threads: If True, skip threads that don't match the selected mode. If False, include all threads regardless of mode. + native: If True, include artificial "" frames to denote calls to + non-Python code. + gc: If True, include artificial "" frames to denote active garbage + collection. The RemoteUnwinder provides functionality to inspect and debug a running Python process, including examining thread states, stack frames and other runtime data. @@ -2848,8 +2908,9 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, int pid, int all_threads, int only_active_thread, int mode, int debug, - int skip_non_matching_threads) -/*[clinic end generated code: output=abf5ea5cd58bcb36 input=08fb6ace023ec3b5]*/ + int skip_non_matching_threads, + int native, int gc) +/*[clinic end generated code: output=e9eb6b4df119f6e0 input=606d099059207df2]*/ { // Validate that all_threads and only_active_thread are not both True if (all_threads && only_active_thread) { @@ -2866,6 +2927,8 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, } #endif + self->native = native; + self->gc = gc; self->debug = debug; self->only_active_thread = only_active_thread; self->mode = mode; @@ -3026,6 +3089,13 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self goto exit; } + uintptr_t gc_frame = 0; + if (self->gc) { + gc_frame = GET_MEMBER(uintptr_t, interp_state_buffer, + self->debug_offsets.interpreter_state.gc + + self->debug_offsets.gc.frame); + } + int64_t interpreter_id = GET_MEMBER(int64_t, interp_state_buffer, self->debug_offsets.interpreter_state.id); @@ -3085,7 +3155,9 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self } while (current_tstate != 0) { - PyObject* frame_info = unwind_stack_for_thread(self, ¤t_tstate, gil_holder_tstate); + PyObject* frame_info = unwind_stack_for_thread(self, ¤t_tstate, + gil_holder_tstate, + gc_frame); if (!frame_info) { // Check if this was an intentional skip due to mode-based filtering if ((self->mode == PROFILING_MODE_CPU || self->mode == PROFILING_MODE_GIL) && !PyErr_Occurred()) { diff --git a/Modules/clinic/_remote_debugging_module.c.h b/Modules/clinic/_remote_debugging_module.c.h index 7dd54e31248..60adb357e32 100644 --- a/Modules/clinic/_remote_debugging_module.c.h +++ b/Modules/clinic/_remote_debugging_module.c.h @@ -11,7 +11,8 @@ preserve PyDoc_STRVAR(_remote_debugging_RemoteUnwinder___init____doc__, "RemoteUnwinder(pid, *, all_threads=False, only_active_thread=False,\n" -" mode=0, debug=False, skip_non_matching_threads=True)\n" +" mode=0, debug=False, skip_non_matching_threads=True,\n" +" native=False, gc=False)\n" "--\n" "\n" "Initialize a new RemoteUnwinder object for debugging a remote Python process.\n" @@ -27,6 +28,10 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder___init____doc__, " lead to the exception.\n" " skip_non_matching_threads: If True, skip threads that don\'t match the selected mode.\n" " If False, include all threads regardless of mode.\n" +" native: If True, include artificial \"\" frames to denote calls to\n" +" non-Python code.\n" +" gc: If True, include artificial \"\" frames to denote active garbage\n" +" collection.\n" "\n" "The RemoteUnwinder provides functionality to inspect and debug a running Python\n" "process, including examining thread states, stack frames and other runtime data.\n" @@ -42,7 +47,8 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, int pid, int all_threads, int only_active_thread, int mode, int debug, - int skip_non_matching_threads); + int skip_non_matching_threads, + int native, int gc); static int _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObject *kwargs) @@ -50,7 +56,7 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje int return_value = -1; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 6 + #define NUM_KEYWORDS 8 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -59,7 +65,7 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(pid), &_Py_ID(all_threads), &_Py_ID(only_active_thread), &_Py_ID(mode), &_Py_ID(debug), &_Py_ID(skip_non_matching_threads), }, + .ob_item = { &_Py_ID(pid), &_Py_ID(all_threads), &_Py_ID(only_active_thread), &_Py_ID(mode), &_Py_ID(debug), &_Py_ID(skip_non_matching_threads), &_Py_ID(native), &_Py_ID(gc), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -68,14 +74,14 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"pid", "all_threads", "only_active_thread", "mode", "debug", "skip_non_matching_threads", NULL}; + static const char * const _keywords[] = {"pid", "all_threads", "only_active_thread", "mode", "debug", "skip_non_matching_threads", "native", "gc", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "RemoteUnwinder", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[6]; + PyObject *argsbuf[8]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; @@ -85,6 +91,8 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje int mode = 0; int debug = 0; int skip_non_matching_threads = 1; + int native = 0; + int gc = 0; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -134,12 +142,30 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje goto skip_optional_kwonly; } } - skip_non_matching_threads = PyObject_IsTrue(fastargs[5]); - if (skip_non_matching_threads < 0) { + if (fastargs[5]) { + skip_non_matching_threads = PyObject_IsTrue(fastargs[5]); + if (skip_non_matching_threads < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[6]) { + native = PyObject_IsTrue(fastargs[6]); + if (native < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + gc = PyObject_IsTrue(fastargs[7]); + if (gc < 0) { goto exit; } skip_optional_kwonly: - return_value = _remote_debugging_RemoteUnwinder___init___impl((RemoteUnwinderObject *)self, pid, all_threads, only_active_thread, mode, debug, skip_non_matching_threads); + return_value = _remote_debugging_RemoteUnwinder___init___impl((RemoteUnwinderObject *)self, pid, all_threads, only_active_thread, mode, debug, skip_non_matching_threads, native, gc); exit: return return_value; @@ -321,4 +347,4 @@ _remote_debugging_RemoteUnwinder_get_async_stack_trace(PyObject *self, PyObject return return_value; } -/*[clinic end generated code: output=2caefeddf7683d32 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=99fed5c94cf36881 input=a9049054013a1b77]*/ diff --git a/Python/gc.c b/Python/gc.c index 03a5d7366ea..064f9406e0a 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -2074,6 +2074,7 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) // Don't start a garbage collection if one is already in progress. return 0; } + gcstate->frame = tstate->current_frame; struct gc_collection_stats stats = { 0 }; if (reason != _Py_GC_REASON_SHUTDOWN) { @@ -2119,6 +2120,7 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) } #endif validate_spaces(gcstate); + gcstate->frame = NULL; _Py_atomic_store_int(&gcstate->collecting, 0); if (gcstate->debug & _PyGC_DEBUG_STATS) { diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index b183062eff7..7724676c242 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -2359,6 +2359,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) _Py_atomic_store_int(&gcstate->collecting, 0); return 0; } + gcstate->frame = tstate->current_frame; assert(generation >= 0 && generation < NUM_GENERATIONS); @@ -2447,6 +2448,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) } assert(!_PyErr_Occurred(tstate)); + gcstate->frame = NULL; _Py_atomic_store_int(&gcstate->collecting, 0); return n + m; } From f6dd9c12a8ba391cbbcc793411ac7dcfa6e01028 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Mon, 17 Nov 2025 05:41:22 -0800 Subject: [PATCH 231/313] GH-139914: Handle stack growth direction on HPPA (GH-140028) Adapted from a patch for Python 3.14 submitted to the Debian BTS by John https://bugs.debian.org/1105111#20 Co-authored-by: John David Anglin --- Include/internal/pycore_ceval.h | 8 ++++ Include/internal/pycore_pystate.h | 4 ++ Include/pyport.h | 6 +++ Lib/test/test_call.py | 9 +++- ...-10-13-13-54-19.gh-issue-139914.M-y_3E.rst | 1 + Modules/_testcapimodule.c | 4 ++ Python/ceval.c | 43 +++++++++++++++++-- configure | 13 ++++++ configure.ac | 8 ++++ pyconfig.h.in | 3 ++ 10 files changed, 94 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 33b9fd053f7..47c42fccdc2 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -217,7 +217,11 @@ extern void _PyEval_DeactivateOpCache(void); static inline int _Py_MakeRecCheck(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; +#if _Py_STACK_GROWS_DOWN return here_addr < _tstate->c_stack_soft_limit; +#else + return here_addr > _tstate->c_stack_soft_limit; +#endif } // Export for '_json' shared extension, used via _Py_EnterRecursiveCall() @@ -249,7 +253,11 @@ static inline int _Py_ReachedRecursionLimit(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; assert(_tstate->c_stack_hard_limit != 0); +#if _Py_STACK_GROWS_DOWN return here_addr <= _tstate->c_stack_soft_limit; +#else + return here_addr >= _tstate->c_stack_soft_limit; +#endif } static inline void _Py_LeaveRecursiveCall(void) { diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index cab458f8402..189a8dde9f0 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -331,7 +331,11 @@ _Py_RecursionLimit_GetMargin(PyThreadState *tstate) _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; assert(_tstate->c_stack_hard_limit != 0); intptr_t here_addr = _Py_get_machine_stack_pointer(); +#if _Py_STACK_GROWS_DOWN return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, here_addr - (intptr_t)_tstate->c_stack_soft_limit, _PyOS_STACK_MARGIN_SHIFT); +#else + return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (intptr_t)_tstate->c_stack_soft_limit - here_addr, _PyOS_STACK_MARGIN_SHIFT); +#endif } #ifdef __cplusplus diff --git a/Include/pyport.h b/Include/pyport.h index e77b39026a5..b250f9e308f 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -677,4 +677,10 @@ extern "C" { #endif +// Assume the stack grows down unless specified otherwise +#ifndef _Py_STACK_GROWS_DOWN +# define _Py_STACK_GROWS_DOWN 1 +#endif + + #endif /* Py_PYPORT_H */ diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 31e58e825be..f42526aee19 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1048,9 +1048,14 @@ def get_sp(): this_sp = _testinternalcapi.get_stack_pointer() lower_sp = _testcapi.pyobject_vectorcall(get_sp, (), ()) - self.assertLess(lower_sp, this_sp) + if _testcapi._Py_STACK_GROWS_DOWN: + self.assertLess(lower_sp, this_sp) + safe_margin = this_sp - lower_sp + else: + self.assertGreater(lower_sp, this_sp) + safe_margin = lower_sp - this_sp # Add an (arbitrary) extra 25% for safety - safe_margin = (this_sp - lower_sp) * 5 / 4 + safe_margin = safe_margin * 5 / 4 self.assertLess(safe_margin, _testinternalcapi.get_stack_margin()) @skip_on_s390x diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst new file mode 100644 index 00000000000..7529108d5d4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst @@ -0,0 +1 @@ +Restore support for HP PA-RISC, which has an upwards-growing stack. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 22cd731d410..c14f925b4e7 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3359,6 +3359,10 @@ _testcapi_exec(PyObject *m) PyModule_AddObject(m, "INT64_MAX", PyLong_FromInt64(INT64_MAX)); PyModule_AddObject(m, "UINT64_MAX", PyLong_FromUInt64(UINT64_MAX)); + if (PyModule_AddIntMacro(m, _Py_STACK_GROWS_DOWN)) { + return -1; + } + if (PyModule_AddIntMacro(m, Py_single_input)) { return -1; } diff --git a/Python/ceval.c b/Python/ceval.c index 31b81a37464..25294ebd993 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -351,13 +351,21 @@ _Py_ReachedRecursionLimitWithMargin(PyThreadState *tstate, int margin_count) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; +#if _Py_STACK_GROWS_DOWN if (here_addr > _tstate->c_stack_soft_limit + margin_count * _PyOS_STACK_MARGIN_BYTES) { +#else + if (here_addr <= _tstate->c_stack_soft_limit - margin_count * _PyOS_STACK_MARGIN_BYTES) { +#endif return 0; } if (_tstate->c_stack_hard_limit == 0) { _Py_InitializeRecursionLimits(tstate); } +#if _Py_STACK_GROWS_DOWN return here_addr <= _tstate->c_stack_soft_limit + margin_count * _PyOS_STACK_MARGIN_BYTES; +#else + return here_addr > _tstate->c_stack_soft_limit - margin_count * _PyOS_STACK_MARGIN_BYTES; +#endif } void @@ -365,7 +373,11 @@ _Py_EnterRecursiveCallUnchecked(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; +#if _Py_STACK_GROWS_DOWN if (here_addr < _tstate->c_stack_hard_limit) { +#else + if (here_addr > _tstate->c_stack_hard_limit) { +#endif Py_FatalError("Unchecked stack overflow."); } } @@ -496,18 +508,33 @@ tstate_set_stack(PyThreadState *tstate, #ifdef _Py_THREAD_SANITIZER // Thread sanitizer crashes if we use more than half the stack. uintptr_t stacksize = top - base; - base += stacksize / 2; +# if _Py_STACK_GROWS_DOWN + base += stacksize/2; +# else + top -= stacksize/2; +# endif #endif _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; +#if _Py_STACK_GROWS_DOWN _tstate->c_stack_top = top; _tstate->c_stack_hard_limit = base + _PyOS_STACK_MARGIN_BYTES; _tstate->c_stack_soft_limit = base + _PyOS_STACK_MARGIN_BYTES * 2; - -#ifndef NDEBUG +# ifndef NDEBUG // Sanity checks _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; assert(ts->c_stack_hard_limit <= ts->c_stack_soft_limit); assert(ts->c_stack_soft_limit < ts->c_stack_top); +# endif +#else + _tstate->c_stack_top = base; + _tstate->c_stack_hard_limit = top - _PyOS_STACK_MARGIN_BYTES; + _tstate->c_stack_soft_limit = top - _PyOS_STACK_MARGIN_BYTES * 2; +# ifndef NDEBUG + // Sanity checks + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(ts->c_stack_hard_limit >= ts->c_stack_soft_limit); + assert(ts->c_stack_soft_limit > ts->c_stack_top); +# endif #endif } @@ -568,9 +595,15 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) uintptr_t here_addr = _Py_get_machine_stack_pointer(); assert(_tstate->c_stack_soft_limit != 0); assert(_tstate->c_stack_hard_limit != 0); +#if _Py_STACK_GROWS_DOWN if (here_addr < _tstate->c_stack_hard_limit) { /* Overflowing while handling an overflow. Give up. */ int kbytes_used = (int)(_tstate->c_stack_top - here_addr)/1024; +#else + if (here_addr > _tstate->c_stack_hard_limit) { + /* Overflowing while handling an overflow. Give up. */ + int kbytes_used = (int)(here_addr - _tstate->c_stack_top)/1024; +#endif char buffer[80]; snprintf(buffer, 80, "Unrecoverable stack overflow (used %d kB)%s", kbytes_used, where); Py_FatalError(buffer); @@ -579,7 +612,11 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) return 0; } else { +#if _Py_STACK_GROWS_DOWN int kbytes_used = (int)(_tstate->c_stack_top - here_addr)/1024; +#else + int kbytes_used = (int)(here_addr - _tstate->c_stack_top)/1024; +#endif tstate->recursion_headroom++; _PyErr_Format(tstate, PyExc_RecursionError, "Stack overflow (used %d kB)%s", diff --git a/configure b/configure index eeb24c1d844..a4514f80c3a 100755 --- a/configure +++ b/configure @@ -967,6 +967,7 @@ LDLIBRARY LIBRARY BUILDEXEEXT NO_AS_NEEDED +_Py_STACK_GROWS_DOWN MULTIARCH_CPPFLAGS PLATFORM_TRIPLET MULTIARCH @@ -7213,6 +7214,18 @@ if test x$MULTIARCH != x; then fi +# Guess C stack direction +case $host in #( + hppa*) : + _Py_STACK_GROWS_DOWN=0 ;; #( + *) : + _Py_STACK_GROWS_DOWN=1 ;; +esac + +printf "%s\n" "#define _Py_STACK_GROWS_DOWN $_Py_STACK_GROWS_DOWN" >>confdefs.h + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for PEP 11 support tier" >&5 printf %s "checking for PEP 11 support tier... " >&6; } case $host/$ac_cv_cc_name in #( diff --git a/configure.ac b/configure.ac index 92adc44da0d..a059a07bec2 100644 --- a/configure.ac +++ b/configure.ac @@ -1202,6 +1202,14 @@ if test x$MULTIARCH != x; then fi AC_SUBST([MULTIARCH_CPPFLAGS]) +# Guess C stack direction +AS_CASE([$host], + [hppa*], [_Py_STACK_GROWS_DOWN=0], + [_Py_STACK_GROWS_DOWN=1]) +AC_DEFINE_UNQUOTED([_Py_STACK_GROWS_DOWN], [$_Py_STACK_GROWS_DOWN], + [Define to 1 if the machine stack grows down (default); 0 if it grows up.]) +AC_SUBST([_Py_STACK_GROWS_DOWN]) + dnl Support tiers according to https://peps.python.org/pep-0011/ dnl dnl NOTE: Windows support tiers are defined in PC/pyconfig.h. diff --git a/pyconfig.h.in b/pyconfig.h.in index fb12079bafa..8a9f5ca8ec8 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -2050,6 +2050,9 @@ /* HACL* library can compile SIMD256 implementations */ #undef _Py_HACL_CAN_COMPILE_VEC256 +/* Define to 1 if the machine stack grows down (default); 0 if it grows up. */ +#undef _Py_STACK_GROWS_DOWN + /* Define if you want to use tail-calling interpreters in CPython. */ #undef _Py_TAIL_CALL_INTERP From 3d148059479b28a21f8eae6abf6d1bcc91ab8cbb Mon Sep 17 00:00:00 2001 From: "R.C.M" Date: Mon, 17 Nov 2025 09:42:26 -0500 Subject: [PATCH 232/313] gh-130693: Support more options for search in tkinter.Text (GH-130848) * Add parameters nolinestop and strictlimits in the tkinter.Text.search() method. * Add the tkinter.Text.search_all() method. * Add more tests for tkinter.Text.search(). * stopindex is now only ignored if it is None. --- Doc/whatsnew/3.15.rst | 13 ++ Lib/test/test_tkinter/test_text.py | 114 +++++++++++++++++- Lib/tkinter/__init__.py | 34 +++++- ...-03-04-17-19-26.gh-issue-130693.Kv01r8.rst | 1 + 4 files changed, 154 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 9393b65ed8e..cf5bef15203 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -734,6 +734,19 @@ timeit :ref:`environment variables `. (Contributed by Yi Hong in :gh:`139374`.) +tkinter +------- + +* The :meth:`!tkinter.Text.search` method now supports two additional + arguments: *nolinestop* which allows the search to + continue across line boundaries; + and *strictlimits* which restricts the search to within the specified range. + (Contributed by Rihaan Meher in :gh:`130848`) + +* A new method :meth:`!tkinter.Text.search_all` has been introduced. + This method allows for searching for all matches of a pattern + using Tcl's ``-all`` and ``-overlap`` options. + (Contributed by Rihaan Meher in :gh:`130848`) types ------ diff --git a/Lib/test/test_tkinter/test_text.py b/Lib/test/test_tkinter/test_text.py index b26956930d3..d579cca95ee 100644 --- a/Lib/test/test_tkinter/test_text.py +++ b/Lib/test/test_tkinter/test_text.py @@ -34,12 +34,116 @@ def test_search(self): # Invalid text index. self.assertRaises(tkinter.TclError, text.search, '', 0) + self.assertRaises(tkinter.TclError, text.search, '', '') + self.assertRaises(tkinter.TclError, text.search, '', 'invalid') + self.assertRaises(tkinter.TclError, text.search, '', '1.0', 0) + self.assertRaises(tkinter.TclError, text.search, '', '1.0', '') + self.assertRaises(tkinter.TclError, text.search, '', '1.0', 'invalid') - # Check if we are getting the indices as strings -- you are likely - # to get Tcl_Obj under Tk 8.5 if Tkinter doesn't convert it. - text.insert('1.0', 'hi-test') - self.assertEqual(text.search('-test', '1.0', 'end'), '1.2') - self.assertEqual(text.search('test', '1.0', 'end'), '1.3') + text.insert('1.0', + 'This is a test. This is only a test.\n' + 'Another line.\n' + 'Yet another line.\n' + '64-bit') + + self.assertEqual(text.search('test', '1.0'), '1.10') + self.assertEqual(text.search('test', '1.0', 'end'), '1.10') + self.assertEqual(text.search('test', '1.0', '1.10'), '') + self.assertEqual(text.search('test', '1.11'), '1.31') + self.assertEqual(text.search('test', '1.32', 'end'), '') + self.assertEqual(text.search('test', '1.32'), '1.10') + + self.assertEqual(text.search('', '1.0'), '1.0') # empty pattern + self.assertEqual(text.search('nonexistent', '1.0'), '') + self.assertEqual(text.search('-bit', '1.0'), '4.2') # starts with a hyphen + + self.assertEqual(text.search('line', '3.0'), '3.12') + self.assertEqual(text.search('line', '3.0', forwards=True), '3.12') + self.assertEqual(text.search('line', '3.0', backwards=True), '2.8') + self.assertEqual(text.search('line', '3.0', forwards=True, backwards=True), '2.8') + + self.assertEqual(text.search('t.', '1.0'), '1.13') + self.assertEqual(text.search('t.', '1.0', exact=True), '1.13') + self.assertEqual(text.search('t.', '1.0', regexp=True), '1.10') + self.assertEqual(text.search('t.', '1.0', exact=True, regexp=True), '1.10') + + self.assertEqual(text.search('TEST', '1.0'), '') + self.assertEqual(text.search('TEST', '1.0', nocase=True), '1.10') + + self.assertEqual(text.search('.*line', '1.0', regexp=True), '2.0') + self.assertEqual(text.search('.*line', '1.0', regexp=True, nolinestop=True), '1.0') + + self.assertEqual(text.search('test', '1.0', '1.13'), '1.10') + self.assertEqual(text.search('test', '1.0', '1.13', strictlimits=True), '') + self.assertEqual(text.search('test', '1.0', '1.14', strictlimits=True), '1.10') + + var = tkinter.Variable(self.root) + self.assertEqual(text.search('test', '1.0', count=var), '1.10') + self.assertEqual(var.get(), 4 if self.wantobjects else '4') + + # TODO: Add test for elide=True + + def test_search_all(self): + text = self.text + + # pattern and index are obligatory arguments. + self.assertRaises(tkinter.TclError, text.search_all, None, '1.0') + self.assertRaises(tkinter.TclError, text.search_all, 'a', None) + self.assertRaises(tkinter.TclError, text.search_all, None, None) + + # Keyword-only arguments + self.assertRaises(TypeError, text.search_all, 'a', '1.0', 'end', None) + + # Invalid text index. + self.assertRaises(tkinter.TclError, text.search_all, '', 0) + self.assertRaises(tkinter.TclError, text.search_all, '', '') + self.assertRaises(tkinter.TclError, text.search_all, '', 'invalid') + self.assertRaises(tkinter.TclError, text.search_all, '', '1.0', 0) + self.assertRaises(tkinter.TclError, text.search_all, '', '1.0', '') + self.assertRaises(tkinter.TclError, text.search_all, '', '1.0', 'invalid') + + def eq(res, expected): + self.assertIsInstance(res, tuple) + self.assertEqual([str(i) for i in res], expected) + + text.insert('1.0', 'ababa\naba\n64-bit') + + eq(text.search_all('aba', '1.0'), ['1.0', '2.0']) + eq(text.search_all('aba', '1.0', 'end'), ['1.0', '2.0']) + eq(text.search_all('aba', '1.1', 'end'), ['1.2', '2.0']) + eq(text.search_all('aba', '1.1'), ['1.2', '2.0', '1.0']) + + res = text.search_all('', '1.0') # empty pattern + eq(res[:5], ['1.0', '1.1', '1.2', '1.3', '1.4']) + eq(res[-5:], ['3.2', '3.3', '3.4', '3.5', '3.6']) + eq(text.search_all('nonexistent', '1.0'), []) + eq(text.search_all('-bit', '1.0'), ['3.2']) # starts with a hyphen + + eq(text.search_all('aba', '1.0', 'end', forwards=True), ['1.0', '2.0']) + eq(text.search_all('aba', 'end', '1.0', backwards=True), ['2.0', '1.2']) + + eq(text.search_all('aba', '1.0', overlap=True), ['1.0', '1.2', '2.0']) + eq(text.search_all('aba', 'end', '1.0', overlap=True, backwards=True), ['2.0', '1.2', '1.0']) + + eq(text.search_all('aba', '1.0', exact=True), ['1.0', '2.0']) + eq(text.search_all('a.a', '1.0', exact=True), []) + eq(text.search_all('a.a', '1.0', regexp=True), ['1.0', '2.0']) + + eq(text.search_all('ABA', '1.0'), []) + eq(text.search_all('ABA', '1.0', nocase=True), ['1.0', '2.0']) + + eq(text.search_all('a.a', '1.0', regexp=True), ['1.0', '2.0']) + eq(text.search_all('a.a', '1.0', regexp=True, nolinestop=True), ['1.0', '1.4']) + + eq(text.search_all('aba', '1.0', '2.2'), ['1.0', '2.0']) + eq(text.search_all('aba', '1.0', '2.2', strictlimits=True), ['1.0']) + eq(text.search_all('aba', '1.0', '2.3', strictlimits=True), ['1.0', '2.0']) + + var = tkinter.Variable(self.root) + eq(text.search_all('aba', '1.0', count=var), ['1.0', '2.0']) + self.assertEqual(var.get(), (3, 3) if self.wantobjects else '3 3') + + # TODO: Add test for elide=True def test_count(self): text = self.text diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index c5453074039..737583a42c6 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -4049,8 +4049,9 @@ def scan_dragto(self, x, y): self.tk.call(self._w, 'scan', 'dragto', x, y) def search(self, pattern, index, stopindex=None, - forwards=None, backwards=None, exact=None, - regexp=None, nocase=None, count=None, elide=None): + forwards=None, backwards=None, exact=None, + regexp=None, nocase=None, count=None, + elide=None, *, nolinestop=None, strictlimits=None): """Search PATTERN beginning from INDEX until STOPINDEX. Return the index of the first character of a match or an empty string.""" @@ -4062,12 +4063,39 @@ def search(self, pattern, index, stopindex=None, if nocase: args.append('-nocase') if elide: args.append('-elide') if count: args.append('-count'); args.append(count) + if nolinestop: args.append('-nolinestop') + if strictlimits: args.append('-strictlimits') if pattern and pattern[0] == '-': args.append('--') args.append(pattern) args.append(index) - if stopindex: args.append(stopindex) + if stopindex is not None: args.append(stopindex) return str(self.tk.call(tuple(args))) + def search_all(self, pattern, index, stopindex=None, *, + forwards=None, backwards=None, exact=None, + regexp=None, nocase=None, count=None, + elide=None, nolinestop=None, overlap=None, + strictlimits=None): + """Search all occurrences of PATTERN from INDEX to STOPINDEX. + Return a tuple of indices where matches begin.""" + args = [self._w, 'search', '-all'] + if forwards: args.append('-forwards') + if backwards: args.append('-backwards') + if exact: args.append('-exact') + if regexp: args.append('-regexp') + if nocase: args.append('-nocase') + if elide: args.append('-elide') + if count: args.append('-count'); args.append(count) + if nolinestop: args.append('-nolinestop') + if overlap: args.append('-overlap') + if strictlimits: args.append('-strictlimits') + if pattern and pattern[0] == '-': args.append('--') + args.append(pattern) + args.append(index) + if stopindex is not None: args.append(stopindex) + result = self.tk.call(tuple(args)) + return self.tk.splitlist(result) + def see(self, index): """Scroll such that the character at INDEX is visible.""" self.tk.call(self._w, 'see', index) diff --git a/Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst b/Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst new file mode 100644 index 00000000000..b175ab7cad4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst @@ -0,0 +1 @@ +Add support for ``-nolinestop``, and ``-strictlimits`` options to :meth:`!tkinter.Text.search`. Also add the :meth:`!tkinter.Text.search_all` method for ``-all`` and ``-overlap`` options. From cc6b62ac561e857a2cc4eb4f43e1e0e9f53c09f1 Mon Sep 17 00:00:00 2001 From: Semyon Moroz Date: Mon, 17 Nov 2025 18:51:21 +0400 Subject: [PATCH 233/313] gh-130160: Add anchors to CLI Usage section for `cmdline` (#133182) --- Doc/library/cmdline.rst | 10 +++++----- Doc/library/ensurepip.rst | 4 +++- Doc/library/gzip.rst | 4 ++-- Doc/library/idle.rst | 4 +++- Doc/library/inspect.rst | 2 +- Doc/library/pdb.rst | 6 +++++- Doc/library/site.rst | 2 +- Doc/library/webbrowser.rst | 7 ++++++- 8 files changed, 26 insertions(+), 13 deletions(-) diff --git a/Doc/library/cmdline.rst b/Doc/library/cmdline.rst index 16c67ddbf7c..c43b10157f9 100644 --- a/Doc/library/cmdline.rst +++ b/Doc/library/cmdline.rst @@ -16,17 +16,17 @@ The following modules have a command-line interface. * :ref:`dis ` * :ref:`doctest ` * :mod:`!encodings.rot_13` -* :mod:`ensurepip` +* :ref:`ensurepip ` * :mod:`filecmp` * :mod:`fileinput` * :mod:`ftplib` * :ref:`gzip ` * :ref:`http.server ` -* :mod:`!idlelib` +* :ref:`idlelib ` * :ref:`inspect ` * :ref:`json ` * :ref:`mimetypes ` -* :mod:`pdb` +* :ref:`pdb ` * :ref:`pickle ` * :ref:`pickletools ` * :ref:`platform ` @@ -52,8 +52,8 @@ The following modules have a command-line interface. * :mod:`turtledemo` * :ref:`unittest ` * :ref:`uuid ` -* :mod:`venv` -* :mod:`webbrowser` +* :ref:`venv ` +* :ref:`webbrowser ` * :ref:`zipapp ` * :ref:`zipfile ` diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index 165b9a9f823..32b92c01570 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -42,7 +42,9 @@ when creating a virtual environment) or after explicitly uninstalling .. include:: ../includes/wasm-mobile-notavail.rst -Command line interface +.. _ensurepip-cli: + +Command-line interface ---------------------- .. program:: ensurepip diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index cb36be42a83..d23c0741ddb 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -283,7 +283,7 @@ Example of how to GZIP compress a binary string:: .. _gzip-cli: -Command Line Interface +Command-line interface ---------------------- The :mod:`gzip` module provides a simple command line interface to compress or @@ -296,7 +296,7 @@ Once executed the :mod:`gzip` module keeps the input file(s). Add a new command line interface with a usage. By default, when you will execute the CLI, the default compression level is 6. -Command line options +Command-line options ^^^^^^^^^^^^^^^^^^^^ .. option:: file diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 52e3726a0f5..a16f46ef812 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -661,7 +661,9 @@ looked for in the user's home directory. Statements in this file will be executed in the Tk namespace, so this file is not useful for importing functions to be used from IDLE's Python shell. -Command line usage +.. _idlelib-cli: + +Command-line usage ^^^^^^^^^^^^^^^^^^ .. program:: idle diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 13a352cbdb2..c00db31a8ec 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -1788,7 +1788,7 @@ Buffer flags .. _inspect-module-cli: -Command Line Interface +Command-line interface ---------------------- The :mod:`inspect` module also provides a basic introspection capability diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 90dc6648045..0bbdc425352 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -76,6 +76,10 @@ The debugger's prompt is ``(Pdb)``, which is the indicator that you are in debug .. _pdb-cli: + +Command-line interface +---------------------- + .. program:: pdb You can also invoke :mod:`pdb` from the command line to debug other scripts. For @@ -334,7 +338,7 @@ access further features, you have to do this yourself: .. _debugger-commands: -Debugger Commands +Debugger commands ----------------- The commands recognized by the debugger are listed below. Most commands can be diff --git a/Doc/library/site.rst b/Doc/library/site.rst index e98dd83b60e..d93e4dc7c75 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -270,7 +270,7 @@ Module contents .. _site-commandline: -Command Line Interface +Command-line interface ---------------------- .. program:: site diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst index fd6abc70261..a2103d8fdd8 100644 --- a/Doc/library/webbrowser.rst +++ b/Doc/library/webbrowser.rst @@ -49,6 +49,11 @@ a new tab, with the browser being brought to the foreground. The use of the :mod:`webbrowser` module on iOS requires the :mod:`ctypes` module. If :mod:`ctypes` isn't available, calls to :func:`.open` will fail. +.. _webbrowser-cli: + +Command-line interface +---------------------- + .. program:: webbrowser The script :program:`webbrowser` can be used as a command-line interface for the @@ -232,7 +237,7 @@ Here are some simple examples:: .. _browser-controllers: -Browser Controller Objects +Browser controller objects -------------------------- Browser controllers provide the :attr:`~controller.name` attribute, From 274a26cca8e3d2f4de0283d4acbc80be391a5f6a Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 17 Nov 2025 16:32:08 +0000 Subject: [PATCH 234/313] gh-135953: Simplify GC markers in the tachyon profiler (#141666) --- Lib/profiling/sampling/gecko_collector.py | 15 +++++++-------- Lib/test/test_profiling/test_sampling_profiler.py | 6 ++---- Modules/_remote_debugging_module.c | 11 ----------- 3 files changed, 9 insertions(+), 23 deletions(-) diff --git a/Lib/profiling/sampling/gecko_collector.py b/Lib/profiling/sampling/gecko_collector.py index 6c6700f1130..21c427b7c86 100644 --- a/Lib/profiling/sampling/gecko_collector.py +++ b/Lib/profiling/sampling/gecko_collector.py @@ -141,7 +141,6 @@ def collect(self, stack_frames): for thread_info in interpreter_info.threads: frames = thread_info.frame_info tid = thread_info.thread_id - gc_collecting = thread_info.gc_collecting # Initialize thread if needed if tid not in self.threads: @@ -197,16 +196,16 @@ def collect(self, stack_frames): self._add_marker(tid, "Waiting for GIL", self.gil_wait_start.pop(tid), current_time, CATEGORY_GIL) - # Track GC events - attribute to all threads that hold the GIL during GC - # (GC is interpreter-wide but runs on whichever thread(s) have the GIL) - # If GIL switches during GC, multiple threads will get GC markers - if gc_collecting and has_gil: - # Start GC marker if not already started for this thread + # Track GC events by detecting frames in the stack trace + # This leverages the improved GC frame tracking from commit 336366fd7ca + # which precisely identifies the thread that initiated GC collection + has_gc_frame = any(frame[2] == "" for frame in frames) + if has_gc_frame: + # This thread initiated GC collection if tid not in self.gc_start_per_thread: self.gc_start_per_thread[tid] = current_time elif tid in self.gc_start_per_thread: - # End GC marker if it was running for this thread - # (either GC finished or thread lost GIL) + # End GC marker when no more GC frames are detected self._add_marker(tid, "GC Collecting", self.gc_start_per_thread.pop(tid), current_time, CATEGORY_GC) diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index a24dbb55cd7..2d00173c22c 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -63,14 +63,13 @@ def __repr__(self): class MockThreadInfo: """Mock ThreadInfo for testing since the real one isn't accessible.""" - def __init__(self, thread_id, frame_info, status=0, gc_collecting=False): # Default to THREAD_STATE_RUNNING (0) + def __init__(self, thread_id, frame_info, status=0): # Default to THREAD_STATE_RUNNING (0) self.thread_id = thread_id self.frame_info = frame_info self.status = status - self.gc_collecting = gc_collecting def __repr__(self): - return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info}, status={self.status}, gc_collecting={self.gc_collecting})" + return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info}, status={self.status})" class MockInterpreterInfo: @@ -2742,7 +2741,6 @@ def __init__(self, thread_id, frame_info, status): self.thread_id = thread_id self.frame_info = frame_info self.status = status - self.gc_collecting = False # Create test data: active thread (HAS_GIL | ON_CPU), idle thread (neither), and another active thread ACTIVE_STATUS = THREAD_STATUS_HAS_GIL | THREAD_STATUS_ON_CPU # Has GIL and on CPU diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index 51b3c6bac02..6544e3a0ce6 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -186,7 +186,6 @@ static PyStructSequence_Field ThreadInfo_fields[] = { {"thread_id", "Thread ID"}, {"status", "Thread status (flags: HAS_GIL, ON_CPU, UNKNOWN or legacy enum)"}, {"frame_info", "Frame information"}, - {"gc_collecting", "Whether GC is collecting (interpreter-level)"}, {NULL} }; @@ -2726,8 +2725,6 @@ unwind_stack_for_thread( goto error; } - int gc_collecting = GET_MEMBER(int, gc_state, unwinder->debug_offsets.gc.collecting); - // Calculate thread status using flags (always) int status_flags = 0; @@ -2827,18 +2824,10 @@ unwind_stack_for_thread( goto error; } - PyObject *py_gc_collecting = PyBool_FromLong(gc_collecting); - if (py_gc_collecting == NULL) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create gc_collecting"); - Py_DECREF(py_status); - goto error; - } - // py_status contains status flags (bitfield) PyStructSequence_SetItem(result, 0, thread_id); PyStructSequence_SetItem(result, 1, py_status); // Steals reference PyStructSequence_SetItem(result, 2, frame_info); // Steals reference - PyStructSequence_SetItem(result, 3, py_gc_collecting); // Steals reference cleanup_stack_chunks(&chunks); return result; From 6b1bdf6c7a6c87f12a247a125e25f8e721cc731e Mon Sep 17 00:00:00 2001 From: Krishna Chaitanya <141550576+XChaitanyaX@users.noreply.github.com> Date: Mon, 17 Nov 2025 22:59:06 +0530 Subject: [PATCH 235/313] gh-141497: Make ipaddress.IP{v4,v6}Network.hosts() always returning an iterator (GH-141547) --- Lib/ipaddress.py | 4 +-- Lib/test/test_ipaddress.py | 34 +++++++++++++++++++ ...-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst | 4 +++ 3 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index aa0cf4a0620..f1062a8cd05 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1546,7 +1546,7 @@ def __init__(self, address, strict=True): if self._prefixlen == (self.max_prefixlen - 1): self.hosts = self.__iter__ elif self._prefixlen == (self.max_prefixlen): - self.hosts = lambda: [IPv4Address(addr)] + self.hosts = lambda: iter((IPv4Address(addr),)) @property @functools.lru_cache() @@ -2337,7 +2337,7 @@ def __init__(self, address, strict=True): if self._prefixlen == (self.max_prefixlen - 1): self.hosts = self.__iter__ elif self._prefixlen == self.max_prefixlen: - self.hosts = lambda: [IPv6Address(addr)] + self.hosts = lambda: iter((IPv6Address(addr),)) def hosts(self): """Generate Iterator over usable hosts in a network. diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index 11721a59972..3f017b97dc2 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -12,6 +12,7 @@ import pickle import ipaddress import weakref +from collections.abc import Iterator from test.support import LARGEST, SMALLEST @@ -1472,18 +1473,27 @@ def testGetSupernet4(self): self.ipv6_scoped_network.supernet(new_prefix=62)) def testHosts(self): + hosts = self.ipv4_network.hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(ipaddress.IPv4Address('1.2.3.1'), next(hosts)) hosts = list(self.ipv4_network.hosts()) self.assertEqual(254, len(hosts)) self.assertEqual(ipaddress.IPv4Address('1.2.3.1'), hosts[0]) self.assertEqual(ipaddress.IPv4Address('1.2.3.254'), hosts[-1]) ipv6_network = ipaddress.IPv6Network('2001:658:22a:cafe::/120') + hosts = ipv6_network.hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), next(hosts)) hosts = list(ipv6_network.hosts()) self.assertEqual(255, len(hosts)) self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), hosts[0]) self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::ff'), hosts[-1]) ipv6_scoped_network = ipaddress.IPv6Network('2001:658:22a:cafe::%scope/120') + hosts = ipv6_scoped_network.hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual((ipaddress.IPv6Address('2001:658:22a:cafe::1')), next(hosts)) hosts = list(ipv6_scoped_network.hosts()) self.assertEqual(255, len(hosts)) self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), hosts[0]) @@ -1494,6 +1504,12 @@ def testHosts(self): ipaddress.IPv4Address('2.0.0.1')] str_args = '2.0.0.0/31' tpl_args = ('2.0.0.0', 31) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -1503,6 +1519,12 @@ def testHosts(self): addrs = [ipaddress.IPv4Address('1.2.3.4')] str_args = '1.2.3.4/32' tpl_args = ('1.2.3.4', 32) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -1512,6 +1534,12 @@ def testHosts(self): ipaddress.IPv6Address('2001:658:22a:cafe::1')] str_args = '2001:658:22a:cafe::/127' tpl_args = ('2001:658:22a:cafe::', 127) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -1520,6 +1548,12 @@ def testHosts(self): addrs = [ipaddress.IPv6Address('2001:658:22a:cafe::1'), ] str_args = '2001:658:22a:cafe::1/128' tpl_args = ('2001:658:22a:cafe::1', 128) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), diff --git a/Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst b/Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst new file mode 100644 index 00000000000..328bfe067ad --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst @@ -0,0 +1,4 @@ +:mod:`ipaddress`: ensure that the methods +:meth:`IPv4Network.hosts() ` and +:meth:`IPv6Network.hosts() ` always return an +iterator. From 5d2eb98a91f2cd703d14f38c751ac7f52b2d7148 Mon Sep 17 00:00:00 2001 From: Louis Date: Mon, 17 Nov 2025 18:47:00 +0100 Subject: [PATCH 236/313] gh-140578: Delete unnecessary NEWS entry (#141427) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .../2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 Misc/NEWS.d/next/Documentation/2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst diff --git a/Misc/NEWS.d/next/Documentation/2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst b/Misc/NEWS.d/next/Documentation/2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst deleted file mode 100644 index 702d38d4d24..00000000000 --- a/Misc/NEWS.d/next/Documentation/2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst +++ /dev/null @@ -1,3 +0,0 @@ -Remove outdated sencence in the documentation for :mod:`multiprocessing`, -that implied that :class:`concurrent.futures.ThreadPoolExecutor` did not -exist. From b3626321b6ebb46dd24acee2aa806450e70febfc Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Mon, 17 Nov 2025 14:40:47 -0500 Subject: [PATCH 237/313] gh-141004: Document `PyODict*` APIs (GH-141136) --- Doc/c-api/dict.rst | 89 ++++++++++++++++++++++++++++++++++++++++++ Doc/c-api/iterator.rst | 1 + 2 files changed, 90 insertions(+) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index b7f201811aa..ede1699cfeb 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -477,3 +477,92 @@ Dictionary View Objects Return true if *op* is an instance of a dictionary items view. This function always succeeds. + + +Ordered Dictionaries +^^^^^^^^^^^^^^^^^^^^ + +Python's C API provides interface for :class:`collections.OrderedDict` from C. +Since Python 3.7, dictionaries are ordered by default, so there is usually +little need for these functions; prefer ``PyDict*`` where possible. + + +.. c:var:: PyTypeObject PyODict_Type + + Type object for ordered dictionaries. This is the same object as + :class:`collections.OrderedDict` in the Python layer. + + +.. c:function:: int PyODict_Check(PyObject *od) + + Return true if *od* is an ordered dictionary object or an instance of a + subtype of the :class:`~collections.OrderedDict` type. This function + always succeeds. + + +.. c:function:: int PyODict_CheckExact(PyObject *od) + + Return true if *od* is an ordered dictionary object, but not an instance of + a subtype of the :class:`~collections.OrderedDict` type. + This function always succeeds. + + +.. c:var:: PyTypeObject PyODictKeys_Type + + Analogous to :c:type:`PyDictKeys_Type` for ordered dictionaries. + + +.. c:var:: PyTypeObject PyODictValues_Type + + Analogous to :c:type:`PyDictValues_Type` for ordered dictionaries. + + +.. c:var:: PyTypeObject PyODictItems_Type + + Analogous to :c:type:`PyDictItems_Type` for ordered dictionaries. + + +.. c:function:: PyObject *PyODict_New(void) + + Return a new empty ordered dictionary, or ``NULL`` on failure. + + This is analogous to :c:func:`PyDict_New`. + + +.. c:function:: int PyODict_SetItem(PyObject *od, PyObject *key, PyObject *value) + + Insert *value* into the ordered dictionary *od* with a key of *key*. + Return ``0`` on success or ``-1`` with an exception set on failure. + + This is analogous to :c:func:`PyDict_SetItem`. + + +.. c:function:: int PyODict_DelItem(PyObject *od, PyObject *key) + + Remove the entry in the ordered dictionary *od* with key *key*. + Return ``0`` on success or ``-1`` with an exception set on failure. + + This is analogous to :c:func:`PyDict_DelItem`. + + +These are :term:`soft deprecated` aliases to ``PyDict`` APIs: + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * ``PyODict`` + * ``PyDict`` + * * .. c:macro:: PyODict_GetItem(od, key) + * :c:func:`PyDict_GetItem` + * * .. c:macro:: PyODict_GetItemWithError(od, key) + * :c:func:`PyDict_GetItemWithError` + * * .. c:macro:: PyODict_GetItemString(od, key) + * :c:func:`PyDict_GetItemString` + * * .. c:macro:: PyODict_Contains(od, key) + * :c:func:`PyDict_Contains` + * * .. c:macro:: PyODict_Size(od) + * :c:func:`PyDict_Size` + * * .. c:macro:: PyODict_SIZE(od) + * :c:func:`PyDict_GET_SIZE` diff --git a/Doc/c-api/iterator.rst b/Doc/c-api/iterator.rst index 7eaf72ec55f..bfbfe3c9279 100644 --- a/Doc/c-api/iterator.rst +++ b/Doc/c-api/iterator.rst @@ -108,6 +108,7 @@ Other Iterator Objects .. c:var:: PyTypeObject PyDictRevIterValue_Type .. c:var:: PyTypeObject PyDictIterItem_Type .. c:var:: PyTypeObject PyDictRevIterItem_Type +.. c:var:: PyTypeObject PyODictIter_Type Type objects for iterators of various built-in objects. From 16ea9505ce690485bab38691e5a83f467757fc03 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Mon, 17 Nov 2025 22:52:13 +0000 Subject: [PATCH 238/313] gh-141004: Document `Py_MEMCPY` (GH-141676) --- Doc/c-api/intro.rst | 8 ++++++++ Misc/NEWS.d/3.14.0a1.rst | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index c76cc2f70ec..bace21b7981 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -183,6 +183,14 @@ complete listing. .. versionadded:: 3.6 +.. c:macro:: Py_MEMCPY(dest, src, n) + + This is a :term:`soft deprecated` alias to :c:func:`!memcpy`. + Use :c:func:`!memcpy` directly instead. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. + .. c:macro:: Py_MIN(x, y) Return the minimum value between ``x`` and ``y``. diff --git a/Misc/NEWS.d/3.14.0a1.rst b/Misc/NEWS.d/3.14.0a1.rst index 305a0b65b98..1938976fa42 100644 --- a/Misc/NEWS.d/3.14.0a1.rst +++ b/Misc/NEWS.d/3.14.0a1.rst @@ -6092,7 +6092,7 @@ Patch by Victor Stinner. .. nonce: qOr9GF .. section: C API -Soft deprecate the :c:macro:`!Py_MEMCPY` macro: use directly ``memcpy()`` +Soft deprecate the :c:macro:`Py_MEMCPY` macro: use directly ``memcpy()`` instead. Patch by Victor Stinner. .. From 4867f717e21c3b5f0ad0e81f950c69dac6c95e6e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 18 Nov 2025 02:26:40 +0000 Subject: [PATCH 239/313] gh-140729: Fix subprocess handling in test_process_pool_executor_pickle (#141688) --- Lib/test/test_profiling/test_sampling_profiler.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index 2d00173c22c..c2cc2ddd48a 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -3311,6 +3311,8 @@ def test_native_frames_disabled(self): self.assertNotIn("", output) +@requires_subprocess() +@skip_if_not_supported class TestProcessPoolExecutorSupport(unittest.TestCase): """ Test that ProcessPoolExecutor works correctly with profiling.sampling. @@ -3339,12 +3341,15 @@ def worker(x): "-d", "5", "-i", "100000", script, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) as proc: - proc.wait(timeout=SHORT_TIMEOUT) - stdout = proc.stdout.read() - stderr = proc.stderr.read() + try: + stdout, stderr = proc.communicate(timeout=SHORT_TIMEOUT) + except subprocess.TimeoutExpired: + proc.kill() + stdout, stderr = proc.communicate() if "PermissionError" in stderr: self.skipTest("Insufficient permissions for remote profiling") From 58f3fe0d9b9882656e629e8caab687c7fcb21b36 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 18 Nov 2025 01:10:32 -0800 Subject: [PATCH 240/313] gh-129005: Remove copies from _pyio using take_bytes (#141539) Memory usage now matches that of _io for large files. --- Lib/_pyio.py | 8 ++++---- Lib/test/test_io/test_bufferedio.py | 3 ++- Lib/test/test_io/test_largefile.py | 6 ++---- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 423178e87a8..69a088df8fc 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -546,7 +546,7 @@ def nreadahead(): res += b if res.endswith(b"\n"): break - return bytes(res) + return res.take_bytes() def __iter__(self): self._checkClosed() @@ -620,7 +620,7 @@ def read(self, size=-1): if n < 0 or n > len(b): raise ValueError(f"readinto returned {n} outside buffer size {len(b)}") del b[n:] - return bytes(b) + return b.take_bytes() def readall(self): """Read until EOF, using multiple read() call.""" @@ -628,7 +628,7 @@ def readall(self): while data := self.read(DEFAULT_BUFFER_SIZE): res += data if res: - return bytes(res) + return res.take_bytes() else: # b'' or None return data @@ -1738,7 +1738,7 @@ def readall(self): assert len(result) - bytes_read >= 1, \ "os.readinto buffer size 0 will result in erroneous EOF / returns 0" result.resize(bytes_read) - return bytes(result) + return result.take_bytes() def readinto(self, buffer): """Same as RawIOBase.readinto().""" diff --git a/Lib/test/test_io/test_bufferedio.py b/Lib/test/test_io/test_bufferedio.py index 30c34e818b1..3278665bdc9 100644 --- a/Lib/test/test_io/test_bufferedio.py +++ b/Lib/test/test_io/test_bufferedio.py @@ -1277,7 +1277,8 @@ def test_flush_and_readinto(self): def _readinto(bufio, n=-1): b = bytearray(n if n >= 0 else 9999) n = bufio.readinto(b) - return bytes(b[:n]) + b.resize(n) + return b.take_bytes() self.check_flush_and_read(_readinto) def test_flush_and_peek(self): diff --git a/Lib/test/test_io/test_largefile.py b/Lib/test/test_io/test_largefile.py index 41f7b70e5cf..438a90a92ed 100644 --- a/Lib/test/test_io/test_largefile.py +++ b/Lib/test/test_io/test_largefile.py @@ -56,9 +56,7 @@ class TestFileMethods(LargeFileTest): (i.e. > 2 GiB) files. """ - # _pyio.FileIO.readall() uses a temporary bytearray then casted to bytes, - # so memuse=2 is needed - @bigmemtest(size=size, memuse=2, dry_run=False) + @bigmemtest(size=size, memuse=1, dry_run=False) def test_large_read(self, _size): # bpo-24658: Test that a read greater than 2GB does not fail. with self.open(TESTFN, "rb") as f: @@ -154,7 +152,7 @@ def test_seekable(self): f.seek(pos) self.assertTrue(f.seekable()) - @bigmemtest(size=size, memuse=2, dry_run=False) + @bigmemtest(size=size, memuse=1, dry_run=False) def test_seek_readall(self, _size): # Seek which doesn't change position should readall successfully. with self.open(TESTFN, 'rb') as f: From 630cd37bfae0fc4021d9e9461b94d36e7ce6b95c Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Tue, 18 Nov 2025 12:17:37 +0300 Subject: [PATCH 241/313] gh-141004: Document Py_HUGE_VAL/IS_FINITE/IS_INFINITE/IS_NAN (#141544) Co-authored-by: Victor Stinner Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/float.rst | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 79de5daaa90..b0d440580b9 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -87,7 +87,7 @@ Floating-Point Objects ```` header. .. deprecated:: 3.15 - The macro is soft deprecated. + The macro is :term:`soft deprecated`. .. c:macro:: Py_NAN @@ -99,6 +99,14 @@ Floating-Point Objects the C11 standard ```` header. +.. c:macro:: Py_HUGE_VAL + + Equivalent to :c:macro:`!INFINITY`. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. + + .. c:macro:: Py_MATH_E The definition (accurate for a :c:expr:`double` type) of the :data:`math.e` constant. @@ -147,6 +155,34 @@ Floating-Point Objects return PyFloat_FromDouble(copysign(INFINITY, sign)); +.. c:macro:: Py_IS_FINITE(X) + + Return ``1`` if the given floating-point number *X* is finite, + that is, it is normal, subnormal or zero, but not infinite or NaN. + Return ``0`` otherwise. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. Use :c:macro:`!isfinite` instead. + + +.. c:macro:: Py_IS_INFINITY(X) + + Return ``1`` if the given floating-point number *X* is positive or negative + infinity. Return ``0`` otherwise. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. Use :c:macro:`!isinf` instead. + + +.. c:macro:: Py_IS_NAN(X) + + Return ``1`` if the given floating-point number *X* is a not-a-number (NaN) + value. Return ``0`` otherwise. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. Use :c:macro:`!isnan` instead. + + Pack and Unpack functions ------------------------- From b87613f21474ea848fec435cbfe63d8cb1c7c44c Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Tue, 18 Nov 2025 11:32:15 +0100 Subject: [PATCH 242/313] Add missing backticks in os and decimal docs (#141699) --- Doc/library/decimal.rst | 2 +- Doc/library/os.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index 985153b5443..ba882f10bbe 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -264,7 +264,7 @@ allows the settings to be changed. This approach meets the needs of most applications. For more advanced work, it may be useful to create alternate contexts using the -Context() constructor. To make an alternate active, use the :func:`setcontext` +:meth:`Context` constructor. To make an alternate active, use the :func:`setcontext` function. In accordance with the standard, the :mod:`decimal` module provides two ready to diff --git a/Doc/library/os.rst b/Doc/library/os.rst index dbc3c92c879..7dc6c177268 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -558,7 +558,7 @@ process and user. .. function:: initgroups(username, gid, /) - Call the system initgroups() to initialize the group access list with all of + Call the system ``initgroups()`` to initialize the group access list with all of the groups of which the specified username is a member, plus the specified group id. From b420f6be53efdf40f552c94f19a7ce85f882b5e2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 18 Nov 2025 13:31:48 +0000 Subject: [PATCH 243/313] GH-139109: Support switch/case dispatch with the tracing interpreter. (GH-141703) --- .github/workflows/jit.yml | 26 +- Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 9 +- Include/internal/pycore_optimizer.h | 2 +- Include/internal/pycore_uop_ids.h | 1 + Include/opcode_ids.h | 47 +- Lib/_opcode_metadata.py | 47 +- Python/bytecodes.c | 9 +- Python/ceval.c | 4 + Python/ceval_macros.h | 10 +- Python/executor_cases.c.h | 2 + Python/generated_cases.c.h | 113 +-- Python/instrumentation.c | 4 +- Python/opcode_targets.h | 910 +++++++++++----------- Python/optimizer_cases.c.h | 2 + Tools/cases_generator/analyzer.py | 7 +- Tools/cases_generator/target_generator.py | 4 +- Tools/cases_generator/tier1_generator.py | 2 +- 18 files changed, 617 insertions(+), 585 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 3349eb04242..62325250bd3 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -57,10 +57,9 @@ jobs: fail-fast: false matrix: target: -# To re-enable later when we support these. -# - i686-pc-windows-msvc/msvc -# - x86_64-pc-windows-msvc/msvc -# - aarch64-pc-windows-msvc/msvc + - i686-pc-windows-msvc/msvc + - x86_64-pc-windows-msvc/msvc + - aarch64-pc-windows-msvc/msvc - x86_64-apple-darwin/clang - aarch64-apple-darwin/clang - x86_64-unknown-linux-gnu/gcc @@ -71,16 +70,15 @@ jobs: llvm: - 21 include: -# To re-enable later when we support these. -# - target: i686-pc-windows-msvc/msvc -# architecture: Win32 -# runner: windows-2022 -# - target: x86_64-pc-windows-msvc/msvc -# architecture: x64 -# runner: windows-2022 -# - target: aarch64-pc-windows-msvc/msvc -# architecture: ARM64 -# runner: windows-11-arm + - target: i686-pc-windows-msvc/msvc + architecture: Win32 + runner: windows-2022 + - target: x86_64-pc-windows-msvc/msvc + architecture: x64 + runner: windows-2022 + - target: aarch64-pc-windows-msvc/msvc + architecture: ARM64 + runner: windows-11-arm - target: x86_64-apple-darwin/clang architecture: x86_64 runner: macos-15-intel diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 7ec7bd1c695..2fb46a6df50 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -286,6 +286,7 @@ Known values: Python 3.15a1 3653 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) Python 3.15a1 3654 (Fix missing exception handlers in logical expression) Python 3.15a1 3655 (Fix miscompilation of some module-level annotations) + Python 3.15a1 3656 (Add TRACE_RECORD instruction, for platforms with switch based interpreter) Python 3.16 will start with 3700 @@ -299,7 +300,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3655 +#define PYC_MAGIC_NUMBER 3656 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 548627dc798..cca88818c57 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -488,6 +488,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case TO_BOOL_STR: return 1; + case TRACE_RECORD: + return 0; case UNARY_INVERT: return 1; case UNARY_NEGATIVE: @@ -971,6 +973,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case TO_BOOL_STR: return 1; + case TRACE_RECORD: + return 0; case UNARY_INVERT: return 1; case UNARY_NEGATIVE: @@ -1287,6 +1291,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [TRACE_RECORD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, @@ -1738,6 +1743,7 @@ const char *_PyOpcode_OpName[267] = { [TO_BOOL_LIST] = "TO_BOOL_LIST", [TO_BOOL_NONE] = "TO_BOOL_NONE", [TO_BOOL_STR] = "TO_BOOL_STR", + [TRACE_RECORD] = "TRACE_RECORD", [UNARY_INVERT] = "UNARY_INVERT", [UNARY_NEGATIVE] = "UNARY_NEGATIVE", [UNARY_NOT] = "UNARY_NOT", @@ -1809,7 +1815,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [230] = 230, [231] = 231, [232] = 232, - [233] = 233, [BINARY_OP] = BINARY_OP, [BINARY_OP_ADD_FLOAT] = BINARY_OP, [BINARY_OP_ADD_INT] = BINARY_OP, @@ -2025,6 +2030,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [TO_BOOL_LIST] = TO_BOOL, [TO_BOOL_NONE] = TO_BOOL, [TO_BOOL_STR] = TO_BOOL, + [TRACE_RECORD] = TRACE_RECORD, [UNARY_INVERT] = UNARY_INVERT, [UNARY_NEGATIVE] = UNARY_NEGATIVE, [UNARY_NOT] = UNARY_NOT, @@ -2070,7 +2076,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 230: \ case 231: \ case 232: \ - case 233: \ ; struct pseudo_targets { uint8_t as_sequence; diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 0307a174e77..e7177552cf6 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -364,7 +364,7 @@ extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int stop_tracing_opcode); -int +PyAPI_FUNC(int) _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *start_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 7a33a5b84fd..c38f28f9db1 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -352,6 +352,7 @@ extern "C" { #define _TO_BOOL_LIST 550 #define _TO_BOOL_NONE TO_BOOL_NONE #define _TO_BOOL_STR 551 +#define _TRACE_RECORD TRACE_RECORD #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 1d5c74adefc..0d066c16901 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -213,28 +213,29 @@ extern "C" { #define UNPACK_SEQUENCE_LIST 207 #define UNPACK_SEQUENCE_TUPLE 208 #define UNPACK_SEQUENCE_TWO_TUPLE 209 -#define INSTRUMENTED_END_FOR 234 -#define INSTRUMENTED_POP_ITER 235 -#define INSTRUMENTED_END_SEND 236 -#define INSTRUMENTED_FOR_ITER 237 -#define INSTRUMENTED_INSTRUCTION 238 -#define INSTRUMENTED_JUMP_FORWARD 239 -#define INSTRUMENTED_NOT_TAKEN 240 -#define INSTRUMENTED_POP_JUMP_IF_TRUE 241 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 242 -#define INSTRUMENTED_POP_JUMP_IF_NONE 243 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 244 -#define INSTRUMENTED_RESUME 245 -#define INSTRUMENTED_RETURN_VALUE 246 -#define INSTRUMENTED_YIELD_VALUE 247 -#define INSTRUMENTED_END_ASYNC_FOR 248 -#define INSTRUMENTED_LOAD_SUPER_ATTR 249 -#define INSTRUMENTED_CALL 250 -#define INSTRUMENTED_CALL_KW 251 -#define INSTRUMENTED_CALL_FUNCTION_EX 252 -#define INSTRUMENTED_JUMP_BACKWARD 253 -#define INSTRUMENTED_LINE 254 -#define ENTER_EXECUTOR 255 +#define INSTRUMENTED_END_FOR 233 +#define INSTRUMENTED_POP_ITER 234 +#define INSTRUMENTED_END_SEND 235 +#define INSTRUMENTED_FOR_ITER 236 +#define INSTRUMENTED_INSTRUCTION 237 +#define INSTRUMENTED_JUMP_FORWARD 238 +#define INSTRUMENTED_NOT_TAKEN 239 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 240 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 241 +#define INSTRUMENTED_POP_JUMP_IF_NONE 242 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 243 +#define INSTRUMENTED_RESUME 244 +#define INSTRUMENTED_RETURN_VALUE 245 +#define INSTRUMENTED_YIELD_VALUE 246 +#define INSTRUMENTED_END_ASYNC_FOR 247 +#define INSTRUMENTED_LOAD_SUPER_ATTR 248 +#define INSTRUMENTED_CALL 249 +#define INSTRUMENTED_CALL_KW 250 +#define INSTRUMENTED_CALL_FUNCTION_EX 251 +#define INSTRUMENTED_JUMP_BACKWARD 252 +#define INSTRUMENTED_LINE 253 +#define ENTER_EXECUTOR 254 +#define TRACE_RECORD 255 #define ANNOTATIONS_PLACEHOLDER 256 #define JUMP 257 #define JUMP_IF_FALSE 258 @@ -249,7 +250,7 @@ extern "C" { #define HAVE_ARGUMENT 43 #define MIN_SPECIALIZED_OPCODE 129 -#define MIN_INSTRUMENTED_OPCODE 234 +#define MIN_INSTRUMENTED_OPCODE 233 #ifdef __cplusplus } diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index f168d169a32..e681cb17e43 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -208,8 +208,9 @@ opmap = { 'CACHE': 0, 'RESERVED': 17, 'RESUME': 128, - 'INSTRUMENTED_LINE': 254, - 'ENTER_EXECUTOR': 255, + 'INSTRUMENTED_LINE': 253, + 'ENTER_EXECUTOR': 254, + 'TRACE_RECORD': 255, 'BINARY_SLICE': 1, 'BUILD_TEMPLATE': 2, 'CALL_FUNCTION_EX': 4, @@ -328,26 +329,26 @@ opmap = { 'UNPACK_EX': 118, 'UNPACK_SEQUENCE': 119, 'YIELD_VALUE': 120, - 'INSTRUMENTED_END_FOR': 234, - 'INSTRUMENTED_POP_ITER': 235, - 'INSTRUMENTED_END_SEND': 236, - 'INSTRUMENTED_FOR_ITER': 237, - 'INSTRUMENTED_INSTRUCTION': 238, - 'INSTRUMENTED_JUMP_FORWARD': 239, - 'INSTRUMENTED_NOT_TAKEN': 240, - 'INSTRUMENTED_POP_JUMP_IF_TRUE': 241, - 'INSTRUMENTED_POP_JUMP_IF_FALSE': 242, - 'INSTRUMENTED_POP_JUMP_IF_NONE': 243, - 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 244, - 'INSTRUMENTED_RESUME': 245, - 'INSTRUMENTED_RETURN_VALUE': 246, - 'INSTRUMENTED_YIELD_VALUE': 247, - 'INSTRUMENTED_END_ASYNC_FOR': 248, - 'INSTRUMENTED_LOAD_SUPER_ATTR': 249, - 'INSTRUMENTED_CALL': 250, - 'INSTRUMENTED_CALL_KW': 251, - 'INSTRUMENTED_CALL_FUNCTION_EX': 252, - 'INSTRUMENTED_JUMP_BACKWARD': 253, + 'INSTRUMENTED_END_FOR': 233, + 'INSTRUMENTED_POP_ITER': 234, + 'INSTRUMENTED_END_SEND': 235, + 'INSTRUMENTED_FOR_ITER': 236, + 'INSTRUMENTED_INSTRUCTION': 237, + 'INSTRUMENTED_JUMP_FORWARD': 238, + 'INSTRUMENTED_NOT_TAKEN': 239, + 'INSTRUMENTED_POP_JUMP_IF_TRUE': 240, + 'INSTRUMENTED_POP_JUMP_IF_FALSE': 241, + 'INSTRUMENTED_POP_JUMP_IF_NONE': 242, + 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 243, + 'INSTRUMENTED_RESUME': 244, + 'INSTRUMENTED_RETURN_VALUE': 245, + 'INSTRUMENTED_YIELD_VALUE': 246, + 'INSTRUMENTED_END_ASYNC_FOR': 247, + 'INSTRUMENTED_LOAD_SUPER_ATTR': 248, + 'INSTRUMENTED_CALL': 249, + 'INSTRUMENTED_CALL_KW': 250, + 'INSTRUMENTED_CALL_FUNCTION_EX': 251, + 'INSTRUMENTED_JUMP_BACKWARD': 252, 'ANNOTATIONS_PLACEHOLDER': 256, 'JUMP': 257, 'JUMP_IF_FALSE': 258, @@ -362,4 +363,4 @@ opmap = { } HAVE_ARGUMENT = 43 -MIN_INSTRUMENTED_OPCODE = 234 +MIN_INSTRUMENTED_OPCODE = 233 diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 565eaa7a599..12ee506e4f2 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5636,10 +5636,12 @@ dummy_func( DISPATCH(); } - label(record_previous_inst) { + inst(TRACE_RECORD, (--)) { #if _Py_TIER2 assert(IS_JIT_TRACING()); - int opcode = next_instr->op.code; + next_instr = this_instr; + frame->instr_ptr = prev_instr; + opcode = next_instr->op.code; bool stop_tracing = (opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); @@ -5675,7 +5677,8 @@ dummy_func( } DISPATCH_GOTO_NON_TRACING(); #else - Py_FatalError("JIT label executed in non-jit build."); + (void)prev_instr; + Py_FatalError("JIT instruction executed in non-jit build."); #endif } diff --git a/Python/ceval.c b/Python/ceval.c index 25294ebd993..14fef42ea96 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1179,6 +1179,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int uint8_t opcode; /* Current opcode */ int oparg; /* Current opcode argument, if any */ assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL); +#if !USE_COMPUTED_GOTOS + uint8_t tracing_mode = 0; + uint8_t dispatch_code; +#endif #endif _PyEntryFrame entry; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 05a2760671e..c30638c221a 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -134,8 +134,8 @@ # define LABEL(name) name: #else # define TARGET(op) case op: TARGET_##op: -# define DISPATCH_GOTO() goto dispatch_opcode -# define DISPATCH_GOTO_NON_TRACING() goto dispatch_opcode +# define DISPATCH_GOTO() dispatch_code = opcode | tracing_mode ; goto dispatch_opcode +# define DISPATCH_GOTO_NON_TRACING() dispatch_code = opcode; goto dispatch_opcode # define JUMP_TO_LABEL(name) goto name; # define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; # define LABEL(name) name: @@ -148,9 +148,9 @@ # define LEAVE_TRACING() \ DISPATCH_TABLE_VAR = DISPATCH_TABLE; #else -# define IS_JIT_TRACING() (0) -# define ENTER_TRACING() -# define LEAVE_TRACING() +# define IS_JIT_TRACING() (tracing_mode != 0) +# define ENTER_TRACING() tracing_mode = 255 +# define LEAVE_TRACING() tracing_mode = 0 #endif /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 6796abf84ac..e1edd20b778 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7579,5 +7579,7 @@ break; } + /* _TRACE_RECORD is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + #undef TIER_TWO diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0d4678df68c..b83b7c528e9 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -11,7 +11,7 @@ #if !_Py_TAIL_CALL_INTERP #if !USE_COMPUTED_GOTOS dispatch_opcode: - switch (opcode) + switch (dispatch_code) #endif { #endif /* _Py_TAIL_CALL_INTERP */ @@ -11683,6 +11683,68 @@ DISPATCH(); } + TARGET(TRACE_RECORD) { + #if _Py_TAIL_CALL_INTERP + int opcode = TRACE_RECORD; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TRACE_RECORD); + opcode = TRACE_RECORD; + #if _Py_TIER2 + assert(IS_JIT_TRACING()); + next_instr = this_instr; + frame->instr_ptr = prev_instr; + opcode = next_instr->op.code; + bool stop_tracing = (opcode == WITH_EXCEPT_START || + opcode == RERAISE || opcode == CLEANUP_THROW || + opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); + _PyFrame_SetStackPointer(frame, stack_pointer); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (full) { + LEAVE_TRACING(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = stop_tracing_and_jit(tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH_GOTO_NON_TRACING(); + } + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if ((_tstate->jit_tracer_state.prev_state.instr->op.code == CALL_LIST_APPEND && + opcode == POP_TOP) || + (_tstate->jit_tracer_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + opcode == STORE_FAST)) { + _tstate->jit_tracer_state.prev_state.instr_is_super = true; + } + else { + _tstate->jit_tracer_state.prev_state.instr = next_instr; + } + PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + if (_tstate->jit_tracer_state.prev_state.instr_code != (PyCodeObject *)prev_code) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_SETREF(_tstate->jit_tracer_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; + _tstate->jit_tracer_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + (&next_instr[1])->counter = trigger_backoff_counter(); + } + DISPATCH_GOTO_NON_TRACING(); + #else + (void)prev_instr; + Py_FatalError("JIT instruction executed in non-jit build."); + #endif + } + TARGET(UNARY_INVERT) { #if _Py_TAIL_CALL_INTERP int opcode = UNARY_INVERT; @@ -12254,55 +12316,6 @@ JUMP_TO_LABEL(error); DISPATCH(); } - LABEL(record_previous_inst) - { - #if _Py_TIER2 - assert(IS_JIT_TRACING()); - int opcode = next_instr->op.code; - bool stop_tracing = (opcode == WITH_EXCEPT_START || - opcode == RERAISE || opcode == CLEANUP_THROW || - opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); - _PyFrame_SetStackPointer(frame, stack_pointer); - int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (full) { - LEAVE_TRACING(); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = stop_tracing_and_jit(tstate, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } - DISPATCH_GOTO_NON_TRACING(); - } - _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - if ((_tstate->jit_tracer_state.prev_state.instr->op.code == CALL_LIST_APPEND && - opcode == POP_TOP) || - (_tstate->jit_tracer_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && - opcode == STORE_FAST)) { - _tstate->jit_tracer_state.prev_state.instr_is_super = true; - } - else { - _tstate->jit_tracer_state.prev_state.instr = next_instr; - } - PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); - if (_tstate->jit_tracer_state.prev_state.instr_code != (PyCodeObject *)prev_code) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_SETREF(_tstate->jit_tracer_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _tstate->jit_tracer_state.prev_state.instr_frame = frame; - _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; - _tstate->jit_tracer_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { - (&next_instr[1])->counter = trigger_backoff_counter(); - } - DISPATCH_GOTO_NON_TRACING(); - #else - Py_FatalError("JIT label executed in non-jit build."); - #endif - } - LABEL(stop_tracing) { #if _Py_TIER2 diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 81e46a331e0..72b7433022f 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -191,7 +191,7 @@ is_instrumented(int opcode) { assert(opcode != 0); assert(opcode != RESERVED); - return opcode != ENTER_EXECUTOR && opcode >= MIN_INSTRUMENTED_OPCODE; + return opcode < ENTER_EXECUTOR && opcode >= MIN_INSTRUMENTED_OPCODE; } #ifndef NDEBUG @@ -526,7 +526,7 @@ valid_opcode(int opcode) if (IS_VALID_OPCODE(opcode) && opcode != CACHE && opcode != RESERVED && - opcode < 255) + opcode < 254) { return true; } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 1b9196503b5..b2fa7d01e8f 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -233,7 +233,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, @@ -256,130 +255,131 @@ static void *opcode_targets_table[256] = { &&TARGET_INSTRUMENTED_JUMP_BACKWARD, &&TARGET_INSTRUMENTED_LINE, &&TARGET_ENTER_EXECUTOR, + &&TARGET_TRACE_RECORD, }; #if _Py_TIER2 static void *opcode_tracing_targets_table[256] = { - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -387,88 +387,88 @@ static void *opcode_tracing_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -492,29 +492,29 @@ static void *opcode_tracing_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, }; #endif #else /* _Py_TAIL_CALL_INTERP */ @@ -528,7 +528,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_record_previous_inst(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); @@ -746,6 +745,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACE_RECORD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); @@ -983,6 +983,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [TO_BOOL_LIST] = _TAIL_CALL_TO_BOOL_LIST, [TO_BOOL_NONE] = _TAIL_CALL_TO_BOOL_NONE, [TO_BOOL_STR] = _TAIL_CALL_TO_BOOL_STR, + [TRACE_RECORD] = _TAIL_CALL_TRACE_RECORD, [UNARY_INVERT] = _TAIL_CALL_UNARY_INVERT, [UNARY_NEGATIVE] = _TAIL_CALL_UNARY_NEGATIVE, [UNARY_NOT] = _TAIL_CALL_UNARY_NOT, @@ -1023,234 +1024,234 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [230] = _TAIL_CALL_UNKNOWN_OPCODE, [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, - [233] = _TAIL_CALL_UNKNOWN_OPCODE, }; static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { - [BINARY_OP] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_ADD_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_EXTEND] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_SLICE] = _TAIL_CALL_record_previous_inst, - [BUILD_INTERPOLATION] = _TAIL_CALL_record_previous_inst, - [BUILD_LIST] = _TAIL_CALL_record_previous_inst, - [BUILD_MAP] = _TAIL_CALL_record_previous_inst, - [BUILD_SET] = _TAIL_CALL_record_previous_inst, - [BUILD_SLICE] = _TAIL_CALL_record_previous_inst, - [BUILD_STRING] = _TAIL_CALL_record_previous_inst, - [BUILD_TEMPLATE] = _TAIL_CALL_record_previous_inst, - [BUILD_TUPLE] = _TAIL_CALL_record_previous_inst, - [CACHE] = _TAIL_CALL_record_previous_inst, - [CALL] = _TAIL_CALL_record_previous_inst, - [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_record_previous_inst, - [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_record_previous_inst, - [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_record_previous_inst, - [CALL_BUILTIN_CLASS] = _TAIL_CALL_record_previous_inst, - [CALL_BUILTIN_FAST] = _TAIL_CALL_record_previous_inst, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_record_previous_inst, - [CALL_BUILTIN_O] = _TAIL_CALL_record_previous_inst, - [CALL_FUNCTION_EX] = _TAIL_CALL_record_previous_inst, - [CALL_INTRINSIC_1] = _TAIL_CALL_record_previous_inst, - [CALL_INTRINSIC_2] = _TAIL_CALL_record_previous_inst, - [CALL_ISINSTANCE] = _TAIL_CALL_record_previous_inst, - [CALL_KW] = _TAIL_CALL_record_previous_inst, - [CALL_KW_BOUND_METHOD] = _TAIL_CALL_record_previous_inst, - [CALL_KW_NON_PY] = _TAIL_CALL_record_previous_inst, - [CALL_KW_PY] = _TAIL_CALL_record_previous_inst, - [CALL_LEN] = _TAIL_CALL_record_previous_inst, - [CALL_LIST_APPEND] = _TAIL_CALL_record_previous_inst, - [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_record_previous_inst, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_record_previous_inst, - [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_record_previous_inst, - [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_record_previous_inst, - [CALL_NON_PY_GENERAL] = _TAIL_CALL_record_previous_inst, - [CALL_PY_EXACT_ARGS] = _TAIL_CALL_record_previous_inst, - [CALL_PY_GENERAL] = _TAIL_CALL_record_previous_inst, - [CALL_STR_1] = _TAIL_CALL_record_previous_inst, - [CALL_TUPLE_1] = _TAIL_CALL_record_previous_inst, - [CALL_TYPE_1] = _TAIL_CALL_record_previous_inst, - [CHECK_EG_MATCH] = _TAIL_CALL_record_previous_inst, - [CHECK_EXC_MATCH] = _TAIL_CALL_record_previous_inst, - [CLEANUP_THROW] = _TAIL_CALL_record_previous_inst, - [COMPARE_OP] = _TAIL_CALL_record_previous_inst, - [COMPARE_OP_FLOAT] = _TAIL_CALL_record_previous_inst, - [COMPARE_OP_INT] = _TAIL_CALL_record_previous_inst, - [COMPARE_OP_STR] = _TAIL_CALL_record_previous_inst, - [CONTAINS_OP] = _TAIL_CALL_record_previous_inst, - [CONTAINS_OP_DICT] = _TAIL_CALL_record_previous_inst, - [CONTAINS_OP_SET] = _TAIL_CALL_record_previous_inst, - [CONVERT_VALUE] = _TAIL_CALL_record_previous_inst, - [COPY] = _TAIL_CALL_record_previous_inst, - [COPY_FREE_VARS] = _TAIL_CALL_record_previous_inst, - [DELETE_ATTR] = _TAIL_CALL_record_previous_inst, - [DELETE_DEREF] = _TAIL_CALL_record_previous_inst, - [DELETE_FAST] = _TAIL_CALL_record_previous_inst, - [DELETE_GLOBAL] = _TAIL_CALL_record_previous_inst, - [DELETE_NAME] = _TAIL_CALL_record_previous_inst, - [DELETE_SUBSCR] = _TAIL_CALL_record_previous_inst, - [DICT_MERGE] = _TAIL_CALL_record_previous_inst, - [DICT_UPDATE] = _TAIL_CALL_record_previous_inst, - [END_ASYNC_FOR] = _TAIL_CALL_record_previous_inst, - [END_FOR] = _TAIL_CALL_record_previous_inst, - [END_SEND] = _TAIL_CALL_record_previous_inst, - [ENTER_EXECUTOR] = _TAIL_CALL_record_previous_inst, - [EXIT_INIT_CHECK] = _TAIL_CALL_record_previous_inst, - [EXTENDED_ARG] = _TAIL_CALL_record_previous_inst, - [FORMAT_SIMPLE] = _TAIL_CALL_record_previous_inst, - [FORMAT_WITH_SPEC] = _TAIL_CALL_record_previous_inst, - [FOR_ITER] = _TAIL_CALL_record_previous_inst, - [FOR_ITER_GEN] = _TAIL_CALL_record_previous_inst, - [FOR_ITER_LIST] = _TAIL_CALL_record_previous_inst, - [FOR_ITER_RANGE] = _TAIL_CALL_record_previous_inst, - [FOR_ITER_TUPLE] = _TAIL_CALL_record_previous_inst, - [GET_AITER] = _TAIL_CALL_record_previous_inst, - [GET_ANEXT] = _TAIL_CALL_record_previous_inst, - [GET_AWAITABLE] = _TAIL_CALL_record_previous_inst, - [GET_ITER] = _TAIL_CALL_record_previous_inst, - [GET_LEN] = _TAIL_CALL_record_previous_inst, - [GET_YIELD_FROM_ITER] = _TAIL_CALL_record_previous_inst, - [IMPORT_FROM] = _TAIL_CALL_record_previous_inst, - [IMPORT_NAME] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_CALL] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_CALL_KW] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_END_FOR] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_END_SEND] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_LINE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_POP_ITER] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_RESUME] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_record_previous_inst, - [INTERPRETER_EXIT] = _TAIL_CALL_record_previous_inst, - [IS_OP] = _TAIL_CALL_record_previous_inst, - [JUMP_BACKWARD] = _TAIL_CALL_record_previous_inst, - [JUMP_BACKWARD_JIT] = _TAIL_CALL_record_previous_inst, - [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_record_previous_inst, - [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_record_previous_inst, - [JUMP_FORWARD] = _TAIL_CALL_record_previous_inst, - [LIST_APPEND] = _TAIL_CALL_record_previous_inst, - [LIST_EXTEND] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_CLASS] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_MODULE] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_PROPERTY] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_SLOT] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_record_previous_inst, - [LOAD_BUILD_CLASS] = _TAIL_CALL_record_previous_inst, - [LOAD_COMMON_CONSTANT] = _TAIL_CALL_record_previous_inst, - [LOAD_CONST] = _TAIL_CALL_record_previous_inst, - [LOAD_DEREF] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST_BORROW] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST_CHECK] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_record_previous_inst, - [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_record_previous_inst, - [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_record_previous_inst, - [LOAD_GLOBAL] = _TAIL_CALL_record_previous_inst, - [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_record_previous_inst, - [LOAD_GLOBAL_MODULE] = _TAIL_CALL_record_previous_inst, - [LOAD_LOCALS] = _TAIL_CALL_record_previous_inst, - [LOAD_NAME] = _TAIL_CALL_record_previous_inst, - [LOAD_SMALL_INT] = _TAIL_CALL_record_previous_inst, - [LOAD_SPECIAL] = _TAIL_CALL_record_previous_inst, - [LOAD_SUPER_ATTR] = _TAIL_CALL_record_previous_inst, - [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_record_previous_inst, - [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_record_previous_inst, - [MAKE_CELL] = _TAIL_CALL_record_previous_inst, - [MAKE_FUNCTION] = _TAIL_CALL_record_previous_inst, - [MAP_ADD] = _TAIL_CALL_record_previous_inst, - [MATCH_CLASS] = _TAIL_CALL_record_previous_inst, - [MATCH_KEYS] = _TAIL_CALL_record_previous_inst, - [MATCH_MAPPING] = _TAIL_CALL_record_previous_inst, - [MATCH_SEQUENCE] = _TAIL_CALL_record_previous_inst, - [NOP] = _TAIL_CALL_record_previous_inst, - [NOT_TAKEN] = _TAIL_CALL_record_previous_inst, - [POP_EXCEPT] = _TAIL_CALL_record_previous_inst, - [POP_ITER] = _TAIL_CALL_record_previous_inst, - [POP_JUMP_IF_FALSE] = _TAIL_CALL_record_previous_inst, - [POP_JUMP_IF_NONE] = _TAIL_CALL_record_previous_inst, - [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_record_previous_inst, - [POP_JUMP_IF_TRUE] = _TAIL_CALL_record_previous_inst, - [POP_TOP] = _TAIL_CALL_record_previous_inst, - [PUSH_EXC_INFO] = _TAIL_CALL_record_previous_inst, - [PUSH_NULL] = _TAIL_CALL_record_previous_inst, - [RAISE_VARARGS] = _TAIL_CALL_record_previous_inst, - [RERAISE] = _TAIL_CALL_record_previous_inst, - [RESERVED] = _TAIL_CALL_record_previous_inst, - [RESUME] = _TAIL_CALL_record_previous_inst, - [RESUME_CHECK] = _TAIL_CALL_record_previous_inst, - [RETURN_GENERATOR] = _TAIL_CALL_record_previous_inst, - [RETURN_VALUE] = _TAIL_CALL_record_previous_inst, - [SEND] = _TAIL_CALL_record_previous_inst, - [SEND_GEN] = _TAIL_CALL_record_previous_inst, - [SETUP_ANNOTATIONS] = _TAIL_CALL_record_previous_inst, - [SET_ADD] = _TAIL_CALL_record_previous_inst, - [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_record_previous_inst, - [SET_UPDATE] = _TAIL_CALL_record_previous_inst, - [STORE_ATTR] = _TAIL_CALL_record_previous_inst, - [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_record_previous_inst, - [STORE_ATTR_SLOT] = _TAIL_CALL_record_previous_inst, - [STORE_ATTR_WITH_HINT] = _TAIL_CALL_record_previous_inst, - [STORE_DEREF] = _TAIL_CALL_record_previous_inst, - [STORE_FAST] = _TAIL_CALL_record_previous_inst, - [STORE_FAST_LOAD_FAST] = _TAIL_CALL_record_previous_inst, - [STORE_FAST_STORE_FAST] = _TAIL_CALL_record_previous_inst, - [STORE_GLOBAL] = _TAIL_CALL_record_previous_inst, - [STORE_NAME] = _TAIL_CALL_record_previous_inst, - [STORE_SLICE] = _TAIL_CALL_record_previous_inst, - [STORE_SUBSCR] = _TAIL_CALL_record_previous_inst, - [STORE_SUBSCR_DICT] = _TAIL_CALL_record_previous_inst, - [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_record_previous_inst, - [SWAP] = _TAIL_CALL_record_previous_inst, - [TO_BOOL] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_BOOL] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_INT] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_LIST] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_NONE] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_STR] = _TAIL_CALL_record_previous_inst, - [UNARY_INVERT] = _TAIL_CALL_record_previous_inst, - [UNARY_NEGATIVE] = _TAIL_CALL_record_previous_inst, - [UNARY_NOT] = _TAIL_CALL_record_previous_inst, - [UNPACK_EX] = _TAIL_CALL_record_previous_inst, - [UNPACK_SEQUENCE] = _TAIL_CALL_record_previous_inst, - [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_record_previous_inst, - [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_record_previous_inst, - [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_record_previous_inst, - [WITH_EXCEPT_START] = _TAIL_CALL_record_previous_inst, - [YIELD_VALUE] = _TAIL_CALL_record_previous_inst, + [BINARY_OP] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_EXTEND] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_SLICE] = _TAIL_CALL_TRACE_RECORD, + [BUILD_INTERPOLATION] = _TAIL_CALL_TRACE_RECORD, + [BUILD_LIST] = _TAIL_CALL_TRACE_RECORD, + [BUILD_MAP] = _TAIL_CALL_TRACE_RECORD, + [BUILD_SET] = _TAIL_CALL_TRACE_RECORD, + [BUILD_SLICE] = _TAIL_CALL_TRACE_RECORD, + [BUILD_STRING] = _TAIL_CALL_TRACE_RECORD, + [BUILD_TEMPLATE] = _TAIL_CALL_TRACE_RECORD, + [BUILD_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [CACHE] = _TAIL_CALL_TRACE_RECORD, + [CALL] = _TAIL_CALL_TRACE_RECORD, + [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_TRACE_RECORD, + [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_CLASS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_FAST] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_O] = _TAIL_CALL_TRACE_RECORD, + [CALL_FUNCTION_EX] = _TAIL_CALL_TRACE_RECORD, + [CALL_INTRINSIC_1] = _TAIL_CALL_TRACE_RECORD, + [CALL_INTRINSIC_2] = _TAIL_CALL_TRACE_RECORD, + [CALL_ISINSTANCE] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_BOUND_METHOD] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_NON_PY] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_PY] = _TAIL_CALL_TRACE_RECORD, + [CALL_LEN] = _TAIL_CALL_TRACE_RECORD, + [CALL_LIST_APPEND] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_TRACE_RECORD, + [CALL_NON_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_PY_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_STR_1] = _TAIL_CALL_TRACE_RECORD, + [CALL_TUPLE_1] = _TAIL_CALL_TRACE_RECORD, + [CALL_TYPE_1] = _TAIL_CALL_TRACE_RECORD, + [CHECK_EG_MATCH] = _TAIL_CALL_TRACE_RECORD, + [CHECK_EXC_MATCH] = _TAIL_CALL_TRACE_RECORD, + [CLEANUP_THROW] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP_INT] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP_STR] = _TAIL_CALL_TRACE_RECORD, + [CONTAINS_OP] = _TAIL_CALL_TRACE_RECORD, + [CONTAINS_OP_DICT] = _TAIL_CALL_TRACE_RECORD, + [CONTAINS_OP_SET] = _TAIL_CALL_TRACE_RECORD, + [CONVERT_VALUE] = _TAIL_CALL_TRACE_RECORD, + [COPY] = _TAIL_CALL_TRACE_RECORD, + [COPY_FREE_VARS] = _TAIL_CALL_TRACE_RECORD, + [DELETE_ATTR] = _TAIL_CALL_TRACE_RECORD, + [DELETE_DEREF] = _TAIL_CALL_TRACE_RECORD, + [DELETE_FAST] = _TAIL_CALL_TRACE_RECORD, + [DELETE_GLOBAL] = _TAIL_CALL_TRACE_RECORD, + [DELETE_NAME] = _TAIL_CALL_TRACE_RECORD, + [DELETE_SUBSCR] = _TAIL_CALL_TRACE_RECORD, + [DICT_MERGE] = _TAIL_CALL_TRACE_RECORD, + [DICT_UPDATE] = _TAIL_CALL_TRACE_RECORD, + [END_ASYNC_FOR] = _TAIL_CALL_TRACE_RECORD, + [END_FOR] = _TAIL_CALL_TRACE_RECORD, + [END_SEND] = _TAIL_CALL_TRACE_RECORD, + [ENTER_EXECUTOR] = _TAIL_CALL_TRACE_RECORD, + [EXIT_INIT_CHECK] = _TAIL_CALL_TRACE_RECORD, + [EXTENDED_ARG] = _TAIL_CALL_TRACE_RECORD, + [FORMAT_SIMPLE] = _TAIL_CALL_TRACE_RECORD, + [FORMAT_WITH_SPEC] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_GEN] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_LIST] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_RANGE] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [GET_AITER] = _TAIL_CALL_TRACE_RECORD, + [GET_ANEXT] = _TAIL_CALL_TRACE_RECORD, + [GET_AWAITABLE] = _TAIL_CALL_TRACE_RECORD, + [GET_ITER] = _TAIL_CALL_TRACE_RECORD, + [GET_LEN] = _TAIL_CALL_TRACE_RECORD, + [GET_YIELD_FROM_ITER] = _TAIL_CALL_TRACE_RECORD, + [IMPORT_FROM] = _TAIL_CALL_TRACE_RECORD, + [IMPORT_NAME] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_CALL] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_CALL_KW] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_END_FOR] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_END_SEND] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_LINE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_ITER] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_RESUME] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, + [INTERPRETER_EXIT] = _TAIL_CALL_TRACE_RECORD, + [IS_OP] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD_JIT] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_TRACE_RECORD, + [JUMP_FORWARD] = _TAIL_CALL_TRACE_RECORD, + [LIST_APPEND] = _TAIL_CALL_TRACE_RECORD, + [LIST_EXTEND] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_CLASS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_MODULE] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_PROPERTY] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_SLOT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_BUILD_CLASS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_COMMON_CONSTANT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_CONST] = _TAIL_CALL_TRACE_RECORD, + [LOAD_DEREF] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_BORROW] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_CHECK] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_GLOBAL] = _TAIL_CALL_TRACE_RECORD, + [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_TRACE_RECORD, + [LOAD_GLOBAL_MODULE] = _TAIL_CALL_TRACE_RECORD, + [LOAD_LOCALS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_NAME] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SMALL_INT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SPECIAL] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SUPER_ATTR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_TRACE_RECORD, + [MAKE_CELL] = _TAIL_CALL_TRACE_RECORD, + [MAKE_FUNCTION] = _TAIL_CALL_TRACE_RECORD, + [MAP_ADD] = _TAIL_CALL_TRACE_RECORD, + [MATCH_CLASS] = _TAIL_CALL_TRACE_RECORD, + [MATCH_KEYS] = _TAIL_CALL_TRACE_RECORD, + [MATCH_MAPPING] = _TAIL_CALL_TRACE_RECORD, + [MATCH_SEQUENCE] = _TAIL_CALL_TRACE_RECORD, + [NOP] = _TAIL_CALL_TRACE_RECORD, + [NOT_TAKEN] = _TAIL_CALL_TRACE_RECORD, + [POP_EXCEPT] = _TAIL_CALL_TRACE_RECORD, + [POP_ITER] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_FALSE] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_NONE] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_TRUE] = _TAIL_CALL_TRACE_RECORD, + [POP_TOP] = _TAIL_CALL_TRACE_RECORD, + [PUSH_EXC_INFO] = _TAIL_CALL_TRACE_RECORD, + [PUSH_NULL] = _TAIL_CALL_TRACE_RECORD, + [RAISE_VARARGS] = _TAIL_CALL_TRACE_RECORD, + [RERAISE] = _TAIL_CALL_TRACE_RECORD, + [RESERVED] = _TAIL_CALL_TRACE_RECORD, + [RESUME] = _TAIL_CALL_TRACE_RECORD, + [RESUME_CHECK] = _TAIL_CALL_TRACE_RECORD, + [RETURN_GENERATOR] = _TAIL_CALL_TRACE_RECORD, + [RETURN_VALUE] = _TAIL_CALL_TRACE_RECORD, + [SEND] = _TAIL_CALL_TRACE_RECORD, + [SEND_GEN] = _TAIL_CALL_TRACE_RECORD, + [SETUP_ANNOTATIONS] = _TAIL_CALL_TRACE_RECORD, + [SET_ADD] = _TAIL_CALL_TRACE_RECORD, + [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_TRACE_RECORD, + [SET_UPDATE] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR_SLOT] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR_WITH_HINT] = _TAIL_CALL_TRACE_RECORD, + [STORE_DEREF] = _TAIL_CALL_TRACE_RECORD, + [STORE_FAST] = _TAIL_CALL_TRACE_RECORD, + [STORE_FAST_LOAD_FAST] = _TAIL_CALL_TRACE_RECORD, + [STORE_FAST_STORE_FAST] = _TAIL_CALL_TRACE_RECORD, + [STORE_GLOBAL] = _TAIL_CALL_TRACE_RECORD, + [STORE_NAME] = _TAIL_CALL_TRACE_RECORD, + [STORE_SLICE] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR_DICT] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_TRACE_RECORD, + [SWAP] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_BOOL] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_INT] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_LIST] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_NONE] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_STR] = _TAIL_CALL_TRACE_RECORD, + [TRACE_RECORD] = _TAIL_CALL_TRACE_RECORD, + [UNARY_INVERT] = _TAIL_CALL_TRACE_RECORD, + [UNARY_NEGATIVE] = _TAIL_CALL_TRACE_RECORD, + [UNARY_NOT] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_EX] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [WITH_EXCEPT_START] = _TAIL_CALL_TRACE_RECORD, + [YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, [121] = _TAIL_CALL_UNKNOWN_OPCODE, [122] = _TAIL_CALL_UNKNOWN_OPCODE, [123] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1281,6 +1282,5 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [230] = _TAIL_CALL_UNKNOWN_OPCODE, [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, - [233] = _TAIL_CALL_UNKNOWN_OPCODE, }; #endif /* _Py_TAIL_CALL_INTERP */ diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 01263fe8c7a..9ebd113df2d 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -3483,3 +3483,5 @@ break; } + /* _TRACE_RECORD is not a viable micro-op for tier 2 */ + diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index d39013db4f7..93aa4899fe6 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -1195,8 +1195,9 @@ def assign_opcodes( # This is an historical oddity. instmap["BINARY_OP_INPLACE_ADD_UNICODE"] = 3 - instmap["INSTRUMENTED_LINE"] = 254 - instmap["ENTER_EXECUTOR"] = 255 + instmap["INSTRUMENTED_LINE"] = 253 + instmap["ENTER_EXECUTOR"] = 254 + instmap["TRACE_RECORD"] = 255 instrumented = [name for name in instructions if name.startswith("INSTRUMENTED")] @@ -1221,7 +1222,7 @@ def assign_opcodes( # Specialized ops appear in their own section # Instrumented opcodes are at the end of the valid range min_internal = instmap["RESUME"] + 1 - min_instrumented = 254 - (len(instrumented) - 1) + min_instrumented = 254 - len(instrumented) assert min_internal + len(specialized) < min_instrumented next_opcode = 1 diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index 36fa1d7fa49..f633f704485 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -34,7 +34,7 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: targets = ["&&_unknown_opcode,\n"] * 256 for name, op in analysis.opmap.items(): if op < 256: - targets[op] = f"&&record_previous_inst,\n" + targets[op] = f"&&TARGET_TRACE_RECORD,\n" out.emit("#if _Py_TIER2\n") out.emit("static void *opcode_tracing_targets_table[256] = {\n") for target in targets: @@ -84,7 +84,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: # Emit the tracing dispatch table. out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {\n") for name in sorted(analysis.instructions.keys()): - out.emit(f"[{name}] = _TAIL_CALL_record_previous_inst,\n") + out.emit(f"[{name}] = _TAIL_CALL_TRACE_RECORD,\n") named_values = analysis.opmap.values() for rest in range(256): if rest not in named_values: diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 94ffb0118f0..c7ff5de681e 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -160,7 +160,7 @@ def generate_tier1( #if !_Py_TAIL_CALL_INTERP #if !USE_COMPUTED_GOTOS dispatch_opcode: - switch (opcode) + switch (dispatch_code) #endif {{ #endif /* _Py_TAIL_CALL_INTERP */ From f46785f8bc118e0efb840af1e520777b1baa03d9 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 18 Nov 2025 16:42:13 +0200 Subject: [PATCH 244/313] gh-133879: Copyedit "What's new in Python 3.15" (#141717) --- .../pending-removal-in-future.rst | 2 +- Doc/whatsnew/3.15.rst | 36 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Doc/deprecations/pending-removal-in-future.rst b/Doc/deprecations/pending-removal-in-future.rst index 7ed430625f3..30186741670 100644 --- a/Doc/deprecations/pending-removal-in-future.rst +++ b/Doc/deprecations/pending-removal-in-future.rst @@ -76,7 +76,7 @@ although there is currently no date scheduled for their removal. * :mod:`mailbox`: Use of StringIO input and text mode is deprecated, use BytesIO and binary mode instead. -* :mod:`os`: Calling :func:`os.register_at_fork` in multi-threaded process. +* :mod:`os`: Calling :func:`os.register_at_fork` in a multi-threaded process. * :class:`!pydoc.ErrorDuringImport`: A tuple value for *exc_info* parameter is deprecated, use an exception instance. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index cf5bef15203..24cc7e2d7eb 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -316,9 +316,7 @@ Other language changes and compression. Common code patterns which can be optimized with :func:`~bytearray.take_bytes` are listed below. - (Contributed by Cody Maloney in :gh:`139871`.) - - .. list-table:: Suggested Optimizing Refactors + .. list-table:: Suggested optimizing refactors :header-rows: 1 * - Description @@ -387,10 +385,12 @@ Other language changes buffer.resize(n) data = buffer.take_bytes() + (Contributed by Cody Maloney in :gh:`139871`.) + * Many functions related to compiling or parsing Python code, such as :func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, - and :func:`importlib.abc.InspectLoader.source_to_code`, now allow to pass - the module name. It is needed to unambiguous :ref:`filter ` + and :func:`importlib.abc.InspectLoader.source_to_code`, now allow the module + name to be passed. It is needed to unambiguously :ref:`filter ` syntax warnings by module name. (Contributed by Serhiy Storchaka in :gh:`135801`.) @@ -776,6 +776,17 @@ unittest (Contributed by Garry Cairns in :gh:`134567`.) +venv +---- + +* On POSIX platforms, platlib directories will be created if needed when + creating virtual environments, instead of using ``lib64 -> lib`` symlink. + This means purelib and platlib of virtual environments no longer share the + same ``lib`` directory on platforms where :data:`sys.platlibdir` is not + equal to ``lib``. + (Contributed by Rui Xi in :gh:`133951`.) + + warnings -------- @@ -788,17 +799,6 @@ warnings (Contributed by Serhiy Storchaka in :gh:`135801`.) -venv ----- - -* On POSIX platforms, platlib directories will be created if needed when - creating virtual environments, instead of using ``lib64 -> lib`` symlink. - This means purelib and platlib of virtual environments no longer share the - same ``lib`` directory on platforms where :data:`sys.platlibdir` is not - equal to ``lib``. - (Contributed by Rui Xi in :gh:`133951`.) - - xml.parsers.expat ----------------- @@ -1242,7 +1242,7 @@ Porting to Python 3.15 This section lists previously described changes and other bugfixes that may require changes to your code. -* :class:`sqlite3.Connection` APIs has been cleaned up. +* :class:`sqlite3.Connection` APIs have been cleaned up. * All parameters of :func:`sqlite3.connect` except *database* are now keyword-only. * The first three parameters of methods :meth:`~sqlite3.Connection.create_function` @@ -1262,7 +1262,7 @@ that may require changes to your code. * :meth:`~mmap.mmap.resize` has been removed on platforms that don't support the underlying syscall, instead of raising a :exc:`SystemError`. -* Resource warning is now emitted for unclosed +* A resource warning is now emitted for an unclosed :func:`xml.etree.ElementTree.iterparse` iterator if it opened a file. Use its :meth:`!close` method or the :func:`contextlib.closing` context manager to close it. From a62562859deea162a36dd5c99f0b87fe09af0292 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 18 Nov 2025 16:50:49 +0200 Subject: [PATCH 245/313] Python 3.15.0a2 --- Doc/c-api/import.rst | 2 +- Doc/c-api/init.rst | 4 +- Doc/library/ast.rst | 2 +- Doc/library/decimal.rst | 2 +- Doc/library/functions.rst | 2 +- Doc/library/functools.rst | 2 +- Doc/library/importlib.rst | 4 +- Doc/library/inspect.rst | 2 +- Doc/library/math.integer.rst | 2 +- Doc/library/math.rst | 2 +- Doc/library/os.rst | 14 +- Doc/library/stat.rst | 2 +- Doc/library/stdtypes.rst | 2 +- Doc/library/symtable.rst | 2 +- Doc/library/unicodedata.rst | 4 +- Doc/library/warnings.rst | 2 +- Doc/library/winreg.rst | 2 +- Doc/library/xml.etree.elementtree.rst | 2 +- Include/patchlevel.h | 4 +- Lib/pydoc_data/topics.py | 51 +- Misc/NEWS.d/3.15.0a2.rst | 1746 +++++++++++++++++ ...-08-10-22-28-06.gh-issue-137618.FdNvIE.rst | 2 - ...-10-16-11-30-53.gh-issue-140189.YCrUyt.rst | 1 - ...-10-17-11-33-45.gh-issue-140239._k-GgW.rst | 1 - ...-10-22-12-44-07.gh-issue-140475.OhzQbR.rst | 1 - ...-10-25-08-07-06.gh-issue-140513.6OhLTs.rst | 2 - ...-10-29-12-30-38.gh-issue-140768.ITYrzw.rst | 1 - ...-10-31-13-20-16.gh-issue-140454.gF6dCe.rst | 3 - ...-10-06-22-17-47.gh-issue-139653.6-1MOd.rst | 4 - ...-10-15-15-59-59.gh-issue-140153.BO7sH4.rst | 2 - ...-10-26-16-45-06.gh-issue-140487.fGOqss.rst | 2 - ...-10-26-16-45-28.gh-issue-140556.s__Dae.rst | 2 - ...-11-05-04-38-16.gh-issue-141004.rJL43P.rst | 1 - ...-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst | 1 - ...-11-06-06-28-14.gh-issue-141042.brOioJ.rst | 3 - ...-11-08-10-51-50.gh-issue-116146.pCmx6L.rst | 2 - ...-11-10-11-26-26.gh-issue-141341.OsO6-y.rst | 2 - ...-06-24-13-12-58.gh-issue-134786.MF0VVk.rst | 2 - ...-07-08-00-41-46.gh-issue-136327.7AiTb_.rst | 2 - ...-07-29-17-51-14.gh-issue-131253.GpRjWy.rst | 1 - ...-09-13-01-23-25.gh-issue-138857.YQ5gdc.rst | 2 - ...-09-15-13-06-11.gh-issue-138944.PeCgLb.rst | 3 - ...-09-23-21-01-12.gh-issue-139269.1rIaxy.rst | 1 - ...-10-03-17-51-43.gh-issue-139475._684ED.rst | 2 - ...-10-06-10-03-37.gh-issue-139640.gY5oTb.rst | 3 - ...10-06-10-03-37.gh-issue-139640.gY5oTb2.rst | 3 - ...-10-06-14-19-47.gh-issue-135801.OhxEZS.rst | 6 - ...-10-12-01-12-12.gh-issue-139817.PAn-8Z.rst | 2 - ...-10-13-13-54-19.gh-issue-139914.M-y_3E.rst | 1 - ...-10-14-17-07-37.gh-issue-140067.ID2gOm.rst | 1 - ...-10-14-18-24-16.gh-issue-139871.SWtuUz.rst | 2 - ...-10-14-20-18-31.gh-issue-140080.8ROjxW.rst | 1 - ...-10-15-00-21-40.gh-issue-140061.J0XeDV.rst | 2 - ...-10-15-17-12-32.gh-issue-140149.cy1m3d.rst | 2 - ...-10-16-21-47-00.gh-issue-140104.A8SQIm.rst | 2 - ...-10-17-14-38-10.gh-issue-140253.gCqFaL.rst | 2 - ...-10-17-18-03-12.gh-issue-139951.IdwM2O.rst | 7 - ...-10-17-20-23-19.gh-issue-140257.8Txmem.rst | 2 - ...-10-18-18-08-36.gh-issue-140301.m-2HxC.rst | 1 - ...-10-18-19-52-20.gh-issue-116738.NLJW0L.rst | 2 - ...-10-18-21-29-45.gh-issue-140306.xS5CcS.rst | 2 - ...-10-18-21-50-44.gh-issue-139109.9QQOzN.rst | 1 - ...-10-19-10-32-28.gh-issue-136895.HfsEh0.rst | 1 - ...-10-20-11-24-36.gh-issue-140358.UQuKdV.rst | 4 - ...-10-21-06-51-50.gh-issue-140406.0gJs8M.rst | 2 - ...-10-21-09-20-03.gh-issue-140398.SoABwJ.rst | 4 - ...-10-22-11-30-16.gh-issue-135904.3WE5oW.rst | 3 - ...-10-22-12-48-05.gh-issue-140476.F3-d1P.rst | 2 - ...-10-22-17-22-22.gh-issue-140431.m8D_A-.rst | 3 - ...-10-22-23-26-37.gh-issue-140443.wT5i1A.rst | 5 - ...-10-23-16-05-50.gh-issue-140471.Ax_aXn.rst | 2 - ...-10-24-14-29-12.gh-issue-133467.A5d6TM.rst | 1 - ...-10-24-20-16-42.gh-issue-140517.cqun-K.rst | 3 - ...-10-24-20-42-33.gh-issue-140551.-9swrl.rst | 2 - ...-10-25-07-25-52.gh-issue-140544.lwjtQe.rst | 1 - ...-10-25-17-36-46.gh-issue-140576.kj0SCY.rst | 2 - ...-10-25-21-31-43.gh-issue-131527.V-JVNP.rst | 2 - ...-10-29-11-31-59.gh-issue-140729.t9JsNt.rst | 2 - ...-10-29-20-59-10.gh-issue-140373.-uoaPP.rst | 2 - ...5-10-31-14-03-42.gh-issue-90344.gvZigO.rst | 1 - ...-11-02-12-47-38.gh-issue-140530.S934bp.rst | 2 - ...-11-02-15-28-33.gh-issue-140260.JNzlGz.rst | 2 - ...-11-03-17-21-38.gh-issue-140939.FVboAw.rst | 2 - ...-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst | 1 - ...-11-04-12-18-06.gh-issue-140942.GYns6n.rst | 2 - ...-11-05-19-50-37.gh-issue-140643.QCEOqG.rst | 3 - ...-11-10-23-07-06.gh-issue-141312.H-58GB.rst | 2 - ...-11-11-13-40-45.gh-issue-141367.I5KY7F.rst | 2 - ...-11-14-00-19-45.gh-issue-141528.VWdax1.rst | 3 - ...-11-14-16-25-15.gh-issue-114203.n3tlQO.rst | 1 - ...-11-15-01-21-00.gh-issue-141579.aB7cD9.rst | 2 - ...9-06-02-13-56-16.gh-issue-81313.axawSH.rst | 1 - ...-03-21-10-59-40.gh-issue-102431.eUDnf4.rst | 2 - ...-05-28-17-14-30.gh-issue-119668.RrIGpn.rst | 1 - ...-06-26-16-16-43.gh-issue-121011.qW54eh.rst | 2 - ...-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst | 4 - ...-03-04-17-19-26.gh-issue-130693.Kv01r8.rst | 1 - ...-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst | 2 - ...-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst | 2 - ...-05-07-22-09-28.gh-issue-133601.9kUL3P.rst | 1 - ...-05-10-15-10-54.gh-issue-133789.I-ZlUX.rst | 1 - ...-06-10-18-02-29.gh-issue-135307.fXGrcK.rst | 2 - ...-06-29-22-01-00.gh-issue-133390.I1DW_3.rst | 2 - ...-07-01-04-57-57.gh-issue-136057.4-t596.rst | 1 - ...5-07-14-09-33-17.gh-issue-55531.Gt2e12.rst | 4 - ...-08-11-04-52-18.gh-issue-137627.Ku5Yi2.rst | 1 - ...5-08-15-20-35-30.gh-issue-69528.qc-Eh_.rst | 2 - ...-08-26-08-17-56.gh-issue-138151.I6CdAk.rst | 3 - ...-09-03-18-26-07.gh-issue-138425.cVE9Ho.rst | 2 - ...5-09-03-20-18-39.gh-issue-98896.tjez89.rst | 2 - ...-09-11-15-03-37.gh-issue-138775.w7rnSx.rst | 2 - ...-09-12-09-34-37.gh-issue-138764.mokHoY.rst | 3 - ...-09-13-12-19-17.gh-issue-138859.PxjIoN.rst | 1 - ...-09-15-21-03-11.gh-issue-138891.oZFdtR.rst | 2 - ...5-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst | 2 - ...-09-23-09-46-46.gh-issue-139246.pzfM-w.rst | 1 - ...-09-25-20-16-10.gh-issue-101828.yTxJlJ.rst | 3 - ...5-09-30-12-52-54.gh-issue-63161.mECM1A.rst | 3 - ...-10-02-22-29-00.gh-issue-139462.VZXUHe.rst | 3 - ...-10-11-09-07-06.gh-issue-139940.g54efZ.rst | 1 - ...-10-13-11-25-41.gh-issue-136702.uvLGK1.rst | 3 - ...5-10-14-20-27-06.gh-issue-76007.2NcUbo.rst | 2 - ...-10-15-02-26-50.gh-issue-140135.54JYfM.rst | 2 - ...-10-15-15-10-34.gh-issue-140166.NtxRez.rst | 1 - ...-10-15-17-23-51.gh-issue-140141.j2mUDB.rst | 5 - ...-10-15-20-47-04.gh-issue-140120.3gffZq.rst | 2 - ...-10-15-21-42-13.gh-issue-140041._Fka2j.rst | 1 - ...-10-16-16-10-11.gh-issue-139707.zR6Qtn.rst | 2 - ...-10-16-17-17-20.gh-issue-135801.faH3fa.rst | 6 - ...-10-16-22-49-16.gh-issue-140212.llBNd0.rst | 5 - ...-10-17-12-33-01.gh-issue-140251.esM-OX.rst | 1 - ...-10-17-20-42-38.gh-issue-129117.X9jr4p.rst | 3 - ...-10-17-23-58-11.gh-issue-140272.lhY8uS.rst | 1 - ...5-10-18-14-30-21.gh-issue-76007.peEgcr.rst | 1 - ...5-10-18-15-20-25.gh-issue-76007.SNUzRq.rst | 2 - ...-10-20-12-33-49.gh-issue-140348.SAKnQZ.rst | 3 - ...-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst | 1 - ...-10-22-12-56-57.gh-issue-140448.GsEkXD.rst | 2 - ...-10-22-20-52-13.gh-issue-140474.xIWlip.rst | 2 - ...-10-23-12-12-22.gh-issue-138774.mnh2gU.rst | 2 - ...-10-23-13-42-15.gh-issue-140481.XKxWpq.rst | 1 - ...-10-23-19-39-16.gh-issue-138162.Znw5DN.rst | 2 - ...-10-25-21-04-00.gh-issue-140607.oOZGxS.rst | 2 - ...-10-25-21-26-16.gh-issue-140593.OxlLc9.rst | 3 - ...-10-25-22-55-07.gh-issue-140601.In3MlS.rst | 4 - ...-10-26-16-24-12.gh-issue-140633.ioayC1.rst | 2 - ...-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst | 3 - ...-10-27-13-49-31.gh-issue-140634.ULng9G.rst | 1 - ...-10-27-16-01-41.gh-issue-125434.qy0uRA.rst | 2 - ...-10-27-18-29-42.gh-issue-140590.LT9HHn.rst | 2 - ...-10-28-02-46-56.gh-issue-139946.aN3_uY.rst | 1 - ...-10-28-17-43-51.gh-issue-140228.8kfHhO.rst | 1 - ...-10-29-09-40-10.gh-issue-140741.L13UCV.rst | 2 - ...-10-29-16-12-41.gh-issue-120057.qGj5Dl.rst | 1 - ...-10-29-16-53-00.gh-issue-140766.CNagKF.rst | 1 - ...-10-30-12-36-19.gh-issue-140790._3T6-N.rst | 1 - ...-10-30-15-33-07.gh-issue-137821.8_Iavt.rst | 2 - ...-10-31-13-57-55.gh-issue-103847.VM7TnW.rst | 1 - ...-10-31-15-06-26.gh-issue-140691.JzHGtg.rst | 3 - ...-10-31-16-25-13.gh-issue-140808.XBiQ4j.rst | 1 - ...-11-01-00-34-53.gh-issue-140826.JEDd7U.rst | 2 - ...-11-01-00-36-14.gh-issue-140874.eAWt3K.rst | 1 - ...-11-01-14-44-09.gh-issue-140873.kfuc9B.rst | 2 - ...-11-02-09-37-22.gh-issue-140734.f8gST9.rst | 2 - ...-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst | 3 - ...-11-02-19-23-32.gh-issue-140815.McEG-T.rst | 2 - ...-11-03-05-38-31.gh-issue-125115.jGS8MN.rst | 1 - ...-11-03-16-23-54.gh-issue-140797.DuFEeR.rst | 2 - ...5-11-04-12-16-13.gh-issue-75593.EFVhKR.rst | 1 - ...-11-04-15-40-35.gh-issue-137969.9VZQVt.rst | 3 - ...-11-04-20-08-41.gh-issue-141018.d_oyOI.rst | 2 - ...-11-06-15-11-50.gh-issue-141141.tgIfgH.rst | 1 - ...5-11-07-12-25-46.gh-issue-85524.9SWFIC.rst | 3 - ...5-11-08-13-03-10.gh-issue-87710.XJeZlP.rst | 1 - ...-11-09-18-55-13.gh-issue-141311.qZ3swc.rst | 2 - ...-11-10-01-47-18.gh-issue-141314.baaa28.rst | 1 - ...-11-12-01-49-03.gh-issue-137109.D6sq2B.rst | 5 - ...-11-12-15-42-47.gh-issue-124111.hTw4OE.rst | 2 - ...-11-13-14-51-30.gh-issue-140938.kXsHHv.rst | 2 - ...-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst | 4 - ...-05-30-22-33-27.gh-issue-136065.bu337o.rst | 1 - ...-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst | 2 - ...-08-15-23-08-44.gh-issue-137836.b55rhh.rst | 3 - ...-07-09-21-45-51.gh-issue-136442.jlbklP.rst | 1 - ...-10-15-00-52-12.gh-issue-140082.fpET50.rst | 3 - ...-10-23-16-39-49.gh-issue-140482.ZMtyeD.rst | 1 - ...-09-20-20-31-54.gh-issue-139188.zfcxkW.rst | 1 - ...-09-21-10-30-08.gh-issue-139198.Fm7NfU.rst | 1 - ...-10-29-15-20-19.gh-issue-140702.ZXtW8h.rst | 2 - ...-11-12-12-54-28.gh-issue-141442.50dS3P.rst | 1 - ...-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst | 1 - README.rst | 2 +- 192 files changed, 1811 insertions(+), 394 deletions(-) create mode 100644 Misc/NEWS.d/3.15.0a2.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-10-16-11-30-53.gh-issue-140189.YCrUyt.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-10-17-11-33-45.gh-issue-140239._k-GgW.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-10-22-12-44-07.gh-issue-140475.OhzQbR.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-10-25-08-07-06.gh-issue-140513.6OhLTs.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-10-29-12-30-38.gh-issue-140768.ITYrzw.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-10-31-13-20-16.gh-issue-140454.gF6dCe.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-10-15-15-59-59.gh-issue-140153.BO7sH4.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-10-26-16-45-06.gh-issue-140487.fGOqss.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-10-26-16-45-28.gh-issue-140556.s__Dae.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-11-10-11-26-26.gh-issue-141341.OsO6-y.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-06-24-13-12-58.gh-issue-134786.MF0VVk.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-00-41-46.gh-issue-136327.7AiTb_.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-29-17-51-14.gh-issue-131253.GpRjWy.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-09-13-01-23-25.gh-issue-138857.YQ5gdc.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-09-15-13-06-11.gh-issue-138944.PeCgLb.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-09-23-21-01-12.gh-issue-139269.1rIaxy.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-03-17-51-43.gh-issue-139475._684ED.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb2.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-12-01-12-12.gh-issue-139817.PAn-8Z.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-17-07-37.gh-issue-140067.ID2gOm.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-20-18-31.gh-issue-140080.8ROjxW.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-00-21-40.gh-issue-140061.J0XeDV.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-17-12-32.gh-issue-140149.cy1m3d.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-14-38-10.gh-issue-140253.gCqFaL.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-18-03-12.gh-issue-139951.IdwM2O.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-20-23-19.gh-issue-140257.8Txmem.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-18-08-36.gh-issue-140301.m-2HxC.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-19-52-20.gh-issue-116738.NLJW0L.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-29-45.gh-issue-140306.xS5CcS.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-19-10-32-28.gh-issue-136895.HfsEh0.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-20-11-24-36.gh-issue-140358.UQuKdV.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-06-51-50.gh-issue-140406.0gJs8M.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-09-20-03.gh-issue-140398.SoABwJ.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-11-30-16.gh-issue-135904.3WE5oW.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-17-22-22.gh-issue-140431.m8D_A-.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-23-26-37.gh-issue-140443.wT5i1A.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-23-16-05-50.gh-issue-140471.Ax_aXn.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-14-29-12.gh-issue-133467.A5d6TM.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-16-42.gh-issue-140517.cqun-K.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-07-25-52.gh-issue-140544.lwjtQe.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-21-31-43.gh-issue-131527.V-JVNP.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-20-59-10.gh-issue-140373.-uoaPP.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-12-47-38.gh-issue-140530.S934bp.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-17-21-38.gh-issue-140939.FVboAw.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst delete mode 100644 Misc/NEWS.d/next/Library/2019-06-02-13-56-16.gh-issue-81313.axawSH.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-21-10-59-40.gh-issue-102431.eUDnf4.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-05-28-17-14-30.gh-issue-119668.RrIGpn.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-07-22-09-28.gh-issue-133601.9kUL3P.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-10-15-10-54.gh-issue-133789.I-ZlUX.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-10-18-02-29.gh-issue-135307.fXGrcK.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-29-22-01-00.gh-issue-133390.I1DW_3.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-07-14-09-33-17.gh-issue-55531.Gt2e12.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-08-11-04-52-18.gh-issue-137627.Ku5Yi2.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-08-15-20-35-30.gh-issue-69528.qc-Eh_.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-08-26-08-17-56.gh-issue-138151.I6CdAk.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-03-18-26-07.gh-issue-138425.cVE9Ho.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-13-12-19-17.gh-issue-138859.PxjIoN.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-15-21-03-11.gh-issue-138891.oZFdtR.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-23-09-46-46.gh-issue-139246.pzfM-w.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-25-20-16-10.gh-issue-101828.yTxJlJ.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-30-12-52-54.gh-issue-63161.mECM1A.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-11-09-07-06.gh-issue-139940.g54efZ.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-13-11-25-41.gh-issue-136702.uvLGK1.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-14-20-27-06.gh-issue-76007.2NcUbo.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-15-02-26-50.gh-issue-140135.54JYfM.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-15-15-10-34.gh-issue-140166.NtxRez.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-15-17-23-51.gh-issue-140141.j2mUDB.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-15-20-47-04.gh-issue-140120.3gffZq.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-15-21-42-13.gh-issue-140041._Fka2j.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-16-16-10-11.gh-issue-139707.zR6Qtn.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-16-17-17-20.gh-issue-135801.faH3fa.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-16-22-49-16.gh-issue-140212.llBNd0.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-17-12-33-01.gh-issue-140251.esM-OX.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-17-20-42-38.gh-issue-129117.X9jr4p.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-17-23-58-11.gh-issue-140272.lhY8uS.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-18-14-30-21.gh-issue-76007.peEgcr.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-18-15-20-25.gh-issue-76007.SNUzRq.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-20-12-33-49.gh-issue-140348.SAKnQZ.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-22-12-56-57.gh-issue-140448.GsEkXD.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-22-20-52-13.gh-issue-140474.xIWlip.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-23-12-12-22.gh-issue-138774.mnh2gU.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-23-13-42-15.gh-issue-140481.XKxWpq.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-23-19-39-16.gh-issue-138162.Znw5DN.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-25-21-04-00.gh-issue-140607.oOZGxS.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-25-21-26-16.gh-issue-140593.OxlLc9.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-26-16-24-12.gh-issue-140633.ioayC1.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-27-13-49-31.gh-issue-140634.ULng9G.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-27-16-01-41.gh-issue-125434.qy0uRA.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-27-18-29-42.gh-issue-140590.LT9HHn.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-28-02-46-56.gh-issue-139946.aN3_uY.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-28-17-43-51.gh-issue-140228.8kfHhO.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-29-09-40-10.gh-issue-140741.L13UCV.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-29-16-12-41.gh-issue-120057.qGj5Dl.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-29-16-53-00.gh-issue-140766.CNagKF.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-30-15-33-07.gh-issue-137821.8_Iavt.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-31-15-06-26.gh-issue-140691.JzHGtg.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-31-16-25-13.gh-issue-140808.XBiQ4j.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-01-00-34-53.gh-issue-140826.JEDd7U.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-01-00-36-14.gh-issue-140874.eAWt3K.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-02-09-37-22.gh-issue-140734.f8gST9.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-02-19-23-32.gh-issue-140815.McEG-T.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-03-16-23-54.gh-issue-140797.DuFEeR.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-06-15-11-50.gh-issue-141141.tgIfgH.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst delete mode 100644 Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst delete mode 100644 Misc/NEWS.d/next/Security/2025-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst delete mode 100644 Misc/NEWS.d/next/Security/2025-08-15-23-08-44.gh-issue-137836.b55rhh.rst delete mode 100644 Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst delete mode 100644 Misc/NEWS.d/next/Tests/2025-10-15-00-52-12.gh-issue-140082.fpET50.rst delete mode 100644 Misc/NEWS.d/next/Tests/2025-10-23-16-39-49.gh-issue-140482.ZMtyeD.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-09-20-20-31-54.gh-issue-139188.zfcxkW.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-09-21-10-30-08.gh-issue-139198.Fm7NfU.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-10-29-15-20-19.gh-issue-140702.ZXtW8h.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst delete mode 100644 Misc/NEWS.d/next/Windows/2025-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 24e673d3d13..971c6a69e5d 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -353,4 +353,4 @@ Importing Modules On error, return NULL with an exception set. - .. versionadded:: next + .. versionadded:: 3.15 diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 18ee1611807..3cac2c8b213 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1390,7 +1390,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. See :c:func:`PyUnstable_ThreadState_ResetStackProtection` for undoing this operation. - .. versionadded:: next + .. versionadded:: 3.15 .. c:function:: void PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) @@ -1400,7 +1400,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. See :c:func:`PyUnstable_ThreadState_SetStackProtection` for an explanation. - .. versionadded:: next + .. versionadded:: 3.15 .. c:function:: PyInterpreterState* PyInterpreterState_Get(void) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 0ea3c3c59a6..2e7d0dbc26e 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2261,7 +2261,7 @@ and classes for traversing abstract syntax trees: The minimum supported version for ``feature_version`` is now ``(3, 7)``. The ``optimize`` argument was added. - .. versionadded:: next + .. versionadded:: 3.15 Added the *module* parameter. diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index ba882f10bbe..05937775699 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -1575,7 +1575,7 @@ Constants Specification that this implementation complies with. See https://speleotrove.com/decimal/decarith.html for the specification. - .. versionadded:: next + .. versionadded:: 3.15 The following constants are only relevant for the C module. They diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 3257daf89d3..8314fed80fa 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -377,7 +377,7 @@ are always available. They are listed here in alphabetical order. ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable support for top-level ``await``, ``async for``, and ``async with``. - .. versionadded:: next + .. versionadded:: 3.15 Added the *module* parameter. diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index b2e2e11c0dc..97136b23408 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -716,7 +716,7 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.8 - .. versionchanged:: next + .. versionchanged:: 3.15 Added support of non-:term:`descriptor` callables. diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 03ba23b6216..3f0a54ac535 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -480,7 +480,7 @@ ABC hierarchy:: .. versionchanged:: 3.5 Made the method static. - .. versionadded:: next + .. versionadded:: 3.15 Added the *fullname* parameter. @@ -1048,7 +1048,7 @@ find and load modules. :meth:`PathFinder.invalidate_caches` invalidates :class:`NamespacePath`, forcing the path value to be recomputed next time it is accessed. - .. versionadded:: next + .. versionadded:: 3.15 .. class:: SourceFileLoader(fullname, path) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index c00db31a8ec..5220c559d3d 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -636,7 +636,7 @@ Retrieving source code .. versionchanged:: 3.5 Documentation strings are now inherited if not overridden. - .. versionchanged:: next + .. versionchanged:: 3.15 Added parameters *inherit_class_doc* and *fallback_to_class_doc*. Documentation strings on :class:`~functools.cached_property` diff --git a/Doc/library/math.integer.rst b/Doc/library/math.integer.rst index 6a9fe74c5e8..0068ae2bdd5 100644 --- a/Doc/library/math.integer.rst +++ b/Doc/library/math.integer.rst @@ -4,7 +4,7 @@ .. module:: math.integer :synopsis: Integer-specific mathematics functions. -.. versionadded:: next +.. versionadded:: 3.15 -------------- diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 54c98346b27..186f99e9591 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -781,7 +781,7 @@ the following functions from the :mod:`math.integer` module: Floats with integral values (like ``5.0``) are no longer accepted in the :func:`factorial` function. -.. deprecated:: next +.. deprecated:: 3.15 These aliases are :term:`soft deprecated` in favor of the :mod:`math.integer` functions. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 7dc6c177268..671270d6112 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -3404,7 +3404,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. class:: statx_result @@ -3661,7 +3661,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: STATX_TYPE @@ -3690,7 +3690,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: AT_STATX_FORCE_SYNC @@ -3700,7 +3700,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: AT_STATX_DONT_SYNC @@ -3709,7 +3709,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: AT_STATX_SYNC_AS_STAT @@ -3721,7 +3721,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: AT_NO_AUTOMOUNT @@ -3733,7 +3733,7 @@ features: .. availability:: Linux. - .. versionadded:: next + .. versionadded:: 3.15 .. function:: statvfs(path) diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst index 1cbec3ab847..82012b31a00 100644 --- a/Doc/library/stat.rst +++ b/Doc/library/stat.rst @@ -511,4 +511,4 @@ meaning of these constants. STATX_ATTR_DAX STATX_ATTR_WRITE_ATOMIC - .. versionadded:: next + .. versionadded:: 3.15 diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index c539345e598..3bcaba0b3e1 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -3191,7 +3191,7 @@ objects. Taking all bytes is a zero-copy operation. - .. versionadded:: next + .. versionadded:: 3.15 See the :ref:`What's New ` entry for common code patterns which can be optimized with diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index c0d9e79197d..f5e6f9f8acf 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -30,7 +30,7 @@ Generating Symbol Tables It is needed to unambiguous :ref:`filter ` syntax warnings by module name. - .. versionadded:: next + .. versionadded:: 3.15 Added the *module* parameter. diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst index fd5f56bd7ea..34f21f49b4b 100644 --- a/Doc/library/unicodedata.rst +++ b/Doc/library/unicodedata.rst @@ -156,7 +156,7 @@ following functions: >>> unicodedata.isxidstart('0') False - .. versionadded:: next + .. versionadded:: 3.15 .. function:: isxidcontinue(chr, /) @@ -171,7 +171,7 @@ following functions: >>> unicodedata.isxidcontinue(' ') False - .. versionadded:: next + .. versionadded:: 3.15 .. function:: decomposition(chr, /) diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 2f3cf6008f5..0de7a90bfcb 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -513,7 +513,7 @@ Available Functions .. versionchanged:: 3.6 Add the *source* parameter. - .. versionchanged:: next + .. versionchanged:: 3.15 If no module is passed, test the filter regular expression against module names created from the path, not only the path itself. diff --git a/Doc/library/winreg.rst b/Doc/library/winreg.rst index b150c53735d..89def6e2afe 100644 --- a/Doc/library/winreg.rst +++ b/Doc/library/winreg.rst @@ -818,6 +818,6 @@ integer handle, and also disconnect the Windows handle from the handle object. will automatically close *key* when control leaves the :keyword:`with` block. -.. versionchanged:: next +.. versionchanged:: 3.15 Handle objects are now compared by their underlying Windows handle value instead of object identity for equality comparisons. diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index cbbc87b4721..e59759683a6 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -656,7 +656,7 @@ Functions .. versionchanged:: 3.13 Added the :meth:`!close` method. - .. versionchanged:: next + .. versionchanged:: 3.15 A :exc:`ResourceWarning` is now emitted if the iterator opened a file and is not explicitly closed. diff --git a/Include/patchlevel.h b/Include/patchlevel.h index e3996ee8679..899c892631f 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -24,10 +24,10 @@ #define PY_MINOR_VERSION 15 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 1 +#define PY_RELEASE_SERIAL 2 /* Version as a string */ -#define PY_VERSION "3.15.0a1+" +#define PY_VERSION "3.15.0a2" /*--end constants--*/ diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 293c3189589..11ffc6bf3a1 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Tue Oct 14 13:46:01 2025 +# Autogenerated by Sphinx on Tue Nov 18 16:51:09 2025 # as part of the release process. topics = { @@ -1098,10 +1098,10 @@ class and instance attributes applies as for regular assignments. 'bltin-ellipsis-object': r'''The Ellipsis Object ******************* -This object is commonly used used to indicate that something is -omitted. It supports no special operations. There is exactly one -ellipsis object, named "Ellipsis" (a built-in name). -"type(Ellipsis)()" produces the "Ellipsis" singleton. +This object is commonly used to indicate that something is omitted. It +supports no special operations. There is exactly one ellipsis object, +named "Ellipsis" (a built-in name). "type(Ellipsis)()" produces the +"Ellipsis" singleton. It is written as "Ellipsis" or "...". @@ -4140,6 +4140,10 @@ def double(x): available for commands and command arguments, e.g. the current global and local names are offered as arguments of the "p" command. + +Command-line interface +====================== + You can also invoke "pdb" from the command line to debug other scripts. For example: @@ -4155,7 +4159,7 @@ def double(x): -c, --command To execute commands as if given in a ".pdbrc" file; see Debugger - Commands. + commands. Changed in version 3.2: Added the "-c" option. @@ -4376,7 +4380,7 @@ class pdb.Pdb(completekey='tab', stdin=None, stdout=None, skip=None, nosigint=Fa See the documentation for the functions explained above. -Debugger Commands +Debugger commands ================= The commands recognized by the debugger are listed below. Most @@ -5616,9 +5620,8 @@ class of the instance or a *non-virtual base class* thereof. The 2.71828 4.0 -Unlike in integer literals, leading zeros are allowed in the numeric -parts. For example, "077.010" is legal, and denotes the same number as -"77.10". +Unlike in integer literals, leading zeros are allowed. For example, +"077.010" is legal, and denotes the same number as "77.01". As in integer literals, single underscores may occur between digits to help readability: @@ -7435,9 +7438,8 @@ class body. A "SyntaxError" is raised if a variable is used or 2.71828 4.0 -Unlike in integer literals, leading zeros are allowed in the numeric -parts. For example, "077.010" is legal, and denotes the same number as -"77.10". +Unlike in integer literals, leading zeros are allowed. For example, +"077.010" is legal, and denotes the same number as "77.01". As in integer literals, single underscores may occur between digits to help readability: @@ -7685,9 +7687,8 @@ class that has an "__rsub__()" method, "type(y).__rsub__(y, x)" is ************************* *Objects* are Python’s abstraction for data. All data in a Python -program is represented by objects or by relations between objects. (In -a sense, and in conformance to Von Neumann’s model of a “stored -program computer”, code is also represented by objects.) +program is represented by objects or by relations between objects. +Even code is represented by objects. Every object has an identity, a type and a value. An object’s *identity* never changes once it has been created; you may think of it @@ -10301,6 +10302,17 @@ class is used in a class pattern with positional arguments, each follow uncased characters and lowercase characters only cased ones. Return "False" otherwise. + For example: + + >>> 'Spam, Spam, Spam'.istitle() + True + >>> 'spam, spam, spam'.istitle() + False + >>> 'SPAM, SPAM, SPAM'.istitle() + False + + See also "title()". + str.isupper() Return "True" if all cased characters [4] in the string are @@ -10663,6 +10675,8 @@ class is used in a class pattern with positional arguments, each >>> titlecase("they're bill's friends.") "They're Bill's Friends." + See also "istitle()". + str.translate(table, /) Return a copy of the string in which each character has been mapped @@ -12362,6 +12376,11 @@ class method object, it is transformed into an instance method object | | "X.__bases__" will be exactly equal to "(A, B, | | | C)". | +----------------------------------------------------+----------------------------------------------------+ +| type.__base__ | **CPython implementation detail:** The single base | +| | class in the inheritance chain that is responsible | +| | for the memory layout of instances. This attribute | +| | corresponds to "tp_base" at the C level. | ++----------------------------------------------------+----------------------------------------------------+ | type.__doc__ | The class’s documentation string, or "None" if | | | undefined. Not inherited by subclasses. | +----------------------------------------------------+----------------------------------------------------+ diff --git a/Misc/NEWS.d/3.15.0a2.rst b/Misc/NEWS.d/3.15.0a2.rst new file mode 100644 index 00000000000..ba82c854fac --- /dev/null +++ b/Misc/NEWS.d/3.15.0a2.rst @@ -0,0 +1,1746 @@ +.. date: 2025-11-04-19-20-05 +.. gh-issue: 140849 +.. nonce: YjB2ZZ +.. release date: 2025-11-18 +.. section: Windows + +Update bundled liblzma to version 5.8.1. + +.. + +.. date: 2025-11-12-12-54-28 +.. gh-issue: 141442 +.. nonce: 50dS3P +.. section: Tools/Demos + +The iOS testbed now correctly handles test arguments that contain spaces. + +.. + +.. date: 2025-10-29-15-20-19 +.. gh-issue: 140702 +.. nonce: ZXtW8h +.. section: Tools/Demos + +The iOS testbed app will now expose the ``GITHUB_ACTIONS`` environment +variable to iOS apps being tested. + +.. + +.. date: 2025-09-21-10-30-08 +.. gh-issue: 139198 +.. nonce: Fm7NfU +.. section: Tools/Demos + +Remove ``Tools/scripts/checkpip.py`` script. + +.. + +.. date: 2025-09-20-20-31-54 +.. gh-issue: 139188 +.. nonce: zfcxkW +.. section: Tools/Demos + +Remove ``Tools/tz/zdump.py`` script. + +.. + +.. date: 2025-10-23-16-39-49 +.. gh-issue: 140482 +.. nonce: ZMtyeD +.. section: Tests + +Preserve and restore the state of ``stty echo`` as part of the test +environment. + +.. + +.. date: 2025-10-15-00-52-12 +.. gh-issue: 140082 +.. nonce: fpET50 +.. section: Tests + +Update ``python -m test`` to set ``FORCE_COLOR=1`` when being run with color +enabled so that :mod:`unittest` which is run by it with redirected output +will output in color. + +.. + +.. date: 2025-07-09-21-45-51 +.. gh-issue: 136442 +.. nonce: jlbklP +.. section: Tests + +Use exitcode ``1`` instead of ``5`` if :func:`unittest.TestCase.setUpClass` +raises an exception + +.. + +.. date: 2025-08-15-23-08-44 +.. gh-issue: 137836 +.. nonce: b55rhh +.. section: Security + +Add support of the "plaintext" element, RAWTEXT elements "xmp", "iframe", +"noembed" and "noframes", and optionally RAWTEXT element "noscript" in +:class:`html.parser.HTMLParser`. + +.. + +.. date: 2025-06-28-13-23-53 +.. gh-issue: 136063 +.. nonce: aGk0Jv +.. section: Security + +:mod:`email.message`: ensure linear complexity for legacy HTTP parameters +parsing. Patch by Bénédikt Tran. + +.. + +.. date: 2025-05-30-22-33-27 +.. gh-issue: 136065 +.. nonce: bu337o +.. section: Security + +Fix quadratic complexity in :func:`os.path.expandvars`. + +.. + +.. date: 2025-11-14-16-24-20 +.. gh-issue: 141497 +.. nonce: L_CxDJ +.. section: Library + +:mod:`ipaddress`: ensure that the methods :meth:`IPv4Network.hosts() +` and :meth:`IPv6Network.hosts() +` always return an iterator. + +.. + +.. date: 2025-11-13-14-51-30 +.. gh-issue: 140938 +.. nonce: kXsHHv +.. section: Library + +The :func:`statistics.stdev` and :func:`statistics.pstdev` functions now +raise a :exc:`ValueError` when the input contains an infinity or a NaN. + +.. + +.. date: 2025-11-12-15-42-47 +.. gh-issue: 124111 +.. nonce: hTw4OE +.. section: Library + +Updated Tcl threading configuration in :mod:`_tkinter` to assume that +threads are always available in Tcl 9 and later. + +.. + +.. date: 2025-11-12-01-49-03 +.. gh-issue: 137109 +.. nonce: D6sq2B +.. section: Library + +The :mod:`os.fork` and related forking APIs will no longer warn in the +common case where Linux or macOS platform APIs return the number of threads +in a process and find the answer to be 1 even when a +:func:`os.register_at_fork` ``after_in_parent=`` callback (re)starts a +thread. + +.. + +.. date: 2025-11-10-01-47-18 +.. gh-issue: 141314 +.. nonce: baaa28 +.. section: Library + +Fix assertion failure in :meth:`io.TextIOWrapper.tell` when reading files +with standalone carriage return (``\r``) line endings. + +.. + +.. date: 2025-11-09-18-55-13 +.. gh-issue: 141311 +.. nonce: qZ3swc +.. section: Library + +Fix assertion failure in :func:`!io.BytesIO.readinto` and undefined behavior +arising when read position is above capcity in :class:`io.BytesIO`. + +.. + +.. date: 2025-11-08-13-03-10 +.. gh-issue: 87710 +.. nonce: XJeZlP +.. section: Library + +:mod:`mimetypes`: Update mime type for ``.ai`` files to ``application/pdf``. + +.. + +.. date: 2025-11-07-12-25-46 +.. gh-issue: 85524 +.. nonce: 9SWFIC +.. section: Library + +Update ``io.FileIO.readall``, an implementation of +:meth:`io.RawIOBase.readall`, to follow :class:`io.IOBase` guidelines and +raise :exc:`io.UnsupportedOperation` when a file is in "w" mode rather than +:exc:`OSError` + +.. + +.. date: 2025-11-06-15-11-50 +.. gh-issue: 141141 +.. nonce: tgIfgH +.. section: Library + +Fix a thread safety issue with :func:`base64.b85decode`. Contributed by +Benel Tayar. + +.. + +.. date: 2025-11-04-20-08-41 +.. gh-issue: 141018 +.. nonce: d_oyOI +.. section: Library + +:mod:`mimetypes`: Update ``.exe``, ``.dll``, ``.rtf`` and (when +``strict=False``) ``.jpg`` to their correct IANA mime type. + +.. + +.. date: 2025-11-04-15-40-35 +.. gh-issue: 137969 +.. nonce: 9VZQVt +.. section: Library + +Fix :meth:`annotationlib.ForwardRef.evaluate` returning +:class:`~annotationlib.ForwardRef` objects which don't update with new +globals. + +.. + +.. date: 2025-11-04-12-16-13 +.. gh-issue: 75593 +.. nonce: EFVhKR +.. section: Library + +Add support of :term:`path-like objects ` and +:term:`bytes-like objects ` in :func:`wave.open`. + +.. + +.. date: 2025-11-03-16-23-54 +.. gh-issue: 140797 +.. nonce: DuFEeR +.. section: Library + +The undocumented :class:`!re.Scanner` class now forbids regular expressions +containing capturing groups in its lexicon patterns. Patterns using +capturing groups could previously lead to crashes with segmentation fault. +Use non-capturing groups (?:...) instead. + +.. + +.. date: 2025-11-03-05-38-31 +.. gh-issue: 125115 +.. nonce: jGS8MN +.. section: Library + +Refactor the :mod:`pdb` parsing issue so positional arguments can pass +through intuitively. + +.. + +.. date: 2025-11-02-19-23-32 +.. gh-issue: 140815 +.. nonce: McEG-T +.. section: Library + +:mod:`faulthandler` now detects if a frame or a code object is invalid or +freed. Patch by Victor Stinner. + +.. + +.. date: 2025-11-02-11-46-00 +.. gh-issue: 100218 +.. nonce: 9Ezfdq +.. section: Library + +Correctly set :attr:`~OSError.errno` when :func:`socket.if_nametoindex` or +:func:`socket.if_indextoname` raise an :exc:`OSError`. Patch by Bénédikt +Tran. + +.. + +.. date: 2025-11-02-09-37-22 +.. gh-issue: 140734 +.. nonce: f8gST9 +.. section: Library + +:mod:`multiprocessing`: fix off-by-one error when checking the length of a +temporary socket file path. Patch by Bénédikt Tran. + +.. + +.. date: 2025-11-01-14-44-09 +.. gh-issue: 140873 +.. nonce: kfuc9B +.. section: Library + +Add support of non-:term:`descriptor` callables in +:func:`functools.singledispatchmethod`. + +.. + +.. date: 2025-11-01-00-36-14 +.. gh-issue: 140874 +.. nonce: eAWt3K +.. section: Library + +Bump the version of pip bundled in ensurepip to version 25.3 + +.. + +.. date: 2025-11-01-00-34-53 +.. gh-issue: 140826 +.. nonce: JEDd7U +.. section: Library + +Now :class:`!winreg.HKEYType` objects are compared by their underlying +Windows registry handle value instead of their object identity. + +.. + +.. date: 2025-10-31-16-25-13 +.. gh-issue: 140808 +.. nonce: XBiQ4j +.. section: Library + +The internal class ``mailbox._ProxyFile`` is no longer a parameterized +generic. + +.. + +.. date: 2025-10-31-15-06-26 +.. gh-issue: 140691 +.. nonce: JzHGtg +.. section: Library + +In :mod:`urllib.request`, when opening a FTP URL fails because a data +connection cannot be made, the control connection's socket is now closed to +avoid a :exc:`ResourceWarning`. + +.. + +.. date: 2025-10-31-13-57-55 +.. gh-issue: 103847 +.. nonce: VM7TnW +.. section: Library + +Fix hang when cancelling process created by +:func:`asyncio.create_subprocess_exec` or +:func:`asyncio.create_subprocess_shell`. Patch by Kumar Aditya. + +.. + +.. date: 2025-10-30-15-33-07 +.. gh-issue: 137821 +.. nonce: 8_Iavt +.. section: Library + +Convert ``_json`` module to use Argument Clinic. Patched by Yoonho Hann. + +.. + +.. date: 2025-10-30-12-36-19 +.. gh-issue: 140790 +.. nonce: _3T6-N +.. section: Library + +Initialize all Pdb's instance variables in ``__init__``, remove some +hasattr/getattr + +.. + +.. date: 2025-10-29-16-53-00 +.. gh-issue: 140766 +.. nonce: CNagKF +.. section: Library + +Add :func:`enum.show_flag_values` and ``enum.bin`` to ``enum.__all__``. + +.. + +.. date: 2025-10-29-16-12-41 +.. gh-issue: 120057 +.. nonce: qGj5Dl +.. section: Library + +Add :func:`os.reload_environ` to ``os.__all__``. + +.. + +.. date: 2025-10-29-09-40-10 +.. gh-issue: 140741 +.. nonce: L13UCV +.. section: Library + +Fix :func:`profiling.sampling.sample` incorrectly handling a +:exc:`FileNotFoundError` or :exc:`PermissionError`. + +.. + +.. date: 2025-10-28-17-43-51 +.. gh-issue: 140228 +.. nonce: 8kfHhO +.. section: Library + +Avoid making unnecessary filesystem calls for frozen modules in +:mod:`linecache` when the global module cache is not present. + +.. + +.. date: 2025-10-28-02-46-56 +.. gh-issue: 139946 +.. nonce: aN3_uY +.. section: Library + +Error and warning keywords in ``argparse.ArgumentParser`` messages are now +colorized when color output is enabled, fixing a visual inconsistency in +which they remained plain text while other output was colorized. + +.. + +.. date: 2025-10-27-18-29-42 +.. gh-issue: 140590 +.. nonce: LT9HHn +.. section: Library + +Fix arguments checking for the :meth:`!functools.partial.__setstate__` that +may lead to internal state corruption and crash. Patch by Sergey Miryanov. + +.. + +.. date: 2025-10-27-16-01-41 +.. gh-issue: 125434 +.. nonce: qy0uRA +.. section: Library + +Display thread name in :mod:`faulthandler` on Windows. Patch by Victor +Stinner. + +.. + +.. date: 2025-10-27-13-49-31 +.. gh-issue: 140634 +.. nonce: ULng9G +.. section: Library + +Fix a reference counting bug in :meth:`!os.sched_param.__reduce__`. + +.. + +.. date: 2025-10-27-00-40-49 +.. gh-issue: 140650 +.. nonce: DYJPJ9 +.. section: Library + +Fix an issue where closing :class:`io.BufferedWriter` could crash if the +closed attribute raised an exception on access or could not be converted to +a boolean. + +.. + +.. date: 2025-10-26-16-24-12 +.. gh-issue: 140633 +.. nonce: ioayC1 +.. section: Library + +Ignore :exc:`AttributeError` when setting a module's ``__file__`` attribute +when loading an extension module packaged as Apple Framework. + +.. + +.. date: 2025-10-25-22-55-07 +.. gh-issue: 140601 +.. nonce: In3MlS +.. section: Library + +:func:`xml.etree.ElementTree.iterparse` now emits a :exc:`ResourceWarning` +when the iterator is not explicitly closed and was opened with a filename. +This helps developers identify and fix resource leaks. Patch by Osama +Abdelkader. + +.. + +.. date: 2025-10-25-21-26-16 +.. gh-issue: 140593 +.. nonce: OxlLc9 +.. section: Library + +:mod:`xml.parsers.expat`: Fix a memory leak that could affect users with +:meth:`~xml.parsers.expat.xmlparser.ElementDeclHandler` set to a custom +element declaration handler. Patch by Sebastian Pipping. + +.. + +.. date: 2025-10-25-21-04-00 +.. gh-issue: 140607 +.. nonce: oOZGxS +.. section: Library + +Inside :meth:`io.RawIOBase.read`, validate that the count of bytes returned +by :meth:`io.RawIOBase.readinto` is valid (inside the provided buffer). + +.. + +.. date: 2025-10-23-19-39-16 +.. gh-issue: 138162 +.. nonce: Znw5DN +.. section: Library + +Fix :class:`logging.LoggerAdapter` with ``merge_extra=True`` and without the +*extra* argument. + +.. + +.. date: 2025-10-23-13-42-15 +.. gh-issue: 140481 +.. nonce: XKxWpq +.. section: Library + +Improve error message when trying to iterate a Tk widget, image or font. + +.. + +.. date: 2025-10-23-12-12-22 +.. gh-issue: 138774 +.. nonce: mnh2gU +.. section: Library + +:func:`ast.unparse` now generates full source code when handling +:class:`ast.Interpolation` nodes that do not have a specified source. + +.. + +.. date: 2025-10-22-20-52-13 +.. gh-issue: 140474 +.. nonce: xIWlip +.. section: Library + +Fix memory leak in :class:`array.array` when creating arrays from an empty +:class:`str` and the ``u`` type code. + +.. + +.. date: 2025-10-22-12-56-57 +.. gh-issue: 140448 +.. nonce: GsEkXD +.. section: Library + +Change the default of ``suggest_on_error`` to ``True`` in +``argparse.ArgumentParser``. + +.. + +.. date: 2025-10-21-15-54-13 +.. gh-issue: 137530 +.. nonce: ZyIVUH +.. section: Library + +:mod:`dataclasses` Fix annotations for generated ``__init__`` methods by +replacing the annotations that were in-line in the generated source code +with ``__annotate__`` functions attached to the methods. + +.. + +.. date: 2025-10-20-12-33-49 +.. gh-issue: 140348 +.. nonce: SAKnQZ +.. section: Library + +Fix regression in Python 3.14.0 where using the ``|`` operator on a +:class:`typing.Union` object combined with an object that is not a type +would raise an error. + +.. + +.. date: 2025-10-18-15-20-25 +.. gh-issue: 76007 +.. nonce: SNUzRq +.. section: Library + +:mod:`decimal`: Deprecate ``__version__`` and replace with +:data:`decimal.SPEC_VERSION`. + +.. + +.. date: 2025-10-18-14-30-21 +.. gh-issue: 76007 +.. nonce: peEgcr +.. section: Library + +Deprecate ``__version__`` from a :mod:`imaplib`. Patch by Hugo van Kemenade. + +.. + +.. date: 2025-10-17-23-58-11 +.. gh-issue: 140272 +.. nonce: lhY8uS +.. section: Library + +Fix memory leak in the :meth:`!clear` method of the :mod:`dbm.gnu` database. + +.. + +.. date: 2025-10-17-20-42-38 +.. gh-issue: 129117 +.. nonce: X9jr4p +.. section: Library + +:mod:`unicodedata`: Add :func:`~unicodedata.isxidstart` and +:func:`~unicodedata.isxidcontinue` functions to check whether a character +can start or continue a `Unicode Standard Annex #31 +`_ identifier. + +.. + +.. date: 2025-10-17-12-33-01 +.. gh-issue: 140251 +.. nonce: esM-OX +.. section: Library + +Colorize the default import statement ``import asyncio`` in asyncio REPL. + +.. + +.. date: 2025-10-16-22-49-16 +.. gh-issue: 140212 +.. nonce: llBNd0 +.. section: Library + +Calendar's HTML formatting now accepts year and month as options. +Previously, running ``python -m calendar -t html 2025 10`` would result in +an error message. It now generates an HTML document displaying the calendar +for the specified month. Contributed by Pål Grønås Drange. + +.. + +.. date: 2025-10-16-17-17-20 +.. gh-issue: 135801 +.. nonce: faH3fa +.. section: Library + +Improve filtering by module in :func:`warnings.warn_explicit` if no *module* +argument is passed. It now tests the module regular expression in the +warnings filter not only against the filename with ``.py`` stripped, but +also against module names constructed starting from different parent +directories of the filename (with ``/__init__.py``, ``.py`` and, on Windows, +``.pyw`` stripped). + +.. + +.. date: 2025-10-16-16-10-11 +.. gh-issue: 139707 +.. nonce: zR6Qtn +.. section: Library + +Improve :exc:`ModuleNotFoundError` error message when a :term:`standard +library` module is missing. + +.. + +.. date: 2025-10-15-21-42-13 +.. gh-issue: 140041 +.. nonce: _Fka2j +.. section: Library + +Fix import of :mod:`ctypes` on Android and Cygwin when ABI flags are +present. + +.. + +.. date: 2025-10-15-20-47-04 +.. gh-issue: 140120 +.. nonce: 3gffZq +.. section: Library + +Fixed a memory leak in :mod:`hmac` when it was using the hacl-star backend. +Discovered by ``@ashm-dev`` using AddressSanitizer. + +.. + +.. date: 2025-10-15-17-23-51 +.. gh-issue: 140141 +.. nonce: j2mUDB +.. section: Library + +The :py:class:`importlib.metadata.PackageNotFoundError` traceback raised +when ``importlib.metadata.Distribution.from_name`` cannot discover a +distribution no longer includes a transient :exc:`StopIteration` exception +trace. + +Contributed by Bartosz Sławecki in :gh:`140142`. + +.. + +.. date: 2025-10-15-15-10-34 +.. gh-issue: 140166 +.. nonce: NtxRez +.. section: Library + +:mod:`mimetypes`: Per the `IANA assignment +`_, update +the MIME type for the ``.texi`` and ``.texinfo`` file formats to +``application/texinfo``, instead of ``application/x-texinfo``. + +.. + +.. date: 2025-10-15-02-26-50 +.. gh-issue: 140135 +.. nonce: 54JYfM +.. section: Library + +Speed up :meth:`io.RawIOBase.readall` by using PyBytesWriter API (about 4x +faster) + +.. + +.. date: 2025-10-14-20-27-06 +.. gh-issue: 76007 +.. nonce: 2NcUbo +.. section: Library + +:mod:`zlib`: Deprecate ``__version__`` and schedule for removal in Python +3.20. + +.. + +.. date: 2025-10-13-11-25-41 +.. gh-issue: 136702 +.. nonce: uvLGK1 +.. section: Library + +:mod:`encodings`: Deprecate passing a non-ascii *encoding* name to +:func:`encodings.normalize_encoding` and schedule removal of support for +Python 3.17. + +.. + +.. date: 2025-10-11-09-07-06 +.. gh-issue: 139940 +.. nonce: g54efZ +.. section: Library + +Print clearer error message when using ``pdb`` to attach to a non-existing +process. + +.. + +.. date: 2025-10-02-22-29-00 +.. gh-issue: 139462 +.. nonce: VZXUHe +.. section: Library + +When a child process in a :class:`concurrent.futures.ProcessPoolExecutor` +terminates abruptly, the resulting traceback will now tell you the PID and +exit code of the terminated process. Contributed by Jonathan Berg. + +.. + +.. date: 2025-09-30-12-52-54 +.. gh-issue: 63161 +.. nonce: mECM1A +.. section: Library + +Fix :func:`tokenize.detect_encoding`. Support non-UTF-8 shebang and comments +if non-UTF-8 encoding is specified. Detect decoding error for non-UTF-8 +encoding. Detect null bytes in source code. + +.. + +.. date: 2025-09-25-20-16-10 +.. gh-issue: 101828 +.. nonce: yTxJlJ +.. section: Library + +Fix ``'shift_jisx0213'``, ``'shift_jis_2004'``, ``'euc_jisx0213'`` and +``'euc_jis_2004'`` codecs truncating null chars as they were treated as part +of multi-character sequences. + +.. + +.. date: 2025-09-23-09-46-46 +.. gh-issue: 139246 +.. nonce: pzfM-w +.. section: Library + +fix: paste zero-width in default repl width is wrong. + +.. + +.. date: 2025-09-18-21-25-41 +.. gh-issue: 83714 +.. nonce: TQjDWZ +.. section: Library + +Implement :func:`os.statx` on Linux kernel versions 4.11 and later with +glibc versions 2.28 and later. Contributed by Jeffrey Bosboom and Victor +Stinner. + +.. + +.. date: 2025-09-15-21-03-11 +.. gh-issue: 138891 +.. nonce: oZFdtR +.. section: Library + +Fix ``SyntaxError`` when ``inspect.get_annotations(f, eval_str=True)`` is +called on a function annotated with a :pep:`646` ``star_expression`` + +.. + +.. date: 2025-09-13-12-19-17 +.. gh-issue: 138859 +.. nonce: PxjIoN +.. section: Library + +Fix generic type parameterization raising a :exc:`TypeError` when omitting a +:class:`ParamSpec` that has a default which is not a list of types. + +.. + +.. date: 2025-09-12-09-34-37 +.. gh-issue: 138764 +.. nonce: mokHoY +.. section: Library + +Prevent :func:`annotationlib.call_annotate_function` from calling +``__annotate__`` functions that don't support ``VALUE_WITH_FAKE_GLOBALS`` in +a fake globals namespace with empty globals. + +Make ``FORWARDREF`` and ``STRING`` annotations fall back to using ``VALUE`` +annotations in the case that neither their own format, nor +``VALUE_WITH_FAKE_GLOBALS`` are supported. + +.. + +.. date: 2025-09-11-15-03-37 +.. gh-issue: 138775 +.. nonce: w7rnSx +.. section: Library + +Use of ``python -m`` with :mod:`base64` has been fixed to detect input from +a terminal so that it properly notices EOF. + +.. + +.. date: 2025-09-03-20-18-39 +.. gh-issue: 98896 +.. nonce: tjez89 +.. section: Library + +Fix a failure in multiprocessing resource_tracker when SharedMemory names +contain colons. Patch by Rani Pinchuk. + +.. + +.. date: 2025-09-03-18-26-07 +.. gh-issue: 138425 +.. nonce: cVE9Ho +.. section: Library + +Fix partial evaluation of :class:`annotationlib.ForwardRef` objects which +rely on names defined as globals. + +.. + +.. date: 2025-08-26-08-17-56 +.. gh-issue: 138151 +.. nonce: I6CdAk +.. section: Library + +In :mod:`annotationlib`, improve evaluation of forward references to +nonlocal variables that are not yet defined when the annotations are +initially evaluated. + +.. + +.. date: 2025-08-15-20-35-30 +.. gh-issue: 69528 +.. nonce: qc-Eh_ +.. section: Library + +The :attr:`~io.FileIO.mode` attribute of files opened in the ``'wb+'`` mode +is now ``'wb+'`` instead of ``'rb+'``. + +.. + +.. date: 2025-08-11-04-52-18 +.. gh-issue: 137627 +.. nonce: Ku5Yi2 +.. section: Library + +Speed up :meth:`csv.Sniffer.sniff` delimiter detection by up to 1.6x. + +.. + +.. date: 2025-07-14-09-33-17 +.. gh-issue: 55531 +.. nonce: Gt2e12 +.. section: Library + +:mod:`encodings`: Improve :func:`~encodings.normalize_encoding` performance +by implementing the function in C using the private +``_Py_normalize_encoding`` which has been modified to make lowercase +conversion optional. + +.. + +.. date: 2025-07-01-04-57-57 +.. gh-issue: 136057 +.. nonce: 4-t596 +.. section: Library + +Fixed the bug in :mod:`pdb` and :mod:`bdb` where ``next`` and ``step`` can't +go over the line if a loop exists in the line. + +.. + +.. date: 2025-06-29-22-01-00 +.. gh-issue: 133390 +.. nonce: I1DW_3 +.. section: Library + +Support table, index, trigger, view, column, function, and schema completion +for :mod:`sqlite3`'s :ref:`command-line interface `. + +.. + +.. date: 2025-06-10-18-02-29 +.. gh-issue: 135307 +.. nonce: fXGrcK +.. section: Library + +:mod:`email`: Fix exception in ``set_content()`` when encoding text and +max_line_length is set to ``0`` or ``None`` (unlimited). + +.. + +.. date: 2025-05-10-15-10-54 +.. gh-issue: 133789 +.. nonce: I-ZlUX +.. section: Library + +Fix unpickling of :mod:`pathlib` objects that were pickled in Python 3.13. + +.. + +.. date: 2025-05-07-22-09-28 +.. gh-issue: 133601 +.. nonce: 9kUL3P +.. section: Library + +Remove deprecated :func:`!typing.no_type_check_decorator`. + +.. + +.. date: 2025-04-18-18-08-05 +.. gh-issue: 132686 +.. nonce: 6kV_Gs +.. section: Library + +Add parameters *inherit_class_doc* and *fallback_to_class_doc* for +:func:`inspect.getdoc`. + +.. + +.. date: 2025-03-12-18-57-10 +.. gh-issue: 131116 +.. nonce: uTpwXZ +.. section: Library + +:func:`inspect.getdoc` now correctly returns an inherited docstring on +:class:`~functools.cached_property` objects if none is given in a subclass. + +.. + +.. date: 2025-03-04-17-19-26 +.. gh-issue: 130693 +.. nonce: Kv01r8 +.. section: Library + +Add support for ``-nolinestop``, and ``-strictlimits`` options to +:meth:`!tkinter.Text.search`. Also add the :meth:`!tkinter.Text.search_all` +method for ``-all`` and ``-overlap`` options. + +.. + +.. date: 2024-08-08-12-39-36 +.. gh-issue: 122255 +.. nonce: J_gU8Y +.. section: Library + +In the :mod:`linecache` module and in the Python implementation of the +:mod:`warnings` module, a ``DeprecationWarning`` is issued when +``mod.__loader__`` differs from ``mod.__spec__.loader`` (like in the C +implementation of the :mod:`!warnings` module). + +.. + +.. date: 2024-06-26-16-16-43 +.. gh-issue: 121011 +.. nonce: qW54eh +.. section: Library + +:func:`math.log` now supports arbitrary large integer-like arguments in the +same way as arbitrary large integer arguments. + +.. + +.. date: 2024-05-28-17-14-30 +.. gh-issue: 119668 +.. nonce: RrIGpn +.. section: Library + +Publicly expose and document :class:`importlib.machinery.NamespacePath`. + +.. + +.. date: 2023-03-21-10-59-40 +.. gh-issue: 102431 +.. nonce: eUDnf4 +.. section: Library + +Clarify constraints for "logical" arguments in methods of +:class:`decimal.Context`. + +.. + +.. date: 2019-06-02-13-56-16 +.. gh-issue: 81313 +.. nonce: axawSH +.. section: Library + +Add the :mod:`math.integer` module (:pep:`791`). + +.. + +.. date: 2025-11-15-01-21-00 +.. gh-issue: 141579 +.. nonce: aB7cD9 +.. section: Core and Builtins + +Fix :func:`sys.activate_stack_trampoline` to properly support the +``perf_jit`` backend. Patch by Pablo Galindo. + +.. + +.. date: 2025-11-14-16-25-15 +.. gh-issue: 114203 +.. nonce: n3tlQO +.. section: Core and Builtins + +Skip locking if object is already locked by two-mutex critical section. + +.. + +.. date: 2025-11-14-00-19-45 +.. gh-issue: 141528 +.. nonce: VWdax1 +.. section: Core and Builtins + +Suggest using :meth:`concurrent.interpreters.Interpreter.close` instead of +the private ``_interpreters.destroy`` function when warning about remaining +subinterpreters. Patch by Sergey Miryanov. + +.. + +.. date: 2025-11-11-13-40-45 +.. gh-issue: 141367 +.. nonce: I5KY7F +.. section: Core and Builtins + +Specialize ``CALL_LIST_APPEND`` instruction only for lists, not for list +subclasses, to avoid unnecessary deopt. Patch by Mikhail Efimov. + +.. + +.. date: 2025-11-10-23-07-06 +.. gh-issue: 141312 +.. nonce: H-58GB +.. section: Core and Builtins + +Fix the assertion failure in the ``__setstate__`` method of the range +iterator when a non-integer argument is passed. Patch by Sergey Miryanov. + +.. + +.. date: 2025-11-05-19-50-37 +.. gh-issue: 140643 +.. nonce: QCEOqG +.. section: Core and Builtins + +Add support for ```` and ```` frames to +:mod:`!profiling.sampling` output to denote active garbage collection and +calls to native code. + +.. + +.. date: 2025-11-04-12-18-06 +.. gh-issue: 140942 +.. nonce: GYns6n +.. section: Core and Builtins + +Add ``.cjs`` to :mod:`mimetypes` to give CommonJS modules a MIME type of +``application/node``. + +.. + +.. date: 2025-11-04-04-57-24 +.. gh-issue: 140479 +.. nonce: lwQ2v2 +.. section: Core and Builtins + +Update JIT compilation to use LLVM 21 at build time. + +.. + +.. date: 2025-11-03-17-21-38 +.. gh-issue: 140939 +.. nonce: FVboAw +.. section: Core and Builtins + +Fix memory leak when :class:`bytearray` or :class:`bytes` is formated with +the ``%*b`` format with a large width that results in a :exc:`MemoryError`. + +.. + +.. date: 2025-11-02-15-28-33 +.. gh-issue: 140260 +.. nonce: JNzlGz +.. section: Core and Builtins + +Fix :mod:`struct` data race in endian table initialization with +subinterpreters. Patch by Shamil Abdulaev. + +.. + +.. date: 2025-11-02-12-47-38 +.. gh-issue: 140530 +.. nonce: S934bp +.. section: Core and Builtins + +Fix a reference leak when ``raise exc from cause`` fails. Patch by Bénédikt +Tran. + +.. + +.. date: 2025-10-31-14-03-42 +.. gh-issue: 90344 +.. nonce: gvZigO +.. section: Core and Builtins + +Replace :class:`io.IncrementalNewlineDecoder` with non incremental newline +decoders in codebase where :meth:`!io.IncrementalNewlineDecoder.decode` was +being called once. + +.. + +.. date: 2025-10-29-20-59-10 +.. gh-issue: 140373 +.. nonce: -uoaPP +.. section: Core and Builtins + +Correctly emit ``PY_UNWIND`` event when generator object is closed. Patch by +Mikhail Efimov. + +.. + +.. date: 2025-10-29-11-31-59 +.. gh-issue: 140729 +.. nonce: t9JsNt +.. section: Core and Builtins + +Fix pickling error in the sampling profiler when using +``concurrent.futures.ProcessPoolExecutor`` script can not be properly +pickled and executed in worker processes. + +.. + +.. date: 2025-10-25-21-31-43 +.. gh-issue: 131527 +.. nonce: V-JVNP +.. section: Core and Builtins + +Dynamic borrow checking for stackrefs is added to ``Py_STACKREF_DEBUG`` +mode. Patch by Mikhail Efimov. + +.. + +.. date: 2025-10-25-17-36-46 +.. gh-issue: 140576 +.. nonce: kj0SCY +.. section: Core and Builtins + +Fixed crash in :func:`tokenize.generate_tokens` in case of specific +incorrect input. Patch by Mikhail Efimov. + +.. + +.. date: 2025-10-25-07-25-52 +.. gh-issue: 140544 +.. nonce: lwjtQe +.. section: Core and Builtins + +Speed up accessing interpreter state by caching it in a thread local +variable. Patch by Kumar Aditya. + +.. + +.. date: 2025-10-24-20-42-33 +.. gh-issue: 140551 +.. nonce: -9swrl +.. section: Core and Builtins + +Fixed crash in :class:`dict` if :meth:`dict.clear` is called at the lookup +stage. Patch by Mikhail Efimov and Inada Naoki. + +.. + +.. date: 2025-10-24-20-16-42 +.. gh-issue: 140517 +.. nonce: cqun-K +.. section: Core and Builtins + +Fixed a reference leak when iterating over the result of :func:`map` with +``strict=True`` when the input iterables have different lengths. Patch by +Mikhail Efimov. + +.. + +.. date: 2025-10-24-14-29-12 +.. gh-issue: 133467 +.. nonce: A5d6TM +.. section: Core and Builtins + +Fix race when updating :attr:`!type.__bases__` that could allow a read of +:attr:`!type.__base__` to observe an inconsistent value on the free threaded +build. + +.. + +.. date: 2025-10-23-16-05-50 +.. gh-issue: 140471 +.. nonce: Ax_aXn +.. section: Core and Builtins + +Fix potential buffer overflow in :class:`ast.AST` node initialization when +encountering malformed :attr:`~ast.AST._fields` containing non-:class:`str`. + +.. + +.. date: 2025-10-22-23-26-37 +.. gh-issue: 140443 +.. nonce: wT5i1A +.. section: Core and Builtins + +The logarithm functions (such as :func:`math.log10` and :func:`math.log`) +may now produce slightly different results for extremely large integers that +cannot be converted to floats without overflow. These results are generally +more accurate, with reduced worst-case error and a tighter overall error +distribution. + +.. + +.. date: 2025-10-22-17-22-22 +.. gh-issue: 140431 +.. nonce: m8D_A- +.. section: Core and Builtins + +Fix a crash in Python's :term:`garbage collector ` due +to partially initialized :term:`coroutine` objects when coroutine origin +tracking depth is enabled (:func:`sys.set_coroutine_origin_tracking_depth`). + +.. + +.. date: 2025-10-22-12-48-05 +.. gh-issue: 140476 +.. nonce: F3-d1P +.. section: Core and Builtins + +Optimize :c:func:`PySet_Add` for :class:`frozenset` in :term:`free threaded +` build. + +.. + +.. date: 2025-10-22-11-30-16 +.. gh-issue: 135904 +.. nonce: 3WE5oW +.. section: Core and Builtins + +Add special labels to the assembly created during stencil creation to +support relocations that the native object file format does not support. +Specifically, 19 bit branches for AArch64 in Mach-O object files. + +.. + +.. date: 2025-10-21-09-20-03 +.. gh-issue: 140398 +.. nonce: SoABwJ +.. section: Core and Builtins + +Fix memory leaks in :mod:`readline` functions +:func:`~readline.read_init_file`, :func:`~readline.read_history_file`, +:func:`~readline.write_history_file`, and +:func:`~readline.append_history_file` when :c:func:`PySys_Audit` fails. + +.. + +.. date: 2025-10-21-06-51-50 +.. gh-issue: 140406 +.. nonce: 0gJs8M +.. section: Core and Builtins + +Fix memory leak when an object's :meth:`~object.__hash__` method returns an +object that isn't an :class:`int`. + +.. + +.. date: 2025-10-20-11-24-36 +.. gh-issue: 140358 +.. nonce: UQuKdV +.. section: Core and Builtins + +Restore elapsed time and unreachable object count in GC debug output. These +were inadvertently removed during a refactor of ``gc.c``. The debug log now +again reports elapsed collection time and the number of unreachable objects. +Contributed by Pål Grønås Drange. + +.. + +.. date: 2025-10-19-10-32-28 +.. gh-issue: 136895 +.. nonce: HfsEh0 +.. section: Core and Builtins + +Update JIT compilation to use LLVM 20 at build time. + +.. + +.. date: 2025-10-18-21-50-44 +.. gh-issue: 139109 +.. nonce: 9QQOzN +.. section: Core and Builtins + +A new tracing frontend for the JIT compiler has been implemented. Patch by +Ken Jin. Design for CPython by Ken Jin, Mark Shannon and Brandt Bucher. + +.. + +.. date: 2025-10-18-21-29-45 +.. gh-issue: 140306 +.. nonce: xS5CcS +.. section: Core and Builtins + +Fix memory leaks in cross-interpreter channel operations and shared +namespace handling. + +.. + +.. date: 2025-10-18-19-52-20 +.. gh-issue: 116738 +.. nonce: NLJW0L +.. section: Core and Builtins + +Make _suggestions module thread-safe on the :term:`free threaded ` build. + +.. + +.. date: 2025-10-18-18-08-36 +.. gh-issue: 140301 +.. nonce: m-2HxC +.. section: Core and Builtins + +Fix memory leak of ``PyConfig`` in subinterpreters. + +.. + +.. date: 2025-10-17-20-23-19 +.. gh-issue: 140257 +.. nonce: 8Txmem +.. section: Core and Builtins + +Fix data race between interpreter_clear() and take_gil() on eval_breaker +during finalization with daemon threads. + +.. + +.. date: 2025-10-17-18-03-12 +.. gh-issue: 139951 +.. nonce: IdwM2O +.. section: Core and Builtins + +Fixes a regression in GC performance for a growing heap composed mostly of +small tuples. + +* Counts number of actually tracked objects, instead of trackable objects. + This ensures that untracking tuples has the desired effect of reducing GC overhead. +* Does not track most untrackable tuples during creation. + This prevents large numbers of small tuples causing excessive GCs. + +.. + +.. date: 2025-10-17-14-38-10 +.. gh-issue: 140253 +.. nonce: gCqFaL +.. section: Core and Builtins + +Wrong placement of a double-star pattern inside a mapping pattern now throws +a specialized syntax error. Contributed by Bartosz Sławecki in :gh:`140253`. + +.. + +.. date: 2025-10-16-21-47-00 +.. gh-issue: 140104 +.. nonce: A8SQIm +.. section: Core and Builtins + +Fix a bug with exception handling in the JIT. Patch by Ken Jin. Bug reported +by Daniel Diniz. + +.. + +.. date: 2025-10-15-17-12-32 +.. gh-issue: 140149 +.. nonce: cy1m3d +.. section: Core and Builtins + +Speed up parsing bytes literals concatenation by using PyBytesWriter API and +a single memory allocation (about 3x faster). + +.. + +.. date: 2025-10-15-00-21-40 +.. gh-issue: 140061 +.. nonce: J0XeDV +.. section: Core and Builtins + +Fixing the checking of whether an object is uniquely referenced to ensure +free-threaded compatibility. Patch by Sergey Miryanov. + +.. + +.. date: 2025-10-14-20-18-31 +.. gh-issue: 140080 +.. nonce: 8ROjxW +.. section: Core and Builtins + +Fix hang during finalization when attempting to call :mod:`atexit` handlers +under no memory. + +.. + +.. date: 2025-10-14-18-24-16 +.. gh-issue: 139871 +.. nonce: SWtuUz +.. section: Core and Builtins + +Update :class:`bytearray` to use a :class:`bytes` under the hood as its +buffer and add :func:`bytearray.take_bytes` to take it out. + +.. + +.. date: 2025-10-14-17-07-37 +.. gh-issue: 140067 +.. nonce: ID2gOm +.. section: Core and Builtins + +Fix memory leak in sub-interpreter creation. + +.. + +.. date: 2025-10-13-13-54-19 +.. gh-issue: 139914 +.. nonce: M-y_3E +.. section: Core and Builtins + +Restore support for HP PA-RISC, which has an upwards-growing stack. + +.. + +.. date: 2025-10-12-01-12-12 +.. gh-issue: 139817 +.. nonce: PAn-8Z +.. section: Core and Builtins + +Attribute ``__qualname__`` is added to :class:`typing.TypeAliasType`. Patch +by Mikhail Efimov. + +.. + +.. date: 2025-10-06-14-19-47 +.. gh-issue: 135801 +.. nonce: OhxEZS +.. section: Core and Builtins + +Many functions related to compiling or parsing Python code, such as +:func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, and +:func:`importlib.abc.InspectLoader.source_to_code` now allow to specify the +module name. It is needed to unambiguous :ref:`filter ` +syntax warnings by module name. + +.. + +.. date: 2025-10-06-10-03-37 +.. gh-issue: 139640 +.. nonce: gY5oTb2 +.. section: Core and Builtins + +:func:`ast.parse` no longer emits syntax warnings for +``return``/``break``/``continue`` in ``finally`` (see :pep:`765`) -- they +are only emitted during compilation. + +.. + +.. date: 2025-10-06-10-03-37 +.. gh-issue: 139640 +.. nonce: gY5oTb +.. section: Core and Builtins + +Fix swallowing some syntax warnings in different modules if they +accidentally have the same message and are emitted from the same line. Fix +duplicated warnings in the ``finally`` block. + +.. + +.. date: 2025-10-03-17-51-43 +.. gh-issue: 139475 +.. nonce: _684ED +.. section: Core and Builtins + +Changes in stackref debugging mode when ``Py_STACKREF_DEBUG`` is set. We use +the same pattern of refcounting for stackrefs as in production build. + +.. + +.. date: 2025-09-23-21-01-12 +.. gh-issue: 139269 +.. nonce: 1rIaxy +.. section: Core and Builtins + +Fix undefined behavior when using unaligned store in JIT's ``patch_*`` +functions. + +.. + +.. date: 2025-09-15-13-06-11 +.. gh-issue: 138944 +.. nonce: PeCgLb +.. section: Core and Builtins + +Fix :exc:`SyntaxError` message when invalid syntax appears on the same line +as a valid ``import ... as ...`` or ``from ... import ... as ...`` +statement. Patch by Brian Schubert. + +.. + +.. date: 2025-09-13-01-23-25 +.. gh-issue: 138857 +.. nonce: YQ5gdc +.. section: Core and Builtins + +Improve :exc:`SyntaxError` message for ``case`` keyword placed outside +:keyword:`match` body. + +.. + +.. date: 2025-07-29-17-51-14 +.. gh-issue: 131253 +.. nonce: GpRjWy +.. section: Core and Builtins + +Support the ``--enable-pystats`` build option for the free-threaded build. + +.. + +.. date: 2025-07-08-00-41-46 +.. gh-issue: 136327 +.. nonce: 7AiTb_ +.. section: Core and Builtins + +Errors when calling functions with invalid values after ``*`` and ``**`` now +do not include the function name. Patch by Ilia Solin. + +.. + +.. date: 2025-06-24-13-12-58 +.. gh-issue: 134786 +.. nonce: MF0VVk +.. section: Core and Builtins + +If :c:macro:`Py_TPFLAGS_MANAGED_DICT` and +:c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` are used, then +:c:macro:`Py_TPFLAGS_HAVE_GC` must be used as well. + +.. + +.. date: 2025-11-10-11-26-26 +.. gh-issue: 141341 +.. nonce: OsO6-y +.. section: C API + +On Windows, rename the ``COMPILER`` macro to ``_Py_COMPILER`` to avoid name +conflicts. Patch by Victor Stinner. + +.. + +.. date: 2025-11-08-10-51-50 +.. gh-issue: 116146 +.. nonce: pCmx6L +.. section: C API + +Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating a +module from a *spec* and *initfunc*. Patch by Itamar Oren. + +.. + +.. date: 2025-11-06-06-28-14 +.. gh-issue: 141042 +.. nonce: brOioJ +.. section: C API + +Make qNaN in :c:func:`PyFloat_Pack2` and :c:func:`PyFloat_Pack4`, if while +conversion to a narrower precision floating-point format --- the remaining +after truncation payload will be zero. Patch by Sergey B Kirpichev. + +.. + +.. date: 2025-11-05-05-45-49 +.. gh-issue: 141004 +.. nonce: N9Ooh9 +.. section: C API + +:c:macro:`!Py_MATH_El` and :c:macro:`!Py_MATH_PIl` are deprecated. + +.. + +.. date: 2025-11-05-04-38-16 +.. gh-issue: 141004 +.. nonce: rJL43P +.. section: C API + +The :c:macro:`!Py_INFINITY` macro is :term:`soft deprecated`. + +.. + +.. date: 2025-10-26-16-45-28 +.. gh-issue: 140556 +.. nonce: s__Dae +.. section: C API + +:pep:`793`: Add a new entry point for C extension modules, +``PyModExport_``. + +.. + +.. date: 2025-10-26-16-45-06 +.. gh-issue: 140487 +.. nonce: fGOqss +.. section: C API + +Fix :c:macro:`Py_RETURN_NOTIMPLEMENTED` in limited C API 3.11 and older: +don't treat ``Py_NotImplemented`` as immortal. Patch by Victor Stinner. + +.. + +.. date: 2025-10-15-15-59-59 +.. gh-issue: 140153 +.. nonce: BO7sH4 +.. section: C API + +Fix :c:func:`Py_REFCNT` definition on limited C API 3.11-3.13. Patch by +Victor Stinner. + +.. + +.. date: 2025-10-06-22-17-47 +.. gh-issue: 139653 +.. nonce: 6-1MOd +.. section: C API + +Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and +:c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set the +stack protection base address and stack protection size of a Python thread +state. Patch by Victor Stinner. + +.. + +.. date: 2025-10-31-13-20-16 +.. gh-issue: 140454 +.. nonce: gF6dCe +.. section: Build + +When building the JIT, match the jit_stencils filename expectations in +Makefile with the generator script. This avoid needless JIT recompilation +during ``make install``. + +.. + +.. date: 2025-10-29-12-30-38 +.. gh-issue: 140768 +.. nonce: ITYrzw +.. section: Build + +Warn when the WASI SDK version doesn't match what's supported. + +.. + +.. date: 2025-10-25-08-07-06 +.. gh-issue: 140513 +.. nonce: 6OhLTs +.. section: Build + +Generate a clear compilation error when ``_Py_TAIL_CALL_INTERP`` is enabled +but either ``preserve_none`` or ``musttail`` is not supported. + +.. + +.. date: 2025-10-22-12-44-07 +.. gh-issue: 140475 +.. nonce: OhzQbR +.. section: Build + +Support WASI SDK 25. + +.. + +.. date: 2025-10-17-11-33-45 +.. gh-issue: 140239 +.. nonce: _k-GgW +.. section: Build + +Check ``statx`` availability only on Linux (including Android). + +.. + +.. date: 2025-10-16-11-30-53 +.. gh-issue: 140189 +.. nonce: YCrUyt +.. section: Build + +iOS builds were added to CI. + +.. + +.. date: 2025-08-10-22-28-06 +.. gh-issue: 137618 +.. nonce: FdNvIE +.. section: Build + +``PYTHON_FOR_REGEN`` now requires Python 3.10 to Python 3.15. Patch by Adam +Turner. diff --git a/Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst b/Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst deleted file mode 100644 index 0b56c4c8f68..00000000000 --- a/Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst +++ /dev/null @@ -1,2 +0,0 @@ -``PYTHON_FOR_REGEN`` now requires Python 3.10 to Python 3.15. -Patch by Adam Turner. diff --git a/Misc/NEWS.d/next/Build/2025-10-16-11-30-53.gh-issue-140189.YCrUyt.rst b/Misc/NEWS.d/next/Build/2025-10-16-11-30-53.gh-issue-140189.YCrUyt.rst deleted file mode 100644 index a1b81659242..00000000000 --- a/Misc/NEWS.d/next/Build/2025-10-16-11-30-53.gh-issue-140189.YCrUyt.rst +++ /dev/null @@ -1 +0,0 @@ -iOS builds were added to CI. diff --git a/Misc/NEWS.d/next/Build/2025-10-17-11-33-45.gh-issue-140239._k-GgW.rst b/Misc/NEWS.d/next/Build/2025-10-17-11-33-45.gh-issue-140239._k-GgW.rst deleted file mode 100644 index 713f022c994..00000000000 --- a/Misc/NEWS.d/next/Build/2025-10-17-11-33-45.gh-issue-140239._k-GgW.rst +++ /dev/null @@ -1 +0,0 @@ -Check ``statx`` availability only on Linux (including Android). diff --git a/Misc/NEWS.d/next/Build/2025-10-22-12-44-07.gh-issue-140475.OhzQbR.rst b/Misc/NEWS.d/next/Build/2025-10-22-12-44-07.gh-issue-140475.OhzQbR.rst deleted file mode 100644 index b4139024761..00000000000 --- a/Misc/NEWS.d/next/Build/2025-10-22-12-44-07.gh-issue-140475.OhzQbR.rst +++ /dev/null @@ -1 +0,0 @@ -Support WASI SDK 25. diff --git a/Misc/NEWS.d/next/Build/2025-10-25-08-07-06.gh-issue-140513.6OhLTs.rst b/Misc/NEWS.d/next/Build/2025-10-25-08-07-06.gh-issue-140513.6OhLTs.rst deleted file mode 100644 index 1035ebf8d78..00000000000 --- a/Misc/NEWS.d/next/Build/2025-10-25-08-07-06.gh-issue-140513.6OhLTs.rst +++ /dev/null @@ -1,2 +0,0 @@ -Generate a clear compilation error when ``_Py_TAIL_CALL_INTERP`` is enabled but -either ``preserve_none`` or ``musttail`` is not supported. diff --git a/Misc/NEWS.d/next/Build/2025-10-29-12-30-38.gh-issue-140768.ITYrzw.rst b/Misc/NEWS.d/next/Build/2025-10-29-12-30-38.gh-issue-140768.ITYrzw.rst deleted file mode 100644 index 0009f83cd20..00000000000 --- a/Misc/NEWS.d/next/Build/2025-10-29-12-30-38.gh-issue-140768.ITYrzw.rst +++ /dev/null @@ -1 +0,0 @@ -Warn when the WASI SDK version doesn't match what's supported. diff --git a/Misc/NEWS.d/next/Build/2025-10-31-13-20-16.gh-issue-140454.gF6dCe.rst b/Misc/NEWS.d/next/Build/2025-10-31-13-20-16.gh-issue-140454.gF6dCe.rst deleted file mode 100644 index 4bb132ce01e..00000000000 --- a/Misc/NEWS.d/next/Build/2025-10-31-13-20-16.gh-issue-140454.gF6dCe.rst +++ /dev/null @@ -1,3 +0,0 @@ -When building the JIT, match the jit_stencils filename expectations in -Makefile with the generator script. This avoid needless JIT recompilation -during ``make install``. diff --git a/Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst b/Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst deleted file mode 100644 index cd3d5262fa0..00000000000 --- a/Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and -:c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set the -stack protection base address and stack protection size of a Python thread -state. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2025-10-15-15-59-59.gh-issue-140153.BO7sH4.rst b/Misc/NEWS.d/next/C_API/2025-10-15-15-59-59.gh-issue-140153.BO7sH4.rst deleted file mode 100644 index 502c48b6842..00000000000 --- a/Misc/NEWS.d/next/C_API/2025-10-15-15-59-59.gh-issue-140153.BO7sH4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :c:func:`Py_REFCNT` definition on limited C API 3.11-3.13. Patch by -Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2025-10-26-16-45-06.gh-issue-140487.fGOqss.rst b/Misc/NEWS.d/next/C_API/2025-10-26-16-45-06.gh-issue-140487.fGOqss.rst deleted file mode 100644 index 16b0d9d4084..00000000000 --- a/Misc/NEWS.d/next/C_API/2025-10-26-16-45-06.gh-issue-140487.fGOqss.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :c:macro:`Py_RETURN_NOTIMPLEMENTED` in limited C API 3.11 and older: -don't treat ``Py_NotImplemented`` as immortal. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2025-10-26-16-45-28.gh-issue-140556.s__Dae.rst b/Misc/NEWS.d/next/C_API/2025-10-26-16-45-28.gh-issue-140556.s__Dae.rst deleted file mode 100644 index 61da60903ee..00000000000 --- a/Misc/NEWS.d/next/C_API/2025-10-26-16-45-28.gh-issue-140556.s__Dae.rst +++ /dev/null @@ -1,2 +0,0 @@ -:pep:`793`: Add a new entry point for C extension modules, -``PyModExport_``. diff --git a/Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst b/Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst deleted file mode 100644 index a054f8eda6f..00000000000 --- a/Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst +++ /dev/null @@ -1 +0,0 @@ -The :c:macro:`!Py_INFINITY` macro is :term:`soft deprecated`. diff --git a/Misc/NEWS.d/next/C_API/2025-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst b/Misc/NEWS.d/next/C_API/2025-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst deleted file mode 100644 index 5f3ccd62016..00000000000 --- a/Misc/NEWS.d/next/C_API/2025-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst +++ /dev/null @@ -1 +0,0 @@ -:c:macro:`!Py_MATH_El` and :c:macro:`!Py_MATH_PIl` are deprecated. diff --git a/Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst b/Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst deleted file mode 100644 index 22a1aa1f405..00000000000 --- a/Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Make qNaN in :c:func:`PyFloat_Pack2` and :c:func:`PyFloat_Pack4`, if while -conversion to a narrower precision floating-point format --- the remaining -after truncation payload will be zero. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst b/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst deleted file mode 100644 index be8043e26dd..00000000000 --- a/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating a -module from a *spec* and *initfunc*. Patch by Itamar Oren. diff --git a/Misc/NEWS.d/next/C_API/2025-11-10-11-26-26.gh-issue-141341.OsO6-y.rst b/Misc/NEWS.d/next/C_API/2025-11-10-11-26-26.gh-issue-141341.OsO6-y.rst deleted file mode 100644 index 460923b4d62..00000000000 --- a/Misc/NEWS.d/next/C_API/2025-11-10-11-26-26.gh-issue-141341.OsO6-y.rst +++ /dev/null @@ -1,2 +0,0 @@ -On Windows, rename the ``COMPILER`` macro to ``_Py_COMPILER`` to avoid name -conflicts. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-24-13-12-58.gh-issue-134786.MF0VVk.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-24-13-12-58.gh-issue-134786.MF0VVk.rst deleted file mode 100644 index 664e4d2db38..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-24-13-12-58.gh-issue-134786.MF0VVk.rst +++ /dev/null @@ -1,2 +0,0 @@ -If :c:macro:`Py_TPFLAGS_MANAGED_DICT` and :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` -are used, then :c:macro:`Py_TPFLAGS_HAVE_GC` must be used as well. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-00-41-46.gh-issue-136327.7AiTb_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-00-41-46.gh-issue-136327.7AiTb_.rst deleted file mode 100644 index 3798e956c95..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-00-41-46.gh-issue-136327.7AiTb_.rst +++ /dev/null @@ -1,2 +0,0 @@ -Errors when calling functions with invalid values after ``*`` and ``**`` now do not -include the function name. Patch by Ilia Solin. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-29-17-51-14.gh-issue-131253.GpRjWy.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-29-17-51-14.gh-issue-131253.GpRjWy.rst deleted file mode 100644 index 2826fad2330..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-29-17-51-14.gh-issue-131253.GpRjWy.rst +++ /dev/null @@ -1 +0,0 @@ -Support the ``--enable-pystats`` build option for the free-threaded build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-13-01-23-25.gh-issue-138857.YQ5gdc.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-13-01-23-25.gh-issue-138857.YQ5gdc.rst deleted file mode 100644 index 93510a9ceaf..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-13-01-23-25.gh-issue-138857.YQ5gdc.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve :exc:`SyntaxError` message for ``case`` keyword placed outside -:keyword:`match` body. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-15-13-06-11.gh-issue-138944.PeCgLb.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-15-13-06-11.gh-issue-138944.PeCgLb.rst deleted file mode 100644 index 248585e2eba..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-15-13-06-11.gh-issue-138944.PeCgLb.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :exc:`SyntaxError` message when invalid syntax appears on the same line -as a valid ``import ... as ...`` or ``from ... import ... as ...`` -statement. Patch by Brian Schubert. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-23-21-01-12.gh-issue-139269.1rIaxy.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-23-21-01-12.gh-issue-139269.1rIaxy.rst deleted file mode 100644 index e36be529d2a..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-23-21-01-12.gh-issue-139269.1rIaxy.rst +++ /dev/null @@ -1 +0,0 @@ -Fix undefined behavior when using unaligned store in JIT's ``patch_*`` functions. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-03-17-51-43.gh-issue-139475._684ED.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-03-17-51-43.gh-issue-139475._684ED.rst deleted file mode 100644 index f4d50b7d020..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-03-17-51-43.gh-issue-139475._684ED.rst +++ /dev/null @@ -1,2 +0,0 @@ -Changes in stackref debugging mode when ``Py_STACKREF_DEBUG`` is set. We use -the same pattern of refcounting for stackrefs as in production build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb.rst deleted file mode 100644 index 396e40f0e13..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix swallowing some syntax warnings in different modules if they -accidentally have the same message and are emitted from the same line. -Fix duplicated warnings in the ``finally`` block. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb2.rst deleted file mode 100644 index b147b430ccc..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb2.rst +++ /dev/null @@ -1,3 +0,0 @@ -:func:`ast.parse` no longer emits syntax warnings for -``return``/``break``/``continue`` in ``finally`` (see :pep:`765`) -- they are -only emitted during compilation. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst deleted file mode 100644 index 96226a7c525..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst +++ /dev/null @@ -1,6 +0,0 @@ -Many functions related to compiling or parsing Python code, such as -:func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, and -:func:`importlib.abc.InspectLoader.source_to_code` now allow to specify -the module name. -It is needed to unambiguous :ref:`filter ` syntax warnings -by module name. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-12-01-12-12.gh-issue-139817.PAn-8Z.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-12-01-12-12.gh-issue-139817.PAn-8Z.rst deleted file mode 100644 index b205d21edfe..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-12-01-12-12.gh-issue-139817.PAn-8Z.rst +++ /dev/null @@ -1,2 +0,0 @@ -Attribute ``__qualname__`` is added to :class:`typing.TypeAliasType`. -Patch by Mikhail Efimov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst deleted file mode 100644 index 7529108d5d4..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst +++ /dev/null @@ -1 +0,0 @@ -Restore support for HP PA-RISC, which has an upwards-growing stack. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-17-07-37.gh-issue-140067.ID2gOm.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-17-07-37.gh-issue-140067.ID2gOm.rst deleted file mode 100644 index 3c5a828101d..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-17-07-37.gh-issue-140067.ID2gOm.rst +++ /dev/null @@ -1 +0,0 @@ -Fix memory leak in sub-interpreter creation. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst deleted file mode 100644 index d4b8578afe3..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update :class:`bytearray` to use a :class:`bytes` under the hood as its buffer -and add :func:`bytearray.take_bytes` to take it out. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-20-18-31.gh-issue-140080.8ROjxW.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-20-18-31.gh-issue-140080.8ROjxW.rst deleted file mode 100644 index 0ddcea57f9d..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-20-18-31.gh-issue-140080.8ROjxW.rst +++ /dev/null @@ -1 +0,0 @@ -Fix hang during finalization when attempting to call :mod:`atexit` handlers under no memory. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-00-21-40.gh-issue-140061.J0XeDV.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-00-21-40.gh-issue-140061.J0XeDV.rst deleted file mode 100644 index 7c3924195eb..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-00-21-40.gh-issue-140061.J0XeDV.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixing the checking of whether an object is uniquely referenced to ensure -free-threaded compatibility. Patch by Sergey Miryanov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-17-12-32.gh-issue-140149.cy1m3d.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-17-12-32.gh-issue-140149.cy1m3d.rst deleted file mode 100644 index e98e28802cf..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-17-12-32.gh-issue-140149.cy1m3d.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up parsing bytes literals concatenation by using PyBytesWriter API and -a single memory allocation (about 3x faster). diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst deleted file mode 100644 index 1c18cbc9ad0..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug with exception handling in the JIT. Patch by Ken Jin. Bug reported -by Daniel Diniz. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-14-38-10.gh-issue-140253.gCqFaL.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-14-38-10.gh-issue-140253.gCqFaL.rst deleted file mode 100644 index 955dcac2e01..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-14-38-10.gh-issue-140253.gCqFaL.rst +++ /dev/null @@ -1,2 +0,0 @@ -Wrong placement of a double-star pattern inside a mapping pattern now throws a specialized syntax error. -Contributed by Bartosz Sławecki in :gh:`140253`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-18-03-12.gh-issue-139951.IdwM2O.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-18-03-12.gh-issue-139951.IdwM2O.rst deleted file mode 100644 index e03996188a7..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-18-03-12.gh-issue-139951.IdwM2O.rst +++ /dev/null @@ -1,7 +0,0 @@ -Fixes a regression in GC performance for a growing heap composed mostly of -small tuples. - -* Counts number of actually tracked objects, instead of trackable objects. - This ensures that untracking tuples has the desired effect of reducing GC overhead. -* Does not track most untrackable tuples during creation. - This prevents large numbers of small tuples causing excessive GCs. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-20-23-19.gh-issue-140257.8Txmem.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-20-23-19.gh-issue-140257.8Txmem.rst deleted file mode 100644 index 50f7e0e48ae..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-20-23-19.gh-issue-140257.8Txmem.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix data race between interpreter_clear() and take_gil() on eval_breaker -during finalization with daemon threads. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-18-08-36.gh-issue-140301.m-2HxC.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-18-08-36.gh-issue-140301.m-2HxC.rst deleted file mode 100644 index 8b1c81c04ec..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-18-08-36.gh-issue-140301.m-2HxC.rst +++ /dev/null @@ -1 +0,0 @@ -Fix memory leak of ``PyConfig`` in subinterpreters. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-19-52-20.gh-issue-116738.NLJW0L.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-19-52-20.gh-issue-116738.NLJW0L.rst deleted file mode 100644 index bf323b870bc..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-19-52-20.gh-issue-116738.NLJW0L.rst +++ /dev/null @@ -1,2 +0,0 @@ -Make _suggestions module thread-safe on the :term:`free threaded ` build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-29-45.gh-issue-140306.xS5CcS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-29-45.gh-issue-140306.xS5CcS.rst deleted file mode 100644 index 2178c496063..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-29-45.gh-issue-140306.xS5CcS.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix memory leaks in cross-interpreter channel operations and shared -namespace handling. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst deleted file mode 100644 index 40b9d19ee42..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst +++ /dev/null @@ -1 +0,0 @@ -A new tracing frontend for the JIT compiler has been implemented. Patch by Ken Jin. Design for CPython by Ken Jin, Mark Shannon and Brandt Bucher. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-19-10-32-28.gh-issue-136895.HfsEh0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-19-10-32-28.gh-issue-136895.HfsEh0.rst deleted file mode 100644 index fffc264a865..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-19-10-32-28.gh-issue-136895.HfsEh0.rst +++ /dev/null @@ -1 +0,0 @@ -Update JIT compilation to use LLVM 20 at build time. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-20-11-24-36.gh-issue-140358.UQuKdV.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-20-11-24-36.gh-issue-140358.UQuKdV.rst deleted file mode 100644 index 739228f7e36..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-20-11-24-36.gh-issue-140358.UQuKdV.rst +++ /dev/null @@ -1,4 +0,0 @@ -Restore elapsed time and unreachable object count in GC debug output. These -were inadvertently removed during a refactor of ``gc.c``. The debug log now -again reports elapsed collection time and the number of unreachable objects. -Contributed by Pål Grønås Drange. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-06-51-50.gh-issue-140406.0gJs8M.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-06-51-50.gh-issue-140406.0gJs8M.rst deleted file mode 100644 index 3506ba42581..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-06-51-50.gh-issue-140406.0gJs8M.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix memory leak when an object's :meth:`~object.__hash__` method returns an -object that isn't an :class:`int`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-09-20-03.gh-issue-140398.SoABwJ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-09-20-03.gh-issue-140398.SoABwJ.rst deleted file mode 100644 index 481dac7f26d..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-09-20-03.gh-issue-140398.SoABwJ.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix memory leaks in :mod:`readline` functions -:func:`~readline.read_init_file`, :func:`~readline.read_history_file`, -:func:`~readline.write_history_file`, and -:func:`~readline.append_history_file` when :c:func:`PySys_Audit` fails. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-11-30-16.gh-issue-135904.3WE5oW.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-11-30-16.gh-issue-135904.3WE5oW.rst deleted file mode 100644 index b52a57dba4a..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-11-30-16.gh-issue-135904.3WE5oW.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add special labels to the assembly created during stencil creation to -support relocations that the native object file format does not support. -Specifically, 19 bit branches for AArch64 in Mach-O object files. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst deleted file mode 100644 index a24033208c5..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst +++ /dev/null @@ -1,2 +0,0 @@ -Optimize :c:func:`PySet_Add` for :class:`frozenset` in :term:`free threaded -` build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-17-22-22.gh-issue-140431.m8D_A-.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-17-22-22.gh-issue-140431.m8D_A-.rst deleted file mode 100644 index 3d62d210f1f..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-17-22-22.gh-issue-140431.m8D_A-.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a crash in Python's :term:`garbage collector ` due to -partially initialized :term:`coroutine` objects when coroutine origin tracking -depth is enabled (:func:`sys.set_coroutine_origin_tracking_depth`). diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-23-26-37.gh-issue-140443.wT5i1A.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-23-26-37.gh-issue-140443.wT5i1A.rst deleted file mode 100644 index a1fff8fef7e..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-23-26-37.gh-issue-140443.wT5i1A.rst +++ /dev/null @@ -1,5 +0,0 @@ -The logarithm functions (such as :func:`math.log10` and :func:`math.log`) may now produce -slightly different results for extremely large integers that cannot be -converted to floats without overflow. These results are generally more -accurate, with reduced worst-case error and a tighter overall error -distribution. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-23-16-05-50.gh-issue-140471.Ax_aXn.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-23-16-05-50.gh-issue-140471.Ax_aXn.rst deleted file mode 100644 index afa9326fff3..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-23-16-05-50.gh-issue-140471.Ax_aXn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix potential buffer overflow in :class:`ast.AST` node initialization when -encountering malformed :attr:`~ast.AST._fields` containing non-:class:`str`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-14-29-12.gh-issue-133467.A5d6TM.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-14-29-12.gh-issue-133467.A5d6TM.rst deleted file mode 100644 index f69786866e9..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-14-29-12.gh-issue-133467.A5d6TM.rst +++ /dev/null @@ -1 +0,0 @@ -Fix race when updating :attr:`!type.__bases__` that could allow a read of :attr:`!type.__base__` to observe an inconsistent value on the free threaded build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-16-42.gh-issue-140517.cqun-K.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-16-42.gh-issue-140517.cqun-K.rst deleted file mode 100644 index 15aaea8ab02..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-16-42.gh-issue-140517.cqun-K.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a reference leak when iterating over the result of :func:`map` -with ``strict=True`` when the input iterables have different lengths. -Patch by Mikhail Efimov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst deleted file mode 100644 index 8fd9b46c0ae..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed crash in :class:`dict` if :meth:`dict.clear` is called at the lookup -stage. Patch by Mikhail Efimov and Inada Naoki. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-07-25-52.gh-issue-140544.lwjtQe.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-07-25-52.gh-issue-140544.lwjtQe.rst deleted file mode 100644 index 51d2b229ee5..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-07-25-52.gh-issue-140544.lwjtQe.rst +++ /dev/null @@ -1 +0,0 @@ -Speed up accessing interpreter state by caching it in a thread local variable. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst deleted file mode 100644 index 2c27525d9f7..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed crash in :func:`tokenize.generate_tokens` in case of -specific incorrect input. Patch by Mikhail Efimov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-21-31-43.gh-issue-131527.V-JVNP.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-21-31-43.gh-issue-131527.V-JVNP.rst deleted file mode 100644 index 9969ea058a3..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-21-31-43.gh-issue-131527.V-JVNP.rst +++ /dev/null @@ -1,2 +0,0 @@ -Dynamic borrow checking for stackrefs is added to ``Py_STACKREF_DEBUG`` -mode. Patch by Mikhail Efimov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst deleted file mode 100644 index 6725547667f..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix pickling error in the sampling profiler when using ``concurrent.futures.ProcessPoolExecutor`` -script can not be properly pickled and executed in worker processes. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-20-59-10.gh-issue-140373.-uoaPP.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-20-59-10.gh-issue-140373.-uoaPP.rst deleted file mode 100644 index c9a97037920..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-20-59-10.gh-issue-140373.-uoaPP.rst +++ /dev/null @@ -1,2 +0,0 @@ -Correctly emit ``PY_UNWIND`` event when generator object is closed. Patch by -Mikhail Efimov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst deleted file mode 100644 index b1d05354f65..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst +++ /dev/null @@ -1 +0,0 @@ -Replace :class:`io.IncrementalNewlineDecoder` with non incremental newline decoders in codebase where :meth:`!io.IncrementalNewlineDecoder.decode` was being called once. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-12-47-38.gh-issue-140530.S934bp.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-12-47-38.gh-issue-140530.S934bp.rst deleted file mode 100644 index e3af493893a..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-12-47-38.gh-issue-140530.S934bp.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a reference leak when ``raise exc from cause`` fails. Patch by Bénédikt -Tran. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst deleted file mode 100644 index 96bf9b51e48..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :mod:`struct` data race in endian table initialization with -subinterpreters. Patch by Shamil Abdulaev. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-17-21-38.gh-issue-140939.FVboAw.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-17-21-38.gh-issue-140939.FVboAw.rst deleted file mode 100644 index a2921761f75..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-17-21-38.gh-issue-140939.FVboAw.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix memory leak when :class:`bytearray` or :class:`bytes` is formated with the -``%*b`` format with a large width that results in a :exc:`MemoryError`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst deleted file mode 100644 index 0a615ed1311..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst +++ /dev/null @@ -1 +0,0 @@ -Update JIT compilation to use LLVM 21 at build time. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst deleted file mode 100644 index 20cfeca1e71..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add ``.cjs`` to :mod:`mimetypes` to give CommonJS modules a MIME type of -``application/node``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst deleted file mode 100644 index e1202dd1a17..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add support for ```` and ```` frames to -:mod:`!profiling.sampling` output to denote active garbage collection and -calls to native code. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst deleted file mode 100644 index fdb136cef3f..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the assertion failure in the ``__setstate__`` method of the range iterator -when a non-integer argument is passed. Patch by Sergey Miryanov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst deleted file mode 100644 index cb830fcd9e1..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst +++ /dev/null @@ -1,2 +0,0 @@ -Specialize ``CALL_LIST_APPEND`` instruction only for lists, not for list -subclasses, to avoid unnecessary deopt. Patch by Mikhail Efimov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst deleted file mode 100644 index a51aa495228..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst +++ /dev/null @@ -1,3 +0,0 @@ -Suggest using :meth:`concurrent.interpreters.Interpreter.close` instead of the -private ``_interpreters.destroy`` function when warning about remaining subinterpreters. -Patch by Sergey Miryanov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst deleted file mode 100644 index 883f9333cae..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst +++ /dev/null @@ -1 +0,0 @@ -Skip locking if object is already locked by two-mutex critical section. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst deleted file mode 100644 index 8ab9979c399..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :func:`sys.activate_stack_trampoline` to properly support the -``perf_jit`` backend. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Library/2019-06-02-13-56-16.gh-issue-81313.axawSH.rst b/Misc/NEWS.d/next/Library/2019-06-02-13-56-16.gh-issue-81313.axawSH.rst deleted file mode 100644 index 2291c938222..00000000000 --- a/Misc/NEWS.d/next/Library/2019-06-02-13-56-16.gh-issue-81313.axawSH.rst +++ /dev/null @@ -1 +0,0 @@ -Add the :mod:`math.integer` module (:pep:`791`). diff --git a/Misc/NEWS.d/next/Library/2023-03-21-10-59-40.gh-issue-102431.eUDnf4.rst b/Misc/NEWS.d/next/Library/2023-03-21-10-59-40.gh-issue-102431.eUDnf4.rst deleted file mode 100644 index e82ddb6e101..00000000000 --- a/Misc/NEWS.d/next/Library/2023-03-21-10-59-40.gh-issue-102431.eUDnf4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Clarify constraints for "logical" arguments in methods of -:class:`decimal.Context`. diff --git a/Misc/NEWS.d/next/Library/2024-05-28-17-14-30.gh-issue-119668.RrIGpn.rst b/Misc/NEWS.d/next/Library/2024-05-28-17-14-30.gh-issue-119668.RrIGpn.rst deleted file mode 100644 index 87cdf8d89d5..00000000000 --- a/Misc/NEWS.d/next/Library/2024-05-28-17-14-30.gh-issue-119668.RrIGpn.rst +++ /dev/null @@ -1 +0,0 @@ -Publicly expose and document :class:`importlib.machinery.NamespacePath`. diff --git a/Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst b/Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst deleted file mode 100644 index aee7fe2bcb5..00000000000 --- a/Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`math.log` now supports arbitrary large integer-like arguments in the -same way as arbitrary large integer arguments. diff --git a/Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst b/Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst deleted file mode 100644 index 63e71c19f8b..00000000000 --- a/Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst +++ /dev/null @@ -1,4 +0,0 @@ -In the :mod:`linecache` module and in the Python implementation of the -:mod:`warnings` module, a ``DeprecationWarning`` is issued when -``mod.__loader__`` differs from ``mod.__spec__.loader`` (like in the C -implementation of the :mod:`!warnings` module). diff --git a/Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst b/Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst deleted file mode 100644 index b175ab7cad4..00000000000 --- a/Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst +++ /dev/null @@ -1 +0,0 @@ -Add support for ``-nolinestop``, and ``-strictlimits`` options to :meth:`!tkinter.Text.search`. Also add the :meth:`!tkinter.Text.search_all` method for ``-all`` and ``-overlap`` options. diff --git a/Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst b/Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst deleted file mode 100644 index f5e60ab6e8c..00000000000 --- a/Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`inspect.getdoc` now correctly returns an inherited docstring on -:class:`~functools.cached_property` objects if none is given in a subclass. diff --git a/Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst b/Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst deleted file mode 100644 index d0c8e2d705c..00000000000 --- a/Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add parameters *inherit_class_doc* and *fallback_to_class_doc* for -:func:`inspect.getdoc`. diff --git a/Misc/NEWS.d/next/Library/2025-05-07-22-09-28.gh-issue-133601.9kUL3P.rst b/Misc/NEWS.d/next/Library/2025-05-07-22-09-28.gh-issue-133601.9kUL3P.rst deleted file mode 100644 index 62f40aee7aa..00000000000 --- a/Misc/NEWS.d/next/Library/2025-05-07-22-09-28.gh-issue-133601.9kUL3P.rst +++ /dev/null @@ -1 +0,0 @@ -Remove deprecated :func:`!typing.no_type_check_decorator`. diff --git a/Misc/NEWS.d/next/Library/2025-05-10-15-10-54.gh-issue-133789.I-ZlUX.rst b/Misc/NEWS.d/next/Library/2025-05-10-15-10-54.gh-issue-133789.I-ZlUX.rst deleted file mode 100644 index d2a4f7f42c3..00000000000 --- a/Misc/NEWS.d/next/Library/2025-05-10-15-10-54.gh-issue-133789.I-ZlUX.rst +++ /dev/null @@ -1 +0,0 @@ -Fix unpickling of :mod:`pathlib` objects that were pickled in Python 3.13. diff --git a/Misc/NEWS.d/next/Library/2025-06-10-18-02-29.gh-issue-135307.fXGrcK.rst b/Misc/NEWS.d/next/Library/2025-06-10-18-02-29.gh-issue-135307.fXGrcK.rst deleted file mode 100644 index 47e1feb5cbf..00000000000 --- a/Misc/NEWS.d/next/Library/2025-06-10-18-02-29.gh-issue-135307.fXGrcK.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`email`: Fix exception in ``set_content()`` when encoding text -and max_line_length is set to ``0`` or ``None`` (unlimited). diff --git a/Misc/NEWS.d/next/Library/2025-06-29-22-01-00.gh-issue-133390.I1DW_3.rst b/Misc/NEWS.d/next/Library/2025-06-29-22-01-00.gh-issue-133390.I1DW_3.rst deleted file mode 100644 index c57f802d4c8..00000000000 --- a/Misc/NEWS.d/next/Library/2025-06-29-22-01-00.gh-issue-133390.I1DW_3.rst +++ /dev/null @@ -1,2 +0,0 @@ -Support table, index, trigger, view, column, function, and schema completion -for :mod:`sqlite3`'s :ref:`command-line interface `. diff --git a/Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst b/Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst deleted file mode 100644 index e237a0e98cc..00000000000 --- a/Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed the bug in :mod:`pdb` and :mod:`bdb` where ``next`` and ``step`` can't go over the line if a loop exists in the line. diff --git a/Misc/NEWS.d/next/Library/2025-07-14-09-33-17.gh-issue-55531.Gt2e12.rst b/Misc/NEWS.d/next/Library/2025-07-14-09-33-17.gh-issue-55531.Gt2e12.rst deleted file mode 100644 index 70e39a4f2c1..00000000000 --- a/Misc/NEWS.d/next/Library/2025-07-14-09-33-17.gh-issue-55531.Gt2e12.rst +++ /dev/null @@ -1,4 +0,0 @@ -:mod:`encodings`: Improve :func:`~encodings.normalize_encoding` performance -by implementing the function in C using the private -``_Py_normalize_encoding`` which has been modified to make lowercase -conversion optional. diff --git a/Misc/NEWS.d/next/Library/2025-08-11-04-52-18.gh-issue-137627.Ku5Yi2.rst b/Misc/NEWS.d/next/Library/2025-08-11-04-52-18.gh-issue-137627.Ku5Yi2.rst deleted file mode 100644 index 855070ed6f4..00000000000 --- a/Misc/NEWS.d/next/Library/2025-08-11-04-52-18.gh-issue-137627.Ku5Yi2.rst +++ /dev/null @@ -1 +0,0 @@ -Speed up :meth:`csv.Sniffer.sniff` delimiter detection by up to 1.6x. diff --git a/Misc/NEWS.d/next/Library/2025-08-15-20-35-30.gh-issue-69528.qc-Eh_.rst b/Misc/NEWS.d/next/Library/2025-08-15-20-35-30.gh-issue-69528.qc-Eh_.rst deleted file mode 100644 index b18781e0dce..00000000000 --- a/Misc/NEWS.d/next/Library/2025-08-15-20-35-30.gh-issue-69528.qc-Eh_.rst +++ /dev/null @@ -1,2 +0,0 @@ -The :attr:`~io.FileIO.mode` attribute of files opened in the ``'wb+'`` mode is -now ``'wb+'`` instead of ``'rb+'``. diff --git a/Misc/NEWS.d/next/Library/2025-08-26-08-17-56.gh-issue-138151.I6CdAk.rst b/Misc/NEWS.d/next/Library/2025-08-26-08-17-56.gh-issue-138151.I6CdAk.rst deleted file mode 100644 index de29f536afc..00000000000 --- a/Misc/NEWS.d/next/Library/2025-08-26-08-17-56.gh-issue-138151.I6CdAk.rst +++ /dev/null @@ -1,3 +0,0 @@ -In :mod:`annotationlib`, improve evaluation of forward references to -nonlocal variables that are not yet defined when the annotations are -initially evaluated. diff --git a/Misc/NEWS.d/next/Library/2025-09-03-18-26-07.gh-issue-138425.cVE9Ho.rst b/Misc/NEWS.d/next/Library/2025-09-03-18-26-07.gh-issue-138425.cVE9Ho.rst deleted file mode 100644 index 328e5988cb0..00000000000 --- a/Misc/NEWS.d/next/Library/2025-09-03-18-26-07.gh-issue-138425.cVE9Ho.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix partial evaluation of :class:`annotationlib.ForwardRef` objects which rely -on names defined as globals. diff --git a/Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst b/Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst deleted file mode 100644 index 6831499c0af..00000000000 --- a/Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a failure in multiprocessing resource_tracker when SharedMemory names contain colons. -Patch by Rani Pinchuk. diff --git a/Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst b/Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst deleted file mode 100644 index 455c1a9925a..00000000000 --- a/Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst +++ /dev/null @@ -1,2 +0,0 @@ -Use of ``python -m`` with :mod:`base64` has been fixed to detect input from a -terminal so that it properly notices EOF. diff --git a/Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst b/Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst deleted file mode 100644 index 85ebef8ff11..00000000000 --- a/Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst +++ /dev/null @@ -1,3 +0,0 @@ -Prevent :func:`annotationlib.call_annotate_function` from calling ``__annotate__`` functions that don't support ``VALUE_WITH_FAKE_GLOBALS`` in a fake globals namespace with empty globals. - -Make ``FORWARDREF`` and ``STRING`` annotations fall back to using ``VALUE`` annotations in the case that neither their own format, nor ``VALUE_WITH_FAKE_GLOBALS`` are supported. diff --git a/Misc/NEWS.d/next/Library/2025-09-13-12-19-17.gh-issue-138859.PxjIoN.rst b/Misc/NEWS.d/next/Library/2025-09-13-12-19-17.gh-issue-138859.PxjIoN.rst deleted file mode 100644 index a5d4dd042fc..00000000000 --- a/Misc/NEWS.d/next/Library/2025-09-13-12-19-17.gh-issue-138859.PxjIoN.rst +++ /dev/null @@ -1 +0,0 @@ -Fix generic type parameterization raising a :exc:`TypeError` when omitting a :class:`ParamSpec` that has a default which is not a list of types. diff --git a/Misc/NEWS.d/next/Library/2025-09-15-21-03-11.gh-issue-138891.oZFdtR.rst b/Misc/NEWS.d/next/Library/2025-09-15-21-03-11.gh-issue-138891.oZFdtR.rst deleted file mode 100644 index f7ecb05d20c..00000000000 --- a/Misc/NEWS.d/next/Library/2025-09-15-21-03-11.gh-issue-138891.oZFdtR.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix ``SyntaxError`` when ``inspect.get_annotations(f, eval_str=True)`` is -called on a function annotated with a :pep:`646` ``star_expression`` diff --git a/Misc/NEWS.d/next/Library/2025-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst b/Misc/NEWS.d/next/Library/2025-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst deleted file mode 100644 index 3653eb9a114..00000000000 --- a/Misc/NEWS.d/next/Library/2025-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Implement :func:`os.statx` on Linux kernel versions 4.11 and later with glibc -versions 2.28 and later. Contributed by Jeffrey Bosboom and Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2025-09-23-09-46-46.gh-issue-139246.pzfM-w.rst b/Misc/NEWS.d/next/Library/2025-09-23-09-46-46.gh-issue-139246.pzfM-w.rst deleted file mode 100644 index a816bda5cfe..00000000000 --- a/Misc/NEWS.d/next/Library/2025-09-23-09-46-46.gh-issue-139246.pzfM-w.rst +++ /dev/null @@ -1 +0,0 @@ -fix: paste zero-width in default repl width is wrong. diff --git a/Misc/NEWS.d/next/Library/2025-09-25-20-16-10.gh-issue-101828.yTxJlJ.rst b/Misc/NEWS.d/next/Library/2025-09-25-20-16-10.gh-issue-101828.yTxJlJ.rst deleted file mode 100644 index 1d100180c07..00000000000 --- a/Misc/NEWS.d/next/Library/2025-09-25-20-16-10.gh-issue-101828.yTxJlJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix ``'shift_jisx0213'``, ``'shift_jis_2004'``, ``'euc_jisx0213'`` and -``'euc_jis_2004'`` codecs truncating null chars -as they were treated as part of multi-character sequences. diff --git a/Misc/NEWS.d/next/Library/2025-09-30-12-52-54.gh-issue-63161.mECM1A.rst b/Misc/NEWS.d/next/Library/2025-09-30-12-52-54.gh-issue-63161.mECM1A.rst deleted file mode 100644 index 3daed20d099..00000000000 --- a/Misc/NEWS.d/next/Library/2025-09-30-12-52-54.gh-issue-63161.mECM1A.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :func:`tokenize.detect_encoding`. Support non-UTF-8 shebang and comments -if non-UTF-8 encoding is specified. Detect decoding error for non-UTF-8 -encoding. Detect null bytes in source code. diff --git a/Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst b/Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst deleted file mode 100644 index 390a6124386..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst +++ /dev/null @@ -1,3 +0,0 @@ -When a child process in a :class:`concurrent.futures.ProcessPoolExecutor` -terminates abruptly, the resulting traceback will now tell you the PID -and exit code of the terminated process. Contributed by Jonathan Berg. diff --git a/Misc/NEWS.d/next/Library/2025-10-11-09-07-06.gh-issue-139940.g54efZ.rst b/Misc/NEWS.d/next/Library/2025-10-11-09-07-06.gh-issue-139940.g54efZ.rst deleted file mode 100644 index 2501135e657..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-11-09-07-06.gh-issue-139940.g54efZ.rst +++ /dev/null @@ -1 +0,0 @@ -Print clearer error message when using ``pdb`` to attach to a non-existing process. diff --git a/Misc/NEWS.d/next/Library/2025-10-13-11-25-41.gh-issue-136702.uvLGK1.rst b/Misc/NEWS.d/next/Library/2025-10-13-11-25-41.gh-issue-136702.uvLGK1.rst deleted file mode 100644 index 88303f017f5..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-13-11-25-41.gh-issue-136702.uvLGK1.rst +++ /dev/null @@ -1,3 +0,0 @@ -:mod:`encodings`: Deprecate passing a non-ascii *encoding* name to -:func:`encodings.normalize_encoding` and schedule removal of support for -Python 3.17. diff --git a/Misc/NEWS.d/next/Library/2025-10-14-20-27-06.gh-issue-76007.2NcUbo.rst b/Misc/NEWS.d/next/Library/2025-10-14-20-27-06.gh-issue-76007.2NcUbo.rst deleted file mode 100644 index 567fb5ef904..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-14-20-27-06.gh-issue-76007.2NcUbo.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`zlib`: Deprecate ``__version__`` and schedule for removal in Python -3.20. diff --git a/Misc/NEWS.d/next/Library/2025-10-15-02-26-50.gh-issue-140135.54JYfM.rst b/Misc/NEWS.d/next/Library/2025-10-15-02-26-50.gh-issue-140135.54JYfM.rst deleted file mode 100644 index 8d5a76af909..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-15-02-26-50.gh-issue-140135.54JYfM.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up :meth:`io.RawIOBase.readall` by using PyBytesWriter API (about 4x -faster) diff --git a/Misc/NEWS.d/next/Library/2025-10-15-15-10-34.gh-issue-140166.NtxRez.rst b/Misc/NEWS.d/next/Library/2025-10-15-15-10-34.gh-issue-140166.NtxRez.rst deleted file mode 100644 index c140db9dcd5..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-15-15-10-34.gh-issue-140166.NtxRez.rst +++ /dev/null @@ -1 +0,0 @@ -:mod:`mimetypes`: Per the `IANA assignment `_, update the MIME type for the ``.texi`` and ``.texinfo`` file formats to ``application/texinfo``, instead of ``application/x-texinfo``. diff --git a/Misc/NEWS.d/next/Library/2025-10-15-17-23-51.gh-issue-140141.j2mUDB.rst b/Misc/NEWS.d/next/Library/2025-10-15-17-23-51.gh-issue-140141.j2mUDB.rst deleted file mode 100644 index 2edadbc3e38..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-15-17-23-51.gh-issue-140141.j2mUDB.rst +++ /dev/null @@ -1,5 +0,0 @@ -The :py:class:`importlib.metadata.PackageNotFoundError` traceback raised when -``importlib.metadata.Distribution.from_name`` cannot discover a -distribution no longer includes a transient :exc:`StopIteration` exception trace. - -Contributed by Bartosz Sławecki in :gh:`140142`. diff --git a/Misc/NEWS.d/next/Library/2025-10-15-20-47-04.gh-issue-140120.3gffZq.rst b/Misc/NEWS.d/next/Library/2025-10-15-20-47-04.gh-issue-140120.3gffZq.rst deleted file mode 100644 index 9eefe140520..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-15-20-47-04.gh-issue-140120.3gffZq.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a memory leak in :mod:`hmac` when it was using the hacl-star backend. -Discovered by ``@ashm-dev`` using AddressSanitizer. diff --git a/Misc/NEWS.d/next/Library/2025-10-15-21-42-13.gh-issue-140041._Fka2j.rst b/Misc/NEWS.d/next/Library/2025-10-15-21-42-13.gh-issue-140041._Fka2j.rst deleted file mode 100644 index 243ff39311c..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-15-21-42-13.gh-issue-140041._Fka2j.rst +++ /dev/null @@ -1 +0,0 @@ -Fix import of :mod:`ctypes` on Android and Cygwin when ABI flags are present. diff --git a/Misc/NEWS.d/next/Library/2025-10-16-16-10-11.gh-issue-139707.zR6Qtn.rst b/Misc/NEWS.d/next/Library/2025-10-16-16-10-11.gh-issue-139707.zR6Qtn.rst deleted file mode 100644 index c5460aae8b3..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-16-16-10-11.gh-issue-139707.zR6Qtn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve :exc:`ModuleNotFoundError` error message when a :term:`standard library` -module is missing. diff --git a/Misc/NEWS.d/next/Library/2025-10-16-17-17-20.gh-issue-135801.faH3fa.rst b/Misc/NEWS.d/next/Library/2025-10-16-17-17-20.gh-issue-135801.faH3fa.rst deleted file mode 100644 index d680312d582..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-16-17-17-20.gh-issue-135801.faH3fa.rst +++ /dev/null @@ -1,6 +0,0 @@ -Improve filtering by module in :func:`warnings.warn_explicit` if no *module* -argument is passed. It now tests the module regular expression in the -warnings filter not only against the filename with ``.py`` stripped, but -also against module names constructed starting from different parent -directories of the filename (with ``/__init__.py``, ``.py`` and, on Windows, -``.pyw`` stripped). diff --git a/Misc/NEWS.d/next/Library/2025-10-16-22-49-16.gh-issue-140212.llBNd0.rst b/Misc/NEWS.d/next/Library/2025-10-16-22-49-16.gh-issue-140212.llBNd0.rst deleted file mode 100644 index 5563d077171..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-16-22-49-16.gh-issue-140212.llBNd0.rst +++ /dev/null @@ -1,5 +0,0 @@ -Calendar's HTML formatting now accepts year and month as options. -Previously, running ``python -m calendar -t html 2025 10`` would result in an -error message. It now generates an HTML document displaying the calendar for -the specified month. -Contributed by Pål Grønås Drange. diff --git a/Misc/NEWS.d/next/Library/2025-10-17-12-33-01.gh-issue-140251.esM-OX.rst b/Misc/NEWS.d/next/Library/2025-10-17-12-33-01.gh-issue-140251.esM-OX.rst deleted file mode 100644 index cb08e02429b..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-17-12-33-01.gh-issue-140251.esM-OX.rst +++ /dev/null @@ -1 +0,0 @@ -Colorize the default import statement ``import asyncio`` in asyncio REPL. diff --git a/Misc/NEWS.d/next/Library/2025-10-17-20-42-38.gh-issue-129117.X9jr4p.rst b/Misc/NEWS.d/next/Library/2025-10-17-20-42-38.gh-issue-129117.X9jr4p.rst deleted file mode 100644 index 8767b1bb483..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-17-20-42-38.gh-issue-129117.X9jr4p.rst +++ /dev/null @@ -1,3 +0,0 @@ -:mod:`unicodedata`: Add :func:`~unicodedata.isxidstart` and -:func:`~unicodedata.isxidcontinue` functions to check whether a character can -start or continue a `Unicode Standard Annex #31 `_ identifier. diff --git a/Misc/NEWS.d/next/Library/2025-10-17-23-58-11.gh-issue-140272.lhY8uS.rst b/Misc/NEWS.d/next/Library/2025-10-17-23-58-11.gh-issue-140272.lhY8uS.rst deleted file mode 100644 index 666a45055f5..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-17-23-58-11.gh-issue-140272.lhY8uS.rst +++ /dev/null @@ -1 +0,0 @@ -Fix memory leak in the :meth:`!clear` method of the :mod:`dbm.gnu` database. diff --git a/Misc/NEWS.d/next/Library/2025-10-18-14-30-21.gh-issue-76007.peEgcr.rst b/Misc/NEWS.d/next/Library/2025-10-18-14-30-21.gh-issue-76007.peEgcr.rst deleted file mode 100644 index be56b2ca6a1..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-18-14-30-21.gh-issue-76007.peEgcr.rst +++ /dev/null @@ -1 +0,0 @@ -Deprecate ``__version__`` from a :mod:`imaplib`. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2025-10-18-15-20-25.gh-issue-76007.SNUzRq.rst b/Misc/NEWS.d/next/Library/2025-10-18-15-20-25.gh-issue-76007.SNUzRq.rst deleted file mode 100644 index 6a91fc41b0a..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-18-15-20-25.gh-issue-76007.SNUzRq.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`decimal`: Deprecate ``__version__`` and replace with -:data:`decimal.SPEC_VERSION`. diff --git a/Misc/NEWS.d/next/Library/2025-10-20-12-33-49.gh-issue-140348.SAKnQZ.rst b/Misc/NEWS.d/next/Library/2025-10-20-12-33-49.gh-issue-140348.SAKnQZ.rst deleted file mode 100644 index 16d5b2a8bf0..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-20-12-33-49.gh-issue-140348.SAKnQZ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix regression in Python 3.14.0 where using the ``|`` operator on a -:class:`typing.Union` object combined with an object that is not a type -would raise an error. diff --git a/Misc/NEWS.d/next/Library/2025-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst b/Misc/NEWS.d/next/Library/2025-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst deleted file mode 100644 index 4ff55b41dea..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst +++ /dev/null @@ -1 +0,0 @@ -:mod:`dataclasses` Fix annotations for generated ``__init__`` methods by replacing the annotations that were in-line in the generated source code with ``__annotate__`` functions attached to the methods. diff --git a/Misc/NEWS.d/next/Library/2025-10-22-12-56-57.gh-issue-140448.GsEkXD.rst b/Misc/NEWS.d/next/Library/2025-10-22-12-56-57.gh-issue-140448.GsEkXD.rst deleted file mode 100644 index db7f92e136d..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-22-12-56-57.gh-issue-140448.GsEkXD.rst +++ /dev/null @@ -1,2 +0,0 @@ -Change the default of ``suggest_on_error`` to ``True`` in -``argparse.ArgumentParser``. diff --git a/Misc/NEWS.d/next/Library/2025-10-22-20-52-13.gh-issue-140474.xIWlip.rst b/Misc/NEWS.d/next/Library/2025-10-22-20-52-13.gh-issue-140474.xIWlip.rst deleted file mode 100644 index aca4e68b1e5..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-22-20-52-13.gh-issue-140474.xIWlip.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix memory leak in :class:`array.array` when creating arrays from an empty -:class:`str` and the ``u`` type code. diff --git a/Misc/NEWS.d/next/Library/2025-10-23-12-12-22.gh-issue-138774.mnh2gU.rst b/Misc/NEWS.d/next/Library/2025-10-23-12-12-22.gh-issue-138774.mnh2gU.rst deleted file mode 100644 index e12f789e674..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-23-12-12-22.gh-issue-138774.mnh2gU.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`ast.unparse` now generates full source code when handling -:class:`ast.Interpolation` nodes that do not have a specified source. diff --git a/Misc/NEWS.d/next/Library/2025-10-23-13-42-15.gh-issue-140481.XKxWpq.rst b/Misc/NEWS.d/next/Library/2025-10-23-13-42-15.gh-issue-140481.XKxWpq.rst deleted file mode 100644 index 1f511c3b9d0..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-23-13-42-15.gh-issue-140481.XKxWpq.rst +++ /dev/null @@ -1 +0,0 @@ -Improve error message when trying to iterate a Tk widget, image or font. diff --git a/Misc/NEWS.d/next/Library/2025-10-23-19-39-16.gh-issue-138162.Znw5DN.rst b/Misc/NEWS.d/next/Library/2025-10-23-19-39-16.gh-issue-138162.Znw5DN.rst deleted file mode 100644 index ef7a90bc37e..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-23-19-39-16.gh-issue-138162.Znw5DN.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :class:`logging.LoggerAdapter` with ``merge_extra=True`` and without the -*extra* argument. diff --git a/Misc/NEWS.d/next/Library/2025-10-25-21-04-00.gh-issue-140607.oOZGxS.rst b/Misc/NEWS.d/next/Library/2025-10-25-21-04-00.gh-issue-140607.oOZGxS.rst deleted file mode 100644 index cc33217c9f5..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-25-21-04-00.gh-issue-140607.oOZGxS.rst +++ /dev/null @@ -1,2 +0,0 @@ -Inside :meth:`io.RawIOBase.read`, validate that the count of bytes returned by -:meth:`io.RawIOBase.readinto` is valid (inside the provided buffer). diff --git a/Misc/NEWS.d/next/Library/2025-10-25-21-26-16.gh-issue-140593.OxlLc9.rst b/Misc/NEWS.d/next/Library/2025-10-25-21-26-16.gh-issue-140593.OxlLc9.rst deleted file mode 100644 index 612ad82dc64..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-25-21-26-16.gh-issue-140593.OxlLc9.rst +++ /dev/null @@ -1,3 +0,0 @@ -:mod:`xml.parsers.expat`: Fix a memory leak that could affect users with -:meth:`~xml.parsers.expat.xmlparser.ElementDeclHandler` set to a custom -element declaration handler. Patch by Sebastian Pipping. diff --git a/Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst b/Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst deleted file mode 100644 index 72666bb8224..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst +++ /dev/null @@ -1,4 +0,0 @@ -:func:`xml.etree.ElementTree.iterparse` now emits a :exc:`ResourceWarning` -when the iterator is not explicitly closed and was opened with a filename. -This helps developers identify and fix resource leaks. Patch by Osama -Abdelkader. diff --git a/Misc/NEWS.d/next/Library/2025-10-26-16-24-12.gh-issue-140633.ioayC1.rst b/Misc/NEWS.d/next/Library/2025-10-26-16-24-12.gh-issue-140633.ioayC1.rst deleted file mode 100644 index 9675a5d427a..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-26-16-24-12.gh-issue-140633.ioayC1.rst +++ /dev/null @@ -1,2 +0,0 @@ -Ignore :exc:`AttributeError` when setting a module's ``__file__`` attribute -when loading an extension module packaged as Apple Framework. diff --git a/Misc/NEWS.d/next/Library/2025-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst b/Misc/NEWS.d/next/Library/2025-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst deleted file mode 100644 index 2ae153a6480..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix an issue where closing :class:`io.BufferedWriter` could crash if -the closed attribute raised an exception on access or could not be -converted to a boolean. diff --git a/Misc/NEWS.d/next/Library/2025-10-27-13-49-31.gh-issue-140634.ULng9G.rst b/Misc/NEWS.d/next/Library/2025-10-27-13-49-31.gh-issue-140634.ULng9G.rst deleted file mode 100644 index b1ba9b26ad5..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-27-13-49-31.gh-issue-140634.ULng9G.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a reference counting bug in :meth:`!os.sched_param.__reduce__`. diff --git a/Misc/NEWS.d/next/Library/2025-10-27-16-01-41.gh-issue-125434.qy0uRA.rst b/Misc/NEWS.d/next/Library/2025-10-27-16-01-41.gh-issue-125434.qy0uRA.rst deleted file mode 100644 index 299e9f04df7..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-27-16-01-41.gh-issue-125434.qy0uRA.rst +++ /dev/null @@ -1,2 +0,0 @@ -Display thread name in :mod:`faulthandler` on Windows. Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/Library/2025-10-27-18-29-42.gh-issue-140590.LT9HHn.rst b/Misc/NEWS.d/next/Library/2025-10-27-18-29-42.gh-issue-140590.LT9HHn.rst deleted file mode 100644 index 802183673cf..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-27-18-29-42.gh-issue-140590.LT9HHn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix arguments checking for the :meth:`!functools.partial.__setstate__` that -may lead to internal state corruption and crash. Patch by Sergey Miryanov. diff --git a/Misc/NEWS.d/next/Library/2025-10-28-02-46-56.gh-issue-139946.aN3_uY.rst b/Misc/NEWS.d/next/Library/2025-10-28-02-46-56.gh-issue-139946.aN3_uY.rst deleted file mode 100644 index 4c68d4cd94b..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-28-02-46-56.gh-issue-139946.aN3_uY.rst +++ /dev/null @@ -1 +0,0 @@ -Error and warning keywords in ``argparse.ArgumentParser`` messages are now colorized when color output is enabled, fixing a visual inconsistency in which they remained plain text while other output was colorized. diff --git a/Misc/NEWS.d/next/Library/2025-10-28-17-43-51.gh-issue-140228.8kfHhO.rst b/Misc/NEWS.d/next/Library/2025-10-28-17-43-51.gh-issue-140228.8kfHhO.rst deleted file mode 100644 index b3b692bae62..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-28-17-43-51.gh-issue-140228.8kfHhO.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid making unnecessary filesystem calls for frozen modules in :mod:`linecache` when the global module cache is not present. diff --git a/Misc/NEWS.d/next/Library/2025-10-29-09-40-10.gh-issue-140741.L13UCV.rst b/Misc/NEWS.d/next/Library/2025-10-29-09-40-10.gh-issue-140741.L13UCV.rst deleted file mode 100644 index 9fa8c561a03..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-29-09-40-10.gh-issue-140741.L13UCV.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :func:`profiling.sampling.sample` incorrectly handling a -:exc:`FileNotFoundError` or :exc:`PermissionError`. diff --git a/Misc/NEWS.d/next/Library/2025-10-29-16-12-41.gh-issue-120057.qGj5Dl.rst b/Misc/NEWS.d/next/Library/2025-10-29-16-12-41.gh-issue-120057.qGj5Dl.rst deleted file mode 100644 index f6b42be1fbf..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-29-16-12-41.gh-issue-120057.qGj5Dl.rst +++ /dev/null @@ -1 +0,0 @@ -Add :func:`os.reload_environ` to ``os.__all__``. diff --git a/Misc/NEWS.d/next/Library/2025-10-29-16-53-00.gh-issue-140766.CNagKF.rst b/Misc/NEWS.d/next/Library/2025-10-29-16-53-00.gh-issue-140766.CNagKF.rst deleted file mode 100644 index fce8dd33757..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-29-16-53-00.gh-issue-140766.CNagKF.rst +++ /dev/null @@ -1 +0,0 @@ -Add :func:`enum.show_flag_values` and ``enum.bin`` to ``enum.__all__``. diff --git a/Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst b/Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst deleted file mode 100644 index 03856f0b9b6..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst +++ /dev/null @@ -1 +0,0 @@ -Initialize all Pdb's instance variables in ``__init__``, remove some hasattr/getattr diff --git a/Misc/NEWS.d/next/Library/2025-10-30-15-33-07.gh-issue-137821.8_Iavt.rst b/Misc/NEWS.d/next/Library/2025-10-30-15-33-07.gh-issue-137821.8_Iavt.rst deleted file mode 100644 index 7ccbfc3cb95..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-30-15-33-07.gh-issue-137821.8_Iavt.rst +++ /dev/null @@ -1,2 +0,0 @@ -Convert ``_json`` module to use Argument Clinic. -Patched by Yoonho Hann. diff --git a/Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst b/Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst deleted file mode 100644 index e14af7d9708..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst +++ /dev/null @@ -1 +0,0 @@ -Fix hang when cancelling process created by :func:`asyncio.create_subprocess_exec` or :func:`asyncio.create_subprocess_shell`. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2025-10-31-15-06-26.gh-issue-140691.JzHGtg.rst b/Misc/NEWS.d/next/Library/2025-10-31-15-06-26.gh-issue-140691.JzHGtg.rst deleted file mode 100644 index 84b6195c926..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-31-15-06-26.gh-issue-140691.JzHGtg.rst +++ /dev/null @@ -1,3 +0,0 @@ -In :mod:`urllib.request`, when opening a FTP URL fails because a data -connection cannot be made, the control connection's socket is now closed to -avoid a :exc:`ResourceWarning`. diff --git a/Misc/NEWS.d/next/Library/2025-10-31-16-25-13.gh-issue-140808.XBiQ4j.rst b/Misc/NEWS.d/next/Library/2025-10-31-16-25-13.gh-issue-140808.XBiQ4j.rst deleted file mode 100644 index 090f39c6e25..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-31-16-25-13.gh-issue-140808.XBiQ4j.rst +++ /dev/null @@ -1 +0,0 @@ -The internal class ``mailbox._ProxyFile`` is no longer a parameterized generic. diff --git a/Misc/NEWS.d/next/Library/2025-11-01-00-34-53.gh-issue-140826.JEDd7U.rst b/Misc/NEWS.d/next/Library/2025-11-01-00-34-53.gh-issue-140826.JEDd7U.rst deleted file mode 100644 index 875d15f2f89..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-01-00-34-53.gh-issue-140826.JEDd7U.rst +++ /dev/null @@ -1,2 +0,0 @@ -Now :class:`!winreg.HKEYType` objects are compared by their underlying Windows -registry handle value instead of their object identity. diff --git a/Misc/NEWS.d/next/Library/2025-11-01-00-36-14.gh-issue-140874.eAWt3K.rst b/Misc/NEWS.d/next/Library/2025-11-01-00-36-14.gh-issue-140874.eAWt3K.rst deleted file mode 100644 index a48162de76b..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-01-00-36-14.gh-issue-140874.eAWt3K.rst +++ /dev/null @@ -1 +0,0 @@ -Bump the version of pip bundled in ensurepip to version 25.3 diff --git a/Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst b/Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst deleted file mode 100644 index e1505764064..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add support of non-:term:`descriptor` callables in -:func:`functools.singledispatchmethod`. diff --git a/Misc/NEWS.d/next/Library/2025-11-02-09-37-22.gh-issue-140734.f8gST9.rst b/Misc/NEWS.d/next/Library/2025-11-02-09-37-22.gh-issue-140734.f8gST9.rst deleted file mode 100644 index 46582f7fcf4..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-02-09-37-22.gh-issue-140734.f8gST9.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`multiprocessing`: fix off-by-one error when checking the length -of a temporary socket file path. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2025-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst b/Misc/NEWS.d/next/Library/2025-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst deleted file mode 100644 index 2f7500d2955..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst +++ /dev/null @@ -1,3 +0,0 @@ -Correctly set :attr:`~OSError.errno` when :func:`socket.if_nametoindex` or -:func:`socket.if_indextoname` raise an :exc:`OSError`. Patch by Bénédikt -Tran. diff --git a/Misc/NEWS.d/next/Library/2025-11-02-19-23-32.gh-issue-140815.McEG-T.rst b/Misc/NEWS.d/next/Library/2025-11-02-19-23-32.gh-issue-140815.McEG-T.rst deleted file mode 100644 index 18c4d3836ef..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-02-19-23-32.gh-issue-140815.McEG-T.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`faulthandler` now detects if a frame or a code object is invalid or -freed. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst b/Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst deleted file mode 100644 index d36debec3ed..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst +++ /dev/null @@ -1 +0,0 @@ -Refactor the :mod:`pdb` parsing issue so positional arguments can pass through intuitively. diff --git a/Misc/NEWS.d/next/Library/2025-11-03-16-23-54.gh-issue-140797.DuFEeR.rst b/Misc/NEWS.d/next/Library/2025-11-03-16-23-54.gh-issue-140797.DuFEeR.rst deleted file mode 100644 index 493b740261e..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-03-16-23-54.gh-issue-140797.DuFEeR.rst +++ /dev/null @@ -1,2 +0,0 @@ -The undocumented :class:`!re.Scanner` class now forbids regular expressions containing capturing groups in its lexicon patterns. Patterns using capturing groups could -previously lead to crashes with segmentation fault. Use non-capturing groups (?:...) instead. diff --git a/Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst b/Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst deleted file mode 100644 index 9a31af9c110..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst +++ /dev/null @@ -1 +0,0 @@ -Add support of :term:`path-like objects ` and :term:`bytes-like objects ` in :func:`wave.open`. diff --git a/Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst b/Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst deleted file mode 100644 index dfa582bdbc8..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :meth:`annotationlib.ForwardRef.evaluate` returning -:class:`~annotationlib.ForwardRef` objects which don't update with new -globals. diff --git a/Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst b/Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst deleted file mode 100644 index e776515a9fb..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`mimetypes`: Update ``.exe``, ``.dll``, ``.rtf`` and (when -``strict=False``) ``.jpg`` to their correct IANA mime type. diff --git a/Misc/NEWS.d/next/Library/2025-11-06-15-11-50.gh-issue-141141.tgIfgH.rst b/Misc/NEWS.d/next/Library/2025-11-06-15-11-50.gh-issue-141141.tgIfgH.rst deleted file mode 100644 index f59ccfb33e7..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-06-15-11-50.gh-issue-141141.tgIfgH.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a thread safety issue with :func:`base64.b85decode`. Contributed by Benel Tayar. diff --git a/Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst b/Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst deleted file mode 100644 index 3e4fd1a5897..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst +++ /dev/null @@ -1,3 +0,0 @@ -Update ``io.FileIO.readall``, an implementation of :meth:`io.RawIOBase.readall`, -to follow :class:`io.IOBase` guidelines and raise :exc:`io.UnsupportedOperation` -when a file is in "w" mode rather than :exc:`OSError` diff --git a/Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst b/Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst deleted file mode 100644 index 62073280e32..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst +++ /dev/null @@ -1 +0,0 @@ -:mod:`mimetypes`: Update mime type for ``.ai`` files to ``application/pdf``. diff --git a/Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst b/Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst deleted file mode 100644 index bb425ce5df3..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix assertion failure in :func:`!io.BytesIO.readinto` and undefined behavior -arising when read position is above capcity in :class:`io.BytesIO`. diff --git a/Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst b/Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst deleted file mode 100644 index 37acaabfa3e..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst +++ /dev/null @@ -1 +0,0 @@ -Fix assertion failure in :meth:`io.TextIOWrapper.tell` when reading files with standalone carriage return (``\r``) line endings. diff --git a/Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst b/Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst deleted file mode 100644 index 32f4e39f6d5..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst +++ /dev/null @@ -1,5 +0,0 @@ -The :mod:`os.fork` and related forking APIs will no longer warn in the -common case where Linux or macOS platform APIs return the number of threads -in a process and find the answer to be 1 even when a -:func:`os.register_at_fork` ``after_in_parent=`` callback (re)starts a -thread. diff --git a/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst b/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst deleted file mode 100644 index 8436cd2415d..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst +++ /dev/null @@ -1,2 +0,0 @@ -Updated Tcl threading configuration in :mod:`_tkinter` to assume that -threads are always available in Tcl 9 and later. diff --git a/Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst b/Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst deleted file mode 100644 index bd3044002a2..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst +++ /dev/null @@ -1,2 +0,0 @@ -The :func:`statistics.stdev` and :func:`statistics.pstdev` functions now raise a -:exc:`ValueError` when the input contains an infinity or a NaN. diff --git a/Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst b/Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst deleted file mode 100644 index 328bfe067ad..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst +++ /dev/null @@ -1,4 +0,0 @@ -:mod:`ipaddress`: ensure that the methods -:meth:`IPv4Network.hosts() ` and -:meth:`IPv6Network.hosts() ` always return an -iterator. diff --git a/Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst b/Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst deleted file mode 100644 index 1d152bb5318..00000000000 --- a/Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst +++ /dev/null @@ -1 +0,0 @@ -Fix quadratic complexity in :func:`os.path.expandvars`. diff --git a/Misc/NEWS.d/next/Security/2025-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst b/Misc/NEWS.d/next/Security/2025-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst deleted file mode 100644 index 940a3ad5a72..00000000000 --- a/Misc/NEWS.d/next/Security/2025-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`email.message`: ensure linear complexity for legacy HTTP parameters -parsing. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Security/2025-08-15-23-08-44.gh-issue-137836.b55rhh.rst b/Misc/NEWS.d/next/Security/2025-08-15-23-08-44.gh-issue-137836.b55rhh.rst deleted file mode 100644 index c30c9439a76..00000000000 --- a/Misc/NEWS.d/next/Security/2025-08-15-23-08-44.gh-issue-137836.b55rhh.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add support of the "plaintext" element, RAWTEXT elements "xmp", "iframe", -"noembed" and "noframes", and optionally RAWTEXT element "noscript" in -:class:`html.parser.HTMLParser`. diff --git a/Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst b/Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst deleted file mode 100644 index f87fb1113ca..00000000000 --- a/Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst +++ /dev/null @@ -1 +0,0 @@ -Use exitcode ``1`` instead of ``5`` if :func:`unittest.TestCase.setUpClass` raises an exception diff --git a/Misc/NEWS.d/next/Tests/2025-10-15-00-52-12.gh-issue-140082.fpET50.rst b/Misc/NEWS.d/next/Tests/2025-10-15-00-52-12.gh-issue-140082.fpET50.rst deleted file mode 100644 index 70e70218254..00000000000 --- a/Misc/NEWS.d/next/Tests/2025-10-15-00-52-12.gh-issue-140082.fpET50.rst +++ /dev/null @@ -1,3 +0,0 @@ -Update ``python -m test`` to set ``FORCE_COLOR=1`` when being run with color -enabled so that :mod:`unittest` which is run by it with redirected output will -output in color. diff --git a/Misc/NEWS.d/next/Tests/2025-10-23-16-39-49.gh-issue-140482.ZMtyeD.rst b/Misc/NEWS.d/next/Tests/2025-10-23-16-39-49.gh-issue-140482.ZMtyeD.rst deleted file mode 100644 index 20747ad7f11..00000000000 --- a/Misc/NEWS.d/next/Tests/2025-10-23-16-39-49.gh-issue-140482.ZMtyeD.rst +++ /dev/null @@ -1 +0,0 @@ -Preserve and restore the state of ``stty echo`` as part of the test environment. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-09-20-20-31-54.gh-issue-139188.zfcxkW.rst b/Misc/NEWS.d/next/Tools-Demos/2025-09-20-20-31-54.gh-issue-139188.zfcxkW.rst deleted file mode 100644 index 9f52d0163ab..00000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2025-09-20-20-31-54.gh-issue-139188.zfcxkW.rst +++ /dev/null @@ -1 +0,0 @@ -Remove ``Tools/tz/zdump.py`` script. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-09-21-10-30-08.gh-issue-139198.Fm7NfU.rst b/Misc/NEWS.d/next/Tools-Demos/2025-09-21-10-30-08.gh-issue-139198.Fm7NfU.rst deleted file mode 100644 index 0dc589c3986..00000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2025-09-21-10-30-08.gh-issue-139198.Fm7NfU.rst +++ /dev/null @@ -1 +0,0 @@ -Remove ``Tools/scripts/checkpip.py`` script. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-10-29-15-20-19.gh-issue-140702.ZXtW8h.rst b/Misc/NEWS.d/next/Tools-Demos/2025-10-29-15-20-19.gh-issue-140702.ZXtW8h.rst deleted file mode 100644 index 9efbf0162dd..00000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2025-10-29-15-20-19.gh-issue-140702.ZXtW8h.rst +++ /dev/null @@ -1,2 +0,0 @@ -The iOS testbed app will now expose the ``GITHUB_ACTIONS`` environment -variable to iOS apps being tested. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst b/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst deleted file mode 100644 index 073c070413f..00000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst +++ /dev/null @@ -1 +0,0 @@ -The iOS testbed now correctly handles test arguments that contain spaces. diff --git a/Misc/NEWS.d/next/Windows/2025-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst b/Misc/NEWS.d/next/Windows/2025-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst deleted file mode 100644 index 6f25b867566..00000000000 --- a/Misc/NEWS.d/next/Windows/2025-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst +++ /dev/null @@ -1 +0,0 @@ -Update bundled liblzma to version 5.8.1. diff --git a/README.rst b/README.rst index a228aafb09c..bc1c1df2069 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.15.0 alpha 1 +This is Python version 3.15.0 alpha 2 ===================================== .. image:: https://github.com/python/cpython/actions/workflows/build.yml/badge.svg?branch=main&event=push From a52c39e2608557a710784d5876150578d2ae5183 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 18 Nov 2025 15:14:16 +0000 Subject: [PATCH 246/313] gh-135953: Refactor test_sampling_profiler into multiple files (#141689) --- .../test_profiling/test_sampling_profiler.py | 3360 ----------------- .../test_sampling_profiler/__init__.py | 9 + .../test_sampling_profiler/helpers.py | 101 + .../test_sampling_profiler/mocks.py | 38 + .../test_sampling_profiler/test_advanced.py | 264 ++ .../test_sampling_profiler/test_cli.py | 664 ++++ .../test_sampling_profiler/test_collectors.py | 896 +++++ .../test_integration.py | 804 ++++ .../test_sampling_profiler/test_modes.py | 514 +++ .../test_sampling_profiler/test_profiler.py | 656 ++++ Makefile.pre.in | 1 + 11 files changed, 3947 insertions(+), 3360 deletions(-) delete mode 100644 Lib/test/test_profiling/test_sampling_profiler.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/__init__.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/helpers.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/mocks.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/test_advanced.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/test_cli.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/test_collectors.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/test_integration.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/test_modes.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/test_profiler.py diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py deleted file mode 100644 index c2cc2ddd48a..00000000000 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ /dev/null @@ -1,3360 +0,0 @@ -"""Tests for the sampling profiler (profiling.sampling).""" - -import contextlib -import io -import json -import marshal -import os -import shutil -import socket -import subprocess -import sys -import tempfile -import unittest -from collections import namedtuple -from unittest import mock - -from profiling.sampling.pstats_collector import PstatsCollector -from profiling.sampling.stack_collector import ( - CollapsedStackCollector, - FlamegraphCollector, -) -from profiling.sampling.gecko_collector import GeckoCollector - -from test.support.os_helper import unlink -from test.support import ( - force_not_colorized_test_class, - SHORT_TIMEOUT, - script_helper, - os_helper, - SuppressCrashReport, -) -from test.support.socket_helper import find_unused_port -from test.support import requires_subprocess, is_emscripten -from test.support import captured_stdout, captured_stderr - -PROCESS_VM_READV_SUPPORTED = False - -try: - from _remote_debugging import PROCESS_VM_READV_SUPPORTED - import _remote_debugging -except ImportError: - raise unittest.SkipTest( - "Test only runs when _remote_debugging is available" - ) -else: - import profiling.sampling - from profiling.sampling.sample import SampleProfiler - - - -class MockFrameInfo: - """Mock FrameInfo for testing since the real one isn't accessible.""" - - def __init__(self, filename, lineno, funcname): - self.filename = filename - self.lineno = lineno - self.funcname = funcname - - def __repr__(self): - return f"MockFrameInfo(filename='{self.filename}', lineno={self.lineno}, funcname='{self.funcname}')" - - -class MockThreadInfo: - """Mock ThreadInfo for testing since the real one isn't accessible.""" - - def __init__(self, thread_id, frame_info, status=0): # Default to THREAD_STATE_RUNNING (0) - self.thread_id = thread_id - self.frame_info = frame_info - self.status = status - - def __repr__(self): - return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info}, status={self.status})" - - -class MockInterpreterInfo: - """Mock InterpreterInfo for testing since the real one isn't accessible.""" - - def __init__(self, interpreter_id, threads): - self.interpreter_id = interpreter_id - self.threads = threads - - def __repr__(self): - return f"MockInterpreterInfo(interpreter_id={self.interpreter_id}, threads={self.threads})" - - -skip_if_not_supported = unittest.skipIf( - ( - sys.platform != "darwin" - and sys.platform != "linux" - and sys.platform != "win32" - ), - "Test only runs on Linux, Windows and MacOS", -) - -SubprocessInfo = namedtuple('SubprocessInfo', ['process', 'socket']) - - -@contextlib.contextmanager -def test_subprocess(script): - # Find an unused port for socket communication - port = find_unused_port() - - # Inject socket connection code at the beginning of the script - socket_code = f''' -import socket -_test_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -_test_sock.connect(('localhost', {port})) -_test_sock.sendall(b"ready") -''' - - # Combine socket code with user script - full_script = socket_code + script - - # Create server socket to wait for process to be ready - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) - - proc = subprocess.Popen( - [sys.executable, "-c", full_script], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - - client_socket = None - try: - # Wait for process to connect and send ready signal - client_socket, _ = server_socket.accept() - server_socket.close() - response = client_socket.recv(1024) - if response != b"ready": - raise RuntimeError(f"Unexpected response from subprocess: {response}") - - yield SubprocessInfo(proc, client_socket) - finally: - if client_socket is not None: - client_socket.close() - if proc.poll() is None: - proc.kill() - proc.wait() - - -def close_and_unlink(file): - file.close() - unlink(file.name) - - -class TestSampleProfilerComponents(unittest.TestCase): - """Unit tests for individual profiler components.""" - - def test_mock_frame_info_with_empty_and_unicode_values(self): - """Test MockFrameInfo handles empty strings, unicode characters, and very long names correctly.""" - # Test with empty strings - frame = MockFrameInfo("", 0, "") - self.assertEqual(frame.filename, "") - self.assertEqual(frame.lineno, 0) - self.assertEqual(frame.funcname, "") - self.assertIn("filename=''", repr(frame)) - - # Test with unicode characters - frame = MockFrameInfo("文件.py", 42, "函数名") - self.assertEqual(frame.filename, "文件.py") - self.assertEqual(frame.funcname, "函数名") - - # Test with very long names - long_filename = "x" * 1000 + ".py" - long_funcname = "func_" + "x" * 1000 - frame = MockFrameInfo(long_filename, 999999, long_funcname) - self.assertEqual(frame.filename, long_filename) - self.assertEqual(frame.lineno, 999999) - self.assertEqual(frame.funcname, long_funcname) - - def test_pstats_collector_with_extreme_intervals_and_empty_data(self): - """Test PstatsCollector handles zero/large intervals, empty frames, None thread IDs, and duplicate frames.""" - # Test with zero interval - collector = PstatsCollector(sample_interval_usec=0) - self.assertEqual(collector.sample_interval_usec, 0) - - # Test with very large interval - collector = PstatsCollector(sample_interval_usec=1000000000) - self.assertEqual(collector.sample_interval_usec, 1000000000) - - # Test collecting empty frames list - collector = PstatsCollector(sample_interval_usec=1000) - collector.collect([]) - self.assertEqual(len(collector.result), 0) - - # Test collecting frames with None thread id - test_frames = [MockInterpreterInfo(0, [MockThreadInfo(None, [MockFrameInfo("file.py", 10, "func")])])] - collector.collect(test_frames) - # Should still process the frames - self.assertEqual(len(collector.result), 1) - - # Test collecting duplicate frames in same sample - test_frames = [ - MockInterpreterInfo( - 0, # interpreter_id - [MockThreadInfo( - 1, - [ - MockFrameInfo("file.py", 10, "func1"), - MockFrameInfo("file.py", 10, "func1"), # Duplicate - ], - )] - ) - ] - collector = PstatsCollector(sample_interval_usec=1000) - collector.collect(test_frames) - # Should count both occurrences - self.assertEqual( - collector.result[("file.py", 10, "func1")]["cumulative_calls"], 2 - ) - - def test_pstats_collector_single_frame_stacks(self): - """Test PstatsCollector with single-frame call stacks to trigger len(frames) <= 1 branch.""" - collector = PstatsCollector(sample_interval_usec=1000) - - # Test with exactly one frame (should trigger the <= 1 condition) - single_frame = [MockInterpreterInfo(0, [MockThreadInfo(1, [MockFrameInfo("single.py", 10, "single_func")])])] - collector.collect(single_frame) - - # Should record the single frame with inline call - self.assertEqual(len(collector.result), 1) - single_key = ("single.py", 10, "single_func") - self.assertIn(single_key, collector.result) - self.assertEqual(collector.result[single_key]["direct_calls"], 1) - self.assertEqual(collector.result[single_key]["cumulative_calls"], 1) - - # Test with empty frames (should also trigger <= 1 condition) - empty_frames = [MockInterpreterInfo(0, [MockThreadInfo(1, [])])] - collector.collect(empty_frames) - - # Should not add any new entries - self.assertEqual( - len(collector.result), 1 - ) # Still just the single frame - - # Test mixed single and multi-frame stacks - mixed_frames = [ - MockInterpreterInfo( - 0, - [ - MockThreadInfo( - 1, - [MockFrameInfo("single2.py", 20, "single_func2")], - ), # Single frame - MockThreadInfo( - 2, - [ # Multi-frame stack - MockFrameInfo("multi.py", 30, "multi_func1"), - MockFrameInfo("multi.py", 40, "multi_func2"), - ], - ), - ] - ), - ] - collector.collect(mixed_frames) - - # Should have recorded all functions - self.assertEqual( - len(collector.result), 4 - ) # single + single2 + multi1 + multi2 - - # Verify single frame handling - single2_key = ("single2.py", 20, "single_func2") - self.assertIn(single2_key, collector.result) - self.assertEqual(collector.result[single2_key]["direct_calls"], 1) - self.assertEqual(collector.result[single2_key]["cumulative_calls"], 1) - - # Verify multi-frame handling still works - multi1_key = ("multi.py", 30, "multi_func1") - multi2_key = ("multi.py", 40, "multi_func2") - self.assertIn(multi1_key, collector.result) - self.assertIn(multi2_key, collector.result) - self.assertEqual(collector.result[multi1_key]["direct_calls"], 1) - self.assertEqual( - collector.result[multi2_key]["cumulative_calls"], 1 - ) # Called from multi1 - - def test_collapsed_stack_collector_with_empty_and_deep_stacks(self): - """Test CollapsedStackCollector handles empty frames, single-frame stacks, and very deep call stacks.""" - collector = CollapsedStackCollector() - - # Test with empty frames - collector.collect([]) - self.assertEqual(len(collector.stack_counter), 0) - - # Test with single frame stack - test_frames = [MockInterpreterInfo(0, [MockThreadInfo(1, [("file.py", 10, "func")])])] - collector.collect(test_frames) - self.assertEqual(len(collector.stack_counter), 1) - ((path, thread_id), count), = collector.stack_counter.items() - self.assertEqual(path, (("file.py", 10, "func"),)) - self.assertEqual(thread_id, 1) - self.assertEqual(count, 1) - - # Test with very deep stack - deep_stack = [(f"file{i}.py", i, f"func{i}") for i in range(100)] - test_frames = [MockInterpreterInfo(0, [MockThreadInfo(1, deep_stack)])] - collector = CollapsedStackCollector() - collector.collect(test_frames) - # One aggregated path with 100 frames (reversed) - ((path_tuple, thread_id),), = (collector.stack_counter.keys(),) - self.assertEqual(len(path_tuple), 100) - self.assertEqual(path_tuple[0], ("file99.py", 99, "func99")) - self.assertEqual(path_tuple[-1], ("file0.py", 0, "func0")) - self.assertEqual(thread_id, 1) - - def test_pstats_collector_basic(self): - """Test basic PstatsCollector functionality.""" - collector = PstatsCollector(sample_interval_usec=1000) - - # Test empty state - self.assertEqual(len(collector.result), 0) - self.assertEqual(len(collector.stats), 0) - - # Test collecting sample data - test_frames = [ - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - MockFrameInfo("file.py", 10, "func1"), - MockFrameInfo("file.py", 20, "func2"), - ], - )] - ) - ] - collector.collect(test_frames) - - # Should have recorded calls for both functions - self.assertEqual(len(collector.result), 2) - self.assertIn(("file.py", 10, "func1"), collector.result) - self.assertIn(("file.py", 20, "func2"), collector.result) - - # Top-level function should have direct call - self.assertEqual( - collector.result[("file.py", 10, "func1")]["direct_calls"], 1 - ) - self.assertEqual( - collector.result[("file.py", 10, "func1")]["cumulative_calls"], 1 - ) - - # Calling function should have cumulative call but no direct calls - self.assertEqual( - collector.result[("file.py", 20, "func2")]["cumulative_calls"], 1 - ) - self.assertEqual( - collector.result[("file.py", 20, "func2")]["direct_calls"], 0 - ) - - def test_pstats_collector_create_stats(self): - """Test PstatsCollector stats creation.""" - collector = PstatsCollector( - sample_interval_usec=1000000 - ) # 1 second intervals - - test_frames = [ - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - MockFrameInfo("file.py", 10, "func1"), - MockFrameInfo("file.py", 20, "func2"), - ], - )] - ) - ] - collector.collect(test_frames) - collector.collect(test_frames) # Collect twice - - collector.create_stats() - - # Check stats format: (direct_calls, cumulative_calls, tt, ct, callers) - func1_stats = collector.stats[("file.py", 10, "func1")] - self.assertEqual(func1_stats[0], 2) # direct_calls (top of stack) - self.assertEqual(func1_stats[1], 2) # cumulative_calls - self.assertEqual( - func1_stats[2], 2.0 - ) # tt (total time - 2 samples * 1 sec) - self.assertEqual(func1_stats[3], 2.0) # ct (cumulative time) - - func2_stats = collector.stats[("file.py", 20, "func2")] - self.assertEqual( - func2_stats[0], 0 - ) # direct_calls (never top of stack) - self.assertEqual( - func2_stats[1], 2 - ) # cumulative_calls (appears in stack) - self.assertEqual(func2_stats[2], 0.0) # tt (no direct calls) - self.assertEqual(func2_stats[3], 2.0) # ct (cumulative time) - - def test_collapsed_stack_collector_basic(self): - collector = CollapsedStackCollector() - - # Test empty state - self.assertEqual(len(collector.stack_counter), 0) - - # Test collecting sample data - test_frames = [ - MockInterpreterInfo(0, [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])]) - ] - collector.collect(test_frames) - - # Should store one reversed path - self.assertEqual(len(collector.stack_counter), 1) - ((path, thread_id), count), = collector.stack_counter.items() - expected_tree = (("file.py", 20, "func2"), ("file.py", 10, "func1")) - self.assertEqual(path, expected_tree) - self.assertEqual(thread_id, 1) - self.assertEqual(count, 1) - - def test_collapsed_stack_collector_export(self): - collapsed_out = tempfile.NamedTemporaryFile(delete=False) - self.addCleanup(close_and_unlink, collapsed_out) - - collector = CollapsedStackCollector() - - test_frames1 = [ - MockInterpreterInfo(0, [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])]) - ] - test_frames2 = [ - MockInterpreterInfo(0, [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])]) - ] # Same stack - test_frames3 = [MockInterpreterInfo(0, [MockThreadInfo(1, [("other.py", 5, "other_func")])])] - - collector.collect(test_frames1) - collector.collect(test_frames2) - collector.collect(test_frames3) - - with (captured_stdout(), captured_stderr()): - collector.export(collapsed_out.name) - # Check file contents - with open(collapsed_out.name, "r") as f: - content = f.read() - - lines = content.strip().split("\n") - self.assertEqual(len(lines), 2) # Two unique stacks - - # Check collapsed format: tid:X;file:func:line;file:func:line count - stack1_expected = "tid:1;file.py:func2:20;file.py:func1:10 2" - stack2_expected = "tid:1;other.py:other_func:5 1" - - self.assertIn(stack1_expected, lines) - self.assertIn(stack2_expected, lines) - - def test_flamegraph_collector_basic(self): - """Test basic FlamegraphCollector functionality.""" - collector = FlamegraphCollector() - - # Empty collector should produce 'No Data' - data = collector._convert_to_flamegraph_format() - # With string table, name is now an index - resolve it using the strings array - strings = data.get("strings", []) - name_index = data.get("name", 0) - resolved_name = strings[name_index] if isinstance(name_index, int) and 0 <= name_index < len(strings) else str(name_index) - self.assertIn(resolved_name, ("No Data", "No significant data")) - - # Test collecting sample data - test_frames = [ - MockInterpreterInfo( - 0, - [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])], - ) - ] - collector.collect(test_frames) - - # Convert and verify structure: func2 -> func1 with counts = 1 - data = collector._convert_to_flamegraph_format() - # Expect promotion: root is the single child (func2), with func1 as its only child - strings = data.get("strings", []) - name_index = data.get("name", 0) - name = strings[name_index] if isinstance(name_index, int) and 0 <= name_index < len(strings) else str(name_index) - self.assertIsInstance(name, str) - self.assertTrue(name.startswith("Program Root: ")) - self.assertIn("func2 (file.py:20)", name) # formatted name - children = data.get("children", []) - self.assertEqual(len(children), 1) - child = children[0] - child_name_index = child.get("name", 0) - child_name = strings[child_name_index] if isinstance(child_name_index, int) and 0 <= child_name_index < len(strings) else str(child_name_index) - self.assertIn("func1 (file.py:10)", child_name) # formatted name - self.assertEqual(child["value"], 1) - - def test_flamegraph_collector_export(self): - """Test flamegraph HTML export functionality.""" - flamegraph_out = tempfile.NamedTemporaryFile( - suffix=".html", delete=False - ) - self.addCleanup(close_and_unlink, flamegraph_out) - - collector = FlamegraphCollector() - - # Create some test data (use Interpreter/Thread objects like runtime) - test_frames1 = [ - MockInterpreterInfo( - 0, - [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])], - ) - ] - test_frames2 = [ - MockInterpreterInfo( - 0, - [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])], - ) - ] # Same stack - test_frames3 = [ - MockInterpreterInfo(0, [MockThreadInfo(1, [("other.py", 5, "other_func")])]) - ] - - collector.collect(test_frames1) - collector.collect(test_frames2) - collector.collect(test_frames3) - - # Export flamegraph - with (captured_stdout(), captured_stderr()): - collector.export(flamegraph_out.name) - - # Verify file was created and contains valid data - self.assertTrue(os.path.exists(flamegraph_out.name)) - self.assertGreater(os.path.getsize(flamegraph_out.name), 0) - - # Check file contains HTML content - with open(flamegraph_out.name, "r", encoding="utf-8") as f: - content = f.read() - - # Should be valid HTML - self.assertIn("", content.lower()) - self.assertIn(" 0) - self.assertGreater(mock_collector.collect.call_count, 0) - self.assertLessEqual(mock_collector.collect.call_count, 3) - - def test_sample_profiler_missed_samples_warning(self): - """Test that the profiler warns about missed samples when sampling is too slow.""" - from profiling.sampling.sample import SampleProfiler - - mock_unwinder = mock.MagicMock() - mock_unwinder.get_stack_trace.return_value = [ - ( - 1, - [ - mock.MagicMock( - filename="test.py", lineno=10, funcname="test_func" - ) - ], - ) - ] - - with mock.patch( - "_remote_debugging.RemoteUnwinder" - ) as mock_unwinder_class: - mock_unwinder_class.return_value = mock_unwinder - - # Use very short interval that we'll miss - profiler = SampleProfiler( - pid=12345, sample_interval_usec=1000, all_threads=False - ) # 1ms interval - - mock_collector = mock.MagicMock() - - # Simulate slow sampling where we miss many samples - times = [ - 0.0, - 0.1, - 0.2, - 0.3, - 0.4, - 0.5, - 0.6, - 0.7, - ] # Extra time points to avoid StopIteration - - with mock.patch("time.perf_counter", side_effect=times): - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - profiler.sample(mock_collector, duration_sec=0.5) - - result = output.getvalue() - - # Should warn about missed samples - self.assertIn("Warning: missed", result) - self.assertIn("samples from the expected total", result) - - -@force_not_colorized_test_class -class TestPrintSampledStats(unittest.TestCase): - """Test the print_sampled_stats function.""" - - def setUp(self): - """Set up test data.""" - # Mock stats data - self.mock_stats = mock.MagicMock() - self.mock_stats.stats = { - ("file1.py", 10, "func1"): ( - 100, - 100, - 0.5, - 0.5, - {}, - ), # cc, nc, tt, ct, callers - ("file2.py", 20, "func2"): (50, 50, 0.25, 0.3, {}), - ("file3.py", 30, "func3"): (200, 200, 1.5, 2.0, {}), - ("file4.py", 40, "func4"): ( - 10, - 10, - 0.001, - 0.001, - {}, - ), # millisecond range - ("file5.py", 50, "func5"): ( - 5, - 5, - 0.000001, - 0.000002, - {}, - ), # microsecond range - } - - def test_print_sampled_stats_basic(self): - """Test basic print_sampled_stats functionality.""" - from profiling.sampling.sample import print_sampled_stats - - # Capture output - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats(self.mock_stats, sample_interval_usec=100) - - result = output.getvalue() - - # Check header is present - self.assertIn("Profile Stats:", result) - self.assertIn("nsamples", result) - self.assertIn("tottime", result) - self.assertIn("cumtime", result) - - # Check functions are present - self.assertIn("func1", result) - self.assertIn("func2", result) - self.assertIn("func3", result) - - def test_print_sampled_stats_sorting(self): - """Test different sorting options.""" - from profiling.sampling.sample import print_sampled_stats - - # Test sort by calls - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - self.mock_stats, sort=0, sample_interval_usec=100 - ) - - result = output.getvalue() - lines = result.strip().split("\n") - - # Find the data lines (skip header) - data_lines = [l for l in lines if "file" in l and ".py" in l] - # func3 should be first (200 calls) - self.assertIn("func3", data_lines[0]) - - # Test sort by time - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - self.mock_stats, sort=1, sample_interval_usec=100 - ) - - result = output.getvalue() - lines = result.strip().split("\n") - - data_lines = [l for l in lines if "file" in l and ".py" in l] - # func3 should be first (1.5s time) - self.assertIn("func3", data_lines[0]) - - def test_print_sampled_stats_limit(self): - """Test limiting output rows.""" - from profiling.sampling.sample import print_sampled_stats - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - self.mock_stats, limit=2, sample_interval_usec=100 - ) - - result = output.getvalue() - - # Count function entries in the main stats section (not in summary) - lines = result.split("\n") - # Find where the main stats section ends (before summary) - main_section_lines = [] - for line in lines: - if "Summary of Interesting Functions:" in line: - break - main_section_lines.append(line) - - # Count function entries only in main section - func_count = sum( - 1 - for line in main_section_lines - if "func" in line and ".py" in line - ) - self.assertEqual(func_count, 2) - - def test_print_sampled_stats_time_units(self): - """Test proper time unit selection.""" - from profiling.sampling.sample import print_sampled_stats - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats(self.mock_stats, sample_interval_usec=100) - - result = output.getvalue() - - # Should use seconds for the header since max time is > 1s - self.assertIn("tottime (s)", result) - self.assertIn("cumtime (s)", result) - - # Test with only microsecond-range times - micro_stats = mock.MagicMock() - micro_stats.stats = { - ("file1.py", 10, "func1"): (100, 100, 0.000005, 0.000010, {}), - } - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats(micro_stats, sample_interval_usec=100) - - result = output.getvalue() - - # Should use microseconds - self.assertIn("tottime (μs)", result) - self.assertIn("cumtime (μs)", result) - - def test_print_sampled_stats_summary(self): - """Test summary section generation.""" - from profiling.sampling.sample import print_sampled_stats - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - self.mock_stats, - show_summary=True, - sample_interval_usec=100, - ) - - result = output.getvalue() - - # Check summary sections are present - self.assertIn("Summary of Interesting Functions:", result) - self.assertIn( - "Functions with Highest Direct/Cumulative Ratio (Hot Spots):", - result, - ) - self.assertIn( - "Functions with Highest Call Frequency (Indirect Calls):", result - ) - self.assertIn( - "Functions with Highest Call Magnification (Cumulative/Direct):", - result, - ) - - def test_print_sampled_stats_no_summary(self): - """Test disabling summary output.""" - from profiling.sampling.sample import print_sampled_stats - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - self.mock_stats, - show_summary=False, - sample_interval_usec=100, - ) - - result = output.getvalue() - - # Summary should not be present - self.assertNotIn("Summary of Interesting Functions:", result) - - def test_print_sampled_stats_empty_stats(self): - """Test with empty stats.""" - from profiling.sampling.sample import print_sampled_stats - - empty_stats = mock.MagicMock() - empty_stats.stats = {} - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats(empty_stats, sample_interval_usec=100) - - result = output.getvalue() - - # Should still print header - self.assertIn("Profile Stats:", result) - - def test_print_sampled_stats_sample_percentage_sorting(self): - """Test sample percentage sorting options.""" - from profiling.sampling.sample import print_sampled_stats - - # Add a function with high sample percentage (more direct calls than func3's 200) - self.mock_stats.stats[("expensive.py", 60, "expensive_func")] = ( - 300, # direct calls (higher than func3's 200) - 300, # cumulative calls - 1.0, # total time - 1.0, # cumulative time - {}, - ) - - # Test sort by sample percentage - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - self.mock_stats, sort=3, sample_interval_usec=100 - ) # sample percentage - - result = output.getvalue() - lines = result.strip().split("\n") - - data_lines = [l for l in lines if ".py" in l and "func" in l] - # expensive_func should be first (highest sample percentage) - self.assertIn("expensive_func", data_lines[0]) - - def test_print_sampled_stats_with_recursive_calls(self): - """Test print_sampled_stats with recursive calls where nc != cc.""" - from profiling.sampling.sample import print_sampled_stats - - # Create stats with recursive calls (nc != cc) - recursive_stats = mock.MagicMock() - recursive_stats.stats = { - # (direct_calls, cumulative_calls, tt, ct, callers) - recursive function - ("recursive.py", 10, "factorial"): ( - 5, # direct_calls - 10, # cumulative_calls (appears more times in stack due to recursion) - 0.5, - 0.6, - {}, - ), - ("normal.py", 20, "normal_func"): ( - 3, # direct_calls - 3, # cumulative_calls (same as direct for non-recursive) - 0.2, - 0.2, - {}, - ), - } - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats(recursive_stats, sample_interval_usec=100) - - result = output.getvalue() - - # Should display recursive calls as "5/10" format - self.assertIn("5/10", result) # nc/cc format for recursive calls - self.assertIn("3", result) # just nc for non-recursive calls - self.assertIn("factorial", result) - self.assertIn("normal_func", result) - - def test_print_sampled_stats_with_zero_call_counts(self): - """Test print_sampled_stats with zero call counts to trigger division protection.""" - from profiling.sampling.sample import print_sampled_stats - - # Create stats with zero call counts - zero_stats = mock.MagicMock() - zero_stats.stats = { - ("file.py", 10, "zero_calls"): (0, 0, 0.0, 0.0, {}), # Zero calls - ("file.py", 20, "normal_func"): ( - 5, - 5, - 0.1, - 0.1, - {}, - ), # Normal function - } - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats(zero_stats, sample_interval_usec=100) - - result = output.getvalue() - - # Should handle zero call counts gracefully - self.assertIn("zero_calls", result) - self.assertIn("zero_calls", result) - self.assertIn("normal_func", result) - - def test_print_sampled_stats_sort_by_name(self): - """Test sort by function name option.""" - from profiling.sampling.sample import print_sampled_stats - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - self.mock_stats, sort=-1, sample_interval_usec=100 - ) # sort by name - - result = output.getvalue() - lines = result.strip().split("\n") - - # Find the data lines (skip header and summary) - # Data lines start with whitespace and numbers, and contain filename:lineno(function) - data_lines = [] - for line in lines: - # Skip header lines and summary sections - if ( - line.startswith(" ") - and "(" in line - and ")" in line - and not line.startswith( - " 1." - ) # Skip summary lines that start with times - and not line.startswith( - " 0." - ) # Skip summary lines that start with times - and not "per call" in line # Skip summary lines - and not "calls" in line # Skip summary lines - and not "total time" in line # Skip summary lines - and not "cumulative time" in line - ): # Skip summary lines - data_lines.append(line) - - # Extract just the function names for comparison - func_names = [] - import re - - for line in data_lines: - # Function name is between the last ( and ), accounting for ANSI color codes - match = re.search(r"\(([^)]+)\)$", line) - if match: - func_name = match.group(1) - # Remove ANSI color codes - func_name = re.sub(r"\x1b\[[0-9;]*m", "", func_name) - func_names.append(func_name) - - # Verify we extracted function names and they are sorted - self.assertGreater( - len(func_names), 0, "Should have extracted some function names" - ) - self.assertEqual( - func_names, - sorted(func_names), - f"Function names {func_names} should be sorted alphabetically", - ) - - def test_print_sampled_stats_with_zero_time_functions(self): - """Test summary sections with functions that have zero time.""" - from profiling.sampling.sample import print_sampled_stats - - # Create stats with zero-time functions - zero_time_stats = mock.MagicMock() - zero_time_stats.stats = { - ("file1.py", 10, "zero_time_func"): ( - 5, - 5, - 0.0, - 0.0, - {}, - ), # Zero time - ("file2.py", 20, "normal_func"): ( - 3, - 3, - 0.1, - 0.1, - {}, - ), # Normal time - } - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - zero_time_stats, - show_summary=True, - sample_interval_usec=100, - ) - - result = output.getvalue() - - # Should handle zero-time functions gracefully in summary - self.assertIn("Summary of Interesting Functions:", result) - self.assertIn("zero_time_func", result) - self.assertIn("normal_func", result) - - def test_print_sampled_stats_with_malformed_qualified_names(self): - """Test summary generation with function names that don't contain colons.""" - from profiling.sampling.sample import print_sampled_stats - - # Create stats with function names that would create malformed qualified names - malformed_stats = mock.MagicMock() - malformed_stats.stats = { - # Function name without clear module separation - ("no_colon_func", 10, "func"): (3, 3, 0.1, 0.1, {}), - ("", 20, "empty_filename_func"): (2, 2, 0.05, 0.05, {}), - ("normal.py", 30, "normal_func"): (5, 5, 0.2, 0.2, {}), - } - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - malformed_stats, - show_summary=True, - sample_interval_usec=100, - ) - - result = output.getvalue() - - # Should handle malformed names gracefully in summary aggregation - self.assertIn("Summary of Interesting Functions:", result) - # All function names should appear somewhere in the output - self.assertIn("func", result) - self.assertIn("empty_filename_func", result) - self.assertIn("normal_func", result) - - def test_print_sampled_stats_with_recursive_call_stats_creation(self): - """Test create_stats with recursive call data to trigger total_rec_calls branch.""" - collector = PstatsCollector(sample_interval_usec=1000000) # 1 second - - # Simulate recursive function data where total_rec_calls would be set - # We need to manually manipulate the collector result to test this branch - collector.result = { - ("recursive.py", 10, "factorial"): { - "total_rec_calls": 3, # Non-zero recursive calls - "direct_calls": 5, - "cumulative_calls": 10, - }, - ("normal.py", 20, "normal_func"): { - "total_rec_calls": 0, # Zero recursive calls - "direct_calls": 2, - "cumulative_calls": 5, - }, - } - - collector.create_stats() - - # Check that recursive calls are handled differently from non-recursive - factorial_stats = collector.stats[("recursive.py", 10, "factorial")] - normal_stats = collector.stats[("normal.py", 20, "normal_func")] - - # factorial should use cumulative_calls (10) as nc - self.assertEqual( - factorial_stats[1], 10 - ) # nc should be cumulative_calls - self.assertEqual(factorial_stats[0], 5) # cc should be direct_calls - - # normal_func should use cumulative_calls as nc - self.assertEqual(normal_stats[1], 5) # nc should be cumulative_calls - self.assertEqual(normal_stats[0], 2) # cc should be direct_calls - - -@skip_if_not_supported -@unittest.skipIf( - sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, - "Test only runs on Linux with process_vm_readv support", -) -class TestRecursiveFunctionProfiling(unittest.TestCase): - """Test profiling of recursive functions and complex call patterns.""" - - def test_recursive_function_call_counting(self): - """Test that recursive function calls are counted correctly.""" - collector = PstatsCollector(sample_interval_usec=1000) - - # Simulate a recursive call pattern: fibonacci(5) calling itself - recursive_frames = [ - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ # First sample: deep in recursion - MockFrameInfo("fib.py", 10, "fibonacci"), - MockFrameInfo("fib.py", 10, "fibonacci"), # recursive call - MockFrameInfo( - "fib.py", 10, "fibonacci" - ), # deeper recursion - MockFrameInfo("fib.py", 10, "fibonacci"), # even deeper - MockFrameInfo("main.py", 5, "main"), # main caller - ], - )] - ), - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ # Second sample: different recursion depth - MockFrameInfo("fib.py", 10, "fibonacci"), - MockFrameInfo("fib.py", 10, "fibonacci"), # recursive call - MockFrameInfo("main.py", 5, "main"), # main caller - ], - )] - ), - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ # Third sample: back to deeper recursion - MockFrameInfo("fib.py", 10, "fibonacci"), - MockFrameInfo("fib.py", 10, "fibonacci"), - MockFrameInfo("fib.py", 10, "fibonacci"), - MockFrameInfo("main.py", 5, "main"), - ], - )] - ), - ] - - for frames in recursive_frames: - collector.collect([frames]) - - collector.create_stats() - - # Check that recursive calls are counted properly - fib_key = ("fib.py", 10, "fibonacci") - main_key = ("main.py", 5, "main") - - self.assertIn(fib_key, collector.stats) - self.assertIn(main_key, collector.stats) - - # Fibonacci should have many calls due to recursion - fib_stats = collector.stats[fib_key] - direct_calls, cumulative_calls, tt, ct, callers = fib_stats - - # Should have recorded multiple calls (9 total appearances in samples) - self.assertEqual(cumulative_calls, 9) - self.assertGreater(tt, 0) # Should have some total time - self.assertGreater(ct, 0) # Should have some cumulative time - - # Main should have fewer calls - main_stats = collector.stats[main_key] - main_direct_calls, main_cumulative_calls = main_stats[0], main_stats[1] - self.assertEqual(main_direct_calls, 0) # Never directly executing - self.assertEqual(main_cumulative_calls, 3) # Appears in all 3 samples - - def test_nested_function_hierarchy(self): - """Test profiling of deeply nested function calls.""" - collector = PstatsCollector(sample_interval_usec=1000) - - # Simulate a deep call hierarchy - deep_call_frames = [ - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - MockFrameInfo("level1.py", 10, "level1_func"), - MockFrameInfo("level2.py", 20, "level2_func"), - MockFrameInfo("level3.py", 30, "level3_func"), - MockFrameInfo("level4.py", 40, "level4_func"), - MockFrameInfo("level5.py", 50, "level5_func"), - MockFrameInfo("main.py", 5, "main"), - ], - )] - ), - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ # Same hierarchy sampled again - MockFrameInfo("level1.py", 10, "level1_func"), - MockFrameInfo("level2.py", 20, "level2_func"), - MockFrameInfo("level3.py", 30, "level3_func"), - MockFrameInfo("level4.py", 40, "level4_func"), - MockFrameInfo("level5.py", 50, "level5_func"), - MockFrameInfo("main.py", 5, "main"), - ], - )] - ), - ] - - for frames in deep_call_frames: - collector.collect([frames]) - - collector.create_stats() - - # All levels should be recorded - for level in range(1, 6): - key = (f"level{level}.py", level * 10, f"level{level}_func") - self.assertIn(key, collector.stats) - - stats = collector.stats[key] - direct_calls, cumulative_calls, tt, ct, callers = stats - - # Each level should appear in stack twice (2 samples) - self.assertEqual(cumulative_calls, 2) - - # Only level1 (deepest) should have direct calls - if level == 1: - self.assertEqual(direct_calls, 2) - else: - self.assertEqual(direct_calls, 0) - - # Deeper levels should have lower cumulative time than higher levels - # (since they don't include time from functions they call) - if level == 1: # Deepest level with most time - self.assertGreater(ct, 0) - - def test_alternating_call_patterns(self): - """Test profiling with alternating call patterns.""" - collector = PstatsCollector(sample_interval_usec=1000) - - # Simulate alternating execution paths - pattern_frames = [ - # Pattern A: path through func_a - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - MockFrameInfo("module.py", 10, "func_a"), - MockFrameInfo("module.py", 30, "shared_func"), - MockFrameInfo("main.py", 5, "main"), - ], - )] - ), - # Pattern B: path through func_b - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - MockFrameInfo("module.py", 20, "func_b"), - MockFrameInfo("module.py", 30, "shared_func"), - MockFrameInfo("main.py", 5, "main"), - ], - )] - ), - # Pattern A again - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - MockFrameInfo("module.py", 10, "func_a"), - MockFrameInfo("module.py", 30, "shared_func"), - MockFrameInfo("main.py", 5, "main"), - ], - )] - ), - # Pattern B again - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - MockFrameInfo("module.py", 20, "func_b"), - MockFrameInfo("module.py", 30, "shared_func"), - MockFrameInfo("main.py", 5, "main"), - ], - )] - ), - ] - - for frames in pattern_frames: - collector.collect([frames]) - - collector.create_stats() - - # Check that both paths are recorded equally - func_a_key = ("module.py", 10, "func_a") - func_b_key = ("module.py", 20, "func_b") - shared_key = ("module.py", 30, "shared_func") - main_key = ("main.py", 5, "main") - - # func_a and func_b should each be directly executing twice - self.assertEqual(collector.stats[func_a_key][0], 2) # direct_calls - self.assertEqual(collector.stats[func_a_key][1], 2) # cumulative_calls - self.assertEqual(collector.stats[func_b_key][0], 2) # direct_calls - self.assertEqual(collector.stats[func_b_key][1], 2) # cumulative_calls - - # shared_func should appear in all samples (4 times) but never directly executing - self.assertEqual(collector.stats[shared_key][0], 0) # direct_calls - self.assertEqual(collector.stats[shared_key][1], 4) # cumulative_calls - - # main should appear in all samples but never directly executing - self.assertEqual(collector.stats[main_key][0], 0) # direct_calls - self.assertEqual(collector.stats[main_key][1], 4) # cumulative_calls - - def test_collapsed_stack_with_recursion(self): - """Test collapsed stack collector with recursive patterns.""" - collector = CollapsedStackCollector() - - # Recursive call pattern - recursive_frames = [ - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - ("factorial.py", 10, "factorial"), - ("factorial.py", 10, "factorial"), # recursive - ("factorial.py", 10, "factorial"), # deeper - ("main.py", 5, "main"), - ], - )] - ), - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - ("factorial.py", 10, "factorial"), - ("factorial.py", 10, "factorial"), # different depth - ("main.py", 5, "main"), - ], - )] - ), - ] - - for frames in recursive_frames: - collector.collect([frames]) - - # Should capture both call paths - self.assertEqual(len(collector.stack_counter), 2) - - # First path should be longer (deeper recursion) than the second - path_tuples = list(collector.stack_counter.keys()) - paths = [p[0] for p in path_tuples] # Extract just the call paths - lengths = [len(p) for p in paths] - self.assertNotEqual(lengths[0], lengths[1]) - - # Both should contain factorial calls - self.assertTrue(any(any(f[2] == "factorial" for f in p) for p in paths)) - - # Verify total occurrences via aggregation - factorial_key = ("factorial.py", 10, "factorial") - main_key = ("main.py", 5, "main") - - def total_occurrences(func): - total = 0 - for (path, thread_id), count in collector.stack_counter.items(): - total += sum(1 for f in path if f == func) * count - return total - - self.assertEqual(total_occurrences(factorial_key), 5) - self.assertEqual(total_occurrences(main_key), 2) - - -@requires_subprocess() -@skip_if_not_supported -class TestSampleProfilerIntegration(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.test_script = ''' -import time -import os - -def slow_fibonacci(n): - """Recursive fibonacci - should show up prominently in profiler.""" - if n <= 1: - return n - return slow_fibonacci(n-1) + slow_fibonacci(n-2) - -def cpu_intensive_work(): - """CPU intensive work that should show in profiler.""" - result = 0 - for i in range(10000): - result += i * i - if i % 100 == 0: - result = result % 1000000 - return result - -def medium_computation(): - """Medium complexity function.""" - result = 0 - for i in range(100): - result += i * i - return result - -def fast_loop(): - """Fast simple loop.""" - total = 0 - for i in range(50): - total += i - return total - -def nested_calls(): - """Test nested function calls.""" - def level1(): - def level2(): - return medium_computation() - return level2() - return level1() - -def main_loop(): - """Main test loop with different execution paths.""" - iteration = 0 - - while True: - iteration += 1 - - # Different execution paths - focus on CPU intensive work - if iteration % 3 == 0: - # Very CPU intensive - result = cpu_intensive_work() - elif iteration % 5 == 0: - # Expensive recursive operation - result = slow_fibonacci(12) - else: - # Medium operation - result = nested_calls() - - # No sleep - keep CPU busy - -if __name__ == "__main__": - main_loop() -''' - - def test_sampling_basic_functionality(self): - with ( - test_subprocess(self.test_script) as subproc, - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=2, - sample_interval_usec=1000, # 1ms - show_summary=False, - ) - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - - output = captured_output.getvalue() - - # Basic checks on output - self.assertIn("Captured", output) - self.assertIn("samples", output) - self.assertIn("Profile Stats", output) - - # Should see some of our test functions - self.assertIn("slow_fibonacci", output) - - def test_sampling_with_pstats_export(self): - pstats_out = tempfile.NamedTemporaryFile( - suffix=".pstats", delete=False - ) - self.addCleanup(close_and_unlink, pstats_out) - - with test_subprocess(self.test_script) as subproc: - # Suppress profiler output when testing file export - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=1, - filename=pstats_out.name, - sample_interval_usec=10000, - ) - except PermissionError: - self.skipTest( - "Insufficient permissions for remote profiling" - ) - - # Verify file was created and contains valid data - self.assertTrue(os.path.exists(pstats_out.name)) - self.assertGreater(os.path.getsize(pstats_out.name), 0) - - # Try to load the stats file - with open(pstats_out.name, "rb") as f: - stats_data = marshal.load(f) - - # Should be a dictionary with the sampled marker - self.assertIsInstance(stats_data, dict) - self.assertIn(("__sampled__",), stats_data) - self.assertTrue(stats_data[("__sampled__",)]) - - # Should have some function data - function_entries = [ - k for k in stats_data.keys() if k != ("__sampled__",) - ] - self.assertGreater(len(function_entries), 0) - - def test_sampling_with_collapsed_export(self): - collapsed_file = tempfile.NamedTemporaryFile( - suffix=".txt", delete=False - ) - self.addCleanup(close_and_unlink, collapsed_file) - - with ( - test_subprocess(self.test_script) as subproc, - ): - # Suppress profiler output when testing file export - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=1, - filename=collapsed_file.name, - output_format="collapsed", - sample_interval_usec=10000, - ) - except PermissionError: - self.skipTest( - "Insufficient permissions for remote profiling" - ) - - # Verify file was created and contains valid data - self.assertTrue(os.path.exists(collapsed_file.name)) - self.assertGreater(os.path.getsize(collapsed_file.name), 0) - - # Check file format - with open(collapsed_file.name, "r") as f: - content = f.read() - - lines = content.strip().split("\n") - self.assertGreater(len(lines), 0) - - # Each line should have format: stack_trace count - for line in lines: - parts = line.rsplit(" ", 1) - self.assertEqual(len(parts), 2) - - stack_trace, count_str = parts - self.assertGreater(len(stack_trace), 0) - self.assertTrue(count_str.isdigit()) - self.assertGreater(int(count_str), 0) - - # Stack trace should contain semicolon-separated entries - if ";" in stack_trace: - stack_parts = stack_trace.split(";") - for part in stack_parts: - # Each part should be file:function:line - self.assertIn(":", part) - - def test_sampling_all_threads(self): - with ( - test_subprocess(self.test_script) as subproc, - # Suppress profiler output - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=1, - all_threads=True, - sample_interval_usec=10000, - show_summary=False, - ) - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - - # Just verify that sampling completed without error - # We're not testing output format here - - def test_sample_target_script(self): - script_file = tempfile.NamedTemporaryFile(delete=False) - script_file.write(self.test_script.encode("utf-8")) - script_file.flush() - self.addCleanup(close_and_unlink, script_file) - - test_args = ["profiling.sampling.sample", "-d", "1", script_file.name] - - with ( - mock.patch("sys.argv", test_args), - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.main() - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - - output = captured_output.getvalue() - - # Basic checks on output - self.assertIn("Captured", output) - self.assertIn("samples", output) - self.assertIn("Profile Stats", output) - - # Should see some of our test functions - self.assertIn("slow_fibonacci", output) - - def test_sample_target_module(self): - tempdir = tempfile.TemporaryDirectory(delete=False) - self.addCleanup(lambda x: shutil.rmtree(x), tempdir.name) - - module_path = os.path.join(tempdir.name, "test_module.py") - - with open(module_path, "w") as f: - f.write(self.test_script) - - test_args = ["profiling.sampling.sample", "-d", "1", "-m", "test_module"] - - with ( - mock.patch("sys.argv", test_args), - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - # Change to temp directory so subprocess can find the module - contextlib.chdir(tempdir.name), - ): - try: - profiling.sampling.sample.main() - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - - output = captured_output.getvalue() - - # Basic checks on output - self.assertIn("Captured", output) - self.assertIn("samples", output) - self.assertIn("Profile Stats", output) - - # Should see some of our test functions - self.assertIn("slow_fibonacci", output) - - -@skip_if_not_supported -@unittest.skipIf( - sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, - "Test only runs on Linux with process_vm_readv support", -) -class TestSampleProfilerErrorHandling(unittest.TestCase): - def test_invalid_pid(self): - with self.assertRaises((OSError, RuntimeError)): - profiling.sampling.sample.sample(-1, duration_sec=1) - - def test_process_dies_during_sampling(self): - with test_subprocess("import time; time.sleep(0.5); exit()") as subproc: - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=2, # Longer than process lifetime - sample_interval_usec=50000, - ) - except PermissionError: - self.skipTest( - "Insufficient permissions for remote profiling" - ) - - output = captured_output.getvalue() - - self.assertIn("Error rate", output) - - def test_invalid_output_format(self): - with self.assertRaises(ValueError): - profiling.sampling.sample.sample( - os.getpid(), - duration_sec=1, - output_format="invalid_format", - ) - - def test_invalid_output_format_with_mocked_profiler(self): - """Test invalid output format with proper mocking to avoid permission issues.""" - with mock.patch( - "profiling.sampling.sample.SampleProfiler" - ) as mock_profiler_class: - mock_profiler = mock.MagicMock() - mock_profiler_class.return_value = mock_profiler - - with self.assertRaises(ValueError) as cm: - profiling.sampling.sample.sample( - 12345, - duration_sec=1, - output_format="unknown_format", - ) - - # Should raise ValueError with the invalid format name - self.assertIn( - "Invalid output format: unknown_format", str(cm.exception) - ) - - def test_is_process_running(self): - with test_subprocess("import time; time.sleep(1000)") as subproc: - try: - profiler = SampleProfiler(pid=subproc.process.pid, sample_interval_usec=1000, all_threads=False) - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) - self.assertTrue(profiler._is_process_running()) - self.assertIsNotNone(profiler.unwinder.get_stack_trace()) - subproc.process.kill() - subproc.process.wait() - self.assertRaises(ProcessLookupError, profiler.unwinder.get_stack_trace) - - # Exit the context manager to ensure the process is terminated - self.assertFalse(profiler._is_process_running()) - self.assertRaises(ProcessLookupError, profiler.unwinder.get_stack_trace) - - @unittest.skipUnless(sys.platform == "linux", "Only valid on Linux") - def test_esrch_signal_handling(self): - with test_subprocess("import time; time.sleep(1000)") as subproc: - try: - unwinder = _remote_debugging.RemoteUnwinder(subproc.process.pid) - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) - initial_trace = unwinder.get_stack_trace() - self.assertIsNotNone(initial_trace) - - subproc.process.kill() - - # Wait for the process to die and try to get another trace - subproc.process.wait() - - with self.assertRaises(ProcessLookupError): - unwinder.get_stack_trace() - - def test_valid_output_formats(self): - """Test that all valid output formats are accepted.""" - valid_formats = ["pstats", "collapsed", "flamegraph", "gecko"] - - tempdir = tempfile.TemporaryDirectory(delete=False) - self.addCleanup(shutil.rmtree, tempdir.name) - - - with (contextlib.chdir(tempdir.name), captured_stdout(), captured_stderr()): - for fmt in valid_formats: - try: - # This will likely fail with permissions, but the format should be valid - profiling.sampling.sample.sample( - os.getpid(), - duration_sec=0.1, - output_format=fmt, - filename=f"test_{fmt}.out", - ) - except (OSError, RuntimeError, PermissionError): - # Expected errors - we just want to test format validation - pass - - def test_script_error_treatment(self): - script_file = tempfile.NamedTemporaryFile("w", delete=False, suffix=".py") - script_file.write("open('nonexistent_file.txt')\n") - script_file.close() - self.addCleanup(os.unlink, script_file.name) - - result = subprocess.run( - [sys.executable, "-m", "profiling.sampling.sample", "-d", "1", script_file.name], - capture_output=True, - text=True, - ) - output = result.stdout + result.stderr - - if "PermissionError" in output: - self.skipTest("Insufficient permissions for remote profiling") - self.assertNotIn("Script file not found", output) - self.assertIn("No such file or directory: 'nonexistent_file.txt'", output) - - -class TestSampleProfilerCLI(unittest.TestCase): - def _setup_sync_mocks(self, mock_socket, mock_popen): - """Helper to set up socket and process mocks for coordinator tests.""" - # Mock the sync socket with context manager support - mock_sock_instance = mock.MagicMock() - mock_sock_instance.getsockname.return_value = ("127.0.0.1", 12345) - - # Mock the connection with context manager support - mock_conn = mock.MagicMock() - mock_conn.recv.return_value = b"ready" - mock_conn.__enter__.return_value = mock_conn - mock_conn.__exit__.return_value = None - - # Mock accept() to return (connection, address) and support indexing - mock_accept_result = mock.MagicMock() - mock_accept_result.__getitem__.return_value = mock_conn # [0] returns the connection - mock_sock_instance.accept.return_value = mock_accept_result - - # Mock socket with context manager support - mock_sock_instance.__enter__.return_value = mock_sock_instance - mock_sock_instance.__exit__.return_value = None - mock_socket.return_value = mock_sock_instance - - # Mock the subprocess - mock_process = mock.MagicMock() - mock_process.pid = 12345 - mock_process.poll.return_value = None - mock_popen.return_value = mock_process - return mock_process - - def _verify_coordinator_command(self, mock_popen, expected_target_args): - """Helper to verify the coordinator command was called correctly.""" - args, kwargs = mock_popen.call_args - coordinator_cmd = args[0] - self.assertEqual(coordinator_cmd[0], sys.executable) - self.assertEqual(coordinator_cmd[1], "-m") - self.assertEqual(coordinator_cmd[2], "profiling.sampling._sync_coordinator") - self.assertEqual(coordinator_cmd[3], "12345") # port - # cwd is coordinator_cmd[4] - self.assertEqual(coordinator_cmd[5:], expected_target_args) - - @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") - def test_cli_module_argument_parsing(self): - test_args = ["profiling.sampling.sample", "-m", "mymodule"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("subprocess.Popen") as mock_popen, - mock.patch("socket.socket") as mock_socket, - ): - self._setup_sync_mocks(mock_socket, mock_popen) - profiling.sampling.sample.main() - - self._verify_coordinator_command(mock_popen, ("-m", "mymodule")) - mock_sample.assert_called_once_with( - 12345, - sort=2, # default sort (sort_value from args.sort) - sample_interval_usec=100, - duration_sec=10, - filename=None, - all_threads=False, - limit=15, - show_summary=True, - output_format="pstats", - realtime_stats=False, - mode=0, - native=False, - gc=True, - ) - - @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") - def test_cli_module_with_arguments(self): - test_args = ["profiling.sampling.sample", "-m", "mymodule", "arg1", "arg2", "--flag"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("subprocess.Popen") as mock_popen, - mock.patch("socket.socket") as mock_socket, - ): - self._setup_sync_mocks(mock_socket, mock_popen) - profiling.sampling.sample.main() - - self._verify_coordinator_command(mock_popen, ("-m", "mymodule", "arg1", "arg2", "--flag")) - mock_sample.assert_called_once_with( - 12345, - sort=2, - sample_interval_usec=100, - duration_sec=10, - filename=None, - all_threads=False, - limit=15, - show_summary=True, - output_format="pstats", - realtime_stats=False, - mode=0, - native=False, - gc=True, - ) - - @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") - def test_cli_script_argument_parsing(self): - test_args = ["profiling.sampling.sample", "myscript.py"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("subprocess.Popen") as mock_popen, - mock.patch("socket.socket") as mock_socket, - ): - self._setup_sync_mocks(mock_socket, mock_popen) - profiling.sampling.sample.main() - - self._verify_coordinator_command(mock_popen, ("myscript.py",)) - mock_sample.assert_called_once_with( - 12345, - sort=2, - sample_interval_usec=100, - duration_sec=10, - filename=None, - all_threads=False, - limit=15, - show_summary=True, - output_format="pstats", - realtime_stats=False, - mode=0, - native=False, - gc=True, - ) - - @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") - def test_cli_script_with_arguments(self): - test_args = ["profiling.sampling.sample", "myscript.py", "arg1", "arg2", "--flag"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("subprocess.Popen") as mock_popen, - mock.patch("socket.socket") as mock_socket, - ): - # Use the helper to set up mocks consistently - mock_process = self._setup_sync_mocks(mock_socket, mock_popen) - # Override specific behavior for this test - mock_process.wait.side_effect = [subprocess.TimeoutExpired(test_args, 0.1), None] - - profiling.sampling.sample.main() - - # Verify the coordinator command was called - args, kwargs = mock_popen.call_args - coordinator_cmd = args[0] - self.assertEqual(coordinator_cmd[0], sys.executable) - self.assertEqual(coordinator_cmd[1], "-m") - self.assertEqual(coordinator_cmd[2], "profiling.sampling._sync_coordinator") - self.assertEqual(coordinator_cmd[3], "12345") # port - # cwd is coordinator_cmd[4] - self.assertEqual(coordinator_cmd[5:], ("myscript.py", "arg1", "arg2", "--flag")) - - def test_cli_mutually_exclusive_pid_module(self): - test_args = ["profiling.sampling.sample", "-p", "12345", "-m", "mymodule"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("sys.stderr", io.StringIO()) as mock_stderr, - self.assertRaises(SystemExit) as cm, - ): - profiling.sampling.sample.main() - - self.assertEqual(cm.exception.code, 2) # argparse error - error_msg = mock_stderr.getvalue() - self.assertIn("not allowed with argument", error_msg) - - def test_cli_mutually_exclusive_pid_script(self): - test_args = ["profiling.sampling.sample", "-p", "12345", "myscript.py"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("sys.stderr", io.StringIO()) as mock_stderr, - self.assertRaises(SystemExit) as cm, - ): - profiling.sampling.sample.main() - - self.assertEqual(cm.exception.code, 2) # argparse error - error_msg = mock_stderr.getvalue() - self.assertIn("only one target type can be specified", error_msg) - - def test_cli_no_target_specified(self): - test_args = ["profiling.sampling.sample", "-d", "5"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("sys.stderr", io.StringIO()) as mock_stderr, - self.assertRaises(SystemExit) as cm, - ): - profiling.sampling.sample.main() - - self.assertEqual(cm.exception.code, 2) # argparse error - error_msg = mock_stderr.getvalue() - self.assertIn("one of the arguments", error_msg) - - @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") - def test_cli_module_with_profiler_options(self): - test_args = [ - "profiling.sampling.sample", "-i", "1000", "-d", "30", "-a", - "--sort-tottime", "-l", "20", "-m", "mymodule", - ] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("subprocess.Popen") as mock_popen, - mock.patch("socket.socket") as mock_socket, - ): - self._setup_sync_mocks(mock_socket, mock_popen) - profiling.sampling.sample.main() - - self._verify_coordinator_command(mock_popen, ("-m", "mymodule")) - mock_sample.assert_called_once_with( - 12345, - sort=1, # sort-tottime - sample_interval_usec=1000, - duration_sec=30, - filename=None, - all_threads=True, - limit=20, - show_summary=True, - output_format="pstats", - realtime_stats=False, - mode=0, - native=False, - gc=True, - ) - - @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") - def test_cli_script_with_profiler_options(self): - """Test script with various profiler options.""" - test_args = [ - "profiling.sampling.sample", "-i", "2000", "-d", "60", - "--collapsed", "-o", "output.txt", - "myscript.py", "scriptarg", - ] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("subprocess.Popen") as mock_popen, - mock.patch("socket.socket") as mock_socket, - ): - self._setup_sync_mocks(mock_socket, mock_popen) - profiling.sampling.sample.main() - - self._verify_coordinator_command(mock_popen, ("myscript.py", "scriptarg")) - # Verify profiler options were passed correctly - mock_sample.assert_called_once_with( - 12345, - sort=2, # default sort - sample_interval_usec=2000, - duration_sec=60, - filename="output.txt", - all_threads=False, - limit=15, - show_summary=True, - output_format="collapsed", - realtime_stats=False, - mode=0, - native=False, - gc=True, - ) - - def test_cli_empty_module_name(self): - test_args = ["profiling.sampling.sample", "-m"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("sys.stderr", io.StringIO()) as mock_stderr, - self.assertRaises(SystemExit) as cm, - ): - profiling.sampling.sample.main() - - self.assertEqual(cm.exception.code, 2) # argparse error - error_msg = mock_stderr.getvalue() - self.assertIn("argument -m/--module: expected one argument", error_msg) - - @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") - def test_cli_long_module_option(self): - test_args = ["profiling.sampling.sample", "--module", "mymodule", "arg1"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("subprocess.Popen") as mock_popen, - mock.patch("socket.socket") as mock_socket, - ): - self._setup_sync_mocks(mock_socket, mock_popen) - profiling.sampling.sample.main() - - self._verify_coordinator_command(mock_popen, ("-m", "mymodule", "arg1")) - - def test_cli_complex_script_arguments(self): - test_args = [ - "profiling.sampling.sample", "script.py", - "--input", "file.txt", "-v", "--output=/tmp/out", "positional" - ] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("profiling.sampling.sample._run_with_sync") as mock_run_with_sync, - ): - mock_process = mock.MagicMock() - mock_process.pid = 12345 - mock_process.wait.side_effect = [subprocess.TimeoutExpired(test_args, 0.1), None] - mock_process.poll.return_value = None - mock_run_with_sync.return_value = mock_process - - profiling.sampling.sample.main() - - mock_run_with_sync.assert_called_once_with(( - sys.executable, "script.py", - "--input", "file.txt", "-v", "--output=/tmp/out", "positional", - )) - - def test_cli_collapsed_format_validation(self): - """Test that CLI properly validates incompatible options with collapsed format.""" - test_cases = [ - # Test sort options are invalid with collapsed - ( - ["profiling.sampling.sample", "--collapsed", "--sort-nsamples", "-p", "12345"], - "sort", - ), - ( - ["profiling.sampling.sample", "--collapsed", "--sort-tottime", "-p", "12345"], - "sort", - ), - ( - [ - "profiling.sampling.sample", - "--collapsed", - "--sort-cumtime", - "-p", - "12345", - ], - "sort", - ), - ( - [ - "profiling.sampling.sample", - "--collapsed", - "--sort-sample-pct", - "-p", - "12345", - ], - "sort", - ), - ( - [ - "profiling.sampling.sample", - "--collapsed", - "--sort-cumul-pct", - "-p", - "12345", - ], - "sort", - ), - ( - ["profiling.sampling.sample", "--collapsed", "--sort-name", "-p", "12345"], - "sort", - ), - # Test limit option is invalid with collapsed - (["profiling.sampling.sample", "--collapsed", "-l", "20", "-p", "12345"], "limit"), - ( - ["profiling.sampling.sample", "--collapsed", "--limit", "20", "-p", "12345"], - "limit", - ), - # Test no-summary option is invalid with collapsed - ( - ["profiling.sampling.sample", "--collapsed", "--no-summary", "-p", "12345"], - "summary", - ), - ] - - for test_args, expected_error_keyword in test_cases: - with ( - mock.patch("sys.argv", test_args), - mock.patch("sys.stderr", io.StringIO()) as mock_stderr, - self.assertRaises(SystemExit) as cm, - ): - profiling.sampling.sample.main() - - self.assertEqual(cm.exception.code, 2) # argparse error code - error_msg = mock_stderr.getvalue() - self.assertIn("error:", error_msg) - self.assertIn("--pstats format", error_msg) - - def test_cli_default_collapsed_filename(self): - """Test that collapsed format gets a default filename when not specified.""" - test_args = ["profiling.sampling.sample", "--collapsed", "-p", "12345"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - ): - profiling.sampling.sample.main() - - # Check that filename was set to default collapsed format - mock_sample.assert_called_once() - call_args = mock_sample.call_args[1] - self.assertEqual(call_args["output_format"], "collapsed") - self.assertEqual(call_args["filename"], "collapsed.12345.txt") - - def test_cli_custom_output_filenames(self): - """Test custom output filenames for both formats.""" - test_cases = [ - ( - ["profiling.sampling.sample", "--pstats", "-o", "custom.pstats", "-p", "12345"], - "custom.pstats", - "pstats", - ), - ( - ["profiling.sampling.sample", "--collapsed", "-o", "custom.txt", "-p", "12345"], - "custom.txt", - "collapsed", - ), - ] - - for test_args, expected_filename, expected_format in test_cases: - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - ): - profiling.sampling.sample.main() - - mock_sample.assert_called_once() - call_args = mock_sample.call_args[1] - self.assertEqual(call_args["filename"], expected_filename) - self.assertEqual(call_args["output_format"], expected_format) - - def test_cli_missing_required_arguments(self): - """Test that CLI requires PID argument.""" - with ( - mock.patch("sys.argv", ["profiling.sampling.sample"]), - mock.patch("sys.stderr", io.StringIO()), - ): - with self.assertRaises(SystemExit): - profiling.sampling.sample.main() - - def test_cli_mutually_exclusive_format_options(self): - """Test that pstats and collapsed options are mutually exclusive.""" - with ( - mock.patch( - "sys.argv", - ["profiling.sampling.sample", "--pstats", "--collapsed", "-p", "12345"], - ), - mock.patch("sys.stderr", io.StringIO()), - ): - with self.assertRaises(SystemExit): - profiling.sampling.sample.main() - - def test_argument_parsing_basic(self): - test_args = ["profiling.sampling.sample", "-p", "12345"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - ): - profiling.sampling.sample.main() - - mock_sample.assert_called_once_with( - 12345, - sample_interval_usec=100, - duration_sec=10, - filename=None, - all_threads=False, - limit=15, - sort=2, - show_summary=True, - output_format="pstats", - realtime_stats=False, - mode=0, - native=False, - gc=True, - ) - - def test_sort_options(self): - sort_options = [ - ("--sort-nsamples", 0), - ("--sort-tottime", 1), - ("--sort-cumtime", 2), - ("--sort-sample-pct", 3), - ("--sort-cumul-pct", 4), - ("--sort-name", -1), - ] - - for option, expected_sort_value in sort_options: - test_args = ["profiling.sampling.sample", option, "-p", "12345"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - ): - profiling.sampling.sample.main() - - mock_sample.assert_called_once() - call_args = mock_sample.call_args[1] - self.assertEqual( - call_args["sort"], - expected_sort_value, - ) - mock_sample.reset_mock() - - -class TestCpuModeFiltering(unittest.TestCase): - """Test CPU mode filtering functionality (--mode=cpu).""" - - def test_mode_validation(self): - """Test that CLI validates mode choices correctly.""" - # Invalid mode choice should raise SystemExit - test_args = ["profiling.sampling.sample", "--mode", "invalid", "-p", "12345"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("sys.stderr", io.StringIO()) as mock_stderr, - self.assertRaises(SystemExit) as cm, - ): - profiling.sampling.sample.main() - - self.assertEqual(cm.exception.code, 2) # argparse error - error_msg = mock_stderr.getvalue() - self.assertIn("invalid choice", error_msg) - - def test_frames_filtered_with_skip_idle(self): - """Test that frames are actually filtered when skip_idle=True.""" - # Import thread status flags - try: - from _remote_debugging import THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU - except ImportError: - THREAD_STATUS_HAS_GIL = (1 << 0) - THREAD_STATUS_ON_CPU = (1 << 1) - - # Create mock frames with different thread statuses - class MockThreadInfoWithStatus: - def __init__(self, thread_id, frame_info, status): - self.thread_id = thread_id - self.frame_info = frame_info - self.status = status - - # Create test data: active thread (HAS_GIL | ON_CPU), idle thread (neither), and another active thread - ACTIVE_STATUS = THREAD_STATUS_HAS_GIL | THREAD_STATUS_ON_CPU # Has GIL and on CPU - IDLE_STATUS = 0 # Neither has GIL nor on CPU - - test_frames = [ - MockInterpreterInfo(0, [ - MockThreadInfoWithStatus(1, [MockFrameInfo("active1.py", 10, "active_func1")], ACTIVE_STATUS), - MockThreadInfoWithStatus(2, [MockFrameInfo("idle.py", 20, "idle_func")], IDLE_STATUS), - MockThreadInfoWithStatus(3, [MockFrameInfo("active2.py", 30, "active_func2")], ACTIVE_STATUS), - ]) - ] - - # Test with skip_idle=True - should only process running threads - collector_skip = PstatsCollector(sample_interval_usec=1000, skip_idle=True) - collector_skip.collect(test_frames) - - # Should only have functions from running threads (status 0) - active1_key = ("active1.py", 10, "active_func1") - active2_key = ("active2.py", 30, "active_func2") - idle_key = ("idle.py", 20, "idle_func") - - self.assertIn(active1_key, collector_skip.result) - self.assertIn(active2_key, collector_skip.result) - self.assertNotIn(idle_key, collector_skip.result) # Idle thread should be filtered out - - # Test with skip_idle=False - should process all threads - collector_no_skip = PstatsCollector(sample_interval_usec=1000, skip_idle=False) - collector_no_skip.collect(test_frames) - - # Should have functions from all threads - self.assertIn(active1_key, collector_no_skip.result) - self.assertIn(active2_key, collector_no_skip.result) - self.assertIn(idle_key, collector_no_skip.result) # Idle thread should be included - - @requires_subprocess() - def test_cpu_mode_integration_filtering(self): - """Integration test: CPU mode should only capture active threads, not idle ones.""" - # Script with one mostly-idle thread and one CPU-active thread - cpu_vs_idle_script = ''' -import time -import threading - -cpu_ready = threading.Event() - -def idle_worker(): - time.sleep(999999) - -def cpu_active_worker(): - cpu_ready.set() - x = 1 - while True: - x += 1 - -def main(): - # Start both threads - idle_thread = threading.Thread(target=idle_worker) - cpu_thread = threading.Thread(target=cpu_active_worker) - idle_thread.start() - cpu_thread.start() - - # Wait for CPU thread to be running, then signal test - cpu_ready.wait() - _test_sock.sendall(b"threads_ready") - - idle_thread.join() - cpu_thread.join() - -main() - -''' - with test_subprocess(cpu_vs_idle_script) as subproc: - # Wait for signal that threads are running - response = subproc.socket.recv(1024) - self.assertEqual(response, b"threads_ready") - - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=2.0, - sample_interval_usec=5000, - mode=1, # CPU mode - show_summary=False, - all_threads=True, - ) - except (PermissionError, RuntimeError) as e: - self.skipTest("Insufficient permissions for remote profiling") - - cpu_mode_output = captured_output.getvalue() - - # Test wall-clock mode (mode=0) - should capture both functions - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=2.0, - sample_interval_usec=5000, - mode=0, # Wall-clock mode - show_summary=False, - all_threads=True, - ) - except (PermissionError, RuntimeError) as e: - self.skipTest("Insufficient permissions for remote profiling") - - wall_mode_output = captured_output.getvalue() - - # Verify both modes captured samples - self.assertIn("Captured", cpu_mode_output) - self.assertIn("samples", cpu_mode_output) - self.assertIn("Captured", wall_mode_output) - self.assertIn("samples", wall_mode_output) - - # CPU mode should strongly favor cpu_active_worker over mostly_idle_worker - self.assertIn("cpu_active_worker", cpu_mode_output) - self.assertNotIn("idle_worker", cpu_mode_output) - - # Wall-clock mode should capture both types of work - self.assertIn("cpu_active_worker", wall_mode_output) - self.assertIn("idle_worker", wall_mode_output) - - def test_cpu_mode_with_no_samples(self): - """Test that CPU mode handles no samples gracefully when no samples are collected.""" - # Mock a collector that returns empty stats - mock_collector = mock.MagicMock() - mock_collector.stats = {} - mock_collector.create_stats = mock.MagicMock() - - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - mock.patch("profiling.sampling.sample.PstatsCollector", return_value=mock_collector), - mock.patch("profiling.sampling.sample.SampleProfiler") as mock_profiler_class, - ): - mock_profiler = mock.MagicMock() - mock_profiler_class.return_value = mock_profiler - - profiling.sampling.sample.sample( - 12345, # dummy PID - duration_sec=0.5, - sample_interval_usec=5000, - mode=1, # CPU mode - show_summary=False, - all_threads=True, - ) - - output = captured_output.getvalue() - - # Should see the "No samples were collected" message - self.assertIn("No samples were collected", output) - self.assertIn("CPU mode", output) - - -class TestGilModeFiltering(unittest.TestCase): - """Test GIL mode filtering functionality (--mode=gil).""" - - def test_gil_mode_validation(self): - """Test that CLI accepts gil mode choice correctly.""" - test_args = ["profiling.sampling.sample", "--mode", "gil", "-p", "12345"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - ): - try: - profiling.sampling.sample.main() - except SystemExit: - pass # Expected due to invalid PID - - # Should have attempted to call sample with mode=2 (GIL mode) - mock_sample.assert_called_once() - call_args = mock_sample.call_args[1] - self.assertEqual(call_args["mode"], 2) # PROFILING_MODE_GIL - - def test_gil_mode_sample_function_call(self): - """Test that sample() function correctly uses GIL mode.""" - with ( - mock.patch("profiling.sampling.sample.SampleProfiler") as mock_profiler, - mock.patch("profiling.sampling.sample.PstatsCollector") as mock_collector, - ): - # Mock the profiler instance - mock_instance = mock.Mock() - mock_profiler.return_value = mock_instance - - # Mock the collector instance - mock_collector_instance = mock.Mock() - mock_collector.return_value = mock_collector_instance - - # Call sample with GIL mode and a filename to avoid pstats creation - profiling.sampling.sample.sample( - 12345, - mode=2, # PROFILING_MODE_GIL - duration_sec=1, - sample_interval_usec=1000, - filename="test_output.txt", - ) - - # Verify SampleProfiler was created with correct mode - mock_profiler.assert_called_once() - call_args = mock_profiler.call_args - self.assertEqual(call_args[1]['mode'], 2) # mode parameter - - # Verify profiler.sample was called - mock_instance.sample.assert_called_once() - - # Verify collector.export was called since we provided a filename - mock_collector_instance.export.assert_called_once_with("test_output.txt") - - def test_gil_mode_collector_configuration(self): - """Test that collectors are configured correctly for GIL mode.""" - with ( - mock.patch("profiling.sampling.sample.SampleProfiler") as mock_profiler, - mock.patch("profiling.sampling.sample.PstatsCollector") as mock_collector, - captured_stdout(), captured_stderr() - ): - # Mock the profiler instance - mock_instance = mock.Mock() - mock_profiler.return_value = mock_instance - - # Call sample with GIL mode - profiling.sampling.sample.sample( - 12345, - mode=2, # PROFILING_MODE_GIL - output_format="pstats", - ) - - # Verify collector was created with skip_idle=True (since mode != WALL) - mock_collector.assert_called_once() - call_args = mock_collector.call_args[1] - self.assertTrue(call_args['skip_idle']) - - def test_gil_mode_with_collapsed_format(self): - """Test GIL mode with collapsed stack format.""" - with ( - mock.patch("profiling.sampling.sample.SampleProfiler") as mock_profiler, - mock.patch("profiling.sampling.sample.CollapsedStackCollector") as mock_collector, - ): - # Mock the profiler instance - mock_instance = mock.Mock() - mock_profiler.return_value = mock_instance - - # Call sample with GIL mode and collapsed format - profiling.sampling.sample.sample( - 12345, - mode=2, # PROFILING_MODE_GIL - output_format="collapsed", - filename="test_output.txt", - ) - - # Verify collector was created with skip_idle=True - mock_collector.assert_called_once() - call_args = mock_collector.call_args[1] - self.assertTrue(call_args['skip_idle']) - - def test_gil_mode_cli_argument_parsing(self): - """Test CLI argument parsing for GIL mode with various options.""" - test_args = [ - "profiling.sampling.sample", - "--mode", "gil", - "--interval", "500", - "--duration", "5", - "-p", "12345" - ] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - ): - try: - profiling.sampling.sample.main() - except SystemExit: - pass # Expected due to invalid PID - - # Verify all arguments were parsed correctly - mock_sample.assert_called_once() - call_args = mock_sample.call_args[1] - self.assertEqual(call_args["mode"], 2) # GIL mode - self.assertEqual(call_args["sample_interval_usec"], 500) - self.assertEqual(call_args["duration_sec"], 5) - - @requires_subprocess() - def test_gil_mode_integration_behavior(self): - """Integration test: GIL mode should capture GIL-holding threads.""" - # Create a test script with GIL-releasing operations - gil_test_script = ''' -import time -import threading - -gil_ready = threading.Event() - -def gil_releasing_work(): - time.sleep(999999) - -def gil_holding_work(): - gil_ready.set() - x = 1 - while True: - x += 1 - -def main(): - # Start both threads - idle_thread = threading.Thread(target=gil_releasing_work) - cpu_thread = threading.Thread(target=gil_holding_work) - idle_thread.start() - cpu_thread.start() - - # Wait for GIL-holding thread to be running, then signal test - gil_ready.wait() - _test_sock.sendall(b"threads_ready") - - idle_thread.join() - cpu_thread.join() - -main() -''' - with test_subprocess(gil_test_script) as subproc: - # Wait for signal that threads are running - response = subproc.socket.recv(1024) - self.assertEqual(response, b"threads_ready") - - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=2.0, - sample_interval_usec=5000, - mode=2, # GIL mode - show_summary=False, - all_threads=True, - ) - except (PermissionError, RuntimeError) as e: - self.skipTest("Insufficient permissions for remote profiling") - - gil_mode_output = captured_output.getvalue() - - # Test wall-clock mode for comparison - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=0.5, - sample_interval_usec=5000, - mode=0, # Wall-clock mode - show_summary=False, - all_threads=True, - ) - except (PermissionError, RuntimeError) as e: - self.skipTest("Insufficient permissions for remote profiling") - - wall_mode_output = captured_output.getvalue() - - # GIL mode should primarily capture GIL-holding work - # (Note: actual behavior depends on threading implementation) - self.assertIn("gil_holding_work", gil_mode_output) - - # Wall-clock mode should capture both types of work - self.assertIn("gil_holding_work", wall_mode_output) - - def test_mode_constants_are_defined(self): - """Test that all profiling mode constants are properly defined.""" - self.assertEqual(profiling.sampling.sample.PROFILING_MODE_WALL, 0) - self.assertEqual(profiling.sampling.sample.PROFILING_MODE_CPU, 1) - self.assertEqual(profiling.sampling.sample.PROFILING_MODE_GIL, 2) - - def test_parse_mode_function(self): - """Test the _parse_mode function with all valid modes.""" - self.assertEqual(profiling.sampling.sample._parse_mode("wall"), 0) - self.assertEqual(profiling.sampling.sample._parse_mode("cpu"), 1) - self.assertEqual(profiling.sampling.sample._parse_mode("gil"), 2) - - # Test invalid mode raises KeyError - with self.assertRaises(KeyError): - profiling.sampling.sample._parse_mode("invalid") - - -@requires_subprocess() -@skip_if_not_supported -class TestGCFrameTracking(unittest.TestCase): - """Tests for GC frame tracking in the sampling profiler.""" - - @classmethod - def setUpClass(cls): - """Create a static test script with GC frames and CPU-intensive work.""" - cls.gc_test_script = ''' -import gc - -class ExpensiveGarbage: - """Class that triggers GC with expensive finalizer (callback).""" - def __init__(self): - self.cycle = self - - def __del__(self): - # CPU-intensive work in the finalizer callback - result = 0 - for i in range(100000): - result += i * i - if i % 1000 == 0: - result = result % 1000000 - -def main_loop(): - """Main loop that triggers GC with expensive callback.""" - while True: - ExpensiveGarbage() - gc.collect() - -if __name__ == "__main__": - main_loop() -''' - - def test_gc_frames_enabled(self): - """Test that GC frames appear when gc tracking is enabled.""" - with ( - test_subprocess(self.gc_test_script) as subproc, - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=1, - sample_interval_usec=5000, - show_summary=False, - native=False, - gc=True, - ) - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - - output = captured_output.getvalue() - - # Should capture samples - self.assertIn("Captured", output) - self.assertIn("samples", output) - - # GC frames should be present - self.assertIn("", output) - - def test_gc_frames_disabled(self): - """Test that GC frames do not appear when gc tracking is disabled.""" - with ( - test_subprocess(self.gc_test_script) as subproc, - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=1, - sample_interval_usec=5000, - show_summary=False, - native=False, - gc=False, - ) - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - - output = captured_output.getvalue() - - # Should capture samples - self.assertIn("Captured", output) - self.assertIn("samples", output) - - # GC frames should NOT be present - self.assertNotIn("", output) - - -@requires_subprocess() -@skip_if_not_supported -class TestNativeFrameTracking(unittest.TestCase): - """Tests for native frame tracking in the sampling profiler.""" - - @classmethod - def setUpClass(cls): - """Create a static test script with native frames and CPU-intensive work.""" - cls.native_test_script = ''' -import operator - -def main_loop(): - while True: - # Native code in the middle of the stack: - operator.call(inner) - -def inner(): - # Python code at the top of the stack: - for _ in range(1_000_0000): - pass - -if __name__ == "__main__": - main_loop() -''' - - def test_native_frames_enabled(self): - """Test that native frames appear when native tracking is enabled.""" - collapsed_file = tempfile.NamedTemporaryFile( - suffix=".txt", delete=False - ) - self.addCleanup(close_and_unlink, collapsed_file) - - with ( - test_subprocess(self.native_test_script) as subproc, - ): - # Suppress profiler output when testing file export - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=1, - filename=collapsed_file.name, - output_format="collapsed", - sample_interval_usec=1000, - native=True, - ) - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - - # Verify file was created and contains valid data - self.assertTrue(os.path.exists(collapsed_file.name)) - self.assertGreater(os.path.getsize(collapsed_file.name), 0) - - # Check file format - with open(collapsed_file.name, "r") as f: - content = f.read() - - lines = content.strip().split("\n") - self.assertGreater(len(lines), 0) - - stacks = [line.rsplit(" ", 1)[0] for line in lines] - - # Most samples should have native code in the middle of the stack: - self.assertTrue(any(";;" in stack for stack in stacks)) - - # No samples should have native code at the top of the stack: - self.assertFalse(any(stack.endswith(";") for stack in stacks)) - - def test_native_frames_disabled(self): - """Test that native frames do not appear when native tracking is disabled.""" - with ( - test_subprocess(self.native_test_script) as subproc, - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=1, - sample_interval_usec=5000, - show_summary=False, - ) - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - output = captured_output.getvalue() - # Native frames should NOT be present: - self.assertNotIn("", output) - - -@requires_subprocess() -@skip_if_not_supported -class TestProcessPoolExecutorSupport(unittest.TestCase): - """ - Test that ProcessPoolExecutor works correctly with profiling.sampling. - """ - - def test_process_pool_executor_pickle(self): - # gh-140729: test use ProcessPoolExecutor.map() can sampling - test_script = ''' -import concurrent.futures - -def worker(x): - return x * 2 - -if __name__ == "__main__": - with concurrent.futures.ProcessPoolExecutor() as executor: - results = list(executor.map(worker, [1, 2, 3])) - print(f"Results: {results}") -''' - with os_helper.temp_dir() as temp_dir: - script = script_helper.make_script( - temp_dir, 'test_process_pool_executor_pickle', test_script - ) - with SuppressCrashReport(): - with script_helper.spawn_python( - "-m", "profiling.sampling.sample", - "-d", "5", - "-i", "100000", - script, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True - ) as proc: - try: - stdout, stderr = proc.communicate(timeout=SHORT_TIMEOUT) - except subprocess.TimeoutExpired: - proc.kill() - stdout, stderr = proc.communicate() - - if "PermissionError" in stderr: - self.skipTest("Insufficient permissions for remote profiling") - - self.assertIn("Results: [2, 4, 6]", stdout) - self.assertNotIn("Can't pickle", stderr) -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_profiling/test_sampling_profiler/__init__.py b/Lib/test/test_profiling/test_sampling_profiler/__init__.py new file mode 100644 index 00000000000..616ae5b49f0 --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/__init__.py @@ -0,0 +1,9 @@ +"""Tests for the sampling profiler (profiling.sampling).""" + +import os +from test.support import load_package_tests + + +def load_tests(*args): + """Load all tests from this subpackage.""" + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_profiling/test_sampling_profiler/helpers.py b/Lib/test/test_profiling/test_sampling_profiler/helpers.py new file mode 100644 index 00000000000..abd5a7377ad --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/helpers.py @@ -0,0 +1,101 @@ +"""Helper utilities for sampling profiler tests.""" + +import contextlib +import socket +import subprocess +import sys +import unittest +from collections import namedtuple + +from test.support import SHORT_TIMEOUT +from test.support.socket_helper import find_unused_port +from test.support.os_helper import unlink + + +PROCESS_VM_READV_SUPPORTED = False + +try: + from _remote_debugging import PROCESS_VM_READV_SUPPORTED # noqa: F401 + import _remote_debugging # noqa: F401 +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) +else: + import profiling.sampling # noqa: F401 + from profiling.sampling.sample import SampleProfiler # noqa: F401 + + +skip_if_not_supported = unittest.skipIf( + ( + sys.platform != "darwin" + and sys.platform != "linux" + and sys.platform != "win32" + ), + "Test only runs on Linux, Windows and MacOS", +) + +SubprocessInfo = namedtuple("SubprocessInfo", ["process", "socket"]) + + +@contextlib.contextmanager +def test_subprocess(script): + """Context manager to create a test subprocess with socket synchronization. + + Args: + script: Python code to execute in the subprocess + + Yields: + SubprocessInfo: Named tuple with process and socket objects + """ + # Find an unused port for socket communication + port = find_unused_port() + + # Inject socket connection code at the beginning of the script + socket_code = f""" +import socket +_test_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +_test_sock.connect(('localhost', {port})) +_test_sock.sendall(b"ready") +""" + + # Combine socket code with user script + full_script = socket_code + script + + # Create server socket to wait for process to be ready + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.bind(("localhost", port)) + server_socket.settimeout(SHORT_TIMEOUT) + server_socket.listen(1) + + proc = subprocess.Popen( + [sys.executable, "-c", full_script], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + client_socket = None + try: + # Wait for process to connect and send ready signal + client_socket, _ = server_socket.accept() + server_socket.close() + response = client_socket.recv(1024) + if response != b"ready": + raise RuntimeError( + f"Unexpected response from subprocess: {response}" + ) + + yield SubprocessInfo(proc, client_socket) + finally: + if client_socket is not None: + client_socket.close() + if proc.poll() is None: + proc.kill() + proc.wait() + + +def close_and_unlink(file): + """Close a file and unlink it from the filesystem.""" + file.close() + unlink(file.name) diff --git a/Lib/test/test_profiling/test_sampling_profiler/mocks.py b/Lib/test/test_profiling/test_sampling_profiler/mocks.py new file mode 100644 index 00000000000..9f1cd5b83e0 --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/mocks.py @@ -0,0 +1,38 @@ +"""Mock classes for sampling profiler tests.""" + + +class MockFrameInfo: + """Mock FrameInfo for testing since the real one isn't accessible.""" + + def __init__(self, filename, lineno, funcname): + self.filename = filename + self.lineno = lineno + self.funcname = funcname + + def __repr__(self): + return f"MockFrameInfo(filename='{self.filename}', lineno={self.lineno}, funcname='{self.funcname}')" + + +class MockThreadInfo: + """Mock ThreadInfo for testing since the real one isn't accessible.""" + + def __init__( + self, thread_id, frame_info, status=0 + ): # Default to THREAD_STATE_RUNNING (0) + self.thread_id = thread_id + self.frame_info = frame_info + self.status = status + + def __repr__(self): + return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info}, status={self.status})" + + +class MockInterpreterInfo: + """Mock InterpreterInfo for testing since the real one isn't accessible.""" + + def __init__(self, interpreter_id, threads): + self.interpreter_id = interpreter_id + self.threads = threads + + def __repr__(self): + return f"MockInterpreterInfo(interpreter_id={self.interpreter_id}, threads={self.threads})" diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_advanced.py b/Lib/test/test_profiling/test_sampling_profiler/test_advanced.py new file mode 100644 index 00000000000..578fb51bc0c --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/test_advanced.py @@ -0,0 +1,264 @@ +"""Tests for advanced sampling profiler features (GC tracking, native frames, ProcessPoolExecutor support).""" + +import io +import os +import subprocess +import tempfile +import unittest +from unittest import mock + +try: + import _remote_debugging # noqa: F401 + import profiling.sampling + import profiling.sampling.sample +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) + +from test.support import ( + SHORT_TIMEOUT, + SuppressCrashReport, + os_helper, + requires_subprocess, + script_helper, +) + +from .helpers import close_and_unlink, skip_if_not_supported, test_subprocess + + +@requires_subprocess() +@skip_if_not_supported +class TestGCFrameTracking(unittest.TestCase): + """Tests for GC frame tracking in the sampling profiler.""" + + @classmethod + def setUpClass(cls): + """Create a static test script with GC frames and CPU-intensive work.""" + cls.gc_test_script = ''' +import gc + +class ExpensiveGarbage: + """Class that triggers GC with expensive finalizer (callback).""" + def __init__(self): + self.cycle = self + + def __del__(self): + # CPU-intensive work in the finalizer callback + result = 0 + for i in range(100000): + result += i * i + if i % 1000 == 0: + result = result % 1000000 + +def main_loop(): + """Main loop that triggers GC with expensive callback.""" + while True: + ExpensiveGarbage() + gc.collect() + +if __name__ == "__main__": + main_loop() +''' + + def test_gc_frames_enabled(self): + """Test that GC frames appear when gc tracking is enabled.""" + with ( + test_subprocess(self.gc_test_script) as subproc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + sample_interval_usec=5000, + show_summary=False, + native=False, + gc=True, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + output = captured_output.getvalue() + + # Should capture samples + self.assertIn("Captured", output) + self.assertIn("samples", output) + + # GC frames should be present + self.assertIn("", output) + + def test_gc_frames_disabled(self): + """Test that GC frames do not appear when gc tracking is disabled.""" + with ( + test_subprocess(self.gc_test_script) as subproc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + sample_interval_usec=5000, + show_summary=False, + native=False, + gc=False, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + output = captured_output.getvalue() + + # Should capture samples + self.assertIn("Captured", output) + self.assertIn("samples", output) + + # GC frames should NOT be present + self.assertNotIn("", output) + + +@requires_subprocess() +@skip_if_not_supported +class TestNativeFrameTracking(unittest.TestCase): + """Tests for native frame tracking in the sampling profiler.""" + + @classmethod + def setUpClass(cls): + """Create a static test script with native frames and CPU-intensive work.""" + cls.native_test_script = """ +import operator + +def main_loop(): + while True: + # Native code in the middle of the stack: + operator.call(inner) + +def inner(): + # Python code at the top of the stack: + for _ in range(1_000_0000): + pass + +if __name__ == "__main__": + main_loop() +""" + + def test_native_frames_enabled(self): + """Test that native frames appear when native tracking is enabled.""" + collapsed_file = tempfile.NamedTemporaryFile( + suffix=".txt", delete=False + ) + self.addCleanup(close_and_unlink, collapsed_file) + + with ( + test_subprocess(self.native_test_script) as subproc, + ): + # Suppress profiler output when testing file export + with ( + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + filename=collapsed_file.name, + output_format="collapsed", + sample_interval_usec=1000, + native=True, + ) + except PermissionError: + self.skipTest( + "Insufficient permissions for remote profiling" + ) + + # Verify file was created and contains valid data + self.assertTrue(os.path.exists(collapsed_file.name)) + self.assertGreater(os.path.getsize(collapsed_file.name), 0) + + # Check file format + with open(collapsed_file.name, "r") as f: + content = f.read() + + lines = content.strip().split("\n") + self.assertGreater(len(lines), 0) + + stacks = [line.rsplit(" ", 1)[0] for line in lines] + + # Most samples should have native code in the middle of the stack: + self.assertTrue(any(";;" in stack for stack in stacks)) + + # No samples should have native code at the top of the stack: + self.assertFalse(any(stack.endswith(";") for stack in stacks)) + + def test_native_frames_disabled(self): + """Test that native frames do not appear when native tracking is disabled.""" + with ( + test_subprocess(self.native_test_script) as subproc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + sample_interval_usec=5000, + show_summary=False, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + output = captured_output.getvalue() + # Native frames should NOT be present: + self.assertNotIn("", output) + + +@requires_subprocess() +@skip_if_not_supported +class TestProcessPoolExecutorSupport(unittest.TestCase): + """ + Test that ProcessPoolExecutor works correctly with profiling.sampling. + """ + + def test_process_pool_executor_pickle(self): + # gh-140729: test use ProcessPoolExecutor.map() can sampling + test_script = """ +import concurrent.futures + +def worker(x): + return x * 2 + +if __name__ == "__main__": + with concurrent.futures.ProcessPoolExecutor() as executor: + results = list(executor.map(worker, [1, 2, 3])) + print(f"Results: {results}") +""" + with os_helper.temp_dir() as temp_dir: + script = script_helper.make_script( + temp_dir, "test_process_pool_executor_pickle", test_script + ) + with SuppressCrashReport(): + with script_helper.spawn_python( + "-m", + "profiling.sampling.sample", + "-d", + "5", + "-i", + "100000", + script, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) as proc: + try: + stdout, stderr = proc.communicate( + timeout=SHORT_TIMEOUT + ) + except subprocess.TimeoutExpired: + proc.kill() + stdout, stderr = proc.communicate() + + if "PermissionError" in stderr: + self.skipTest("Insufficient permissions for remote profiling") + + self.assertIn("Results: [2, 4, 6]", stdout) + self.assertNotIn("Can't pickle", stderr) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_cli.py b/Lib/test/test_profiling/test_sampling_profiler/test_cli.py new file mode 100644 index 00000000000..5833920d1b9 --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/test_cli.py @@ -0,0 +1,664 @@ +"""Tests for sampling profiler CLI argument parsing and functionality.""" + +import io +import subprocess +import sys +import unittest +from unittest import mock + +try: + import _remote_debugging # noqa: F401 + import profiling.sampling + import profiling.sampling.sample +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) + +from test.support import is_emscripten + + +class TestSampleProfilerCLI(unittest.TestCase): + def _setup_sync_mocks(self, mock_socket, mock_popen): + """Helper to set up socket and process mocks for coordinator tests.""" + # Mock the sync socket with context manager support + mock_sock_instance = mock.MagicMock() + mock_sock_instance.getsockname.return_value = ("127.0.0.1", 12345) + + # Mock the connection with context manager support + mock_conn = mock.MagicMock() + mock_conn.recv.return_value = b"ready" + mock_conn.__enter__.return_value = mock_conn + mock_conn.__exit__.return_value = None + + # Mock accept() to return (connection, address) and support indexing + mock_accept_result = mock.MagicMock() + mock_accept_result.__getitem__.return_value = ( + mock_conn # [0] returns the connection + ) + mock_sock_instance.accept.return_value = mock_accept_result + + # Mock socket with context manager support + mock_sock_instance.__enter__.return_value = mock_sock_instance + mock_sock_instance.__exit__.return_value = None + mock_socket.return_value = mock_sock_instance + + # Mock the subprocess + mock_process = mock.MagicMock() + mock_process.pid = 12345 + mock_process.poll.return_value = None + mock_popen.return_value = mock_process + return mock_process + + def _verify_coordinator_command(self, mock_popen, expected_target_args): + """Helper to verify the coordinator command was called correctly.""" + args, kwargs = mock_popen.call_args + coordinator_cmd = args[0] + self.assertEqual(coordinator_cmd[0], sys.executable) + self.assertEqual(coordinator_cmd[1], "-m") + self.assertEqual( + coordinator_cmd[2], "profiling.sampling._sync_coordinator" + ) + self.assertEqual(coordinator_cmd[3], "12345") # port + # cwd is coordinator_cmd[4] + self.assertEqual(coordinator_cmd[5:], expected_target_args) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_module_argument_parsing(self): + test_args = ["profiling.sampling.sample", "-m", "mymodule"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profiling.sampling.sample.main() + + self._verify_coordinator_command(mock_popen, ("-m", "mymodule")) + mock_sample.assert_called_once_with( + 12345, + sort=2, # default sort (sort_value from args.sort) + sample_interval_usec=100, + duration_sec=10, + filename=None, + all_threads=False, + limit=15, + show_summary=True, + output_format="pstats", + realtime_stats=False, + mode=0, + native=False, + gc=True, + ) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_module_with_arguments(self): + test_args = [ + "profiling.sampling.sample", + "-m", + "mymodule", + "arg1", + "arg2", + "--flag", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profiling.sampling.sample.main() + + self._verify_coordinator_command( + mock_popen, ("-m", "mymodule", "arg1", "arg2", "--flag") + ) + mock_sample.assert_called_once_with( + 12345, + sort=2, + sample_interval_usec=100, + duration_sec=10, + filename=None, + all_threads=False, + limit=15, + show_summary=True, + output_format="pstats", + realtime_stats=False, + mode=0, + native=False, + gc=True, + ) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_script_argument_parsing(self): + test_args = ["profiling.sampling.sample", "myscript.py"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profiling.sampling.sample.main() + + self._verify_coordinator_command(mock_popen, ("myscript.py",)) + mock_sample.assert_called_once_with( + 12345, + sort=2, + sample_interval_usec=100, + duration_sec=10, + filename=None, + all_threads=False, + limit=15, + show_summary=True, + output_format="pstats", + realtime_stats=False, + mode=0, + native=False, + gc=True, + ) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_script_with_arguments(self): + test_args = [ + "profiling.sampling.sample", + "myscript.py", + "arg1", + "arg2", + "--flag", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + # Use the helper to set up mocks consistently + mock_process = self._setup_sync_mocks(mock_socket, mock_popen) + # Override specific behavior for this test + mock_process.wait.side_effect = [ + subprocess.TimeoutExpired(test_args, 0.1), + None, + ] + + profiling.sampling.sample.main() + + # Verify the coordinator command was called + args, kwargs = mock_popen.call_args + coordinator_cmd = args[0] + self.assertEqual(coordinator_cmd[0], sys.executable) + self.assertEqual(coordinator_cmd[1], "-m") + self.assertEqual( + coordinator_cmd[2], "profiling.sampling._sync_coordinator" + ) + self.assertEqual(coordinator_cmd[3], "12345") # port + # cwd is coordinator_cmd[4] + self.assertEqual( + coordinator_cmd[5:], ("myscript.py", "arg1", "arg2", "--flag") + ) + + def test_cli_mutually_exclusive_pid_module(self): + test_args = [ + "profiling.sampling.sample", + "-p", + "12345", + "-m", + "mymodule", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + profiling.sampling.sample.main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("not allowed with argument", error_msg) + + def test_cli_mutually_exclusive_pid_script(self): + test_args = ["profiling.sampling.sample", "-p", "12345", "myscript.py"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + profiling.sampling.sample.main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("only one target type can be specified", error_msg) + + def test_cli_no_target_specified(self): + test_args = ["profiling.sampling.sample", "-d", "5"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + profiling.sampling.sample.main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("one of the arguments", error_msg) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_module_with_profiler_options(self): + test_args = [ + "profiling.sampling.sample", + "-i", + "1000", + "-d", + "30", + "-a", + "--sort-tottime", + "-l", + "20", + "-m", + "mymodule", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profiling.sampling.sample.main() + + self._verify_coordinator_command(mock_popen, ("-m", "mymodule")) + mock_sample.assert_called_once_with( + 12345, + sort=1, # sort-tottime + sample_interval_usec=1000, + duration_sec=30, + filename=None, + all_threads=True, + limit=20, + show_summary=True, + output_format="pstats", + realtime_stats=False, + mode=0, + native=False, + gc=True, + ) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_script_with_profiler_options(self): + """Test script with various profiler options.""" + test_args = [ + "profiling.sampling.sample", + "-i", + "2000", + "-d", + "60", + "--collapsed", + "-o", + "output.txt", + "myscript.py", + "scriptarg", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profiling.sampling.sample.main() + + self._verify_coordinator_command( + mock_popen, ("myscript.py", "scriptarg") + ) + # Verify profiler options were passed correctly + mock_sample.assert_called_once_with( + 12345, + sort=2, # default sort + sample_interval_usec=2000, + duration_sec=60, + filename="output.txt", + all_threads=False, + limit=15, + show_summary=True, + output_format="collapsed", + realtime_stats=False, + mode=0, + native=False, + gc=True, + ) + + def test_cli_empty_module_name(self): + test_args = ["profiling.sampling.sample", "-m"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + profiling.sampling.sample.main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("argument -m/--module: expected one argument", error_msg) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_long_module_option(self): + test_args = [ + "profiling.sampling.sample", + "--module", + "mymodule", + "arg1", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profiling.sampling.sample.main() + + self._verify_coordinator_command( + mock_popen, ("-m", "mymodule", "arg1") + ) + + def test_cli_complex_script_arguments(self): + test_args = [ + "profiling.sampling.sample", + "script.py", + "--input", + "file.txt", + "-v", + "--output=/tmp/out", + "positional", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch( + "profiling.sampling.sample._run_with_sync" + ) as mock_run_with_sync, + ): + mock_process = mock.MagicMock() + mock_process.pid = 12345 + mock_process.wait.side_effect = [ + subprocess.TimeoutExpired(test_args, 0.1), + None, + ] + mock_process.poll.return_value = None + mock_run_with_sync.return_value = mock_process + + profiling.sampling.sample.main() + + mock_run_with_sync.assert_called_once_with( + ( + sys.executable, + "script.py", + "--input", + "file.txt", + "-v", + "--output=/tmp/out", + "positional", + ) + ) + + def test_cli_collapsed_format_validation(self): + """Test that CLI properly validates incompatible options with collapsed format.""" + test_cases = [ + # Test sort options are invalid with collapsed + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--sort-nsamples", + "-p", + "12345", + ], + "sort", + ), + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--sort-tottime", + "-p", + "12345", + ], + "sort", + ), + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--sort-cumtime", + "-p", + "12345", + ], + "sort", + ), + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--sort-sample-pct", + "-p", + "12345", + ], + "sort", + ), + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--sort-cumul-pct", + "-p", + "12345", + ], + "sort", + ), + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--sort-name", + "-p", + "12345", + ], + "sort", + ), + # Test limit option is invalid with collapsed + ( + [ + "profiling.sampling.sample", + "--collapsed", + "-l", + "20", + "-p", + "12345", + ], + "limit", + ), + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--limit", + "20", + "-p", + "12345", + ], + "limit", + ), + # Test no-summary option is invalid with collapsed + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--no-summary", + "-p", + "12345", + ], + "summary", + ), + ] + + for test_args, expected_error_keyword in test_cases: + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + profiling.sampling.sample.main() + + self.assertEqual(cm.exception.code, 2) # argparse error code + error_msg = mock_stderr.getvalue() + self.assertIn("error:", error_msg) + self.assertIn("--pstats format", error_msg) + + def test_cli_default_collapsed_filename(self): + """Test that collapsed format gets a default filename when not specified.""" + test_args = ["profiling.sampling.sample", "--collapsed", "-p", "12345"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + ): + profiling.sampling.sample.main() + + # Check that filename was set to default collapsed format + mock_sample.assert_called_once() + call_args = mock_sample.call_args[1] + self.assertEqual(call_args["output_format"], "collapsed") + self.assertEqual(call_args["filename"], "collapsed.12345.txt") + + def test_cli_custom_output_filenames(self): + """Test custom output filenames for both formats.""" + test_cases = [ + ( + [ + "profiling.sampling.sample", + "--pstats", + "-o", + "custom.pstats", + "-p", + "12345", + ], + "custom.pstats", + "pstats", + ), + ( + [ + "profiling.sampling.sample", + "--collapsed", + "-o", + "custom.txt", + "-p", + "12345", + ], + "custom.txt", + "collapsed", + ), + ] + + for test_args, expected_filename, expected_format in test_cases: + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + ): + profiling.sampling.sample.main() + + mock_sample.assert_called_once() + call_args = mock_sample.call_args[1] + self.assertEqual(call_args["filename"], expected_filename) + self.assertEqual(call_args["output_format"], expected_format) + + def test_cli_missing_required_arguments(self): + """Test that CLI requires PID argument.""" + with ( + mock.patch("sys.argv", ["profiling.sampling.sample"]), + mock.patch("sys.stderr", io.StringIO()), + ): + with self.assertRaises(SystemExit): + profiling.sampling.sample.main() + + def test_cli_mutually_exclusive_format_options(self): + """Test that pstats and collapsed options are mutually exclusive.""" + with ( + mock.patch( + "sys.argv", + [ + "profiling.sampling.sample", + "--pstats", + "--collapsed", + "-p", + "12345", + ], + ), + mock.patch("sys.stderr", io.StringIO()), + ): + with self.assertRaises(SystemExit): + profiling.sampling.sample.main() + + def test_argument_parsing_basic(self): + test_args = ["profiling.sampling.sample", "-p", "12345"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + ): + profiling.sampling.sample.main() + + mock_sample.assert_called_once_with( + 12345, + sample_interval_usec=100, + duration_sec=10, + filename=None, + all_threads=False, + limit=15, + sort=2, + show_summary=True, + output_format="pstats", + realtime_stats=False, + mode=0, + native=False, + gc=True, + ) + + def test_sort_options(self): + sort_options = [ + ("--sort-nsamples", 0), + ("--sort-tottime", 1), + ("--sort-cumtime", 2), + ("--sort-sample-pct", 3), + ("--sort-cumul-pct", 4), + ("--sort-name", -1), + ] + + for option, expected_sort_value in sort_options: + test_args = ["profiling.sampling.sample", option, "-p", "12345"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + ): + profiling.sampling.sample.main() + + mock_sample.assert_called_once() + call_args = mock_sample.call_args[1] + self.assertEqual( + call_args["sort"], + expected_sort_value, + ) + mock_sample.reset_mock() diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py new file mode 100644 index 00000000000..4a24256203c --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py @@ -0,0 +1,896 @@ +"""Tests for sampling profiler collector components.""" + +import json +import marshal +import os +import tempfile +import unittest + +try: + import _remote_debugging # noqa: F401 + from profiling.sampling.pstats_collector import PstatsCollector + from profiling.sampling.stack_collector import ( + CollapsedStackCollector, + FlamegraphCollector, + ) + from profiling.sampling.gecko_collector import GeckoCollector +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) + +from test.support import captured_stdout, captured_stderr + +from .mocks import MockFrameInfo, MockThreadInfo, MockInterpreterInfo +from .helpers import close_and_unlink + + +class TestSampleProfilerComponents(unittest.TestCase): + """Unit tests for individual profiler components.""" + + def test_mock_frame_info_with_empty_and_unicode_values(self): + """Test MockFrameInfo handles empty strings, unicode characters, and very long names correctly.""" + # Test with empty strings + frame = MockFrameInfo("", 0, "") + self.assertEqual(frame.filename, "") + self.assertEqual(frame.lineno, 0) + self.assertEqual(frame.funcname, "") + self.assertIn("filename=''", repr(frame)) + + # Test with unicode characters + frame = MockFrameInfo("文件.py", 42, "函数名") + self.assertEqual(frame.filename, "文件.py") + self.assertEqual(frame.funcname, "函数名") + + # Test with very long names + long_filename = "x" * 1000 + ".py" + long_funcname = "func_" + "x" * 1000 + frame = MockFrameInfo(long_filename, 999999, long_funcname) + self.assertEqual(frame.filename, long_filename) + self.assertEqual(frame.lineno, 999999) + self.assertEqual(frame.funcname, long_funcname) + + def test_pstats_collector_with_extreme_intervals_and_empty_data(self): + """Test PstatsCollector handles zero/large intervals, empty frames, None thread IDs, and duplicate frames.""" + # Test with zero interval + collector = PstatsCollector(sample_interval_usec=0) + self.assertEqual(collector.sample_interval_usec, 0) + + # Test with very large interval + collector = PstatsCollector(sample_interval_usec=1000000000) + self.assertEqual(collector.sample_interval_usec, 1000000000) + + # Test collecting empty frames list + collector = PstatsCollector(sample_interval_usec=1000) + collector.collect([]) + self.assertEqual(len(collector.result), 0) + + # Test collecting frames with None thread id + test_frames = [ + MockInterpreterInfo( + 0, + [MockThreadInfo(None, [MockFrameInfo("file.py", 10, "func")])], + ) + ] + collector.collect(test_frames) + # Should still process the frames + self.assertEqual(len(collector.result), 1) + + # Test collecting duplicate frames in same sample + test_frames = [ + MockInterpreterInfo( + 0, # interpreter_id + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 10, "func1"), # Duplicate + ], + ) + ], + ) + ] + collector = PstatsCollector(sample_interval_usec=1000) + collector.collect(test_frames) + # Should count both occurrences + self.assertEqual( + collector.result[("file.py", 10, "func1")]["cumulative_calls"], 2 + ) + + def test_pstats_collector_single_frame_stacks(self): + """Test PstatsCollector with single-frame call stacks to trigger len(frames) <= 1 branch.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Test with exactly one frame (should trigger the <= 1 condition) + single_frame = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [MockFrameInfo("single.py", 10, "single_func")] + ) + ], + ) + ] + collector.collect(single_frame) + + # Should record the single frame with inline call + self.assertEqual(len(collector.result), 1) + single_key = ("single.py", 10, "single_func") + self.assertIn(single_key, collector.result) + self.assertEqual(collector.result[single_key]["direct_calls"], 1) + self.assertEqual(collector.result[single_key]["cumulative_calls"], 1) + + # Test with empty frames (should also trigger <= 1 condition) + empty_frames = [MockInterpreterInfo(0, [MockThreadInfo(1, [])])] + collector.collect(empty_frames) + + # Should not add any new entries + self.assertEqual( + len(collector.result), 1 + ) # Still just the single frame + + # Test mixed single and multi-frame stacks + mixed_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [MockFrameInfo("single2.py", 20, "single_func2")], + ), # Single frame + MockThreadInfo( + 2, + [ # Multi-frame stack + MockFrameInfo("multi.py", 30, "multi_func1"), + MockFrameInfo("multi.py", 40, "multi_func2"), + ], + ), + ], + ), + ] + collector.collect(mixed_frames) + + # Should have recorded all functions + self.assertEqual( + len(collector.result), 4 + ) # single + single2 + multi1 + multi2 + + # Verify single frame handling + single2_key = ("single2.py", 20, "single_func2") + self.assertIn(single2_key, collector.result) + self.assertEqual(collector.result[single2_key]["direct_calls"], 1) + self.assertEqual(collector.result[single2_key]["cumulative_calls"], 1) + + # Verify multi-frame handling still works + multi1_key = ("multi.py", 30, "multi_func1") + multi2_key = ("multi.py", 40, "multi_func2") + self.assertIn(multi1_key, collector.result) + self.assertIn(multi2_key, collector.result) + self.assertEqual(collector.result[multi1_key]["direct_calls"], 1) + self.assertEqual( + collector.result[multi2_key]["cumulative_calls"], 1 + ) # Called from multi1 + + def test_collapsed_stack_collector_with_empty_and_deep_stacks(self): + """Test CollapsedStackCollector handles empty frames, single-frame stacks, and very deep call stacks.""" + collector = CollapsedStackCollector() + + # Test with empty frames + collector.collect([]) + self.assertEqual(len(collector.stack_counter), 0) + + # Test with single frame stack + test_frames = [ + MockInterpreterInfo( + 0, [MockThreadInfo(1, [("file.py", 10, "func")])] + ) + ] + collector.collect(test_frames) + self.assertEqual(len(collector.stack_counter), 1) + (((path, thread_id), count),) = collector.stack_counter.items() + self.assertEqual(path, (("file.py", 10, "func"),)) + self.assertEqual(thread_id, 1) + self.assertEqual(count, 1) + + # Test with very deep stack + deep_stack = [(f"file{i}.py", i, f"func{i}") for i in range(100)] + test_frames = [MockInterpreterInfo(0, [MockThreadInfo(1, deep_stack)])] + collector = CollapsedStackCollector() + collector.collect(test_frames) + # One aggregated path with 100 frames (reversed) + (((path_tuple, thread_id),),) = (collector.stack_counter.keys(),) + self.assertEqual(len(path_tuple), 100) + self.assertEqual(path_tuple[0], ("file99.py", 99, "func99")) + self.assertEqual(path_tuple[-1], ("file0.py", 0, "func0")) + self.assertEqual(thread_id, 1) + + def test_pstats_collector_basic(self): + """Test basic PstatsCollector functionality.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Test empty state + self.assertEqual(len(collector.result), 0) + self.assertEqual(len(collector.stats), 0) + + # Test collecting sample data + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ], + ) + ], + ) + ] + collector.collect(test_frames) + + # Should have recorded calls for both functions + self.assertEqual(len(collector.result), 2) + self.assertIn(("file.py", 10, "func1"), collector.result) + self.assertIn(("file.py", 20, "func2"), collector.result) + + # Top-level function should have direct call + self.assertEqual( + collector.result[("file.py", 10, "func1")]["direct_calls"], 1 + ) + self.assertEqual( + collector.result[("file.py", 10, "func1")]["cumulative_calls"], 1 + ) + + # Calling function should have cumulative call but no direct calls + self.assertEqual( + collector.result[("file.py", 20, "func2")]["cumulative_calls"], 1 + ) + self.assertEqual( + collector.result[("file.py", 20, "func2")]["direct_calls"], 0 + ) + + def test_pstats_collector_create_stats(self): + """Test PstatsCollector stats creation.""" + collector = PstatsCollector( + sample_interval_usec=1000000 + ) # 1 second intervals + + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ], + ) + ], + ) + ] + collector.collect(test_frames) + collector.collect(test_frames) # Collect twice + + collector.create_stats() + + # Check stats format: (direct_calls, cumulative_calls, tt, ct, callers) + func1_stats = collector.stats[("file.py", 10, "func1")] + self.assertEqual(func1_stats[0], 2) # direct_calls (top of stack) + self.assertEqual(func1_stats[1], 2) # cumulative_calls + self.assertEqual( + func1_stats[2], 2.0 + ) # tt (total time - 2 samples * 1 sec) + self.assertEqual(func1_stats[3], 2.0) # ct (cumulative time) + + func2_stats = collector.stats[("file.py", 20, "func2")] + self.assertEqual( + func2_stats[0], 0 + ) # direct_calls (never top of stack) + self.assertEqual( + func2_stats[1], 2 + ) # cumulative_calls (appears in stack) + self.assertEqual(func2_stats[2], 0.0) # tt (no direct calls) + self.assertEqual(func2_stats[3], 2.0) # ct (cumulative time) + + def test_collapsed_stack_collector_basic(self): + collector = CollapsedStackCollector() + + # Test empty state + self.assertEqual(len(collector.stack_counter), 0) + + # Test collecting sample data + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] + collector.collect(test_frames) + + # Should store one reversed path + self.assertEqual(len(collector.stack_counter), 1) + (((path, thread_id), count),) = collector.stack_counter.items() + expected_tree = (("file.py", 20, "func2"), ("file.py", 10, "func1")) + self.assertEqual(path, expected_tree) + self.assertEqual(thread_id, 1) + self.assertEqual(count, 1) + + def test_collapsed_stack_collector_export(self): + collapsed_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, collapsed_out) + + collector = CollapsedStackCollector() + + test_frames1 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] + test_frames2 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] # Same stack + test_frames3 = [ + MockInterpreterInfo( + 0, [MockThreadInfo(1, [("other.py", 5, "other_func")])] + ) + ] + + collector.collect(test_frames1) + collector.collect(test_frames2) + collector.collect(test_frames3) + + with captured_stdout(), captured_stderr(): + collector.export(collapsed_out.name) + # Check file contents + with open(collapsed_out.name, "r") as f: + content = f.read() + + lines = content.strip().split("\n") + self.assertEqual(len(lines), 2) # Two unique stacks + + # Check collapsed format: tid:X;file:func:line;file:func:line count + stack1_expected = "tid:1;file.py:func2:20;file.py:func1:10 2" + stack2_expected = "tid:1;other.py:other_func:5 1" + + self.assertIn(stack1_expected, lines) + self.assertIn(stack2_expected, lines) + + def test_flamegraph_collector_basic(self): + """Test basic FlamegraphCollector functionality.""" + collector = FlamegraphCollector() + + # Empty collector should produce 'No Data' + data = collector._convert_to_flamegraph_format() + # With string table, name is now an index - resolve it using the strings array + strings = data.get("strings", []) + name_index = data.get("name", 0) + resolved_name = ( + strings[name_index] + if isinstance(name_index, int) and 0 <= name_index < len(strings) + else str(name_index) + ) + self.assertIn(resolved_name, ("No Data", "No significant data")) + + # Test collecting sample data + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] + collector.collect(test_frames) + + # Convert and verify structure: func2 -> func1 with counts = 1 + data = collector._convert_to_flamegraph_format() + # Expect promotion: root is the single child (func2), with func1 as its only child + strings = data.get("strings", []) + name_index = data.get("name", 0) + name = ( + strings[name_index] + if isinstance(name_index, int) and 0 <= name_index < len(strings) + else str(name_index) + ) + self.assertIsInstance(name, str) + self.assertTrue(name.startswith("Program Root: ")) + self.assertIn("func2 (file.py:20)", name) # formatted name + children = data.get("children", []) + self.assertEqual(len(children), 1) + child = children[0] + child_name_index = child.get("name", 0) + child_name = ( + strings[child_name_index] + if isinstance(child_name_index, int) + and 0 <= child_name_index < len(strings) + else str(child_name_index) + ) + self.assertIn("func1 (file.py:10)", child_name) # formatted name + self.assertEqual(child["value"], 1) + + def test_flamegraph_collector_export(self): + """Test flamegraph HTML export functionality.""" + flamegraph_out = tempfile.NamedTemporaryFile( + suffix=".html", delete=False + ) + self.addCleanup(close_and_unlink, flamegraph_out) + + collector = FlamegraphCollector() + + # Create some test data (use Interpreter/Thread objects like runtime) + test_frames1 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] + test_frames2 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] # Same stack + test_frames3 = [ + MockInterpreterInfo( + 0, [MockThreadInfo(1, [("other.py", 5, "other_func")])] + ) + ] + + collector.collect(test_frames1) + collector.collect(test_frames2) + collector.collect(test_frames3) + + # Export flamegraph + with captured_stdout(), captured_stderr(): + collector.export(flamegraph_out.name) + + # Verify file was created and contains valid data + self.assertTrue(os.path.exists(flamegraph_out.name)) + self.assertGreater(os.path.getsize(flamegraph_out.name), 0) + + # Check file contains HTML content + with open(flamegraph_out.name, "r", encoding="utf-8") as f: + content = f.read() + + # Should be valid HTML + self.assertIn("", content.lower()) + self.assertIn(" 0) + self.assertGreater(mock_collector.collect.call_count, 0) + self.assertLessEqual(mock_collector.collect.call_count, 3) + + def test_sample_profiler_missed_samples_warning(self): + """Test that the profiler warns about missed samples when sampling is too slow.""" + + mock_unwinder = mock.MagicMock() + mock_unwinder.get_stack_trace.return_value = [ + ( + 1, + [ + mock.MagicMock( + filename="test.py", lineno=10, funcname="test_func" + ) + ], + ) + ] + + with mock.patch( + "_remote_debugging.RemoteUnwinder" + ) as mock_unwinder_class: + mock_unwinder_class.return_value = mock_unwinder + + # Use very short interval that we'll miss + profiler = SampleProfiler( + pid=12345, sample_interval_usec=1000, all_threads=False + ) # 1ms interval + + mock_collector = mock.MagicMock() + + # Simulate slow sampling where we miss many samples + times = [ + 0.0, + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + ] # Extra time points to avoid StopIteration + + with mock.patch("time.perf_counter", side_effect=times): + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + profiler.sample(mock_collector, duration_sec=0.5) + + result = output.getvalue() + + # Should warn about missed samples + self.assertIn("Warning: missed", result) + self.assertIn("samples from the expected total", result) + + +@force_not_colorized_test_class +class TestPrintSampledStats(unittest.TestCase): + """Test the print_sampled_stats function.""" + + def setUp(self): + """Set up test data.""" + # Mock stats data + self.mock_stats = mock.MagicMock() + self.mock_stats.stats = { + ("file1.py", 10, "func1"): ( + 100, + 100, + 0.5, + 0.5, + {}, + ), # cc, nc, tt, ct, callers + ("file2.py", 20, "func2"): (50, 50, 0.25, 0.3, {}), + ("file3.py", 30, "func3"): (200, 200, 1.5, 2.0, {}), + ("file4.py", 40, "func4"): ( + 10, + 10, + 0.001, + 0.001, + {}, + ), # millisecond range + ("file5.py", 50, "func5"): ( + 5, + 5, + 0.000001, + 0.000002, + {}, + ), # microsecond range + } + + def test_print_sampled_stats_basic(self): + """Test basic print_sampled_stats functionality.""" + + # Capture output + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(self.mock_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Check header is present + self.assertIn("Profile Stats:", result) + self.assertIn("nsamples", result) + self.assertIn("tottime", result) + self.assertIn("cumtime", result) + + # Check functions are present + self.assertIn("func1", result) + self.assertIn("func2", result) + self.assertIn("func3", result) + + def test_print_sampled_stats_sorting(self): + """Test different sorting options.""" + + # Test sort by calls + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, sort=0, sample_interval_usec=100 + ) + + result = output.getvalue() + lines = result.strip().split("\n") + + # Find the data lines (skip header) + data_lines = [l for l in lines if "file" in l and ".py" in l] + # func3 should be first (200 calls) + self.assertIn("func3", data_lines[0]) + + # Test sort by time + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, sort=1, sample_interval_usec=100 + ) + + result = output.getvalue() + lines = result.strip().split("\n") + + data_lines = [l for l in lines if "file" in l and ".py" in l] + # func3 should be first (1.5s time) + self.assertIn("func3", data_lines[0]) + + def test_print_sampled_stats_limit(self): + """Test limiting output rows.""" + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, limit=2, sample_interval_usec=100 + ) + + result = output.getvalue() + + # Count function entries in the main stats section (not in summary) + lines = result.split("\n") + # Find where the main stats section ends (before summary) + main_section_lines = [] + for line in lines: + if "Summary of Interesting Functions:" in line: + break + main_section_lines.append(line) + + # Count function entries only in main section + func_count = sum( + 1 + for line in main_section_lines + if "func" in line and ".py" in line + ) + self.assertEqual(func_count, 2) + + def test_print_sampled_stats_time_units(self): + """Test proper time unit selection.""" + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(self.mock_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should use seconds for the header since max time is > 1s + self.assertIn("tottime (s)", result) + self.assertIn("cumtime (s)", result) + + # Test with only microsecond-range times + micro_stats = mock.MagicMock() + micro_stats.stats = { + ("file1.py", 10, "func1"): (100, 100, 0.000005, 0.000010, {}), + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(micro_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should use microseconds + self.assertIn("tottime (μs)", result) + self.assertIn("cumtime (μs)", result) + + def test_print_sampled_stats_summary(self): + """Test summary section generation.""" + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, + show_summary=True, + sample_interval_usec=100, + ) + + result = output.getvalue() + + # Check summary sections are present + self.assertIn("Summary of Interesting Functions:", result) + self.assertIn( + "Functions with Highest Direct/Cumulative Ratio (Hot Spots):", + result, + ) + self.assertIn( + "Functions with Highest Call Frequency (Indirect Calls):", result + ) + self.assertIn( + "Functions with Highest Call Magnification (Cumulative/Direct):", + result, + ) + + def test_print_sampled_stats_no_summary(self): + """Test disabling summary output.""" + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, + show_summary=False, + sample_interval_usec=100, + ) + + result = output.getvalue() + + # Summary should not be present + self.assertNotIn("Summary of Interesting Functions:", result) + + def test_print_sampled_stats_empty_stats(self): + """Test with empty stats.""" + + empty_stats = mock.MagicMock() + empty_stats.stats = {} + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(empty_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should still print header + self.assertIn("Profile Stats:", result) + + def test_print_sampled_stats_sample_percentage_sorting(self): + """Test sample percentage sorting options.""" + + # Add a function with high sample percentage (more direct calls than func3's 200) + self.mock_stats.stats[("expensive.py", 60, "expensive_func")] = ( + 300, # direct calls (higher than func3's 200) + 300, # cumulative calls + 1.0, # total time + 1.0, # cumulative time + {}, + ) + + # Test sort by sample percentage + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, sort=3, sample_interval_usec=100 + ) # sample percentage + + result = output.getvalue() + lines = result.strip().split("\n") + + data_lines = [l for l in lines if ".py" in l and "func" in l] + # expensive_func should be first (highest sample percentage) + self.assertIn("expensive_func", data_lines[0]) + + def test_print_sampled_stats_with_recursive_calls(self): + """Test print_sampled_stats with recursive calls where nc != cc.""" + + # Create stats with recursive calls (nc != cc) + recursive_stats = mock.MagicMock() + recursive_stats.stats = { + # (direct_calls, cumulative_calls, tt, ct, callers) - recursive function + ("recursive.py", 10, "factorial"): ( + 5, # direct_calls + 10, # cumulative_calls (appears more times in stack due to recursion) + 0.5, + 0.6, + {}, + ), + ("normal.py", 20, "normal_func"): ( + 3, # direct_calls + 3, # cumulative_calls (same as direct for non-recursive) + 0.2, + 0.2, + {}, + ), + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(recursive_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should display recursive calls as "5/10" format + self.assertIn("5/10", result) # nc/cc format for recursive calls + self.assertIn("3", result) # just nc for non-recursive calls + self.assertIn("factorial", result) + self.assertIn("normal_func", result) + + def test_print_sampled_stats_with_zero_call_counts(self): + """Test print_sampled_stats with zero call counts to trigger division protection.""" + + # Create stats with zero call counts + zero_stats = mock.MagicMock() + zero_stats.stats = { + ("file.py", 10, "zero_calls"): (0, 0, 0.0, 0.0, {}), # Zero calls + ("file.py", 20, "normal_func"): ( + 5, + 5, + 0.1, + 0.1, + {}, + ), # Normal function + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(zero_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should handle zero call counts gracefully + self.assertIn("zero_calls", result) + self.assertIn("zero_calls", result) + self.assertIn("normal_func", result) + + def test_print_sampled_stats_sort_by_name(self): + """Test sort by function name option.""" + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, sort=-1, sample_interval_usec=100 + ) # sort by name + + result = output.getvalue() + lines = result.strip().split("\n") + + # Find the data lines (skip header and summary) + # Data lines start with whitespace and numbers, and contain filename:lineno(function) + data_lines = [] + for line in lines: + # Skip header lines and summary sections + if ( + line.startswith(" ") + and "(" in line + and ")" in line + and not line.startswith( + " 1." + ) # Skip summary lines that start with times + and not line.startswith( + " 0." + ) # Skip summary lines that start with times + and not "per call" in line # Skip summary lines + and not "calls" in line # Skip summary lines + and not "total time" in line # Skip summary lines + and not "cumulative time" in line + ): # Skip summary lines + data_lines.append(line) + + # Extract just the function names for comparison + func_names = [] + import re + + for line in data_lines: + # Function name is between the last ( and ), accounting for ANSI color codes + match = re.search(r"\(([^)]+)\)$", line) + if match: + func_name = match.group(1) + # Remove ANSI color codes + func_name = re.sub(r"\x1b\[[0-9;]*m", "", func_name) + func_names.append(func_name) + + # Verify we extracted function names and they are sorted + self.assertGreater( + len(func_names), 0, "Should have extracted some function names" + ) + self.assertEqual( + func_names, + sorted(func_names), + f"Function names {func_names} should be sorted alphabetically", + ) + + def test_print_sampled_stats_with_zero_time_functions(self): + """Test summary sections with functions that have zero time.""" + + # Create stats with zero-time functions + zero_time_stats = mock.MagicMock() + zero_time_stats.stats = { + ("file1.py", 10, "zero_time_func"): ( + 5, + 5, + 0.0, + 0.0, + {}, + ), # Zero time + ("file2.py", 20, "normal_func"): ( + 3, + 3, + 0.1, + 0.1, + {}, + ), # Normal time + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + zero_time_stats, + show_summary=True, + sample_interval_usec=100, + ) + + result = output.getvalue() + + # Should handle zero-time functions gracefully in summary + self.assertIn("Summary of Interesting Functions:", result) + self.assertIn("zero_time_func", result) + self.assertIn("normal_func", result) + + def test_print_sampled_stats_with_malformed_qualified_names(self): + """Test summary generation with function names that don't contain colons.""" + + # Create stats with function names that would create malformed qualified names + malformed_stats = mock.MagicMock() + malformed_stats.stats = { + # Function name without clear module separation + ("no_colon_func", 10, "func"): (3, 3, 0.1, 0.1, {}), + ("", 20, "empty_filename_func"): (2, 2, 0.05, 0.05, {}), + ("normal.py", 30, "normal_func"): (5, 5, 0.2, 0.2, {}), + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + malformed_stats, + show_summary=True, + sample_interval_usec=100, + ) + + result = output.getvalue() + + # Should handle malformed names gracefully in summary aggregation + self.assertIn("Summary of Interesting Functions:", result) + # All function names should appear somewhere in the output + self.assertIn("func", result) + self.assertIn("empty_filename_func", result) + self.assertIn("normal_func", result) + + def test_print_sampled_stats_with_recursive_call_stats_creation(self): + """Test create_stats with recursive call data to trigger total_rec_calls branch.""" + collector = PstatsCollector(sample_interval_usec=1000000) # 1 second + + # Simulate recursive function data where total_rec_calls would be set + # We need to manually manipulate the collector result to test this branch + collector.result = { + ("recursive.py", 10, "factorial"): { + "total_rec_calls": 3, # Non-zero recursive calls + "direct_calls": 5, + "cumulative_calls": 10, + }, + ("normal.py", 20, "normal_func"): { + "total_rec_calls": 0, # Zero recursive calls + "direct_calls": 2, + "cumulative_calls": 5, + }, + } + + collector.create_stats() + + # Check that recursive calls are handled differently from non-recursive + factorial_stats = collector.stats[("recursive.py", 10, "factorial")] + normal_stats = collector.stats[("normal.py", 20, "normal_func")] + + # factorial should use cumulative_calls (10) as nc + self.assertEqual( + factorial_stats[1], 10 + ) # nc should be cumulative_calls + self.assertEqual(factorial_stats[0], 5) # cc should be direct_calls + + # normal_func should use cumulative_calls as nc + self.assertEqual(normal_stats[1], 5) # nc should be cumulative_calls + self.assertEqual(normal_stats[0], 2) # cc should be direct_calls diff --git a/Makefile.pre.in b/Makefile.pre.in index dd28ff5d2a3..59c3c808794 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2692,6 +2692,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_pathlib/support \ test/test_peg_generator \ test/test_profiling \ + test/test_profiling/test_sampling_profiler \ test/test_pydoc \ test/test_pyrepl \ test/test_string \ From 4695ec109d07c9bfd9eb7d91d6285c974a4331a7 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 18 Nov 2025 16:33:52 +0100 Subject: [PATCH 247/313] gh-138189: Link references to type slots (GH-141410) Link references to type slots --- Doc/c-api/structures.rst | 8 +++----- Doc/c-api/type.rst | 16 ++++++++-------- Doc/c-api/typeobj.rst | 2 +- Doc/howto/isolating-extensions.rst | 2 +- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 414dfdc84e6..b4e7cb1d77e 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -698,14 +698,12 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: entry indicates an offset from the subclass-specific data, rather than from ``PyObject``. - Can only be used as part of :c:member:`Py_tp_members ` + Can only be used as part of the :c:data:`Py_tp_members` :c:type:`slot ` when creating a class using negative :c:member:`~PyType_Spec.basicsize`. It is mandatory in that case. - - This flag is only used in :c:type:`PyType_Slot`. - When setting :c:member:`~PyTypeObject.tp_members` during - class creation, Python clears it and sets + When setting :c:member:`~PyTypeObject.tp_members` from the slot during + class creation, Python clears the flag and sets :c:member:`PyMemberDef.offset` to the offset from the ``PyObject`` struct. .. index:: diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index b608f815160..c7946e3190f 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -383,8 +383,8 @@ The following functions and structs are used to create The *bases* argument can be used to specify base classes; it can either be only one class or a tuple of classes. - If *bases* is ``NULL``, the *Py_tp_bases* slot is used instead. - If that also is ``NULL``, the *Py_tp_base* slot is used instead. + If *bases* is ``NULL``, the :c:data:`Py_tp_bases` slot is used instead. + If that also is ``NULL``, the :c:data:`Py_tp_base` slot is used instead. If that also is ``NULL``, the new type derives from :class:`object`. The *module* argument can be used to record the module in which the new @@ -590,9 +590,9 @@ The following functions and structs are used to create :c:type:`PyAsyncMethods` with an added ``Py_`` prefix. For example, use: - * ``Py_tp_dealloc`` to set :c:member:`PyTypeObject.tp_dealloc` - * ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add` - * ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length` + * :c:data:`Py_tp_dealloc` to set :c:member:`PyTypeObject.tp_dealloc` + * :c:data:`Py_nb_add` to set :c:member:`PyNumberMethods.nb_add` + * :c:data:`Py_sq_length` to set :c:member:`PySequenceMethods.sq_length` An additional slot is supported that does not correspond to a :c:type:`!PyTypeObject` struct field: @@ -611,7 +611,7 @@ The following functions and structs are used to create If it is not possible to switch to a ``MANAGED`` flag (for example, for vectorcall or to support Python older than 3.12), specify the - offset in :c:member:`Py_tp_members `. + offset in :c:data:`Py_tp_members`. See :ref:`PyMemberDef documentation ` for details. @@ -639,7 +639,7 @@ The following functions and structs are used to create .. versionchanged:: 3.14 The field :c:member:`~PyTypeObject.tp_vectorcall` can now set - using ``Py_tp_vectorcall``. See the field's documentation + using :c:data:`Py_tp_vectorcall`. See the field's documentation for details. .. c:member:: void *pfunc @@ -649,7 +649,7 @@ The following functions and structs are used to create *pfunc* values may not be ``NULL``, except for the following slots: - * ``Py_tp_doc`` + * :c:data:`Py_tp_doc` * :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` rather than ``NULL``) diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 34d19acdf17..49fe02d919d 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -2273,7 +2273,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) This field should be set to ``NULL`` and treated as read-only. Python will fill it in when the type is :c:func:`initialized `. - For dynamically created classes, the ``Py_tp_bases`` + For dynamically created classes, the :c:data:`Py_tp_bases` :c:type:`slot ` can be used instead of the *bases* argument of :c:func:`PyType_FromSpecWithBases`. The argument form is preferred. diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index 7da6dc8a397..6092c75f48f 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -353,7 +353,7 @@ garbage collection protocol. That is, heap types should: - Have the :c:macro:`Py_TPFLAGS_HAVE_GC` flag. -- Define a traverse function using ``Py_tp_traverse``, which +- Define a traverse function using :c:data:`Py_tp_traverse`, which visits the type (e.g. using ``Py_VISIT(Py_TYPE(self))``). Please refer to the documentation of From 600f3feb234219c9a9998e30ea653a2afb1f8116 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 18 Nov 2025 17:13:13 +0100 Subject: [PATCH 248/313] gh-141070: Add PyUnstable_Object_Dump() function (#141072) * Promote _PyObject_Dump() as a public function. * Keep _PyObject_Dump() alias to PyUnstable_Object_Dump() for backward compatibility. * Replace _PyObject_Dump() with PyUnstable_Object_Dump(). Co-authored-by: Peter Bierma Co-authored-by: Kumar Aditya Co-authored-by: Petr Viktorin --- Doc/c-api/object.rst | 29 +++++++++++ Doc/whatsnew/3.15.rst | 12 +++-- Include/cpython/object.h | 14 +++-- .../pycore_global_objects_fini_generated.h | 2 +- Lib/test/test_capi/test_object.py | 52 +++++++++++++++++++ ...-11-05-21-48-31.gh-issue-141070.mkrhjQ.rst | 2 + Modules/_testcapi/object.c | 25 +++++++++ Objects/object.c | 4 +- Objects/unicodeobject.c | 3 +- Python/gc.c | 2 +- Python/pythonrun.c | 8 +-- 11 files changed, 135 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-11-05-21-48-31.gh-issue-141070.mkrhjQ.rst diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 96353266ac7..76971c46c16 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -85,6 +85,35 @@ Object Protocol instead of the :func:`repr`. +.. c:function:: void PyUnstable_Object_Dump(PyObject *op) + + Dump an object *op* to ``stderr``. This should only be used for debugging. + + The output is intended to try dumping objects even after memory corruption: + + * Information is written starting with fields that are the least likely to + crash when accessed. + * This function can be called without an :term:`attached thread state`, but + it's not recommended to do so: it can cause deadlocks. + * An object that does not belong to the current interpreter may be dumped, + but this may also cause crashes or unintended behavior. + * Implement a heuristic to detect if the object memory has been freed. Don't + display the object contents in this case, only its memory address. + * The output format may change at any time. + + Example of output: + + .. code-block:: output + + object address : 0x7f80124702c0 + object refcount : 2 + object type : 0x9902e0 + object type name: str + object repr : 'abcdef' + + .. versionadded:: next + + .. c:function:: int PyObject_HasAttrWithError(PyObject *o, PyObject *attr_name) Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 24cc7e2d7eb..5a98297d3f8 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1084,19 +1084,23 @@ New features (Contributed by Victor Stinner in :gh:`129813`.) +* Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating + a module from a *spec* and *initfunc*. + (Contributed by Itamar Oren in :gh:`116146`.) + * Add :c:func:`PyTuple_FromArray` to create a :class:`tuple` from an array. (Contributed by Victor Stinner in :gh:`111489`.) +* Add :c:func:`PyUnstable_Object_Dump` to dump an object to ``stderr``. + It should only be used for debugging. + (Contributed by Victor Stinner in :gh:`141070`.) + * Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and :c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set the stack protection base address and stack protection size of a Python thread state. (Contributed by Victor Stinner in :gh:`139653`.) -* Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating - a module from a *spec* and *initfunc*. - (Contributed by Itamar Oren in :gh:`116146`.) - Changed C APIs -------------- diff --git a/Include/cpython/object.h b/Include/cpython/object.h index d64298232e7..130a105de42 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -295,7 +295,10 @@ PyAPI_FUNC(PyObject *) PyType_GetDict(PyTypeObject *); PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); PyAPI_FUNC(void) _Py_BreakPoint(void); -PyAPI_FUNC(void) _PyObject_Dump(PyObject *); +PyAPI_FUNC(void) PyUnstable_Object_Dump(PyObject *); + +// Alias for backward compatibility +#define _PyObject_Dump PyUnstable_Object_Dump PyAPI_FUNC(PyObject*) _PyObject_GetAttrId(PyObject *, _Py_Identifier *); @@ -387,10 +390,11 @@ PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *); process with a message on stderr if the given condition fails to hold, but compile away to nothing if NDEBUG is defined. - However, before aborting, Python will also try to call _PyObject_Dump() on - the given object. This may be of use when investigating bugs in which a - particular object is corrupt (e.g. buggy a tp_visit method in an extension - module breaking the garbage collector), to help locate the broken objects. + However, before aborting, Python will also try to call + PyUnstable_Object_Dump() on the given object. This may be of use when + investigating bugs in which a particular object is corrupt (e.g. buggy a + tp_visit method in an extension module breaking the garbage collector), to + help locate the broken objects. The WITH_MSG variant allows you to supply an additional message that Python will attempt to print to stderr, after the object dump. */ diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index ecef4364cc3..c3968aff8f3 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -13,7 +13,7 @@ static inline void _PyStaticObject_CheckRefcnt(PyObject *obj) { if (!_Py_IsImmortal(obj)) { fprintf(stderr, "Immortal Object has less refcnt than expected.\n"); - _PyObject_Dump(obj); + PyUnstable_Object_Dump(obj); } } #endif diff --git a/Lib/test/test_capi/test_object.py b/Lib/test/test_capi/test_object.py index d4056727d07..c5040913e9e 100644 --- a/Lib/test/test_capi/test_object.py +++ b/Lib/test/test_capi/test_object.py @@ -1,4 +1,5 @@ import enum +import os import sys import textwrap import unittest @@ -13,6 +14,9 @@ _testcapi = import_helper.import_module('_testcapi') _testinternalcapi = import_helper.import_module('_testinternalcapi') +NULL = None +STDERR_FD = 2 + class Constant(enum.IntEnum): Py_CONSTANT_NONE = 0 @@ -247,5 +251,53 @@ def func(x): func(object()) + def pyobject_dump(self, obj, release_gil=False): + pyobject_dump = _testcapi.pyobject_dump + + try: + old_stderr = os.dup(STDERR_FD) + except OSError as exc: + # os.dup(STDERR_FD) is not supported on WASI + self.skipTest(f"os.dup() failed with {exc!r}") + + filename = os_helper.TESTFN + try: + try: + with open(filename, "wb") as fp: + fd = fp.fileno() + os.dup2(fd, STDERR_FD) + pyobject_dump(obj, release_gil) + finally: + os.dup2(old_stderr, STDERR_FD) + os.close(old_stderr) + + with open(filename) as fp: + return fp.read().rstrip() + finally: + os_helper.unlink(filename) + + def test_pyobject_dump(self): + # test string object + str_obj = 'test string' + output = self.pyobject_dump(str_obj) + hex_regex = r'(0x)?[0-9a-fA-F]+' + regex = ( + fr"object address : {hex_regex}\n" + r"object refcount : [0-9]+\n" + fr"object type : {hex_regex}\n" + r"object type name: str\n" + r"object repr : 'test string'" + ) + self.assertRegex(output, regex) + + # release the GIL + output = self.pyobject_dump(str_obj, release_gil=True) + self.assertRegex(output, regex) + + # test NULL object + output = self.pyobject_dump(NULL) + self.assertRegex(output, r'') + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/C_API/2025-11-05-21-48-31.gh-issue-141070.mkrhjQ.rst b/Misc/NEWS.d/next/C_API/2025-11-05-21-48-31.gh-issue-141070.mkrhjQ.rst new file mode 100644 index 00000000000..39cfcf73404 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-05-21-48-31.gh-issue-141070.mkrhjQ.rst @@ -0,0 +1,2 @@ +Add :c:func:`PyUnstable_Object_Dump` to dump an object to ``stderr``. It should +only be used for debugging. Patch by Victor Stinner. diff --git a/Modules/_testcapi/object.c b/Modules/_testcapi/object.c index 798ef97c495..4c9632c07a9 100644 --- a/Modules/_testcapi/object.c +++ b/Modules/_testcapi/object.c @@ -485,6 +485,30 @@ is_uniquely_referenced(PyObject *self, PyObject *op) } +static PyObject * +pyobject_dump(PyObject *self, PyObject *args) +{ + PyObject *op; + int release_gil = 0; + + if (!PyArg_ParseTuple(args, "O|i", &op, &release_gil)) { + return NULL; + } + NULLABLE(op); + + if (release_gil) { + Py_BEGIN_ALLOW_THREADS + PyUnstable_Object_Dump(op); + Py_END_ALLOW_THREADS + + } + else { + PyUnstable_Object_Dump(op); + } + Py_RETURN_NONE; +} + + static PyMethodDef test_methods[] = { {"call_pyobject_print", call_pyobject_print, METH_VARARGS}, {"pyobject_print_null", pyobject_print_null, METH_VARARGS}, @@ -511,6 +535,7 @@ static PyMethodDef test_methods[] = { {"test_py_is_funcs", test_py_is_funcs, METH_NOARGS}, {"clear_managed_dict", clear_managed_dict, METH_O, NULL}, {"is_uniquely_referenced", is_uniquely_referenced, METH_O}, + {"pyobject_dump", pyobject_dump, METH_VARARGS}, {NULL}, }; diff --git a/Objects/object.c b/Objects/object.c index 0540112d7d2..0a80c6edcf1 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -713,7 +713,7 @@ _PyObject_IsFreed(PyObject *op) /* For debugging convenience. See Misc/gdbinit for some useful gdb hooks */ void -_PyObject_Dump(PyObject* op) +PyUnstable_Object_Dump(PyObject* op) { if (_PyObject_IsFreed(op)) { /* It seems like the object memory has been freed: @@ -3150,7 +3150,7 @@ _PyObject_AssertFailed(PyObject *obj, const char *expr, const char *msg, /* This might succeed or fail, but we're about to abort, so at least try to provide any extra info we can: */ - _PyObject_Dump(obj); + PyUnstable_Object_Dump(obj); fprintf(stderr, "\n"); fflush(stderr); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 4e8c132327b..7f9f75126a9 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -547,7 +547,8 @@ unicode_check_encoding_errors(const char *encoding, const char *errors) } /* Disable checks during Python finalization. For example, it allows to - call _PyObject_Dump() during finalization for debugging purpose. */ + * call PyUnstable_Object_Dump() during finalization for debugging purpose. + */ if (_PyInterpreterState_GetFinalizing(interp) != NULL) { return 0; } diff --git a/Python/gc.c b/Python/gc.c index 064f9406e0a..27364ecfdcd 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -2237,7 +2237,7 @@ _PyGC_Fini(PyInterpreterState *interp) void _PyGC_Dump(PyGC_Head *g) { - _PyObject_Dump(FROM_GC(g)); + PyUnstable_Object_Dump(FROM_GC(g)); } diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 49ce0a97d47..272be504a68 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1181,7 +1181,7 @@ _PyErr_Display(PyObject *file, PyObject *unused, PyObject *value, PyObject *tb) } if (print_exception_recursive(&ctx, value) < 0) { PyErr_Clear(); - _PyObject_Dump(value); + PyUnstable_Object_Dump(value); fprintf(stderr, "lost sys.stderr\n"); } Py_XDECREF(ctx.seen); @@ -1199,14 +1199,14 @@ PyErr_Display(PyObject *unused, PyObject *value, PyObject *tb) PyObject *file; if (PySys_GetOptionalAttr(&_Py_ID(stderr), &file) < 0) { PyObject *exc = PyErr_GetRaisedException(); - _PyObject_Dump(value); + PyUnstable_Object_Dump(value); fprintf(stderr, "lost sys.stderr\n"); - _PyObject_Dump(exc); + PyUnstable_Object_Dump(exc); Py_DECREF(exc); return; } if (file == NULL) { - _PyObject_Dump(value); + PyUnstable_Object_Dump(value); fprintf(stderr, "lost sys.stderr\n"); return; } From daafacf0053e9c329b0f96447258f628dd0bd6f1 Mon Sep 17 00:00:00 2001 From: Shamil Date: Tue, 18 Nov 2025 19:34:58 +0300 Subject: [PATCH 249/313] gh-42400: Fix buffer overflow in _Py_wrealpath() for very long paths (#141529) Co-authored-by: Victor Stinner --- .../Security/2025-11-13-22-31-56.gh-issue-42400.pqB5Kq.rst | 3 +++ Python/fileutils.c | 7 ++++--- 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-11-13-22-31-56.gh-issue-42400.pqB5Kq.rst diff --git a/Misc/NEWS.d/next/Security/2025-11-13-22-31-56.gh-issue-42400.pqB5Kq.rst b/Misc/NEWS.d/next/Security/2025-11-13-22-31-56.gh-issue-42400.pqB5Kq.rst new file mode 100644 index 00000000000..17dc241aef9 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-11-13-22-31-56.gh-issue-42400.pqB5Kq.rst @@ -0,0 +1,3 @@ +Fix buffer overflow in ``_Py_wrealpath()`` for paths exceeding ``MAXPATHLEN`` bytes +by using dynamic memory allocation instead of fixed-size buffer. +Patch by Shamil Abdulaev. diff --git a/Python/fileutils.c b/Python/fileutils.c index 93abd70a34d..0c1766b8804 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2118,7 +2118,6 @@ _Py_wrealpath(const wchar_t *path, wchar_t *resolved_path, size_t resolved_path_len) { char *cpath; - char cresolved_path[MAXPATHLEN]; wchar_t *wresolved_path; char *res; size_t r; @@ -2127,12 +2126,14 @@ _Py_wrealpath(const wchar_t *path, errno = EINVAL; return NULL; } - res = realpath(cpath, cresolved_path); + res = realpath(cpath, NULL); PyMem_RawFree(cpath); if (res == NULL) return NULL; - wresolved_path = Py_DecodeLocale(cresolved_path, &r); + wresolved_path = Py_DecodeLocale(res, &r); + free(res); + if (wresolved_path == NULL) { errno = EINVAL; return NULL; From 4cfa695c953e5dfdab99ade81cee960ddf4b106d Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 18 Nov 2025 09:51:18 -0800 Subject: [PATCH 250/313] GH-141686: Break cycles created by JSONEncoder.iterencode (GH-141687) --- Lib/json/encoder.py | 30 +++++++++---------- ...-11-17-16-53-49.gh-issue-141686.V-xaoI.rst | 2 ++ 2 files changed, 16 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-17-16-53-49.gh-issue-141686.V-xaoI.rst diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 5cf6d64f3ea..4c70e8b75ed 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -264,17 +264,6 @@ def floatstr(o, allow_nan=self.allow_nan, def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot, - ## HACK: hand-optimized bytecode; turn globals into locals - ValueError=ValueError, - dict=dict, - float=float, - id=id, - int=int, - isinstance=isinstance, - list=list, - str=str, - tuple=tuple, - _intstr=int.__repr__, ): def _iterencode_list(lst, _current_indent_level): @@ -311,7 +300,7 @@ def _iterencode_list(lst, _current_indent_level): # Subclasses of int/float may override __repr__, but we still # want to encode them as integers/floats in JSON. One example # within the standard library is IntEnum. - yield buf + _intstr(value) + yield buf + int.__repr__(value) elif isinstance(value, float): # see comment above for int yield buf + _floatstr(value) @@ -374,7 +363,7 @@ def _iterencode_dict(dct, _current_indent_level): key = 'null' elif isinstance(key, int): # see comment for int/float in _make_iterencode - key = _intstr(key) + key = int.__repr__(key) elif _skipkeys: continue else: @@ -399,7 +388,7 @@ def _iterencode_dict(dct, _current_indent_level): yield 'false' elif isinstance(value, int): # see comment for int/float in _make_iterencode - yield _intstr(value) + yield int.__repr__(value) elif isinstance(value, float): # see comment for int/float in _make_iterencode yield _floatstr(value) @@ -434,7 +423,7 @@ def _iterencode(o, _current_indent_level): yield 'false' elif isinstance(o, int): # see comment for int/float in _make_iterencode - yield _intstr(o) + yield int.__repr__(o) elif isinstance(o, float): # see comment for int/float in _make_iterencode yield _floatstr(o) @@ -458,4 +447,13 @@ def _iterencode(o, _current_indent_level): raise if markers is not None: del markers[markerid] - return _iterencode + + def _iterencode_once(o, _current_indent_level): + nonlocal _iterencode, _iterencode_dict, _iterencode_list + try: + yield from _iterencode(o, _current_indent_level) + finally: + # Break reference cycles due to mutually recursive closures: + del _iterencode, _iterencode_dict, _iterencode_list + + return _iterencode_once diff --git a/Misc/NEWS.d/next/Library/2025-11-17-16-53-49.gh-issue-141686.V-xaoI.rst b/Misc/NEWS.d/next/Library/2025-11-17-16-53-49.gh-issue-141686.V-xaoI.rst new file mode 100644 index 00000000000..87e9cb8d69b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-17-16-53-49.gh-issue-141686.V-xaoI.rst @@ -0,0 +1,2 @@ +Break reference cycles created by each call to :func:`json.dump` or +:meth:`json.JSONEncoder.iterencode`. From 17636ba48ce535fc1b1926c0bab26339da50631a Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 19 Nov 2025 06:39:21 +0800 Subject: [PATCH 251/313] gh-141691: Apply ruff rules to Apple folder. (#141694) Add ruff configuration to run over the Apple build tooling and testbed script. --- .pre-commit-config.yaml | 8 ++ Apple/.ruff.toml | 22 ++++++ Apple/__main__.py | 154 +++++++++++++++++++------------------- Apple/testbed/__main__.py | 33 ++++---- 4 files changed, 126 insertions(+), 91 deletions(-) create mode 100644 Apple/.ruff.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b0311f05279..c5767ee841e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,6 +2,10 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.13.2 hooks: + - id: ruff-check + name: Run Ruff (lint) on Apple/ + args: [--exit-non-zero-on-fix, --config=Apple/.ruff.toml] + files: ^Apple/ - id: ruff-check name: Run Ruff (lint) on Doc/ args: [--exit-non-zero-on-fix] @@ -30,6 +34,10 @@ repos: name: Run Ruff (lint) on Tools/wasm/ args: [--exit-non-zero-on-fix, --config=Tools/wasm/.ruff.toml] files: ^Tools/wasm/ + - id: ruff-format + name: Run Ruff (format) on Apple/ + args: [--exit-non-zero-on-fix, --config=Apple/.ruff.toml] + files: ^Apple - id: ruff-format name: Run Ruff (format) on Doc/ args: [--check] diff --git a/Apple/.ruff.toml b/Apple/.ruff.toml new file mode 100644 index 00000000000..4cdc39ebee4 --- /dev/null +++ b/Apple/.ruff.toml @@ -0,0 +1,22 @@ +extend = "../.ruff.toml" # Inherit the project-wide settings + +[format] +preview = true +docstring-code-format = true + +[lint] +select = [ + "C4", # flake8-comprehensions + "E", # pycodestyle + "F", # pyflakes + "I", # isort + "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "PYI", # flake8-pyi + "RUF100", # Ban unused `# noqa` comments + "UP", # pyupgrade + "W", # pycodestyle + "YTT", # flake8-2020 +] diff --git a/Apple/__main__.py b/Apple/__main__.py index e76fc351798..1c588c23d6b 100644 --- a/Apple/__main__.py +++ b/Apple/__main__.py @@ -46,13 +46,12 @@ import sys import sysconfig import time -from collections.abc import Sequence +from collections.abc import Callable, Sequence from contextlib import contextmanager from datetime import datetime, timezone from os.path import basename, relpath from pathlib import Path from subprocess import CalledProcessError -from typing import Callable EnvironmentT = dict[str, str] ArgsT = Sequence[str | Path] @@ -140,17 +139,15 @@ def print_env(env: EnvironmentT) -> None: def apple_env(host: str) -> EnvironmentT: """Construct an Apple development environment for the given host.""" env = { - "PATH": ":".join( - [ - str(PYTHON_DIR / "Apple/iOS/Resources/bin"), - str(subdir(host) / "prefix"), - "/usr/bin", - "/bin", - "/usr/sbin", - "/sbin", - "/Library/Apple/usr/bin", - ] - ), + "PATH": ":".join([ + str(PYTHON_DIR / "Apple/iOS/Resources/bin"), + str(subdir(host) / "prefix"), + "/usr/bin", + "/bin", + "/usr/sbin", + "/sbin", + "/Library/Apple/usr/bin", + ]), } return env @@ -196,14 +193,10 @@ def clean(context: argparse.Namespace, target: str = "all") -> None: paths.append(target) if target in {"all", "hosts", "test"}: - paths.extend( - [ - path.name - for path in CROSS_BUILD_DIR.glob( - f"{context.platform}-testbed.*" - ) - ] - ) + paths.extend([ + path.name + for path in CROSS_BUILD_DIR.glob(f"{context.platform}-testbed.*") + ]) for path in paths: delete_path(path) @@ -352,18 +345,16 @@ def download(url: str, target_dir: Path) -> Path: out_path = target_path / basename(url) if not Path(out_path).is_file(): - run( - [ - "curl", - "-Lf", - "--retry", - "5", - "--retry-all-errors", - "-o", - out_path, - url, - ] - ) + run([ + "curl", + "-Lf", + "--retry", + "5", + "--retry-all-errors", + "-o", + out_path, + url, + ]) else: print(f"Using cached version of {basename(url)}") return out_path @@ -468,8 +459,7 @@ def package_version(prefix_path: Path) -> str: def lib_platform_files(dirname, names): - """A file filter that ignores platform-specific files in the lib directory. - """ + """A file filter that ignores platform-specific files in lib.""" path = Path(dirname) if ( path.parts[-3] == "lib" @@ -478,7 +468,7 @@ def lib_platform_files(dirname, names): ): return names elif path.parts[-2] == "lib" and path.parts[-1].startswith("python"): - ignored_names = set( + ignored_names = { name for name in names if ( @@ -486,7 +476,7 @@ def lib_platform_files(dirname, names): or name.startswith("_sysconfig_vars_") or name == "build-details.json" ) - ) + } else: ignored_names = set() @@ -499,7 +489,9 @@ def lib_non_platform_files(dirname, names): """ path = Path(dirname) if path.parts[-2] == "lib" and path.parts[-1].startswith("python"): - return set(names) - lib_platform_files(dirname, names) - {"lib-dynload"} + return ( + set(names) - lib_platform_files(dirname, names) - {"lib-dynload"} + ) else: return set() @@ -514,7 +506,8 @@ def create_xcframework(platform: str) -> str: package_path.mkdir() except FileExistsError: raise RuntimeError( - f"{platform} XCframework already exists; do you need to run with --clean?" + f"{platform} XCframework already exists; do you need to run " + "with --clean?" ) from None frameworks = [] @@ -607,7 +600,7 @@ def create_xcframework(platform: str) -> str: print(f" - {slice_name} binaries") shutil.copytree(first_path / "bin", slice_path / "bin") - # Copy the include path (this will be a symlink to the framework headers) + # Copy the include path (a symlink to the framework headers) print(f" - {slice_name} include files") shutil.copytree( first_path / "include", @@ -659,7 +652,8 @@ def create_xcframework(platform: str) -> str: # statically link those libraries into a Framework, you become # responsible for providing a privacy manifest for that framework. xcprivacy_file = { - "OpenSSL": subdir(host_triple) / "prefix/share/OpenSSL.xcprivacy" + "OpenSSL": subdir(host_triple) + / "prefix/share/OpenSSL.xcprivacy" } print(f" - {multiarch} xcprivacy files") for module, lib in [ @@ -669,7 +663,8 @@ def create_xcframework(platform: str) -> str: shutil.copy( xcprivacy_file[lib], slice_path - / f"lib-{arch}/python{version_tag}/lib-dynload/{module}.xcprivacy", + / f"lib-{arch}/python{version_tag}" + / f"lib-dynload/{module}.xcprivacy", ) print(" - build tools") @@ -692,18 +687,16 @@ def package(context: argparse.Namespace) -> None: # Clone testbed print() - run( - [ - sys.executable, - "Apple/testbed", - "clone", - "--platform", - context.platform, - "--framework", - CROSS_BUILD_DIR / context.platform / "Python.xcframework", - CROSS_BUILD_DIR / context.platform / "testbed", - ] - ) + run([ + sys.executable, + "Apple/testbed", + "clone", + "--platform", + context.platform, + "--framework", + CROSS_BUILD_DIR / context.platform / "Python.xcframework", + CROSS_BUILD_DIR / context.platform / "testbed", + ]) # Build the final archive archive_name = ( @@ -757,7 +750,7 @@ def build(context: argparse.Namespace, host: str | None = None) -> None: package(context) -def test(context: argparse.Namespace, host: str | None = None) -> None: +def test(context: argparse.Namespace, host: str | None = None) -> None: # noqa: PT028 """The implementation of the "test" command.""" if host is None: host = context.host @@ -795,18 +788,16 @@ def test(context: argparse.Namespace, host: str | None = None) -> None: / f"Frameworks/{apple_multiarch(host)}" ) - run( - [ - sys.executable, - "Apple/testbed", - "clone", - "--platform", - context.platform, - "--framework", - framework_path, - testbed_dir, - ] - ) + run([ + sys.executable, + "Apple/testbed", + "clone", + "--platform", + context.platform, + "--framework", + framework_path, + testbed_dir, + ]) run( [ @@ -840,7 +831,7 @@ def apple_sim_host(platform_name: str) -> str: """Determine the native simulator target for this platform.""" for _, slice_parts in HOSTS[platform_name].items(): for host_triple in slice_parts: - parts = host_triple.split('-') + parts = host_triple.split("-") if parts[0] == platform.machine() and parts[-1] == "simulator": return host_triple @@ -968,20 +959,29 @@ def parse_args() -> argparse.Namespace: cmd.add_argument( "--simulator", help=( - "The name of the simulator to use (eg: 'iPhone 16e'). Defaults to " - "the most recently released 'entry level' iPhone device. Device " - "architecture and OS version can also be specified; e.g., " - "`--simulator 'iPhone 16 Pro,arch=arm64,OS=26.0'` would run on " - "an ARM64 iPhone 16 Pro simulator running iOS 26.0." + "The name of the simulator to use (eg: 'iPhone 16e'). " + "Defaults to the most recently released 'entry level' " + "iPhone device. Device architecture and OS version can also " + "be specified; e.g., " + "`--simulator 'iPhone 16 Pro,arch=arm64,OS=26.0'` would " + "run on an ARM64 iPhone 16 Pro simulator running iOS 26.0." ), ) group = cmd.add_mutually_exclusive_group() group.add_argument( - "--fast-ci", action="store_const", dest="ci_mode", const="fast", - help="Add test arguments for GitHub Actions") + "--fast-ci", + action="store_const", + dest="ci_mode", + const="fast", + help="Add test arguments for GitHub Actions", + ) group.add_argument( - "--slow-ci", action="store_const", dest="ci_mode", const="slow", - help="Add test arguments for buildbots") + "--slow-ci", + action="store_const", + dest="ci_mode", + const="slow", + help="Add test arguments for buildbots", + ) for subcommand in [configure_build, configure_host, build, ci]: subcommand.add_argument( diff --git a/Apple/testbed/__main__.py b/Apple/testbed/__main__.py index 49974cb1428..0dd77ab8b82 100644 --- a/Apple/testbed/__main__.py +++ b/Apple/testbed/__main__.py @@ -32,15 +32,15 @@ def select_simulator_device(platform): json_data = json.loads(raw_json) if platform == "iOS": - # Any iOS device will do; we'll look for "SE" devices - but the name isn't - # consistent over time. Older Xcode versions will use "iPhone SE (Nth - # generation)"; As of 2025, they've started using "iPhone 16e". + # Any iOS device will do; we'll look for "SE" devices - but the name + # isn't consistent over time. Older Xcode versions will use "iPhone SE + # (Nth generation)"; As of 2025, they've started using "iPhone 16e". # - # When Xcode is updated after a new release, new devices will be available - # and old ones will be dropped from the set available on the latest iOS - # version. Select the one with the highest minimum runtime version - this - # is an indicator of the "newest" released device, which should always be - # supported on the "most recent" iOS version. + # When Xcode is updated after a new release, new devices will be + # available and old ones will be dropped from the set available on the + # latest iOS version. Select the one with the highest minimum runtime + # version - this is an indicator of the "newest" released device, which + # should always be supported on the "most recent" iOS version. se_simulators = sorted( (devicetype["minRuntimeVersion"], devicetype["name"]) for devicetype in json_data["devicetypes"] @@ -295,7 +295,8 @@ def main(): parser = argparse.ArgumentParser( description=( - "Manages the process of testing an Apple Python project through Xcode." + "Manages the process of testing an Apple Python project " + "through Xcode." ), ) @@ -336,7 +337,10 @@ def main(): run = subcommands.add_parser( "run", - usage="%(prog)s [-h] [--simulator SIMULATOR] -- [ ...]", + usage=( + "%(prog)s [-h] [--simulator SIMULATOR] -- " + " [ ...]" + ), description=( "Run a testbed project. The arguments provided after `--` will be " "passed to the running iOS process as if they were arguments to " @@ -397,9 +401,9 @@ def main(): / "bin" ).is_dir(): print( - f"Testbed does not contain a compiled Python framework. Use " - f"`python {sys.argv[0]} clone ...` to create a runnable " - f"clone of this testbed." + "Testbed does not contain a compiled Python framework. " + f"Use `python {sys.argv[0]} clone ...` to create a " + "runnable clone of this testbed." ) sys.exit(20) @@ -411,7 +415,8 @@ def main(): ) else: print( - f"Must specify test arguments (e.g., {sys.argv[0]} run -- test)" + "Must specify test arguments " + f"(e.g., {sys.argv[0]} run -- test)" ) print() parser.print_help(sys.stderr) From 652c764a59913327b28b32016405696a620d969e Mon Sep 17 00:00:00 2001 From: Thierry Martos <81799048+ThierryMT@users.noreply.github.com> Date: Tue, 18 Nov 2025 16:01:09 -0800 Subject: [PATCH 252/313] gh-140381: Increase slow_fibonacci call frequency in test_profiling (#140673) --- .../test_profiling/test_sampling_profiler/test_integration.py | 4 ++-- .../next/Tests/2025-10-27-15-53-47.gh-issue-140381.N5o3pa.rst | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2025-10-27-15-53-47.gh-issue-140381.N5o3pa.rst diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_integration.py b/Lib/test/test_profiling/test_sampling_profiler/test_integration.py index 4fb2c595bbe..e1c80fa6d5d 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_integration.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_integration.py @@ -414,8 +414,8 @@ def main_loop(): if iteration % 3 == 0: # Very CPU intensive result = cpu_intensive_work() - elif iteration % 5 == 0: - # Expensive recursive operation + elif iteration % 2 == 0: + # Expensive recursive operation (increased frequency for slower machines) result = slow_fibonacci(12) else: # Medium operation diff --git a/Misc/NEWS.d/next/Tests/2025-10-27-15-53-47.gh-issue-140381.N5o3pa.rst b/Misc/NEWS.d/next/Tests/2025-10-27-15-53-47.gh-issue-140381.N5o3pa.rst new file mode 100644 index 00000000000..568a2b65d7d --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2025-10-27-15-53-47.gh-issue-140381.N5o3pa.rst @@ -0,0 +1 @@ +Fix flaky test_profiling tests on i686 and s390x architectures by increasing slow_fibonacci call frequency from every 5th iteration to every 2nd iteration. From ce791541769a41beabec0f515cd62e504d46ff1c Mon Sep 17 00:00:00 2001 From: Edward Xu Date: Wed, 19 Nov 2025 08:57:59 +0800 Subject: [PATCH 253/313] gh-139103: fix free-threading `dataclass.__init__` perf issue (gh-141596) The dataclasses `__init__` function is generated dynamically by a call to `exec()` and so doesn't have deferred reference counting enabled. Enable deferred reference counting on functions when assigned as an attribute to type objects to avoid reference count contention when creating dataclass instances. --- .../2025-11-15-23-58-23.gh-issue-139103.9cVYJ0.rst | 1 + Objects/typeobject.c | 12 ++++++++++++ Tools/ftscalingbench/ftscalingbench.py | 12 ++++++++++++ 3 files changed, 25 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-23-58-23.gh-issue-139103.9cVYJ0.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-23-58-23.gh-issue-139103.9cVYJ0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-23-58-23.gh-issue-139103.9cVYJ0.rst new file mode 100644 index 00000000000..c038dc742cc --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-23-58-23.gh-issue-139103.9cVYJ0.rst @@ -0,0 +1 @@ +Improve multithreaded scaling of dataclasses on the free-threaded build. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 61bcc21ce13..c99c6b3f637 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6546,6 +6546,18 @@ type_setattro(PyObject *self, PyObject *name, PyObject *value) assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_INLINE_VALUES)); assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_MANAGED_DICT)); +#ifdef Py_GIL_DISABLED + // gh-139103: Enable deferred refcounting for functions assigned + // to type objects. This is important for `dataclass.__init__`, + // which is generated dynamically. + if (value != NULL && + PyFunction_Check(value) && + !_PyObject_HasDeferredRefcount(value)) + { + PyUnstable_Object_EnableDeferredRefcount(value); + } +#endif + PyObject *old_value = NULL; PyObject *descr = _PyType_LookupRef(metatype, name); if (descr != NULL) { diff --git a/Tools/ftscalingbench/ftscalingbench.py b/Tools/ftscalingbench/ftscalingbench.py index 1a59e25189d..097a065f368 100644 --- a/Tools/ftscalingbench/ftscalingbench.py +++ b/Tools/ftscalingbench/ftscalingbench.py @@ -27,6 +27,7 @@ import sys import threading import time +from dataclasses import dataclass from operator import methodcaller # The iterations in individual benchmarks are scaled by this factor. @@ -202,6 +203,17 @@ def method_caller(): for i in range(1000 * WORK_SCALE): mc(obj) +@dataclass +class MyDataClass: + x: int + y: int + z: int + +@register_benchmark +def instantiate_dataclass(): + for _ in range(1000 * WORK_SCALE): + obj = MyDataClass(x=1, y=2, z=3) + def bench_one_thread(func): t0 = time.perf_counter_ns() func() From 7b0b70867586ef7109de60ccce94d13164dbb776 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 19 Nov 2025 09:48:51 +0800 Subject: [PATCH 254/313] gh-141692: Add a slice-specific lib folder to iOS XCframeworks. (#141693) Modifies the iOS XCframework to include a lib folder for each slice that contains a symlinked version of the libPython dynamic library. --- Apple/__main__.py | 14 ++++++++++++++ Apple/testbed/Python.xcframework/build/utils.sh | 3 ++- Makefile.pre.in | 3 +++ .../2025-11-18-13-55-47.gh-issue-141692.tud9if.rst | 3 +++ 4 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-11-18-13-55-47.gh-issue-141692.tud9if.rst diff --git a/Apple/__main__.py b/Apple/__main__.py index 1c588c23d6b..256966e76c2 100644 --- a/Apple/__main__.py +++ b/Apple/__main__.py @@ -477,6 +477,12 @@ def lib_platform_files(dirname, names): or name == "build-details.json" ) } + elif path.parts[-1] == "lib": + ignored_names = { + name + for name in names + if name.startswith("libpython") and name.endswith(".dylib") + } else: ignored_names = set() @@ -614,6 +620,12 @@ def create_xcframework(platform: str) -> str: slice_framework / "Headers/pyconfig.h", ) + print(f" - {slice_name} shared library") + # Create a simlink for the fat library + shared_lib = slice_path / f"lib/libpython{version_tag}.dylib" + shared_lib.parent.mkdir() + shared_lib.symlink_to("../Python.framework/Python") + print(f" - {slice_name} architecture-specific files") for host_triple, multiarch in slice_parts.items(): print(f" - {multiarch} standard library") @@ -625,6 +637,7 @@ def create_xcframework(platform: str) -> str: framework_path(host_triple, multiarch) / "lib", package_path / "Python.xcframework/lib", ignore=lib_platform_files, + symlinks=True, ) has_common_stdlib = True @@ -632,6 +645,7 @@ def create_xcframework(platform: str) -> str: framework_path(host_triple, multiarch) / "lib", slice_path / f"lib-{arch}", ignore=lib_non_platform_files, + symlinks=True, ) # Copy the host's pyconfig.h to an architecture-specific name. diff --git a/Apple/testbed/Python.xcframework/build/utils.sh b/Apple/testbed/Python.xcframework/build/utils.sh index 961c46d014b..e7155d8b30e 100755 --- a/Apple/testbed/Python.xcframework/build/utils.sh +++ b/Apple/testbed/Python.xcframework/build/utils.sh @@ -46,7 +46,8 @@ install_stdlib() { rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" rsync -au "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/lib-$ARCHS/" "$CODESIGNING_FOLDER_PATH/python/lib/" else - rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" + # A single-arch framework will have a libpython symlink; that can't be included at runtime + rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" --exclude 'libpython*.dylib' fi } diff --git a/Makefile.pre.in b/Makefile.pre.in index 59c3c808794..13108b1baf9 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -3050,6 +3050,9 @@ frameworkinstallunversionedstructure: $(LDLIBRARY) $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR) sed 's/%VERSION%/'"`$(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import platform; print(platform.python_version())'`"'/g' < $(RESSRCDIR)/Info.plist > $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Info.plist $(INSTALL_SHARED) $(LDLIBRARY) $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/$(LDLIBRARY) + $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(LIBDIR) + $(LN) -fs "../$(LDLIBRARY)" "$(DESTDIR)$(prefix)/lib/libpython$(LDVERSION).dylib" + $(LN) -fs "../$(LDLIBRARY)" "$(DESTDIR)$(prefix)/lib/libpython$(VERSION).dylib" $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(BINDIR) for file in $(srcdir)/$(RESSRCDIR)/bin/* ; do \ $(INSTALL) -m $(EXEMODE) $$file $(DESTDIR)$(BINDIR); \ diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-11-18-13-55-47.gh-issue-141692.tud9if.rst b/Misc/NEWS.d/next/Tools-Demos/2025-11-18-13-55-47.gh-issue-141692.tud9if.rst new file mode 100644 index 00000000000..d85c54db364 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-11-18-13-55-47.gh-issue-141692.tud9if.rst @@ -0,0 +1,3 @@ +Each slice of an iOS XCframework now contains a ``lib`` folder that contains +a symlink to the libpython dylib. This allows binary modules to be compiled +for iOS using dynamic libreary linking, rather than Framework linking. From 92c5de73b8d7526326c865b1a669b868f0d40c1e Mon Sep 17 00:00:00 2001 From: Ayappan Perumal Date: Wed, 19 Nov 2025 13:07:09 +0530 Subject: [PATCH 255/313] gh-141659: Fix bad file descriptor error in subprocess on AIX (GH-141660) /proc/self does not exist on AIX. --- .../2025-11-17-08-16-30.gh-issue-141659.QNi9Aj.rst | 1 + Modules/_posixsubprocess.c | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-08-16-30.gh-issue-141659.QNi9Aj.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-08-16-30.gh-issue-141659.QNi9Aj.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-08-16-30.gh-issue-141659.QNi9Aj.rst new file mode 100644 index 00000000000..eeb055c6012 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-08-16-30.gh-issue-141659.QNi9Aj.rst @@ -0,0 +1 @@ +Fix bad file descriptor errors from ``_posixsubprocess`` on AIX. diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 0219a3360fd..6f0a6d1d4e3 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -514,7 +514,13 @@ _close_open_fds_maybe_unsafe(int start_fd, int *fds_to_keep, proc_fd_dir = NULL; else #endif +#if defined(_AIX) + char fd_path[PATH_MAX]; + snprintf(fd_path, sizeof(fd_path), "/proc/%ld/fd", (long)getpid()); + proc_fd_dir = opendir(fd_path); +#else proc_fd_dir = opendir(FD_DIR); +#endif if (!proc_fd_dir) { /* No way to get a list of open fds. */ _close_range_except(start_fd, -1, fds_to_keep, fds_to_keep_len, From dbbf4b2e21d4aa73f54361ecda12187bacd7f6d3 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Wed, 19 Nov 2025 11:42:16 +0200 Subject: [PATCH 256/313] Post 3.15.0a2 --- Include/patchlevel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 899c892631f..804aa1a0427 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -27,7 +27,7 @@ #define PY_RELEASE_SERIAL 2 /* Version as a string */ -#define PY_VERSION "3.15.0a2" +#define PY_VERSION "3.15.0a2+" /*--end constants--*/ From c25a070759952b13f97ecc37ca2991c2669aee47 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 19 Nov 2025 10:16:24 +0000 Subject: [PATCH 257/313] GH-139653: Only raise an exception (or fatal error) when the stack pointer is about to overflow the stack. (GH-141711) Only raises if the stack pointer is both below the limit *and* above the stack base. This prevents false positives for user-space threads, as the stack pointer will be outside those bounds if the stack has been swapped. --- Include/internal/pycore_ceval.h | 7 +++-- InternalDocs/stack_protection.md | 9 +++++- ...-11-17-14-40-45.gh-issue-139653.LzOy1M.rst | 4 +++ Python/ceval.c | 28 ++++++++++++++----- 4 files changed, 38 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-14-40-45.gh-issue-139653.LzOy1M.rst diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 47c42fccdc2..2ae84be7b33 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -217,10 +217,13 @@ extern void _PyEval_DeactivateOpCache(void); static inline int _Py_MakeRecCheck(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + // Overflow if stack pointer is between soft limit and the base of the hardware stack. + // If it is below the hardware stack base, assume that we have the wrong stack limits, and do nothing. + // We could have the wrong stack limits because of limited platform support, or user-space threads. #if _Py_STACK_GROWS_DOWN - return here_addr < _tstate->c_stack_soft_limit; + return here_addr < _tstate->c_stack_soft_limit && here_addr >= _tstate->c_stack_soft_limit - 2 * _PyOS_STACK_MARGIN_BYTES; #else - return here_addr > _tstate->c_stack_soft_limit; + return here_addr > _tstate->c_stack_soft_limit && here_addr <= _tstate->c_stack_soft_limit + 2 * _PyOS_STACK_MARGIN_BYTES; #endif } diff --git a/InternalDocs/stack_protection.md b/InternalDocs/stack_protection.md index fa025bd930f..14802e57d09 100644 --- a/InternalDocs/stack_protection.md +++ b/InternalDocs/stack_protection.md @@ -38,12 +38,19 @@ # Stack Protection ```python kb_used = (stack_top - stack_pointer)>>10 -if stack_pointer < hard_limit: +if stack_pointer < bottom_of_machine_stack: + pass # Our stack limits could be wrong so it is safest to do nothing. +elif stack_pointer < hard_limit: FatalError(f"Unrecoverable stack overflow (used {kb_used} kB)") elif stack_pointer < soft_limit: raise RecursionError(f"Stack overflow (used {kb_used} kB)") ``` +### User space threads and other oddities + +Some libraries provide user-space threads. These will change the C stack at runtime. +To guard against this we only raise if the stack pointer is in the window between the expected stack base and the soft limit. + ### Diagnosing and fixing stack overflows For stack protection to work correctly the amount of stack consumed between calls to `_Py_EnterRecursiveCall()` must be less than `_PyOS_STACK_MARGIN_BYTES`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-14-40-45.gh-issue-139653.LzOy1M.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-14-40-45.gh-issue-139653.LzOy1M.rst new file mode 100644 index 00000000000..c3ae0e8adab --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-14-40-45.gh-issue-139653.LzOy1M.rst @@ -0,0 +1,4 @@ +Only raise a ``RecursionError`` or trigger a fatal error if the stack +pointer is both below the limit pointer *and* above the stack base. If +outside of these bounds assume that it is OK. This prevents false positives +when user-space threads swap stacks. diff --git a/Python/ceval.c b/Python/ceval.c index 14fef42ea96..5381cd826df 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -362,9 +362,11 @@ _Py_ReachedRecursionLimitWithMargin(PyThreadState *tstate, int margin_count) _Py_InitializeRecursionLimits(tstate); } #if _Py_STACK_GROWS_DOWN - return here_addr <= _tstate->c_stack_soft_limit + margin_count * _PyOS_STACK_MARGIN_BYTES; + return here_addr <= _tstate->c_stack_soft_limit + margin_count * _PyOS_STACK_MARGIN_BYTES && + here_addr >= _tstate->c_stack_soft_limit - 2 * _PyOS_STACK_MARGIN_BYTES; #else - return here_addr > _tstate->c_stack_soft_limit - margin_count * _PyOS_STACK_MARGIN_BYTES; + return here_addr > _tstate->c_stack_soft_limit - margin_count * _PyOS_STACK_MARGIN_BYTES && + here_addr <= _tstate->c_stack_soft_limit + 2 * _PyOS_STACK_MARGIN_BYTES; #endif } @@ -455,7 +457,7 @@ int pthread_attr_destroy(pthread_attr_t *a) #endif static void -hardware_stack_limits(uintptr_t *base, uintptr_t *top) +hardware_stack_limits(uintptr_t *base, uintptr_t *top, uintptr_t sp) { #ifdef WIN32 ULONG_PTR low, high; @@ -491,10 +493,19 @@ hardware_stack_limits(uintptr_t *base, uintptr_t *top) return; } # endif - uintptr_t here_addr = _Py_get_machine_stack_pointer(); - uintptr_t top_addr = _Py_SIZE_ROUND_UP(here_addr, 4096); + // Add some space for caller function then round to minimum page size + // This is a guess at the top of the stack, but should be a reasonably + // good guess if called from _PyThreadState_Attach when creating a thread. + // If the thread is attached deep in a call stack, then the guess will be poor. +#if _Py_STACK_GROWS_DOWN + uintptr_t top_addr = _Py_SIZE_ROUND_UP(sp + 8*sizeof(void*), SYSTEM_PAGE_SIZE); *top = top_addr; *base = top_addr - Py_C_STACK_SIZE; +# else + uintptr_t base_addr = _Py_SIZE_ROUND_DOWN(sp - 8*sizeof(void*), SYSTEM_PAGE_SIZE); + *base = base_addr; + *top = base_addr + Py_C_STACK_SIZE; +#endif #endif } @@ -543,7 +554,8 @@ void _Py_InitializeRecursionLimits(PyThreadState *tstate) { uintptr_t base, top; - hardware_stack_limits(&base, &top); + uintptr_t here_addr = _Py_get_machine_stack_pointer(); + hardware_stack_limits(&base, &top, here_addr); assert(top != 0); tstate_set_stack(tstate, base, top); @@ -587,7 +599,7 @@ PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) /* The function _Py_EnterRecursiveCallTstate() only calls _Py_CheckRecursiveCall() - if the recursion_depth reaches recursion_limit. */ + if the stack pointer is between the stack base and c_stack_hard_limit. */ int _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) { @@ -596,10 +608,12 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) assert(_tstate->c_stack_soft_limit != 0); assert(_tstate->c_stack_hard_limit != 0); #if _Py_STACK_GROWS_DOWN + assert(here_addr >= _tstate->c_stack_hard_limit - _PyOS_STACK_MARGIN_BYTES); if (here_addr < _tstate->c_stack_hard_limit) { /* Overflowing while handling an overflow. Give up. */ int kbytes_used = (int)(_tstate->c_stack_top - here_addr)/1024; #else + assert(here_addr <= _tstate->c_stack_hard_limit + _PyOS_STACK_MARGIN_BYTES); if (here_addr > _tstate->c_stack_hard_limit) { /* Overflowing while handling an overflow. Give up. */ int kbytes_used = (int)(here_addr - _tstate->c_stack_top)/1024; From 52f70a6f60254fec5297d1ff731b6c1ebc52ec24 Mon Sep 17 00:00:00 2001 From: Guo Ci Date: Wed, 19 Nov 2025 05:30:53 -0500 Subject: [PATCH 258/313] Correct class name from PullDom to PullDOM (#141207) --- Doc/library/xml.dom.pulldom.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/xml.dom.pulldom.rst b/Doc/library/xml.dom.pulldom.rst index 8bceeecd463..a21cfaa4645 100644 --- a/Doc/library/xml.dom.pulldom.rst +++ b/Doc/library/xml.dom.pulldom.rst @@ -74,7 +74,7 @@ given point) or to make use of the :func:`DOMEventStream.expandNode` method and switch to DOM-related processing. -.. class:: PullDom(documentFactory=None) +.. class:: PullDOM(documentFactory=None) Subclass of :class:`xml.sax.handler.ContentHandler`. From afa0badcc587ea7500e2b4dd2ea269f8bbda5fb2 Mon Sep 17 00:00:00 2001 From: da-woods Date: Wed, 19 Nov 2025 11:38:10 +0000 Subject: [PATCH 259/313] gh-141726: Add PyDict_SetDefaultRef() to the Stable ABI (#141727) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/data/stable_abi.dat | 1 + Include/cpython/dictobject.h | 10 ---------- Include/dictobject.h | 12 ++++++++++++ Lib/test/test_stable_abi_ctypes.py | 1 + .../2025-11-18-18-36-15.gh-issue-141726.ILrhyK.rst | 1 + Misc/stable_abi.toml | 2 ++ PC/python3dll.c | 1 + 7 files changed, 18 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-11-18-18-36-15.gh-issue-141726.ILrhyK.rst diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 5cbf3771950..95e032655cf 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -160,6 +160,7 @@ func,PyDict_Merge,3.2,, func,PyDict_MergeFromSeq2,3.2,, func,PyDict_New,3.2,, func,PyDict_Next,3.2,, +func,PyDict_SetDefaultRef,3.15,, func,PyDict_SetItem,3.2,, func,PyDict_SetItemString,3.2,, func,PyDict_Size,3.2,, diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index df9ec7050fc..5f2f7b6d4f5 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -39,16 +39,6 @@ Py_DEPRECATED(3.14) PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObje PyAPI_FUNC(PyObject *) PyDict_SetDefault( PyObject *mp, PyObject *key, PyObject *defaultobj); -// Inserts `key` with a value `default_value`, if `key` is not already present -// in the dictionary. If `result` is not NULL, then the value associated -// with `key` is returned in `*result` (either the existing value, or the now -// inserted `default_value`). -// Returns: -// -1 on error -// 0 if `key` was not present and `default_value` was inserted -// 1 if `key` was present and `default_value` was not inserted -PyAPI_FUNC(int) PyDict_SetDefaultRef(PyObject *mp, PyObject *key, PyObject *default_value, PyObject **result); - /* Get the number of items of a dictionary. */ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { PyDictObject *mp; diff --git a/Include/dictobject.h b/Include/dictobject.h index 1bbeec1ab69..0384e3131dc 100644 --- a/Include/dictobject.h +++ b/Include/dictobject.h @@ -68,6 +68,18 @@ PyAPI_FUNC(int) PyDict_GetItemRef(PyObject *mp, PyObject *key, PyObject **result PyAPI_FUNC(int) PyDict_GetItemStringRef(PyObject *mp, const char *key, PyObject **result); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030F0000 +// Inserts `key` with a value `default_value`, if `key` is not already present +// in the dictionary. If `result` is not NULL, then the value associated +// with `key` is returned in `*result` (either the existing value, or the now +// inserted `default_value`). +// Returns: +// -1 on error +// 0 if `key` was not present and `default_value` was inserted +// 1 if `key` was present and `default_value` was not inserted +PyAPI_FUNC(int) PyDict_SetDefaultRef(PyObject *mp, PyObject *key, PyObject *default_value, PyObject **result); +#endif + #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 PyAPI_FUNC(PyObject *) PyObject_GenericGetDict(PyObject *, void *); #endif diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 7167646ecc6..bc834f5a681 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -165,6 +165,7 @@ SYMBOL_NAMES = ( "PyDict_MergeFromSeq2", "PyDict_New", "PyDict_Next", + "PyDict_SetDefaultRef", "PyDict_SetItem", "PyDict_SetItemString", "PyDict_Size", diff --git a/Misc/NEWS.d/next/C_API/2025-11-18-18-36-15.gh-issue-141726.ILrhyK.rst b/Misc/NEWS.d/next/C_API/2025-11-18-18-36-15.gh-issue-141726.ILrhyK.rst new file mode 100644 index 00000000000..3fdad5c6b3e --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-18-18-36-15.gh-issue-141726.ILrhyK.rst @@ -0,0 +1 @@ +Add :c:func:`PyDict_SetDefaultRef` to the Stable ABI. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 7ee6cf1dae5..5c503f81d32 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2639,3 +2639,5 @@ added = '3.15' [const.Py_mod_token] added = '3.15' +[function.PyDict_SetDefaultRef] + added = '3.15' diff --git a/PC/python3dll.c b/PC/python3dll.c index 99e0f05fe03..35db1a660a7 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -191,6 +191,7 @@ EXPORT_FUNC(PyDict_Merge) EXPORT_FUNC(PyDict_MergeFromSeq2) EXPORT_FUNC(PyDict_New) EXPORT_FUNC(PyDict_Next) +EXPORT_FUNC(PyDict_SetDefaultRef) EXPORT_FUNC(PyDict_SetItem) EXPORT_FUNC(PyDict_SetItemString) EXPORT_FUNC(PyDict_Size) From 95296a9d40aa2d58502a09e86e2a93c03df23366 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 19 Nov 2025 13:55:10 +0200 Subject: [PATCH 260/313] gh-140875: Fix handling of unclosed charrefs before EOF in HTMLParser (GH-140904) --- Lib/html/parser.py | 29 +++-- Lib/test/test_htmlparser.py | 110 ++++++++++++++---- ...-11-02-10-44-23.gh-issue-140875.wt6B37.rst | 3 + 3 files changed, 109 insertions(+), 33 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-02-10-44-23.gh-issue-140875.wt6B37.rst diff --git a/Lib/html/parser.py b/Lib/html/parser.py index e50620de800..80fb8c3f929 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -24,6 +24,7 @@ entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]') charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]') +incomplete_charref = re.compile('&#(?:[0-9]|[xX][0-9a-fA-F])') attr_charref = re.compile(r'&(#[0-9]+|#[xX][0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*)[;=]?') starttagopen = re.compile('<[a-zA-Z]') @@ -304,10 +305,20 @@ def goahead(self, end): k = k - 1 i = self.updatepos(i, k) continue + match = incomplete_charref.match(rawdata, i) + if match: + if end: + self.handle_charref(rawdata[i+2:]) + i = self.updatepos(i, n) + break + # incomplete + break + elif i + 3 < n: # larger than "&#x" + # not the end of the buffer, and can't be confused + # with some other construct + self.handle_data("&#") + i = self.updatepos(i, i + 2) else: - if ";" in rawdata[i:]: # bail by consuming &# - self.handle_data(rawdata[i:i+2]) - i = self.updatepos(i, i+2) break elif startswith('&', i): match = entityref.match(rawdata, i) @@ -321,15 +332,13 @@ def goahead(self, end): continue match = incomplete.match(rawdata, i) if match: - # match.group() will contain at least 2 chars - if end and match.group() == rawdata[i:]: - k = match.end() - if k <= i: - k = n - i = self.updatepos(i, i + 1) + if end: + self.handle_entityref(rawdata[i+1:]) + i = self.updatepos(i, n) + break # incomplete break - elif (i + 1) < n: + elif i + 1 < n: # not the end of the buffer, and can't be confused # with some other construct self.handle_data("&") diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index 19dde9362a4..e4eff1ea17a 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -109,12 +109,13 @@ def get_events(self): class TestCaseBase(unittest.TestCase): - def get_collector(self): - return EventCollector(convert_charrefs=False) + def get_collector(self, convert_charrefs=False): + return EventCollector(convert_charrefs=convert_charrefs) - def _run_check(self, source, expected_events, collector=None): + def _run_check(self, source, expected_events, + *, collector=None, convert_charrefs=False): if collector is None: - collector = self.get_collector() + collector = self.get_collector(convert_charrefs=convert_charrefs) parser = collector for s in source: parser.feed(s) @@ -128,7 +129,7 @@ def _run_check(self, source, expected_events, collector=None): def _run_check_extra(self, source, events): self._run_check(source, events, - EventCollectorExtra(convert_charrefs=False)) + collector=EventCollectorExtra(convert_charrefs=False)) class HTMLParserTestCase(TestCaseBase): @@ -187,10 +188,87 @@ def test_malformatted_charref(self): ]) def test_unclosed_entityref(self): - self._run_check("&entityref foo", [ - ("entityref", "entityref"), - ("data", " foo"), - ]) + self._run_check('> <', [('entityref', 'gt'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('> <', [('data', '> <')], convert_charrefs=True) + + self._run_check('&undefined <', + [('entityref', 'undefined'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('&undefined <', [('data', '&undefined <')], + convert_charrefs=True) + + self._run_check('>undefined <', + [('entityref', 'gtundefined'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('>undefined <', [('data', '>undefined <')], + convert_charrefs=True) + + self._run_check('& <', [('data', '& '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('& <', [('data', '& <')], convert_charrefs=True) + + def test_eof_in_entityref(self): + self._run_check('>', [('entityref', 'gt')], convert_charrefs=False) + self._run_check('>', [('data', '>')], convert_charrefs=True) + + self._run_check('&g', [('entityref', 'g')], convert_charrefs=False) + self._run_check('&g', [('data', '&g')], convert_charrefs=True) + + self._run_check('&undefined', [('entityref', 'undefined')], + convert_charrefs=False) + self._run_check('&undefined', [('data', '&undefined')], + convert_charrefs=True) + + self._run_check('>undefined', [('entityref', 'gtundefined')], + convert_charrefs=False) + self._run_check('>undefined', [('data', '>undefined')], + convert_charrefs=True) + + self._run_check('&', [('data', '&')], convert_charrefs=False) + self._run_check('&', [('data', '&')], convert_charrefs=True) + + def test_unclosed_charref(self): + self._run_check('{ <', [('charref', '123'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('{ <', [('data', '{ <')], convert_charrefs=True) + self._run_check('« <', [('charref', 'xab'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('« <', [('data', '\xab <')], convert_charrefs=True) + + self._run_check('� <', + [('charref', '123456789'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('� <', [('data', '\ufffd <')], + convert_charrefs=True) + self._run_check('� <', + [('charref', 'x123456789'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('� <', [('data', '\ufffd <')], + convert_charrefs=True) + + self._run_check('&# <', [('data', '&# '), ('entityref', 'lt')], convert_charrefs=False) + self._run_check('&# <', [('data', '&# <')], convert_charrefs=True) + self._run_check('&#x <', [('data', '&#x '), ('entityref', 'lt')], convert_charrefs=False) + self._run_check('&#x <', [('data', '&#x <')], convert_charrefs=True) + + def test_eof_in_charref(self): + self._run_check('{', [('charref', '123')], convert_charrefs=False) + self._run_check('{', [('data', '{')], convert_charrefs=True) + self._run_check('«', [('charref', 'xab')], convert_charrefs=False) + self._run_check('«', [('data', '\xab')], convert_charrefs=True) + + self._run_check('�', [('charref', '123456789')], + convert_charrefs=False) + self._run_check('�', [('data', '\ufffd')], convert_charrefs=True) + self._run_check('�', [('charref', 'x123456789')], + convert_charrefs=False) + self._run_check('�', [('data', '\ufffd')], convert_charrefs=True) + + self._run_check('&#', [('data', '&#')], convert_charrefs=False) + self._run_check('&#', [('data', '&#')], convert_charrefs=True) + self._run_check('&#x', [('data', '&#x')], convert_charrefs=False) + self._run_check('&#x', [('data', '&#x')], convert_charrefs=True) def test_bad_nesting(self): # Strangely, this *is* supposed to test that overlapping @@ -762,20 +840,6 @@ def test_correct_detection_of_start_tags(self): ] self._run_check(html, expected) - def test_EOF_in_charref(self): - # see #17802 - # This test checks that the UnboundLocalError reported in the issue - # is not raised, however I'm not sure the returned values are correct. - # Maybe HTMLParser should use self.unescape for these - data = [ - ('a&', [('data', 'a&')]), - ('a&b', [('data', 'ab')]), - ('a&b ', [('data', 'a'), ('entityref', 'b'), ('data', ' ')]), - ('a&b;', [('data', 'a'), ('entityref', 'b')]), - ] - for html, expected in data: - self._run_check(html, expected) - def test_eof_in_comments(self): data = [ ('