diff --git a/.gitattributes b/.gitattributes index b8189f12ded..f4d65dfd1df 100644 --- a/.gitattributes +++ b/.gitattributes @@ -34,6 +34,9 @@ Lib/test/xmltestdata/* noeol Lib/venv/scripts/common/activate text eol=lf Lib/venv/scripts/posix/* text eol=lf +# Prevent GitHub's web conflict editor from converting LF to CRLF +*.rst text eol=lf + # CRLF files [attr]dos text eol=crlf diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml deleted file mode 100644 index 19314dd0c88..00000000000 --- a/.github/workflows/documentation-links.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Read the Docs PR preview -# Automatically edits a pull request's descriptions with a link -# to the documentation's preview on Read the Docs. - -on: - pull_request_target: - types: - - opened - paths: - - 'Doc/**' - - '.github/workflows/doc.yml' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - documentation-links: - runs-on: ubuntu-latest - permissions: - pull-requests: write - timeout-minutes: 5 - - steps: - - uses: readthedocs/actions/preview@b8bba1484329bda1a3abe986df7ebc80a8950333 # v1.5 - with: - project-slug: "cpython-previews" - single-version: "true" diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index e5a5b3939e5..7f6571ef954 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -19,6 +19,7 @@ on: - "Tools/build/consts_getter.py" - "Tools/build/deepfreeze.py" - "Tools/build/generate-build-details.py" + - "Tools/build/generate_levenshtein_examples.py" - "Tools/build/generate_sbom.py" - "Tools/build/generate_stdlib_module_names.py" - "Tools/build/mypy.ini" diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index 7994a01ee46..6201e719ca8 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -26,9 +26,16 @@ apt-get -yq --no-install-recommends install \ xvfb \ zlib1g-dev -# Workaround missing libmpdec-dev on ubuntu 24.04: -# https://launchpad.net/~ondrej/+archive/ubuntu/php -# https://deb.sury.org/ -sudo add-apt-repository ppa:ondrej/php -apt-get update -apt-get -yq --no-install-recommends install libmpdec-dev +# Workaround missing libmpdec-dev on ubuntu 24.04 by building mpdecimal +# from source. ppa:ondrej/php (launchpad.net) are unreliable +# (https://status.canonical.com) so fetch the tarball directly +# from the upstream host. +# https://www.bytereef.org/mpdecimal/ +MPDECIMAL_VERSION=4.0.1 +curl -fsSL "https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-${MPDECIMAL_VERSION}.tar.gz" \ + | tar -xz -C /tmp +(cd "/tmp/mpdecimal-${MPDECIMAL_VERSION}" \ + && ./configure --prefix=/usr/local \ + && make -j"$(nproc)" \ + && make install) +ldconfig diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 0453b6ab555..7b524569f85 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -56,7 +56,7 @@ jobs: with: python-version: '3' cache: 'pip' - cache-dependency-path: 'Doc/requirements.txt' + cache-dependency-path: 'Doc/pylock.toml' - name: 'Install build dependencies' run: make -C Doc/ venv diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index 9d4f412cfcf..33f6f0ef455 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -40,17 +40,15 @@ jobs: # Install clang wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh + sudo ./llvm.sh 20 + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-20 100 + sudo update-alternatives --set clang /usr/bin/clang-20 + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-20 100 + sudo update-alternatives --set clang++ /usr/bin/clang++-20 if [ "${SANITIZER}" = "TSan" ]; then - sudo ./llvm.sh 17 # gh-121946: llvm-18 package is temporarily broken - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 - sudo update-alternatives --set clang /usr/bin/clang-17 - sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 - sudo update-alternatives --set clang++ /usr/bin/clang++-17 # Reduce ASLR to avoid TSan crashing sudo sysctl -w vm.mmap_rnd_bits=28 - else - sudo ./llvm.sh 20 fi - name: Sanitizer option setup diff --git a/Doc/Makefile b/Doc/Makefile index 7bdabd8bf16..60970d50833 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -13,7 +13,7 @@ JOBS = auto PAPER = SOURCES = DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py) -REQUIREMENTS = requirements.txt +REQUIREMENTS = pylock.toml SPHINXERRORHANDLING = --fail-on-warning # Internal variables. diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 59044d2d88c..09c9ed3ca54 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -2,7 +2,7 @@ .. _allocating-objects: -Allocating Objects on the Heap +Allocating objects on the heap ============================== @@ -153,10 +153,12 @@ Allocating Objects on the Heap To allocate and create extension modules. -Deprecated aliases -^^^^^^^^^^^^^^^^^^ +Soft-deprecated aliases +^^^^^^^^^^^^^^^^^^^^^^^ -These are :term:`soft deprecated` aliases to existing functions and macros. +.. soft-deprecated:: 3.15 + +These are aliases to existing functions and macros. They exist solely for backwards compatibility. @@ -164,7 +166,7 @@ They exist solely for backwards compatibility. :widths: auto :header-rows: 1 - * * Deprecated alias + * * Soft-deprecated alias * Function * * .. c:macro:: PyObject_NEW(type, typeobj) * :c:macro:`PyObject_New` diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index fe950196297..dc3e0f37c36 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -258,7 +258,9 @@ readonly, format .. c:macro:: PyBUF_WRITEABLE - This is a :term:`soft deprecated` alias to :c:macro:`PyBUF_WRITABLE`. + This is an alias to :c:macro:`PyBUF_WRITABLE`. + + .. soft-deprecated:: 3.13 .. c:macro:: PyBUF_FORMAT diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index d1fde1baf71..f56bcd6333a 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -47,9 +47,9 @@ called with a non-bytes parameter. *len* on success, and ``NULL`` on failure. If *v* is ``NULL``, the contents of the bytes object are uninitialized. - .. deprecated:: 3.15 - ``PyBytes_FromStringAndSize(NULL, len)`` is :term:`soft deprecated`, - use the :c:type:`PyBytesWriter` API instead. + .. soft-deprecated:: 3.15 + Use the :c:type:`PyBytesWriter` API instead of + ``PyBytes_FromStringAndSize(NULL, len)``. .. c:function:: PyObject* PyBytes_FromFormat(const char *format, ...) @@ -238,9 +238,8 @@ called with a non-bytes parameter. *\*bytes* is set to ``NULL``, :exc:`MemoryError` is set, and ``-1`` is returned. - .. deprecated:: 3.15 - The function is :term:`soft deprecated`, - use the :c:type:`PyBytesWriter` API instead. + .. soft-deprecated:: 3.15 + Use the :c:type:`PyBytesWriter` API instead. .. c:function:: PyObject *PyBytes_Repr(PyObject *bytes, int smartquotes) diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index be2c85ec974..57b77f92a7d 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -212,7 +212,7 @@ bound into a function. .. c:function:: PyObject *PyCode_Optimize(PyObject *code, PyObject *consts, PyObject *names, PyObject *lnotab_obj) - This is a :term:`soft deprecated` function that does nothing. + This is a function that does nothing. Prior to Python 3.10, this function would perform basic optimizations to a code object. @@ -220,6 +220,8 @@ bound into a function. .. versionchanged:: 3.10 This function now does nothing. + .. soft-deprecated:: 3.13 + .. _c_codeobject_flags: diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst index 1746fe95eaa..3f38411a52d 100644 --- a/Doc/c-api/concrete.rst +++ b/Doc/c-api/concrete.rst @@ -112,6 +112,7 @@ Other Objects picklebuffer.rst weakref.rst capsule.rst + sentinel.rst frame.rst gen.rst coro.rst diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index b913e24b3c7..539c4610ce4 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -140,7 +140,7 @@ found in the dictionary of type objects. .. c:macro:: PyDescr_COMMON - This is a :term:`soft deprecated` macro including the common fields for a + This is a macro including the common fields for a descriptor object. This was included in Python's C API by mistake; do not use it in extensions. @@ -148,6 +148,8 @@ found in the dictionary of type objects. descriptor protocol (:c:member:`~PyTypeObject.tp_descr_get` and :c:member:`~PyTypeObject.tp_descr_set`). + .. soft-deprecated:: 3.15 + Built-in descriptors ^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 8ecd7c62517..7a07818b7b4 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -818,7 +818,7 @@ Exception Classes .. c:macro:: PyException_HEAD - This is a :term:`soft deprecated` macro including the base fields for an + This is a macro including the base fields for an exception object. This was included in Python's C API by mistake and is not designed for use @@ -826,6 +826,8 @@ Exception Classes :c:func:`PyErr_NewException` or otherwise create a class inheriting from :c:data:`PyExc_BaseException`. + .. soft-deprecated:: 3.15 + Exception Objects ================= diff --git a/Doc/c-api/extension-modules.rst b/Doc/c-api/extension-modules.rst index 92b531665e1..7bc04970b19 100644 --- a/Doc/c-api/extension-modules.rst +++ b/Doc/c-api/extension-modules.rst @@ -191,10 +191,10 @@ the :c:data:`Py_mod_multiple_interpreters` slot. ``PyInit`` function ................... -.. deprecated:: 3.15 +.. soft-deprecated:: 3.15 - This functionality is :term:`soft deprecated`. - It will not get new features, but there are no plans to remove it. + This functionality will not get new features, + but there are no plans to remove it. Instead of :c:func:`PyModExport_modulename`, an extension module can define an older-style :dfn:`initialization function` with the signature: @@ -272,10 +272,9 @@ For example, a module called ``spam`` would be defined like this:: Legacy single-phase initialization ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. deprecated:: 3.15 +.. soft-deprecated:: 3.15 - Single-phase initialization is :term:`soft deprecated`. - It is a legacy mechanism to initialize extension + Single-phase initialization is a legacy mechanism to initialize extension modules, with known drawbacks and design flaws. Extension module authors are encouraged to use multi-phase initialization instead. diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index d89072ab24e..dcafefdc045 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -2,7 +2,7 @@ .. _fileobjects: -File Objects +File objects ------------ .. index:: pair: object; file @@ -136,11 +136,12 @@ the :mod:`io` APIs instead. failure; the appropriate exception will be set. -Deprecated API -^^^^^^^^^^^^^^ +Soft-deprecated API +^^^^^^^^^^^^^^^^^^^ +.. soft-deprecated:: 3.15 -These are :term:`soft deprecated` APIs that were included in Python's C API +These are APIs that were included in Python's C API by mistake. They are documented solely for completeness; use other ``PyFile*`` APIs instead. diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 929b56bd8e8..a12ad11abb1 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -86,8 +86,7 @@ Floating-Point Objects It is equivalent to the :c:macro:`!INFINITY` macro from the C11 standard ```` header. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_NAN @@ -103,8 +102,7 @@ Floating-Point Objects Equivalent to :c:macro:`!INFINITY`. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.14 .. c:macro:: Py_MATH_E @@ -161,8 +159,8 @@ Floating-Point Objects that is, it is normal, subnormal or zero, but not infinite or NaN. Return ``0`` otherwise. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. Use :c:macro:`!isfinite` instead. + .. soft-deprecated:: 3.14 + Use :c:macro:`!isfinite` instead. .. c:macro:: Py_IS_INFINITY(X) @@ -170,8 +168,8 @@ Floating-Point Objects Return ``1`` if the given floating-point number *X* is positive or negative infinity. Return ``0`` otherwise. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. Use :c:macro:`!isinf` instead. + .. soft-deprecated:: 3.14 + Use :c:macro:`!isinf` instead. .. c:macro:: Py_IS_NAN(X) @@ -179,8 +177,8 @@ Floating-Point Objects Return ``1`` if the given floating-point number *X* is a not-a-number (NaN) value. Return ``0`` otherwise. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. Use :c:macro:`!isnan` instead. + .. soft-deprecated:: 3.14 + Use :c:macro:`!isnan` instead. Pack and Unpack functions diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 967cfc72765..4159ff6e596 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -1,6 +1,6 @@ .. highlight:: c -Frame Objects +Frame objects ------------- .. c:type:: PyFrameObject @@ -147,7 +147,7 @@ See also :ref:`Reflection `. Return the line number that *frame* is currently executing. -Frame Locals Proxies +Frame locals proxies ^^^^^^^^^^^^^^^^^^^^ .. versionadded:: 3.13 @@ -169,7 +169,7 @@ See :pep:`667` for more information. Return non-zero if *obj* is a frame :func:`locals` proxy. -Legacy Local Variable APIs +Legacy local variable APIs ^^^^^^^^^^^^^^^^^^^^^^^^^^ These APIs are :term:`soft deprecated`. As of Python 3.13, they do nothing. @@ -178,40 +178,34 @@ They exist solely for backwards compatibility. .. c:function:: void PyFrame_LocalsToFast(PyFrameObject *f, int clear) - This function is :term:`soft deprecated` and does nothing. - Prior to Python 3.13, this function would copy the :attr:`~frame.f_locals` attribute of *f* to the internal "fast" array of local variables, allowing changes in frame objects to be visible to the interpreter. If *clear* was true, this function would process variables that were unset in the locals dictionary. - .. versionchanged:: 3.13 + .. soft-deprecated:: 3.13 This function now does nothing. .. c:function:: void PyFrame_FastToLocals(PyFrameObject *f) - This function is :term:`soft deprecated` and does nothing. - Prior to Python 3.13, this function would copy the internal "fast" array of local variables (which is used by the interpreter) to the :attr:`~frame.f_locals` attribute of *f*, allowing changes in local variables to be visible to frame objects. - .. versionchanged:: 3.13 + .. soft-deprecated:: 3.13 This function now does nothing. .. c:function:: int PyFrame_FastToLocalsWithError(PyFrameObject *f) - This function is :term:`soft deprecated` and does nothing. - Prior to Python 3.13, this function was similar to :c:func:`PyFrame_FastToLocals`, but would return ``0`` on success, and ``-1`` with an exception set on failure. - .. versionchanged:: 3.13 + .. soft-deprecated:: 3.13 This function now does nothing. @@ -219,7 +213,7 @@ They exist solely for backwards compatibility. :pep:`667` -Internal Frames +Internal frames ^^^^^^^^^^^^^^^ Unless using :pep:`523`, you will not need this. @@ -249,5 +243,3 @@ Unless using :pep:`523`, you will not need this. Return the currently executing line number, or -1 if there is no line number. .. versionadded:: 3.12 - - diff --git a/Doc/c-api/gen.rst b/Doc/c-api/gen.rst index 74db49a6814..ed121726b89 100644 --- a/Doc/c-api/gen.rst +++ b/Doc/c-api/gen.rst @@ -90,7 +90,9 @@ Deprecated API .. c:macro:: PyAsyncGenASend_CheckExact(op) - This is a :term:`soft deprecated` API that was included in Python's C API + This is an API that was included in Python's C API by mistake. It is solely here for completeness; do not use this API. + + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 2a22a023bda..500f2818e2e 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -536,16 +536,14 @@ have been standardized in C11 (or previous standards). Use the standard ``alignas`` specifier rather than this macro. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: PY_FORMAT_SIZE_T The :c:func:`printf` formatting modifier for :c:type:`size_t`. Use ``"z"`` directly instead. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_LL(number) Py_ULL(number) @@ -558,8 +556,7 @@ have been standardized in C11 (or previous standards). Consider using the C99 standard suffixes ``LL`` and ``LLU`` directly. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: PY_LONG_LONG PY_INT32_T @@ -572,8 +569,7 @@ have been standardized in C11 (or previous standards). respectively. Historically, these types needed compiler-specific extensions. - .. deprecated:: 3.15 - These macros are :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: PY_LLONG_MIN PY_LLONG_MAX @@ -587,16 +583,14 @@ have been standardized in C11 (or previous standards). The required header, ````, :ref:`is included ` in ``Python.h``. - .. deprecated:: 3.15 - These macros are :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_MEMCPY(dest, src, n) - This is a :term:`soft deprecated` alias to :c:func:`!memcpy`. - Use :c:func:`!memcpy` directly instead. + This is an alias to :c:func:`!memcpy`. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.14 + Use :c:func:`!memcpy` directly instead. .. c:macro:: Py_UNICODE_SIZE @@ -606,29 +600,25 @@ have been standardized in C11 (or previous standards). The required header for the latter, ````, :ref:`is included ` in ``Python.h``. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_UNICODE_WIDE Defined if ``wchar_t`` can hold a Unicode character (UCS-4). Use ``sizeof(wchar_t) >= 4`` instead - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_VA_COPY - This is a :term:`soft deprecated` alias to the C99-standard ``va_copy`` - function. + This is an alias to the C99-standard ``va_copy`` function. Historically, this would use a compiler-specific method to copy a ``va_list``. .. versionchanged:: 3.6 This is now an alias to ``va_copy``. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. _api-objects: diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 790ec8da109..60e3ae4a064 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -197,12 +197,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: long PyLong_AS_LONG(PyObject *obj) - A :term:`soft deprecated` alias. Exactly equivalent to the preferred ``PyLong_AsLong``. In particular, it can fail with :exc:`OverflowError` or another exception. - .. deprecated:: 3.14 - The function is soft deprecated. + .. soft-deprecated:: 3.14 .. c:function:: int PyLong_AsInt(PyObject *obj) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index a66a1bfd7f8..b67ca671a2a 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -965,9 +965,7 @@ or code that creates modules dynamically. // PyModule_AddObject() stole a reference to obj: // Py_XDECREF(obj) is not needed here. - .. deprecated:: 3.13 - - :c:func:`PyModule_AddObject` is :term:`soft deprecated`. + .. soft-deprecated:: 3.13 .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst index b0227c2f4fa..4bfcb86abf5 100644 --- a/Doc/c-api/monitoring.rst +++ b/Doc/c-api/monitoring.rst @@ -205,6 +205,4 @@ would typically correspond to a Python function. .. versionadded:: 3.13 - .. deprecated:: 3.14 - - This function is :term:`soft deprecated`. + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/perfmaps.rst b/Doc/c-api/perfmaps.rst index 76a1e9f528d..bd05e628faa 100644 --- a/Doc/c-api/perfmaps.rst +++ b/Doc/c-api/perfmaps.rst @@ -31,7 +31,7 @@ Note that holding an :term:`attached thread state` is not required for these API or ``-2`` on failure to create a lock. Check ``errno`` for more information about the cause of a failure. -.. c:function:: int PyUnstable_WritePerfMapEntry(const void *code_addr, unsigned int code_size, const char *entry_name) +.. c:function:: int PyUnstable_WritePerfMapEntry(const void *code_addr, size_t code_size, const char *entry_name) Write one single entry to the ``/tmp/perf-$pid.map`` file. This function is thread safe. Here is what an example entry looks like:: diff --git a/Doc/c-api/sentinel.rst b/Doc/c-api/sentinel.rst new file mode 100644 index 00000000000..710ded56e2a --- /dev/null +++ b/Doc/c-api/sentinel.rst @@ -0,0 +1,35 @@ +.. highlight:: c + +.. _sentinelobjects: + +Sentinel objects +---------------- + +.. c:var:: PyTypeObject PySentinel_Type + + This instance of :c:type:`PyTypeObject` represents the Python + :class:`sentinel` type. This is the same object as :class:`sentinel`. + + .. versionadded:: next + +.. c:function:: int PySentinel_Check(PyObject *o) + + Return true if *o* is a :class:`sentinel` object. The :class:`sentinel` type + does not allow subclasses, so this check is exact. + + .. versionadded:: next + +.. c:function:: PyObject* PySentinel_New(const char *name, const char *module_name) + + Return a new :class:`sentinel` object with :attr:`~sentinel.__name__` set to + *name* and :attr:`~sentinel.__module__` set to *module_name*. + *name* must not be ``NULL``. If *module_name* is ``NULL``, :attr:`~sentinel.__module__` + is set to ``None``. + Return ``NULL`` with an exception set on failure. + + For pickling to work, *module_name* must be the name of an importable + module, and the sentinel must be accessible from that module under a + path matching *name*. Pickle treats *name* as a global variable name + in *module_name* (see :meth:`object.__reduce__`). + + .. versionadded:: next diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index df5bf6b64a9..6bae8f25ad7 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -109,9 +109,8 @@ Sequence Protocol Alias for :c:func:`PySequence_Contains`. - .. deprecated:: 3.14 - The function is :term:`soft deprecated` and should no longer be used to - write new code. + .. soft-deprecated:: 3.14 + The function should no longer be used to write new code. .. c:function:: Py_ssize_t PySequence_Index(PyObject *o, PyObject *value) diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index 53febd0c4c1..db537aff2e6 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -201,7 +201,7 @@ Deprecated API .. c:macro:: PySet_MINSIZE - A :term:`soft deprecated` constant representing the size of an internal + A constant representing the size of an internal preallocated table inside :c:type:`PySetObject` instances. This is documented solely for completeness, as there are no guarantees @@ -211,3 +211,5 @@ Deprecated API :c:macro:`!PySet_MINSIZE` can be replaced with a small constant like ``8``. If looking for the size of a set, use :c:func:`PySet_Size` instead. + + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index c3960d6ff87..d3d8239365f 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1391,8 +1391,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. versionchanged:: 3.9 - Renamed to the current name, without the leading underscore. - The old provisional name is :term:`soft deprecated`. + Renamed to the current name, without the leading underscore. + The old provisional name is :term:`soft deprecated`. .. versionchanged:: 3.12 @@ -1501,11 +1501,13 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_HAVE_VERSION_TAG - This is a :term:`soft deprecated` macro that does nothing. + This macro does nothing. Historically, this would indicate that the :c:member:`~PyTypeObject.tp_version_tag` field was available and initialized. + .. soft-deprecated:: 3.13 + .. c:macro:: Py_TPFLAGS_INLINE_VALUES diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 2a6e6b96313..663b79e45ee 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -2037,6 +2037,10 @@ PySeqIter_Check:PyObject *:op:0: PySeqIter_New:PyObject*::+1: PySeqIter_New:PyObject*:seq:0: +PySentinel_New:PyObject*::+1: +PySentinel_New:const char*:name:: +PySentinel_New:const char*:module_name:: + PySequence_Check:int::: PySequence_Check:PyObject*:o:0: diff --git a/Doc/deprecations/pending-removal-in-3.15.rst b/Doc/deprecations/pending-removal-in-3.15.rst index e7f27f73664..1d9a3095813 100644 --- a/Doc/deprecations/pending-removal-in-3.15.rst +++ b/Doc/deprecations/pending-removal-in-3.15.rst @@ -60,7 +60,7 @@ Pending removal in Python 3.15 * :mod:`types`: - * :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was + * :class:`types.CodeType`: Accessing :attr:`!codeobject.co_lnotab` was deprecated in :pep:`626` since 3.10 and was planned to be removed in 3.12, but it only got a proper :exc:`DeprecationWarning` in 3.12. diff --git a/Doc/deprecations/pending-removal-in-3.17.rst b/Doc/deprecations/pending-removal-in-3.17.rst index ea9fb93ddd8..952ffad6435 100644 --- a/Doc/deprecations/pending-removal-in-3.17.rst +++ b/Doc/deprecations/pending-removal-in-3.17.rst @@ -35,7 +35,7 @@ Pending removal in Python 3.17 - Passing non-ascii *encoding* names to :func:`encodings.normalize_encoding` is deprecated and scheduled for removal in Python 3.17. - (Contributed by Stan Ulbrych in :gh:`136702`) + (Contributed by Stan Ulbrych in :gh:`136702`.) * :mod:`typing`: diff --git a/Doc/deprecations/pending-removal-in-future.rst b/Doc/deprecations/pending-removal-in-future.rst index e8306b8efee..74f98d33a4b 100644 --- a/Doc/deprecations/pending-removal-in-future.rst +++ b/Doc/deprecations/pending-removal-in-future.rst @@ -47,7 +47,7 @@ although there is currently no date scheduled for their removal. * :mod:`codecs`: use :func:`open` instead of :func:`codecs.open`. (:gh:`133038`) -* :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method +* :attr:`!codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method instead. * :mod:`datetime`: diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index ff34bb5d71c..591565cbc01 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1924,7 +1924,7 @@ correctly using identity tests: .. code-block:: python - _sentinel = object() + _sentinel = sentinel('_sentinel') def pop(self, key, default=_sentinel): if key in self: diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 9d5a9ac8b71..a7a68281860 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -594,7 +594,7 @@ a pure Python equivalent: def object_getattribute(obj, name): "Emulate PyObject_GenericGetAttr() in Objects/object.c" - null = object() + null = sentinel('null') objtype = type(obj) cls_var = find_name_in_mro(objtype, name, null) descr_get = getattr(type(cls_var), '__get__', null) @@ -1635,12 +1635,12 @@ by member descriptors: .. testcode:: - null = object() + null = sentinel('null') class Member: def __init__(self, name, clsname, offset): - 'Emulate PyMemberDef in Include/structmember.h' + 'Emulate PyMemberDef in Include/descrobject.h' # Also see descr_new() in Objects/descrobject.c self.name = name self.clsname = clsname diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 5260c2ca4ad..2fe5814bb04 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -371,7 +371,7 @@ Equality comparisons are defined though:: >>> Color.BLUE == Color.BLUE True -Comparisons against non-enumeration values will always compare not equal +Equality comparisons against non-enumeration values will always return ``False`` (again, :class:`IntEnum` was explicitly designed to behave differently, see below):: diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index 2f089a3d896..b21ed1c8f37 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -416,11 +416,9 @@ C API extensions need to be built specifically for the free-threaded build. The wheels, shared libraries, and binaries are indicated by a ``t`` suffix. * `pypa/manylinux `_ supports the - free-threaded build, with the ``t`` suffix, such as ``python3.13t``. -* `pypa/cibuildwheel `_ supports the - free-threaded build on Python 3.13 and 3.14. On Python 3.14, free-threaded - wheels will be built by default. On Python 3.13, you will need to set - `CIBW_ENABLE to cpython-freethreading `_. + free-threaded build, with the ``t`` suffix, such as ``python3.14t``. +* `pypa/cibuildwheel `_ supports + building wheels for the free-threaded build of Python 3.14 and newer. Limited C API and Stable ABI ............................ diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index fc4772bbcca..653f28ddbab 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -217,8 +217,9 @@ Example, using the :mod:`sys` APIs in file :file:`example.py`: How to obtain the best results ------------------------------ -For best results, Python should be compiled with -``CFLAGS="-fno-omit-frame-pointer -mno-omit-leaf-frame-pointer"`` as this allows +For best results, keep frame pointers enabled. On supported GCC-compatible +toolchains, CPython builds itself with ``-fno-omit-frame-pointer`` and, when +available, ``-mno-omit-leaf-frame-pointer`` by default. These flags allow profilers to unwind using only the frame pointer and not on DWARF debug information. This is because as the code that is interposed to allow ``perf`` support is dynamically generated it doesn't have any DWARF debugging information diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 9b4e7ae1834..3c6e8745474 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -35,6 +35,8 @@ The abstract grammar is currently defined as follows: :language: asdl +.. _ast_nodes: + Node classes ------------ @@ -164,8 +166,7 @@ Node classes Previous versions of Python allowed the creation of AST nodes that were missing required fields. Similarly, AST node constructors allowed arbitrary keyword arguments that were set as attributes of the AST node, even if they did not - match any of the fields of the AST node. This behavior is deprecated and will - be removed in Python 3.15. + match any of the fields of the AST node. These cases now raise a :exc:`TypeError`. .. note:: The descriptions of the specific node classes displayed here @@ -2480,7 +2481,7 @@ and classes for traversing abstract syntax trees: node = YourTransformer().visit(node) -.. function:: dump(node, annotate_fields=True, include_attributes=False, *, indent=None, show_empty=False) +.. function:: dump(node, annotate_fields=True, include_attributes=False, *, color=False, indent=None, show_empty=False) Return a formatted dump of the tree in *node*. This is mainly useful for debugging purposes. If *annotate_fields* is true (by default), @@ -2490,6 +2491,10 @@ and classes for traversing abstract syntax trees: numbers and column offsets are not dumped by default. If this is wanted, *include_attributes* can be set to true. + If *color* is ``True``, the returned string is syntax highlighted using + ANSI escape sequences. + If ``False`` (the default), colored output is always disabled. + If *indent* is a non-negative integer or string, then the tree will be pretty-printed with that indent level. An indent level of 0, negative, or ``""`` will only insert newlines. ``None`` (the default) @@ -2527,6 +2532,9 @@ and classes for traversing abstract syntax trees: .. versionchanged:: 3.15 Omit optional ``Load()`` values by default. + .. versionchanged:: next + Added the *color* parameter. + .. _ast-compiler-flags: @@ -2584,6 +2592,10 @@ Command-line usage .. versionadded:: 3.9 +.. versionchanged:: next + The output is now syntax highlighted by default. This can be + :ref:`controlled using environment variables `. + The :mod:`!ast` module can be executed as a script from the command line. It is as simple as: diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index f3409bcd2df..713b40d7466 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -304,7 +304,7 @@ generator can occur in an unexpected order:: try: yield 2 finally: - await asyncio.sleep(0.1) # immitate some async work + await asyncio.sleep(0.1) # imitate some async work work_done = True diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 4e60eee4429..2e17d0dc70c 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -355,6 +355,34 @@ and reliable way to wait for all tasks in the group to finish. Passes on all *kwargs* to :meth:`loop.create_task` + .. method:: cancel() + + Cancel the task group. This is a non-exceptional, early exit of the + task group's lifetime -- useful once the group's goal has been met or + its services no longer needed. + + :meth:`~asyncio.Task.cancel` will be called on any tasks in the group that + aren't yet done, as well as the parent (body) of the group. The task group + context manager will exit *without* :exc:`asyncio.CancelledError` being raised. + + If :meth:`cancel` is called before entering the task group, the group will be + cancelled upon entry. This is useful for patterns where one piece of + code passes an unused :class:`asyncio.TaskGroup` instance to another in order to have + the ability to cancel anything run within the group. + + :meth:`cancel` is idempotent and may be called after the task group has + already exited. + + Some ways to use :meth:`cancel`: + + * call it from the task group body based on some condition or event + * pass the task group instance to child tasks via :meth:`create_task`, allowing a child + task to conditionally cancel the entire entire group + * pass the task group instance or bound :meth:`cancel` method to some other task *before* + opening the task group, allowing remote cancellation + + .. versionadded:: next + Example:: async def main(): @@ -366,7 +394,8 @@ Example:: The ``async with`` statement will wait for all tasks in the group to finish. While waiting, new tasks may still be added to the group (for example, by passing ``tg`` into one of the coroutines -and calling ``tg.create_task()`` in that coroutine). +and calling ``tg.create_task()`` in that coroutine). There is also opportunity to +request termination of the entire task group with ``tg.cancel()``, based on some condition. Once the last task has finished and the ``async with`` block is exited, no new tasks may be added to the group. @@ -427,53 +456,6 @@ reported by :meth:`asyncio.Task.cancelling`. Improved handling of simultaneous internal and external cancellations and correct preservation of cancellation counts. -Terminating a task group ------------------------- - -While terminating a task group is not natively supported by the standard -library, termination can be achieved by adding an exception-raising task -to the task group and ignoring the raised exception: - -.. code-block:: python - - import asyncio - from asyncio import TaskGroup - - class TerminateTaskGroup(Exception): - """Exception raised to terminate a task group.""" - - async def force_terminate_task_group(): - """Used to force termination of a task group.""" - raise TerminateTaskGroup() - - async def job(task_id, sleep_time): - print(f'Task {task_id}: start') - await asyncio.sleep(sleep_time) - print(f'Task {task_id}: done') - - async def main(): - try: - async with TaskGroup() as group: - # spawn some tasks - group.create_task(job(1, 0.5)) - group.create_task(job(2, 1.5)) - # sleep for 1 second - await asyncio.sleep(1) - # add an exception-raising task to force the group to terminate - group.create_task(force_terminate_task_group()) - except* TerminateTaskGroup: - pass - - asyncio.run(main()) - -Expected output: - -.. code-block:: text - - Task 1: start - Task 2: start - Task 1: done - Sleeping ======== diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index 5e08d56fd66..32da8294c5a 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -73,8 +73,8 @@ POST request. Added the *padded* and *wrapcol* parameters. -.. function:: b64decode(s, altchars=None, validate=False, *, padded=True) - b64decode(s, altchars=None, validate=True, *, ignorechars, padded=True) +.. function:: b64decode(s, altchars=None, validate=False, *, padded=True, canonical=False) + b64decode(s, altchars=None, validate=True, *, ignorechars, padded=True, canonical=False) Decode the Base64 encoded :term:`bytes-like object` or ASCII string *s* and return the decoded :class:`bytes`. @@ -112,10 +112,13 @@ POST request. If *validate* is true, these non-alphabet characters in the input result in a :exc:`binascii.Error`. + If *canonical* is true, non-zero padding bits are rejected. + See :func:`binascii.a2b_base64` for details. + For more information about the strict base64 check, see :func:`binascii.a2b_base64` .. versionchanged:: 3.15 - Added the *ignorechars* and *padded* parameters. + Added the *canonical*, *ignorechars*, and *padded* parameters. .. deprecated:: 3.15 Accepting the ``+`` and ``/`` characters with an alternative alphabet @@ -179,7 +182,7 @@ POST request. Added the *padded* and *wrapcol* parameters. -.. function:: b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b'') +.. function:: b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b'', canonical=False) Decode the Base32 encoded :term:`bytes-like object` or ASCII string *s* and return the decoded :class:`bytes`. @@ -205,12 +208,15 @@ POST request. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. + If *canonical* is true, non-zero padding bits are rejected. + See :func:`binascii.a2b_base32` for details. + A :exc:`binascii.Error` is raised if *s* is incorrectly padded or if there are non-alphabet characters present in the input. .. versionchanged:: 3.15 - Added the *ignorechars* and *padded* parameters. + Added the *canonical*, *ignorechars*, and *padded* parameters. .. function:: b32hexencode(s, *, padded=True, wrapcol=0) @@ -224,7 +230,7 @@ POST request. Added the *padded* and *wrapcol* parameters. -.. function:: b32hexdecode(s, casefold=False, *, padded=True, ignorechars=b'') +.. function:: b32hexdecode(s, casefold=False, *, padded=True, ignorechars=b'', canonical=False) Similar to :func:`b32decode` but uses the Extended Hex Alphabet, as defined in :rfc:`4648`. @@ -237,7 +243,7 @@ POST request. .. versionadded:: 3.10 .. versionchanged:: 3.15 - Added the *ignorechars* and *padded* parameters. + Added the *canonical*, *ignorechars*, and *padded* parameters. .. function:: b16encode(s, *, wrapcol=0) @@ -317,7 +323,7 @@ Refer to the documentation of the individual functions for more information. .. versionadded:: 3.4 -.. function:: a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v') +.. function:: a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v', canonical=False) Decode the Ascii85 encoded :term:`bytes-like object` or ASCII string *b* and return the decoded :class:`bytes`. @@ -334,8 +340,16 @@ Refer to the documentation of the individual functions for more information. This should only contain whitespace characters, and by default contains all whitespace characters in ASCII. + If *canonical* is true, non-canonical encodings are rejected. + See :func:`binascii.a2b_ascii85` for details. + .. versionadded:: 3.4 + .. versionchanged:: next + Added the *canonical* parameter. + Single-character final groups are now always rejected as encoding + violations. + .. function:: b85encode(b, pad=False, *, wrapcol=0) @@ -355,7 +369,7 @@ Refer to the documentation of the individual functions for more information. Added the *wrapcol* parameter. -.. function:: b85decode(b, *, ignorechars=b'') +.. function:: b85decode(b, *, ignorechars=b'', canonical=False) Decode the base85-encoded :term:`bytes-like object` or ASCII string *b* and return the decoded :class:`bytes`. Padding is implicitly removed, if @@ -364,10 +378,15 @@ Refer to the documentation of the individual functions for more information. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. + If *canonical* is true, non-canonical encodings are rejected. + See :func:`binascii.a2b_base85` for details. + .. versionadded:: 3.4 .. versionchanged:: 3.15 - Added the *ignorechars* parameter. + Added the *canonical* and *ignorechars* parameters. + Single-character final groups are now always rejected as encoding + violations. .. function:: z85encode(s, pad=False, *, wrapcol=0) @@ -392,7 +411,7 @@ Refer to the documentation of the individual functions for more information. Added the *wrapcol* parameter. -.. function:: z85decode(s, *, ignorechars=b'') +.. function:: z85decode(s, *, ignorechars=b'', canonical=False) Decode the Z85-encoded :term:`bytes-like object` or ASCII string *s* and return the decoded :class:`bytes`. See `Z85 specification @@ -401,10 +420,15 @@ Refer to the documentation of the individual functions for more information. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. + If *canonical* is true, non-canonical encodings are rejected. + See :func:`binascii.a2b_base85` for details. + .. versionadded:: 3.13 .. versionchanged:: 3.15 - Added the *ignorechars* parameter. + Added the *canonical* and *ignorechars* parameters. + Single-character final groups are now always rejected as encoding + violations. .. _base64-legacy: diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 08a82cc4b5f..8b4ba6ae9fb 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -48,8 +48,8 @@ The :mod:`!binascii` module defines the following functions: Added the *backtick* parameter. -.. function:: a2b_base64(string, /, *, padded=True, alphabet=BASE64_ALPHABET, strict_mode=False) - a2b_base64(string, /, *, ignorechars, padded=True, alphabet=BASE64_ALPHABET, strict_mode=True) +.. function:: a2b_base64(string, /, *, padded=True, alphabet=BASE64_ALPHABET, strict_mode=False, canonical=False) + a2b_base64(string, /, *, ignorechars, padded=True, alphabet=BASE64_ALPHABET, strict_mode=True, canonical=False) Convert a block of base64 data back to binary and return the binary data. More than one line may be passed at a time. @@ -83,11 +83,15 @@ The :mod:`!binascii` module defines the following functions: * Contains no excess data after padding (including excess padding, newlines, etc.). * Does not start with a padding. + If *canonical* is true, non-zero padding bits in the last group are rejected + with :exc:`binascii.Error`, enforcing canonical encoding as defined in + :rfc:`4648` section 3.5. This check is independent of *strict_mode*. + .. versionchanged:: 3.11 Added the *strict_mode* parameter. .. versionchanged:: 3.15 - Added the *alphabet*, *ignorechars* and *padded* parameters. + Added the *alphabet*, *canonical*, *ignorechars*, and *padded* parameters. .. function:: b2a_base64(data, *, padded=True, alphabet=BASE64_ALPHABET, wrapcol=0, newline=True) @@ -113,7 +117,7 @@ The :mod:`!binascii` module defines the following functions: Added the *alphabet*, *padded* and *wrapcol* parameters. -.. function:: a2b_ascii85(string, /, *, foldspaces=False, adobe=False, ignorechars=b'') +.. function:: a2b_ascii85(string, /, *, foldspaces=False, adobe=False, ignorechars=b'', canonical=False) Convert Ascii85 data back to binary and return the binary data. @@ -122,7 +126,8 @@ The :mod:`!binascii` module defines the following functions: characters). Each group encodes 32 bits of binary data in the range from ``0`` to ``2 ** 32 - 1``, inclusive. The special character ``z`` is accepted as a short form of the group ``!!!!!``, which encodes four - consecutive null bytes. + consecutive null bytes. A single-character final group is always rejected + as an encoding violation. *foldspaces* is a flag that specifies whether the 'y' short sequence should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). @@ -135,6 +140,12 @@ The :mod:`!binascii` module defines the following functions: to ignore from the input. This should only contain whitespace characters. + If *canonical* is true, non-canonical encodings are rejected with + :exc:`binascii.Error`. Here "canonical" means the encoding that + :func:`b2a_ascii85` would produce: the ``z`` abbreviation must be used + for all-zero groups (rather than ``!!!!!``), and partial final groups + must use the same padding digits as the encoder. + Invalid Ascii85 data will raise :exc:`binascii.Error`. .. versionadded:: 3.15 @@ -163,7 +174,7 @@ The :mod:`!binascii` module defines the following functions: .. versionadded:: 3.15 -.. function:: a2b_base85(string, /, *, alphabet=BASE85_ALPHABET, ignorechars=b'') +.. function:: a2b_base85(string, /, *, alphabet=BASE85_ALPHABET, ignorechars=b'', canonical=False) Convert Base85 data back to binary and return the binary data. More than one line may be passed at a time. @@ -171,7 +182,8 @@ The :mod:`!binascii` module defines the following functions: Valid Base85 data contains characters from the Base85 alphabet in groups of five (except for the final group, which may have from two to five characters). Each group encodes 32 bits of binary data in the range from - ``0`` to ``2 ** 32 - 1``, inclusive. + ``0`` to ``2 ** 32 - 1``, inclusive. A single-character final group is + always rejected as an encoding violation. Optional *alphabet* must be a :class:`bytes` object of length 85 which specifies an alternative alphabet. @@ -179,6 +191,11 @@ The :mod:`!binascii` module defines the following functions: *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. + If *canonical* is true, non-canonical encodings are rejected with + :exc:`binascii.Error`. Here "canonical" means the encoding that + :func:`b2a_base85` would produce: partial final groups must use the + same padding digits as the encoder. + Invalid Base85 data will raise :exc:`binascii.Error`. .. versionadded:: 3.15 @@ -202,7 +219,7 @@ The :mod:`!binascii` module defines the following functions: .. versionadded:: 3.15 -.. function:: a2b_base32(string, /, *, padded=True, alphabet=BASE32_ALPHABET, ignorechars=b'') +.. function:: a2b_base32(string, /, *, padded=True, alphabet=BASE32_ALPHABET, ignorechars=b'', canonical=False) Convert base32 data back to binary and return the binary data. @@ -231,6 +248,10 @@ The :mod:`!binascii` module defines the following functions: presented before the end of the encoded data and the excess pad characters will be ignored. + If *canonical* is true, non-zero padding bits in the last group are rejected + with :exc:`binascii.Error`, enforcing canonical encoding as defined in + :rfc:`4648` section 3.5. + Invalid base32 data will raise :exc:`binascii.Error`. .. versionadded:: 3.15 diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 54cafaf4fe4..1c8f25e96dc 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -54,13 +54,13 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is .. method:: setfirstweekday(firstweekday) - Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6) + Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6). Identical to setting the :attr:`~Calendar.firstweekday` property. .. method:: iterweekdays() - Return an iterator for the week day numbers that will be used for one + Return an iterator for the weekday numbers that will be used for one week. The first value from the iterator will be the same as the value of the :attr:`~Calendar.firstweekday` property. @@ -86,7 +86,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is Return an iterator for the month *month* in the year *year* similar to :meth:`itermonthdates`, but not restricted by the :class:`datetime.date` range. Days returned will be tuples consisting of a day of the month - number and a week day number. + number and a weekday number. .. method:: itermonthdays3(year, month) @@ -408,7 +408,7 @@ For simple text calendars this module provides the following functions. .. function:: monthrange(year, month) - Returns weekday of first day of the month and number of days in month, for the + Returns weekday of first day of the month and number of days in month, for the specified *year* and *month*. @@ -446,7 +446,7 @@ For simple text calendars this module provides the following functions. An unrelated but handy function that takes a time tuple such as returned by the :func:`~time.gmtime` function in the :mod:`time` module, and returns the corresponding Unix timestamp value, assuming an epoch of 1970, and the POSIX - encoding. In fact, :func:`time.gmtime` and :func:`timegm` are each others' + encoding. In fact, :func:`time.gmtime` and :func:`timegm` are each other's inverse. @@ -580,9 +580,14 @@ The :mod:`!calendar` module defines the following exceptions: .. exception:: IllegalMonthError(month) - A subclass of :exc:`ValueError`, + A subclass of :exc:`ValueError` and :exc:`IndexError`, raised when the given month number is outside of the range 1-12 (inclusive). + .. versionchanged:: 3.12 + :exc:`IllegalMonthError` is now also a subclass of + :exc:`ValueError`. New code should avoid catching + :exc:`IndexError`. + .. attribute:: month The invalid month number. diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index cb9300f072b..e42bdc06be0 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -326,7 +326,7 @@ For example:: .. versionadded:: 3.10 The usual dictionary methods are available for :class:`Counter` objects - except for two which work differently for counters. + except for these two which work differently for counters: .. method:: fromkeys(iterable) diff --git a/Doc/library/compression.zstd.rst b/Doc/library/compression.zstd.rst index 7ca843f27f5..6d99e36e1e5 100644 --- a/Doc/library/compression.zstd.rst +++ b/Doc/library/compression.zstd.rst @@ -331,10 +331,14 @@ Compressing and decompressing data in memory If *max_length* is non-negative, the method returns at most *max_length* bytes of decompressed data. If this limit is reached and further - output can be produced, the :attr:`~.needs_input` attribute will - be set to ``False``. In this case, the next call to + output can be produced (or EOF is reached), the :attr:`~.needs_input` + attribute will be set to ``False``. In this case, the next call to :meth:`~.decompress` may provide *data* as ``b''`` to obtain - more of the output. + more of the output. The full content can thus be read like:: + + process_output(d.decompress(data, max_length)) + while not d.eof and not d.needs_input: + process_output(d.decompress(b"", max_length)) If all of the input data was decompressed and returned (either because this was less than *max_length* bytes, or because diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 5c6403879ab..77bac8dcc3a 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -467,12 +467,40 @@ Functions and classes provided: statements. If this is not the case, then the original construct with the explicit :keyword:`!with` statement inside the function should be used. + When the decorated callable is a generator function, coroutine function, or + asynchronous generator function, the returned wrapper is of the same kind + and keeps the context manager open for the lifetime of the iteration or + await rather than only for the call that creates the generator or coroutine + object. Wrapped generators and asynchronous generators are explicitly + closed when iteration ends, as if by :func:`closing` or :func:`aclosing`. + + .. note:: + For asynchronous generators the wrapper re-yields each value with + ``async for``; values sent with :meth:`~agen.asend` and exceptions + thrown with :meth:`~agen.athrow` are not forwarded to the wrapped + generator. + .. versionadded:: 3.2 + .. versionchanged:: next + Decorating a generator function, coroutine function, or asynchronous + generator function now keeps the context manager open across iteration + or await. Previously the context manager exited as soon as the + generator or coroutine object was created. + .. class:: AsyncContextDecorator - Similar to :class:`ContextDecorator` but only for asynchronous functions. + Similar to :class:`ContextDecorator`, but the context manager is entered + and exited with :keyword:`async with`. Decorate coroutine functions and + asynchronous generator functions with this class; the returned wrapper is + of the same kind. + + .. note:: + Synchronous functions and generators are accepted, but the wrapper is + always asynchronous, so the decorated callable must then be awaited or + iterated with ``async for``. If that change of calling convention is + not intended, use :class:`ContextDecorator` instead. Example of ``AsyncContextDecorator``:: @@ -510,6 +538,13 @@ Functions and classes provided: .. versionadded:: 3.10 + .. versionchanged:: next + Decorating an asynchronous generator function now keeps the context + manager open across iteration. Previously the context manager exited + as soon as the generator object was created. Synchronous functions + and synchronous generator functions are also now accepted, with an + asynchronous wrapper returned. + .. class:: ExitStack() diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index c4fac8c2622..3db66ad8ea6 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1739,7 +1739,7 @@ If wrapping a shared library with :mod:`!ctypes`, consider determining the shared library name at development time, and hardcoding it into the wrapper module instead of using :func:`!find_library` to locate the library at runtime. -Also consider addding a configuration option or environment variable to let +Also consider adding a configuration option or environment variable to let users select a library to use, and then perhaps use :func:`!find_library` as a default or fallback. @@ -1760,11 +1760,10 @@ as a default or fallback. (or by) Python. It is recommended to only use this function as a default or fallback, - .. deprecated:: 3.15 + .. soft-deprecated:: 3.15 - This function is :term:`soft deprecated`. - It is kept for use in cases where it works, but not expected to be - updated for additional platforms and configurations. + This function is kept for use in cases where it works, but not expected to + be updated for additional platforms and configurations. On Linux, :func:`!find_library` tries to run external programs (``/sbin/ldconfig``, ``gcc``, ``objdump`` and ``ld``) to find the @@ -3195,8 +3194,8 @@ Arrays and pointers Equivalent to ``type * length``, where *type* is a :mod:`!ctypes` data type and *length* an integer. - This function is :term:`soft deprecated` in favor of multiplication. - There are no plans to remove it. + .. soft-deprecated:: 3.14 + In favor of multiplication. .. class:: _Pointer diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index fd8e0c0bea1..0bce3e5b762 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -371,8 +371,8 @@ Module contents Converts the dataclass *obj* to a dict (by using the factory function *dict_factory*). Each dataclass is converted to a dict of its fields, as ``name: value`` pairs. dataclasses, dicts, - lists, and tuples are recursed into. Other objects are copied with - :func:`copy.deepcopy`. + frozendicts, lists, and tuples are recursed into. Other objects are copied + with :func:`copy.deepcopy`. Example of using :func:`!asdict` on nested dataclasses:: @@ -402,8 +402,8 @@ Module contents Converts the dataclass *obj* to a tuple (by using the factory function *tuple_factory*). Each dataclass is converted - to a tuple of its field values. dataclasses, dicts, lists, and - tuples are recursed into. Other objects are copied with + to a tuple of its field values. dataclasses, dicts, frozendicts, lists, + and tuples are recursed into. Other objects are copied with :func:`copy.deepcopy`. Continuing from the previous example:: diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 1f7014e9cd4..3e7ae509fed 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -400,7 +400,7 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.10 The :pep:`626` :meth:`~codeobject.co_lines` method is used instead of the - :attr:`~codeobject.co_firstlineno` and :attr:`~codeobject.co_lnotab` + :attr:`~codeobject.co_firstlineno` and :attr:`!codeobject.co_lnotab` attributes of the :ref:`code object `. .. versionchanged:: 3.13 diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index 8f6e4218c97..816d02d86f4 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -403,11 +403,26 @@ added matters. To illustrate:: .. attribute:: utf8 If ``False``, follow :rfc:`5322`, supporting non-ASCII characters in - headers by encoding them as "encoded words". If ``True``, follow - :rfc:`6532` and use ``utf-8`` encoding for headers. Messages + headers by encoding them as :rfc:`2047` "encoded words". If ``True``, + follow :rfc:`6532` and use ``utf-8`` encoding for headers. Messages formatted in this way may be passed to SMTP servers that support the ``SMTPUTF8`` extension (:rfc:`6531`). + When ``False``, the generator will raise + :exc:`~email.errors.HeaderWriteError` if any header includes non-ASCII + characters in a context where :rfc:`2047` does not permit encoded words. + This particularly applies to mailboxes ("addr-spec") with non-ASCII + characters, which can be created via + :class:`~email.headerregistry.Address`. To use a mailbox with a non-ASCII + domain name with ``utf8=False``, first encode the domain using the + third-party :pypi:`idna` or :pypi:`uts46` module or with + :mod:`encodings.idna`. It is not possible to use a non-ASCII username + ("local-part") in a mailbox when ``utf8=False``. + + .. versionchanged:: 3.15 + Can trigger the raising of :exc:`~email.errors.HeaderWriteError`. + (Earlier versions incorrectly applied :rfc:`2047` in certain contexts, + mostly notably in addr-specs.) .. attribute:: refold_source diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst index 677966a8b2e..529e97bae6d 100644 --- a/Doc/library/faulthandler.rst +++ b/Doc/library/faulthandler.rst @@ -31,7 +31,8 @@ tracebacks: * Each string is limited to 500 characters. * Only the filename, the function name and the line number are displayed. (no source code) -* It is limited to 100 frames and 100 threads. +* It is limited to 100 frames per thread, and 100 threads + (configurable via *max_threads*). * The order is reversed: the most recent call is shown first. By default, the Python traceback is written to :data:`sys.stderr`. To see @@ -55,16 +56,20 @@ at Python startup. Dumping the traceback --------------------- -.. function:: dump_traceback(file=sys.stderr, all_threads=True) +.. function:: dump_traceback(file=sys.stderr, all_threads=True, *, max_threads=100) Dump the tracebacks of all threads into *file*. If *all_threads* is - ``False``, dump only the current thread. + ``False``, dump only the current thread. *max_threads* caps the number + of threads dumped. .. seealso:: :func:`traceback.print_tb`, which can be used to print a traceback object. .. versionchanged:: 3.5 Added support for passing file descriptor to this function. + .. versionchanged:: next + Added the *max_threads* keyword argument. + Dumping the C stack ------------------- @@ -100,7 +105,7 @@ instead of the stack, even if the operating system supports dumping stacks. Fault handler state ------------------- -.. function:: enable(file=sys.stderr, all_threads=True, c_stack=True) +.. function:: enable(file=sys.stderr, all_threads=True, c_stack=True, *, max_threads=100) Enable the fault handler: install handlers for the :const:`~signal.SIGSEGV`, :const:`~signal.SIGFPE`, :const:`~signal.SIGABRT`, :const:`~signal.SIGBUS` @@ -116,6 +121,8 @@ Fault handler state traceback, unless the system does not support it. See :func:`dump_c_stack` for more information on compatibility. + *max_threads* caps the number of threads dumped when a fatal signal fires. + .. versionchanged:: 3.5 Added support for passing file descriptor to this function. @@ -133,6 +140,9 @@ Fault handler state .. versionchanged:: 3.14 The dump now displays the C stack trace if *c_stack* is true. + .. versionchanged:: next + Added the *max_threads* keyword argument. + .. function:: disable() Disable the fault handler: uninstall the signal handlers installed by @@ -146,7 +156,7 @@ Fault handler state Dumping the tracebacks after a timeout -------------------------------------- -.. function:: dump_traceback_later(timeout, repeat=False, file=sys.stderr, exit=False) +.. function:: dump_traceback_later(timeout, repeat=False, file=sys.stderr, exit=False, *, max_threads=100) Dump the tracebacks of all threads, after a timeout of *timeout* seconds, or every *timeout* seconds if *repeat* is ``True``. If *exit* is ``True``, call @@ -154,7 +164,7 @@ Dumping the tracebacks after a timeout :c:func:`!_exit` exits the process immediately, which means it doesn't do any cleanup like flushing file buffers.) If the function is called twice, the new call replaces previous parameters and resets the timeout. The timer has a - sub-second resolution. + sub-second resolution. *max_threads* caps the number of threads dumped. The *file* must be kept open until the traceback is dumped or :func:`cancel_dump_traceback_later` is called: see :ref:`issue with file @@ -168,6 +178,9 @@ Dumping the tracebacks after a timeout .. versionchanged:: 3.7 This function is now always available. + .. versionchanged:: next + Added the *max_threads* keyword argument. + .. function:: cancel_dump_traceback_later() Cancel the last call to :func:`dump_traceback_later`. @@ -176,11 +189,12 @@ Dumping the tracebacks after a timeout Dumping the traceback on a user signal -------------------------------------- -.. function:: register(signum, file=sys.stderr, all_threads=True, chain=False) +.. function:: register(signum, file=sys.stderr, all_threads=True, chain=False, *, max_threads=100) Register a user signal: install a handler for the *signum* signal to dump the traceback of all threads, or of the current thread if *all_threads* is ``False``, into *file*. Call the previous handler if chain is ``True``. + *max_threads* caps the number of threads dumped. The *file* must be kept open until the signal is unregistered by :func:`unregister`: see :ref:`issue with file descriptors `. @@ -190,6 +204,9 @@ Dumping the traceback on a user signal .. versionchanged:: 3.5 Added support for passing file descriptor to this function. + .. versionchanged:: next + Added the *max_threads* keyword argument. + .. function:: unregister(signum) Unregister a user signal: uninstall the handler of the *signum* signal diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 119141d2e6d..06fd5cdc7be 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -19,13 +19,13 @@ are always available. They are listed here in alphabetical order. | | :func:`ascii` | | :func:`filter` | | :func:`map` | | **S** | | | | | :func:`float` | | :func:`max` | | |func-set|_ | | | **B** | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` | -| | :func:`bin` | | |func-frozenset|_ | | :func:`min` | | :func:`slice` | -| | :func:`bool` | | | | | | :func:`sorted` | -| | :func:`breakpoint` | | **G** | | **N** | | :func:`staticmethod` | -| | |func-bytearray|_ | | :func:`getattr` | | :func:`next` | | |func-str|_ | -| | |func-bytes|_ | | :func:`globals` | | | | :func:`sum` | -| | | | | | **O** | | :func:`super` | -| | **C** | | **H** | | :func:`object` | | | +| | :func:`bin` | | |func-frozenset|_ | | :func:`min` | | :func:`sentinel` | +| | :func:`bool` | | | | | | :func:`slice` | +| | :func:`breakpoint` | | **G** | | **N** | | :func:`sorted` | +| | |func-bytearray|_ | | :func:`getattr` | | :func:`next` | | :func:`staticmethod` | +| | |func-bytes|_ | | :func:`globals` | | | | |func-str|_ | +| | | | | | **O** | | :func:`sum` | +| | **C** | | **H** | | :func:`object` | | :func:`super` | | | :func:`callable` | | :func:`hasattr` | | :func:`oct` | | **T** | | | :func:`chr` | | :func:`hash` | | :func:`open` | | |func-tuple|_ | | | :func:`classmethod` | | :func:`help` | | :func:`ord` | | :func:`type` | @@ -1827,6 +1827,63 @@ are always available. They are listed here in alphabetical order. :func:`setattr`. +.. class:: sentinel(name, /) + + Return a new unique sentinel object. *name* must be a :class:`str`, and is + used as the returned object's representation:: + + >>> MISSING = sentinel("MISSING") + >>> MISSING + MISSING + + Sentinel objects are truthy and compare equal only to themselves. They are + intended to be compared with the :keyword:`is` operator. + + ``sentinel`` does not support subclassing. + + Shallow and deep copies of a sentinel object return the object itself. + + Sentinels are conventionally assigned to a variable with a matching name. + Sentinels defined in this way can be used in :term:`type hints `:: + + MISSING = sentinel("MISSING") + + def next_value(default: int | MISSING = MISSING): + ... + + Sentinel objects support the :ref:`| ` operator for use in type expressions. + + :mod:`Pickling ` is supported for sentinel objects that are + placed in the global scope of a module under a name matching the sentinel's + name, and for sentinels placed in class scopes with a name matching the + :term:`qualified name` of the sentinel. Other sentinels, such as those + defined in a function scope, are not picklable. The identity of the sentinel is preserved + after pickling:: + + import pickle + + PICKLABLE = sentinel("PICKLABLE") + + assert pickle.loads(pickle.dumps(PICKLABLE)) is PICKLABLE + + class Cls: + PICKLABLE = sentinel("Cls.PICKLABLE") + + assert pickle.loads(pickle.dumps(Cls.PICKLABLE)) is Cls.PICKLABLE + + Sentinel objects have the following attributes: + + .. attribute:: __name__ + + The sentinel's name. + + .. attribute:: __module__ + + The name of the module where the sentinel was created. + + .. versionadded:: next + + .. class:: slice(stop, /) slice(start, stop, step=None, /) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 265610db3ca..7da59cba517 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -468,7 +468,7 @@ The :mod:`!functools` module defines the following functions: Roughly equivalent to:: - initial_missing = object() + initial_missing = sentinel('initial_missing') def reduce(function, iterable, /, initial=initial_missing): it = iter(iterable) diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 652475886fc..701af579453 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -37,18 +37,11 @@ The :mod:`!gc` module provides the following functions: .. function:: collect(generation=2) - Perform a collection. The optional argument *generation* + With no arguments, run a full collection. The optional argument *generation* may be an integer specifying which generation to collect (from 0 to 2). A :exc:`ValueError` is raised if the generation number is invalid. The sum of collected objects and uncollectable objects is returned. - Calling ``gc.collect(0)`` will perform a GC collection on the young generation. - - Calling ``gc.collect(1)`` will perform a GC collection on the young generation - and an increment of the old generation. - - Calling ``gc.collect(2)`` or ``gc.collect()`` performs a full collection - The free lists maintained for a number of built-in types are cleared whenever a full collection or collection of the highest generation (2) is run. Not all items in some free lists may be freed due to the @@ -60,6 +53,9 @@ The :mod:`!gc` module provides the following functions: .. versionchanged:: 3.14 ``generation=1`` performs an increment of collection. + .. versionchanged:: 3.14.5 + ``generation=1`` performs collection of the middle generation. + .. function:: set_debug(flags) @@ -75,13 +71,9 @@ The :mod:`!gc` module provides the following functions: .. function:: get_objects(generation=None) - Returns a list of all objects tracked by the collector, excluding the list - returned. If *generation* is not ``None``, return only the objects as follows: - - * 0: All objects in the young generation - * 1: No objects, as there is no generation 1 (as of Python 3.14) - * 2: All objects in the old generation + returned. If *generation* is not ``None``, return only the objects tracked by + the collector that are in that generation. .. versionchanged:: 3.8 New *generation* parameter. @@ -89,6 +81,9 @@ The :mod:`!gc` module provides the following functions: .. versionchanged:: 3.14 Generation 1 is removed + .. versionchanged:: 3.14.5 + Generation 1 is reintroduced to maintain GC behavior from 3.13. + .. audit-event:: gc.get_objects generation gc.get_objects .. function:: get_stats() @@ -124,33 +119,33 @@ The :mod:`!gc` module provides the following functions: Set the garbage collection thresholds (the collection frequency). Setting *threshold0* to zero disables collection. - The GC classifies objects into two generations depending on whether they have - survived a collection. New objects are placed in the young generation. If an - object survives a collection it is moved into the old generation. - - In order to decide when to run, the collector keeps track of the number of object + The GC classifies objects into three generations depending on how many + collection sweeps they have survived. New objects are placed in the youngest + generation (generation ``0``). If an object survives a collection it is moved + into the next older generation. Since generation ``2`` is the oldest + generation, objects in that generation remain there after a collection. In + order to decide when to run, the collector keeps track of the number object allocations and deallocations since the last collection. When the number of allocations minus the number of deallocations exceeds *threshold0*, collection - starts. For each collection, all the objects in the young generation and some - fraction of the old generation is collected. + starts. Initially only generation ``0`` is examined. If generation ``0`` has + been examined more than *threshold1* times since generation ``1`` has been + examined, then generation ``1`` is examined as well. + With the third generation, things are a bit more complicated, + see `Collecting the oldest generation `_ for more information. In the free-threaded build, the increase in process memory usage is also checked before running the collector. If the memory usage has not increased by 10% since the last collection and the net number of object allocations has not exceeded 40 times *threshold0*, the collection is not run. - The fraction of the old generation that is collected is **inversely** proportional - to *threshold1*. The larger *threshold1* is, the slower objects in the old generation - are collected. - For the default value of 10, 1% of the old generation is scanned during each collection. - - *threshold2* is ignored. - - See `Garbage collector design `_ for more information. + See `Garbage collector design `_ for more information. .. versionchanged:: 3.14 *threshold2* is ignored + .. versionchanged:: 3.14.5 + *threshold2* is restored to match Python 3.13 behavior. + .. function:: get_count() diff --git a/Doc/library/html.parser.rst b/Doc/library/html.parser.rst index 341a8337ba2..11f851d4f6c 100644 --- a/Doc/library/html.parser.rst +++ b/Doc/library/html.parser.rst @@ -141,7 +141,7 @@ implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`): argument is a list of ``(name, value)`` pairs containing the attributes found inside the tag's ``<>`` brackets. The *name* will be translated to lower case, and quotes in the *value* have been removed, and character and entity references - have been replaced. + have been replaced. For empty attributes, *value* is ``None``. For instance, for the tag ````, this method would be called as ``handle_starttag('a', [('href', 'https://www.cwi.nl/')])``. @@ -317,6 +317,18 @@ without further parsing: Data : alert("hello! ☺"); End tag : script +Attribute names are converted to lowercase, quotes from attribute values removed, +and ``None`` is returned as *value* for empty attributes (such as ``checked``): + +.. doctest:: + + >>> parser.feed("") + Start tag: input + attr: ('type', 'checkbox') + attr: ('checked', None) + attr: ('required', '') + attr: ('disabled', 'disabled') + Parsing comments: .. doctest:: diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst index b3fcd21c7e2..1122b30d29d 100644 --- a/Doc/library/http.cookies.rst +++ b/Doc/library/http.cookies.rst @@ -25,10 +25,8 @@ The character set, :data:`string.ascii_letters`, :data:`string.digits` and in a cookie name (as :attr:`~Morsel.key`). .. versionchanged:: 3.3 - Allowed '``:``' as a valid cookie name character. + Allowed ':' as a valid cookie name character. -.. versionchanged:: 3.15 - Allowed '``"``' as a valid cookie value character. .. note:: @@ -313,10 +311,3 @@ The following example demonstrates how to use the :mod:`!http.cookies` module. >>> print(C) Set-Cookie: number=7 Set-Cookie: string=seven - >>> import json - >>> C = cookies.SimpleCookie() - >>> C.load(f'cookies=7; mixins="{json.dumps({"chips": "dark chocolate"})}"; state=gooey') - >>> print(C) - Set-Cookie: cookies=7 - Set-Cookie: mixins="{"chips": "dark chocolate"}" - Set-Cookie: state=gooey diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 33ecaae5c87..5f325df5570 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -390,6 +390,14 @@ instantiation, of which this module provides three different variants: This will be ``"SimpleHTTP/" + __version__``, where ``__version__`` is defined at the module level. + .. attribute:: default_content_type + + Specifies the Content-Type header value sent when the MIME type + cannot be guessed from the file extension of the requested URL. + By default, it is set to ``'application/octet-stream'``. + + .. versionadded:: next + .. attribute:: extensions_map A dictionary mapping suffixes into MIME types, contains custom overrides @@ -528,6 +536,18 @@ The following options are accepted: .. versionadded:: 3.11 +.. option:: --content-type + + Specifies the default Content-Type HTTP header used when the MIME type + cannot be guessed from the URL's file extension. By default, the server + uses ``'application/octet-stream'``: + + .. code-block:: bash + + python -m http.server --content-type text/html + + .. versionadded:: next + .. option:: --tls-cert Specifies a TLS certificate chain for HTTPS connections: diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 785f6c614b4..0b76020eacc 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -286,7 +286,7 @@ ABC hierarchy:: This method can potentially yield a very large number of objects, and it may carry out IO operations when computing these values. - Because of this, it will generaly be desirable to compute the result + Because of this, it will generally be desirable to compute the result values on-the-fly, as they are needed. As such, the returned object is only guaranteed to be an :class:`iterable `, instead of a :class:`list` or other @@ -340,7 +340,7 @@ ABC hierarchy:: This method can potentially yield a very large number of objects, and it may carry out IO operations when computing these values. - Because of this, it will generaly be desirable to compute the result + Because of this, it will generally be desirable to compute the result values on-the-fly, as they are needed. As such, the returned object is only guaranteed to be an :class:`iterable `, instead of a :class:`list` or other diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index ff893a45139..e23449886a3 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -195,10 +195,6 @@ attributes (see :ref:`import-mod-attrs` for module attributes): | | | read more :ref:`here | | | | `| +-----------------+-------------------+---------------------------+ -| | co_lnotab | encoded mapping of line | -| | | numbers to bytecode | -| | | indices | -+-----------------+-------------------+---------------------------+ | | co_freevars | tuple of names of free | | | | variables (referenced via | | | | a function's closure) | diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 5a0ac60ab7d..06f8bf2a8b6 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -833,6 +833,7 @@ and :term:`generators ` which incur interpreter overhead. from collections import Counter, deque from contextlib import suppress from functools import reduce + from heapq import heappush, heappushpop, heappush_max, heappushpop_max from math import comb, isqrt, prod, sumprod from operator import getitem, is_not, itemgetter, mul, neg, truediv @@ -848,11 +849,6 @@ and :term:`generators ` which incur interpreter overhead. # prepend(1, [2, 3, 4]) → 1 2 3 4 return chain([value], iterable) - def running_mean(iterable): - "Yield the average of all values seen so far." - # running_mean([8.5, 9.5, 7.5, 6.5]) → 8.5 9.0 8.5 8.0 - return map(truediv, accumulate(iterable), count(1)) - def repeatfunc(function, times=None, *args): "Repeat calls to a function with specified arguments." if times is None: @@ -1150,6 +1146,49 @@ and :term:`generators ` which incur interpreter overhead. return n + # ==== Running statistics ==== + + def running_mean(iterable): + "Average of values seen so far." + # running_mean([37, 33, 38, 28]) → 37 35 36 34 + return map(truediv, accumulate(iterable), count(1)) + + def running_min(iterable): + "Smallest of values seen so far." + # running_min([37, 33, 38, 28]) → 37 33 33 28 + return accumulate(iterable, func=min) + + def running_max(iterable): + "Largest of values seen so far." + # running_max([37, 33, 38, 28]) → 37 37 38 38 + return accumulate(iterable, func=max) + + def running_median(iterable): + "Median of values seen so far." + # running_median([37, 33, 38, 28]) → 37 35 37 35 + read = iter(iterable).__next__ + lo = [] # max-heap + hi = [] # min-heap the same size as or one smaller than lo + with suppress(StopIteration): + while True: + heappush_max(lo, heappushpop(hi, read())) + yield lo[0] + heappush(hi, heappushpop_max(lo, read())) + yield (lo[0] + hi[0]) / 2 + + def running_statistics(iterable): + "Aggregate statistics for values seen so far." + # Generate tuples: (size, minimum, median, maximum, mean) + t0, t1, t2, t3 = tee(iterable, 4) + return zip( + count(1), + running_min(t0), + running_median(t1), + running_max(t2), + running_mean(t3), + ) + + .. doctest:: :hide: @@ -1226,10 +1265,6 @@ and :term:`generators ` which incur interpreter overhead. [(0, 'a'), (1, 'b'), (2, 'c')] - >>> list(running_mean([8.5, 9.5, 7.5, 6.5])) - [8.5, 9.0, 8.5, 8.0] - - >>> for _ in loops(5): ... print('hi') ... @@ -1789,6 +1824,28 @@ and :term:`generators ` which incur interpreter overhead. True + >>> list(running_mean([8.5, 9.5, 7.5, 6.5])) + [8.5, 9.0, 8.5, 8.0] + >>> list(running_mean([37, 33, 38, 28])) + [37.0, 35.0, 36.0, 34.0] + + + >>> list(running_min([37, 33, 38, 28])) + [37, 33, 33, 28] + + + >>> list(running_max([37, 33, 38, 28])) + [37, 37, 38, 38] + + + >>> list(running_median([37, 33, 38, 28])) + [37, 35.0, 37, 35.0] + + + >>> list(running_statistics([37, 33, 38, 28])) + [(1, 37, 37, 37, 37.0), (2, 33, 35.0, 37, 35.0), (3, 33, 37, 38, 36.0), (4, 28, 35.0, 38, 34.0)] + + .. testcode:: :hide: diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 4a11aec15df..9cc8c5d6886 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -781,9 +781,8 @@ the following functions from the :mod:`math.integer` module: Floats with integral values (like ``5.0``) are no longer accepted in the :func:`factorial` function. -.. deprecated:: 3.15 - These aliases are :term:`soft deprecated` in favor of the - :mod:`math.integer` functions. +.. soft-deprecated:: 3.15 + Use the :mod:`math.integer` functions instead of these aliases. Constants diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 1e599bde8bc..0facacd50fd 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -54,8 +54,8 @@ the information :func:`init` sets up. .. versionchanged:: 3.8 Added support for *url* being a :term:`path-like object`. - .. deprecated:: 3.13 - Passing a file path instead of URL is :term:`soft deprecated`. + .. soft-deprecated:: 3.13 + Passing a file path instead of URL. Use :func:`guess_file_type` for this. diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 63bc252e129..187143d02cd 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1336,12 +1336,12 @@ Connection objects are usually created using Note that multiple connection objects may be polled at once by using :func:`multiprocessing.connection.wait`. - .. method:: send_bytes(buffer[, offset[, size]]) + .. method:: send_bytes(buf[, offset[, size]]) Send byte data from a :term:`bytes-like object` as a complete message. - If *offset* is given then data is read from that position in *buffer*. If - *size* is given then that many bytes will be read from buffer. Very large + If *offset* is given then data is read from that position in *buf*. If + *size* is given then that many bytes will be read from *buf*. Very large buffers (approximately 32 MiB+, though it depends on the OS) may raise a :exc:`ValueError` exception @@ -1361,18 +1361,18 @@ Connection objects are usually created using alias of :exc:`OSError`. - .. method:: recv_bytes_into(buffer[, offset]) + .. method:: recv_bytes_into(buf[, offset]) - Read into *buffer* a complete message of byte data sent from the other end + Read into *buf* a complete message of byte data sent from the other end of the connection and return the number of bytes in the message. Blocks until there is something to receive. Raises :exc:`EOFError` if there is nothing left to receive and the other end was closed. - *buffer* must be a writable :term:`bytes-like object`. If + *buf* must be a writable :term:`bytes-like object`. If *offset* is given then the message will be written into the buffer from that position. Offset must be a non-negative integer less than the - length of *buffer* (in bytes). + length of *buf* (in bytes). If the buffer is too short then a :exc:`BufferTooShort` exception is raised and the complete message is available as ``e.args[0]`` where ``e`` @@ -2917,6 +2917,16 @@ between themselves. Suitable authentication keys can also be generated by using :func:`os.urandom`. +This authentication protects :class:`Listener` and :func:`Client` connections, +which are reachable by address. It is not applied to the anonymous pipes +created by :func:`~multiprocessing.Pipe` or used internally by +:class:`~multiprocessing.Queue`. +:mod:`multiprocessing` treats all local processes running as the same user as +trusted; on most operating systems such processes can access each other's pipe +file descriptors regardless. Applications that require isolation between +processes of the same user must arrange it at the operating-system level -- +for example, by running workers under a different user account or in a sandbox. + Logging ^^^^^^^ diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 7547967c6b3..d2534b3e974 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -5110,9 +5110,8 @@ written in Python, such as a mail server's external command delivery program. Use :class:`subprocess.Popen` or :func:`subprocess.run` to control options like encodings. - .. deprecated:: 3.14 - The function is :term:`soft deprecated` and should no longer be used to - write new code. The :mod:`subprocess` module is recommended instead. + .. soft-deprecated:: 3.14 + The :mod:`subprocess` module is recommended instead. .. function:: posix_spawn(path, argv, env, *, file_actions=None, \ @@ -5340,9 +5339,8 @@ written in Python, such as a mail server's external command delivery program. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. - .. deprecated:: 3.14 - These functions are :term:`soft deprecated` and should no longer be used - to write new code. The :mod:`subprocess` module is recommended instead. + .. soft-deprecated:: 3.14 + The :mod:`subprocess` module is recommended instead. .. data:: P_NOWAIT diff --git a/Doc/library/pickletools.rst b/Doc/library/pickletools.rst index 7a771ea3ab9..e753ad3b08b 100644 --- a/Doc/library/pickletools.rst +++ b/Doc/library/pickletools.rst @@ -79,6 +79,9 @@ Command-line options A pickle file to read, or ``-`` to indicate reading from standard input. +.. versionadded:: next + Output is in color by default and can be + :ref:`controlled using environment variables `. Programmatic interface diff --git a/Doc/library/profiling.sampling.rst b/Doc/library/profiling.sampling.rst index a6ce2f30ead..790d3600180 100644 --- a/Doc/library/profiling.sampling.rst +++ b/Doc/library/profiling.sampling.rst @@ -17,7 +17,7 @@ -------------- -.. image:: tachyon-logo.png +.. image:: ../../Lib/profiling/sampling/_assets/tachyon-logo.png :alt: Tachyon logo :align: center :width: 300px diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 7e0a00cba2f..a46fd424581 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -931,7 +931,6 @@ Functions .. function:: prefixmatch(pattern, string, flags=0) -.. function:: match(pattern, string, flags=0) If zero or more characters at the beginning of *string* match the regular expression *pattern*, return a corresponding :class:`~re.Match`. Return @@ -954,7 +953,11 @@ Functions :func:`~re.match`. Use that name when you need to retain compatibility with older Python versions. - .. deprecated:: 3.15 + .. versionadded:: 3.15 + +.. function:: match(pattern, string, flags=0) + + .. soft-deprecated:: 3.15 :func:`~re.match` has been :term:`soft deprecated` in favor of the alternate :func:`~re.prefixmatch` name of this API which is more explicitly descriptive. Use it to better @@ -1285,7 +1288,6 @@ Regular expression objects .. method:: Pattern.prefixmatch(string[, pos[, endpos]]) -.. method:: Pattern.match(string[, pos[, endpos]]) If zero or more characters at the *beginning* of *string* match this regular expression, return a corresponding :class:`~re.Match`. Return ``None`` if the @@ -1310,7 +1312,11 @@ Regular expression objects :meth:`~Pattern.match`. Use that name when you need to retain compatibility with older Python versions. - .. deprecated:: 3.15 + .. versionadded:: 3.15 + +.. method:: Pattern.match(string[, pos[, endpos]]) + + .. soft-deprecated:: 3.15 :meth:`~Pattern.match` has been :term:`soft deprecated` in favor of the alternate :meth:`~Pattern.prefixmatch` name of this API which is more explicitly descriptive. Use it to @@ -1794,8 +1800,8 @@ while new code should prefer :func:`!prefixmatch`. .. versionadded:: 3.15 :func:`!prefixmatch` -.. deprecated:: 3.15 - :func:`!match` is :term:`soft deprecated` +.. soft-deprecated:: 3.15 + :func:`!match` Making a phonebook ^^^^^^^^^^^^^^^^^^ @@ -1947,7 +1953,7 @@ successive matches:: class Token(NamedTuple): type: str - value: str + value: int | float | str line: int column: int diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 71747d5f515..96bc9e7a0d6 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -486,6 +486,7 @@ The AF_* and SOCK_* constants are now :class:`AddressFamily` and .. versionchanged:: 3.15 ``IPV6_HDRINCL`` was added. + Added support for ``SO_PASSRIGHTS`` on Linux platforms when available. .. data:: AF_CAN diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index cbb131855dc..dba0e26787d 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -713,7 +713,7 @@ However, for reading convenience, most of the examples show sorted sequences. .. function:: covariance(x, y, /) - Return the sample covariance of two inputs *x* and *y*. Covariance + Return the sample covariance of two sequence inputs *x* and *y*. Covariance is a measure of the joint variability of two inputs. Both inputs must be of the same length (no less than two), otherwise @@ -739,7 +739,7 @@ However, for reading convenience, most of the examples show sorted sequences. Return the `Pearson's correlation coefficient `_ - for two inputs. Pearson's correlation coefficient *r* takes values + for two sequence inputs. Pearson's correlation coefficient *r* takes values between -1 and +1. It measures the strength and direction of a linear relationship. @@ -802,7 +802,7 @@ However, for reading convenience, most of the examples show sorted sequences. (it is equal to the difference between predicted and actual values of the dependent variable). - Both inputs must be of the same length (no less than two), and + Both inputs must be sequences of the same length (no less than two), and the independent variable *x* cannot be constant; otherwise a :exc:`StatisticsError` is raised. diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index a828ad4e49f..bf5f754e156 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -227,32 +227,32 @@ platform-dependent. +--------+--------------------------+--------------------+----------------+------------+ | ``c`` | :c:expr:`char` | bytes of length 1 | 1 | | +--------+--------------------------+--------------------+----------------+------------+ -| ``b`` | :c:expr:`signed char` | integer | 1 | \(1), \(2) | +| ``b`` | :c:expr:`signed char` | int | 1 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``B`` | :c:expr:`unsigned char` | integer | 1 | \(2) | +| ``B`` | :c:expr:`unsigned char` | int | 1 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ | ``?`` | :c:expr:`_Bool` | bool | 1 | \(1) | +--------+--------------------------+--------------------+----------------+------------+ -| ``h`` | :c:expr:`short` | integer | 2 | \(2) | +| ``h`` | :c:expr:`short` | int | 2 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``H`` | :c:expr:`unsigned short` | integer | 2 | \(2) | +| ``H`` | :c:expr:`unsigned short` | int | 2 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``i`` | :c:expr:`int` | integer | 4 | \(2) | +| ``i`` | :c:expr:`int` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``I`` | :c:expr:`unsigned int` | integer | 4 | \(2) | +| ``I`` | :c:expr:`unsigned int` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``l`` | :c:expr:`long` | integer | 4 | \(2) | +| ``l`` | :c:expr:`long` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``L`` | :c:expr:`unsigned long` | integer | 4 | \(2) | +| ``L`` | :c:expr:`unsigned long` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``q`` | :c:expr:`long long` | integer | 8 | \(2) | +| ``q`` | :c:expr:`long long` | int | 8 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``Q`` | :c:expr:`unsigned long | integer | 8 | \(2) | +| ``Q`` | :c:expr:`unsigned long | int | 8 | \(2) | | | long` | | | | +--------+--------------------------+--------------------+----------------+------------+ -| ``n`` | :c:type:`ssize_t` | integer | | \(3) | +| ``n`` | :c:type:`ssize_t` | int | | \(2), \(3) | +--------+--------------------------+--------------------+----------------+------------+ -| ``N`` | :c:type:`size_t` | integer | | \(3) | +| ``N`` | :c:type:`size_t` | int | | \(2), \(3) | +--------+--------------------------+--------------------+----------------+------------+ | ``e`` | :c:expr:`_Float16` | float | 2 | \(4), \(6) | +--------+--------------------------+--------------------+----------------+------------+ @@ -272,7 +272,7 @@ platform-dependent. +--------+--------------------------+--------------------+----------------+------------+ | ``p`` | :c:expr:`char[]` | bytes | | \(8) | +--------+--------------------------+--------------------+----------------+------------+ -| ``P`` | :c:expr:`void \*` | integer | | \(5) | +| ``P`` | :c:expr:`void \*` | int | | \(2), \(5) | +--------+--------------------------+--------------------+----------------+------------+ .. versionchanged:: 3.3 @@ -349,27 +349,31 @@ Notes: The ``'p'`` format character encodes a "Pascal string", meaning a short variable-length string stored in a *fixed number of bytes*, given by the count. The first byte stored is the length of the string, or 255, whichever is - smaller. The bytes of the string follow. If the string passed in to + smaller. The bytes of the string follow. If the byte string passed in to :func:`pack` is too long (longer than the count minus 1), only the leading - ``count-1`` bytes of the string are stored. If the string is shorter than + ``count-1`` bytes of the string are stored. If the byte string is shorter than ``count-1``, it is padded with null bytes so that exactly count bytes in all are used. Note that for :func:`unpack`, the ``'p'`` format character consumes - ``count`` bytes, but that the string returned can never contain more than 255 + ``count`` bytes, but that the :class:`!bytes` object returned can never contain more than 255 bytes. + When packing, arguments of types :class:`bytes` and :class:`bytearray` + are accepted. (9) For the ``'s'`` format character, the count is interpreted as the length of the - bytes, not a repeat count like for the other format characters; for example, + byte string, not a repeat count like for the other format characters; for example, ``'10s'`` means a single 10-byte string mapping to or from a single Python byte string, while ``'10c'`` means 10 separate one byte character elements (e.g., ``cccccccccc``) mapping to or from ten different Python byte objects. (See :ref:`struct-examples` for a concrete demonstration of the difference.) - If a count is not given, it defaults to 1. For packing, the string is + If a count is not given, it defaults to 1. For packing, the byte string is truncated or padded with null bytes as appropriate to make it fit. For - unpacking, the resulting bytes object always has exactly the specified number - of bytes. As a special case, ``'0s'`` means a single, empty string (while + unpacking, the resulting :class:`!bytes` object always has exactly the specified number + of bytes. As a special case, ``'0s'`` means a single, empty byte string (while ``'0c'`` means 0 characters). + When packing, arguments of types :class:`bytes` and :class:`bytearray` + are accepted. (10) For the ``'F'`` and ``'D'`` format characters, the packed representation uses diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 16e6b1d6dc7..7cca6f2bcda 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -180,8 +180,8 @@ Local events '''''''''''' Local events are associated with normal execution of the program and happen -at clearly defined locations. All local events can be disabled. -The local events are: +at clearly defined locations. All local events can be disabled +per location. The local events are: * :monitoring-event:`PY_START` * :monitoring-event:`PY_RESUME` @@ -205,6 +205,8 @@ Using :monitoring-event:`BRANCH_LEFT` and :monitoring-event:`BRANCH_RIGHT` events will give much better performance as they can be disabled independently. +.. _monitoring-ancillary-events: + Ancillary events '''''''''''''''' @@ -226,7 +228,7 @@ Other events '''''''''''' Other events are not necessarily tied to a specific location in the -program and cannot be individually disabled via :data:`DISABLE`. +program and cannot be individually disabled per location. The other events that can be monitored are: @@ -234,6 +236,12 @@ The other events that can be monitored are: * :monitoring-event:`PY_UNWIND` * :monitoring-event:`RAISE` * :monitoring-event:`EXCEPTION_HANDLED` +* :monitoring-event:`RERAISE` + +.. versionchanged:: 3.15 + Other events can now be turned on and disabled on a per code object + basis. Returning :data:`DISABLE` from a callback disables the event + for the entire code object (for the current tool). The STOP_ITERATION event @@ -247,8 +255,7 @@ raise an exception unless it would be visible to other code. To allow tools to monitor for real exceptions without slowing down generators and coroutines, the :monitoring-event:`STOP_ITERATION` event is provided. -:monitoring-event:`STOP_ITERATION` can be locally disabled, unlike -:monitoring-event:`RAISE`. +:monitoring-event:`STOP_ITERATION` can be locally disabled. Note that the :monitoring-event:`STOP_ITERATION` event and the :monitoring-event:`RAISE` event for a :exc:`StopIteration` exception are @@ -314,15 +321,14 @@ location by returning :data:`sys.monitoring.DISABLE` from a callback function. This does not change which events are set, or any other code locations for the same event. -Disabling events for specific locations is very important for high -performance monitoring. For example, a program can be run under a -debugger with no overhead if the debugger disables all monitoring -except for a few breakpoints. +:ref:`Other events ` can be disabled on a per code +object basis by returning :data:`sys.monitoring.DISABLE` from a callback +function. This disables the event for the entire code object (for the current +tool). -If :data:`DISABLE` is returned by a callback for a -:ref:`global event `, :exc:`ValueError` will be raised -by the interpreter in a non-specific location (that is, no traceback will be -provided). +Disabling events for specific locations is very important for high performance +monitoring. For example, a program can be run under a debugger with no overhead +if the debugger disables all monitoring except for a few breakpoints. .. function:: restart_events() -> None diff --git a/Doc/library/tachyon-logo.png b/Doc/library/tachyon-logo.png deleted file mode 100644 index bf0901ec9f3..00000000000 Binary files a/Doc/library/tachyon-logo.png and /dev/null differ diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index a86469bb9ad..6f1e01cf5aa 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -142,6 +142,10 @@ Some facts and figures: a Zstandard dictionary used to improve compression of smaller amounts of data. + For modes ``'w:gz'`` and ``'w|gz'``, :func:`tarfile.open` accepts the + keyword argument *mtime* to create a gzip archive header with that mtime. By + default, the mtime is set to the time of creation of the archive. + For special purposes, there is a second format for *mode*: ``'filemode|[compression]'``. :func:`tarfile.open` will return a :class:`TarFile` object that processes its data as a stream of blocks. No random seeking will diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 19cc4f191df..fbe3951e034 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -1436,3 +1436,159 @@ is equivalent to:: Currently, :class:`Lock`, :class:`RLock`, :class:`Condition`, :class:`Semaphore`, and :class:`BoundedSemaphore` objects may be used as :keyword:`with` statement context managers. + + +Iterator synchronization +------------------------ + +By default, Python iterators do not support concurrent access. Most iterators make +no guarantees when accessed simultaneously from multiple threads. Generator +iterators, for example, raise :exc:`ValueError` if one of their iterator methods +is called while the generator is already executing. The tools in this section +allow reliable concurrency support to be added to ordinary iterators and +iterator-producing callables. + +The :class:`serialize_iterator` wrapper lets multiple threads share a single iterator and +take turns consuming from it. While one thread is running ``__next__()``, the +others block until the iterator becomes available. Each value produced by the +underlying iterator is delivered to exactly one caller. + +The :func:`concurrent_tee` function lets multiple threads each receive the full +stream of values from one underlying iterator. It creates independent iterators +that all draw from the same source. Values are buffered until consumed by all +of the derived iterators. + +.. class:: serialize_iterator(iterable) + + Return an iterator wrapper that serializes concurrent calls to + :meth:`~iterator.__next__` using a lock. + + If the wrapped iterator also defines :meth:`~generator.send`, + :meth:`~generator.throw`, or :meth:`~generator.close`, those calls + are serialized as well. + + This makes it possible to share a single iterator, including a generator + iterator, between multiple threads. A lock ensures that calls are handled + one at a time. No values are duplicated or skipped by the wrapper itself. + Each item from the underlying iterator is given to exactly one caller. + + This wrapper does not copy or buffer values. Threads that call + :func:`next` while another thread is already advancing the iterator will + block until the active call completes. + + Example: + + .. code-block:: python + + import threading + + def squares(n): + for x in range(n): + yield x * x + + def consume(name, iterable): + for item in iterable: + print(name, item) + + source = threading.serialize_iterator(squares(5)) + + t1 = threading.Thread(target=consume, args=("left", source)) + t2 = threading.Thread(target=consume, args=("right", source)) + t1.start() + t2.start() + t1.join() + t2.join() + + In this example, each number is printed exactly once, but the work is shared + between the two threads. + + .. versionadded:: next + + +.. function:: synchronized_iterator(func) + + Wrap an iterator-producing callable so that each iterator it returns is + automatically passed through :class:`serialize_iterator`. + + This is especially useful as a :term:`decorator` for generator functions, + allowing their generator-iterators to be consumed from multiple threads. + + Example: + + .. code-block:: python + + import threading + + @threading.synchronized_iterator + def squares(n): + for x in range(n): + yield x * x + + def consume(name, iterable): + for item in iterable: + print(name, item) + + source = squares(5) + + t1 = threading.Thread(target=consume, args=("left", source)) + t2 = threading.Thread(target=consume, args=("right", source)) + t1.start() + t2.start() + t1.join() + t2.join() + + The returned wrapper preserves the metadata of *func*, such as its name and + wrapped function reference. + + .. versionadded:: next + + +.. function:: concurrent_tee(iterable, n=2) + + Return *n* independent iterators from a single input *iterable*, with + guaranteed behavior when the derived iterators are consumed concurrently. + + This function is similar to :func:`itertools.tee`, but is intended for cases + where the source iterator may feed consumers running in different threads. + Each returned iterator yields every value from the underlying iterable, in + the same order. + + Internally, values are buffered until every derived iterator has consumed + them. + + The returned iterators share the same underlying synchronization lock. Each + individual derived iterator is intended to be consumed by one thread at a + time. If a single derived iterator must itself be shared by multiple + threads, wrap it with :class:`serialize_iterator`. + + If *n* is ``0``, return an empty tuple. If *n* is negative, raise + :exc:`ValueError`. + + Example: + + .. code-block:: python + + import threading + + def squares(n): + for x in range(n): + yield x * x + + def consume(name, iterable): + for item in iterable: + print(name, item) + + source = squares(5) + left, right = threading.concurrent_tee(source) + + t1 = threading.Thread(target=consume, args=("left", left)) + t2 = threading.Thread(target=consume, args=("right", right)) + t1.start() + t2.start() + t1.join() + t2.join() + + In this example, both consumer threads see the full sequence of squares + from a single generator expression. + + .. versionadded:: next diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst index 3db4cf42c17..72fbcaba160 100644 --- a/Doc/library/tokenize.rst +++ b/Doc/library/tokenize.rst @@ -28,7 +28,7 @@ type can be determined by checking the ``exact_type`` property on the **undefined** when providing invalid Python code and it can change at any point. -Tokenizing Input +Tokenizing input ---------------- The primary entry point is a :term:`generator`: @@ -146,7 +146,7 @@ function it uses to do this is available: .. _tokenize-cli: -Command-Line Usage +Command-line usage ------------------ .. versionadded:: 3.3 @@ -173,8 +173,12 @@ The following options are accepted: If :file:`filename.py` is specified its contents are tokenized to stdout. Otherwise, tokenization is performed on stdin. +.. versionadded:: next + Output is in color by default and can be + :ref:`controlled using environment variables `. + Examples ------------------- +-------- Example of a script rewriter that transforms float literals into Decimal objects:: @@ -227,7 +231,7 @@ Example of tokenizing from the command line. The script:: will be tokenized to the following output where the first column is the range of the line/column coordinates where the token is found, the second column is -the name of the token, and the final column is the value of the token (if any) +the name of the token, and the final column is the value of the token (if any): .. code-block:: shell-session diff --git a/Doc/library/tomllib.rst b/Doc/library/tomllib.rst index 2bac968c2be..55610784362 100644 --- a/Doc/library/tomllib.rst +++ b/Doc/library/tomllib.rst @@ -19,6 +19,12 @@ support writing TOML. Added TOML 1.1.0 support. See the :ref:`What's New ` for details. +.. warning:: + + Be cautious when parsing data from untrusted sources. + A malicious TOML string may cause the decoder to consume considerable + CPU and memory resources. + Limiting the size of data to be parsed is recommended. .. seealso:: diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 2ce868cf84d..1957cadcbb1 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1174,7 +1174,8 @@ These can be used as types in annotations. They all support subscription using or transforms parameters of another callable. Usage is in the form ``Concatenate[Arg1Type, Arg2Type, ..., ParamSpecVariable]``. ``Concatenate`` - is currently only valid when used as the first argument to a :ref:`Callable `. + is valid when used in :ref:`Callable ` type hints + and when instantiating user-defined generic classes with :class:`ParamSpec` parameters. The last parameter to ``Concatenate`` must be a :class:`ParamSpec` or ellipsis (``...``). @@ -1980,7 +1981,7 @@ without the dedicated syntax, as documented below. .. _typevartuple: -.. class:: TypeVarTuple(name, *, default=typing.NoDefault) +.. class:: TypeVarTuple(name, *, bound=None, covariant=False, contravariant=False, infer_variance=False, default=typing.NoDefault) Type variable tuple. A specialized form of :ref:`type variable ` that enables *variadic* generics. @@ -2090,6 +2091,24 @@ without the dedicated syntax, as documented below. The name of the type variable tuple. + .. attribute:: __covariant__ + + Whether the type variable tuple has been explicitly marked as covariant. + + .. versionadded:: 3.15 + + .. attribute:: __contravariant__ + + Whether the type variable tuple has been explicitly marked as contravariant. + + .. versionadded:: 3.15 + + .. attribute:: __infer_variance__ + + Whether the type variable tuple's variance should be inferred by type checkers. + + .. versionadded:: 3.15 + .. attribute:: __default__ The default value of the type variable tuple, or :data:`typing.NoDefault` if it @@ -2116,6 +2135,11 @@ without the dedicated syntax, as documented below. .. versionadded:: 3.13 + Type variable tuples created with ``covariant=True`` or + ``contravariant=True`` can be used to declare covariant or contravariant + generic types. The ``bound`` argument is also accepted, similar to + :class:`TypeVar`, but its actual semantics are yet to be decided. + .. versionadded:: 3.11 .. versionchanged:: 3.12 @@ -2127,6 +2151,11 @@ without the dedicated syntax, as documented below. Support for default values was added. + .. versionchanged:: 3.15 + + Added support for the ``bound``, ``covariant``, ``contravariant``, and + ``infer_variance`` parameters. + .. class:: ParamSpec(name, *, bound=None, covariant=False, contravariant=False, default=typing.NoDefault) Parameter specification variable. A specialized version of @@ -2196,6 +2225,20 @@ without the dedicated syntax, as documented below. The name of the parameter specification. + .. attribute:: __covariant__ + + Whether the parameter specification has been explicitly marked as covariant. + + .. attribute:: __contravariant__ + + Whether the parameter specification has been explicitly marked as contravariant. + + .. attribute:: __infer_variance__ + + Whether the parameter specification's variance should be inferred by type checkers. + + .. versionadded:: 3.12 + .. attribute:: __default__ The default value of the parameter specification, or :data:`typing.NoDefault` if it @@ -3358,6 +3401,36 @@ Functions and decorators .. versionadded:: 3.12 +.. decorator:: disjoint_base + + Decorator to mark a class as a disjoint base. + + Type checkers do not allow child classes of a disjoint base ``C`` to + inherit from other disjoint bases that are not parent or child classes of ``C``. + + For example:: + + @disjoint_base + class Disjoint1: pass + + @disjoint_base + class Disjoint2: pass + + class Disjoint3(Disjoint1, Disjoint2): pass # Type checker error + + Type checkers can use knowledge of disjoint bases to detect unreachable code + and determine when two types can overlap. + + The corresponding runtime concept is a solid base (see :ref:`multiple-inheritance`). + Classes that are solid bases at runtime can be marked with ``@disjoint_base`` in stub files. + Users may also mark other classes as disjoint bases to indicate to type checkers that + multiple inheritance with other disjoint bases should not be allowed. + + Note that the concept of a solid base is a CPython implementation + detail, and the exact set of standard library classes that are + disjoint bases at runtime may change in future versions of Python. + + .. versionadded:: next .. decorator:: type_check_only @@ -3380,13 +3453,13 @@ Functions and decorators Introspection helpers --------------------- -.. function:: get_type_hints(obj, globalns=None, localns=None, include_extras=False) +.. function:: get_type_hints(obj, globalns=None, localns=None, include_extras=False, *, format=Format.VALUE) Return a dictionary containing type hints for a function, method, module, class object, or other callable object. - This is often the same as ``obj.__annotations__``, but this function makes - the following changes to the annotations dictionary: + This is often the same as :func:`annotationlib.get_annotations`, but this + function makes the following changes to the annotations dictionary: * Forward references encoded as string literals or :class:`ForwardRef` objects are handled by evaluating them in *globalns*, *localns*, and @@ -3400,17 +3473,15 @@ Introspection helpers annotations from ``C``'s base classes with those on ``C`` directly. This is done by traversing :attr:`C.__mro__ ` and iteratively combining - ``__annotations__`` dictionaries. Annotations on classes appearing - earlier in the :term:`method resolution order` always take precedence over - annotations on classes appearing later in the method resolution order. + :term:`annotations ` of each base class. Annotations + on classes appearing earlier in the :term:`method resolution order` always + take precedence over annotations on classes appearing later in the method + resolution order. * The function recursively replaces all occurrences of ``Annotated[T, ...]``, ``Required[T]``, ``NotRequired[T]``, and ``ReadOnly[T]`` with ``T``, unless *include_extras* is set to ``True`` (see :class:`Annotated` for more information). - See also :func:`annotationlib.get_annotations`, a lower-level function that - returns annotations more directly. - .. caution:: This function may execute arbitrary code contained in annotations. @@ -3418,11 +3489,12 @@ Introspection helpers .. note:: - If any forward references in the annotations of *obj* are not resolvable - or are not valid Python code, this function will raise an exception - such as :exc:`NameError`. For example, this can happen with imported - :ref:`type aliases ` that include forward references, - or with names imported under :data:`if TYPE_CHECKING `. + If :attr:`Format.VALUE ` is used and any + forward references in the annotations of *obj* are not resolvable, a + :exc:`NameError` exception is raised. For example, this can happen + with names imported under :data:`if TYPE_CHECKING `. + More generally, any kind of exception can be raised if an annotation + contains invalid Python code. .. note:: @@ -3440,6 +3512,10 @@ Introspection helpers if a default value equal to ``None`` was set. Now the annotation is returned unchanged. + .. versionchanged:: 3.14 + Added the ``format`` parameter. See the documentation on + :func:`annotationlib.get_annotations` for more information. + .. versionchanged:: 3.14 Calling :func:`get_type_hints` on instances is no longer supported. Some instances were accepted in earlier versions as an undocumented diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index ce0a22b9456..f043915c0f4 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -308,6 +308,11 @@ Decompression objects support the following methods and attributes: :attr:`unconsumed_tail`. This bytestring must be passed to a subsequent call to :meth:`decompress` if decompression is to continue. If *max_length* is zero then the whole input is decompressed, and :attr:`unconsumed_tail` is empty. + For example, the full content could be read like:: + + process_output(d.decompress(data, max_length)) + while chunk := d.decompress(d.unconsumed_tail, max_length): + process_output(chunk) .. versionchanged:: 3.6 *max_length* can be used as a keyword argument. diff --git a/Doc/pylock.toml b/Doc/pylock.toml new file mode 100644 index 00000000000..f1febe21c23 --- /dev/null +++ b/Doc/pylock.toml @@ -0,0 +1,256 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile Doc/requirements.txt --exclude-newer P14D --exclude-newer-package linklint=PT0S --exclude-newer-package python-docs-theme=PT0S --no-cache --output-file Doc/pylock.toml --python-version 3.12 --universal +lock-version = "1.0" +created-by = "uv" + +[[packages]] +name = "alabaster" +version = "1.0.0" +sdist = { url = "https://files.pythonhosted.org/packages/a6/f8/d9c74d0daf3f742840fd818d69cfae176fa332022fd44e3469487d5a9420/alabaster-1.0.0.tar.gz", upload-time = 2024-07-26T18:15:03Z, size = 24210, hashes = { sha256 = "c00dca57bca26fa62a6d7d0a9fcce65f3e026e9bfe33e9c538fd3fbb2144fd9e" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl", upload-time = 2024-07-26T18:15:02Z, size = 13929, hashes = { sha256 = "fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b" } }] + +[[packages]] +name = "babel" +version = "2.18.0" +sdist = { url = "https://files.pythonhosted.org/packages/7d/b2/51899539b6ceeeb420d40ed3cd4b7a40519404f9baf3d4ac99dc413a834b/babel-2.18.0.tar.gz", upload-time = 2026-02-01T12:30:56Z, size = 9959554, hashes = { sha256 = "b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", upload-time = 2026-02-01T12:30:53Z, size = 10196845, hashes = { sha256 = "e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35" } }] + +[[packages]] +name = "blurb" +version = "2.0.0" +sdist = { url = "https://files.pythonhosted.org/packages/d7/82/8597d891f4b03f3eaefcb4213a811643d558350cac9a69864d127832cc4f/blurb-2.0.0.tar.gz", upload-time = 2025-01-15T12:48:53Z, size = 24666, hashes = { sha256 = "c78d8114294225a4f7a2eabba6e05d36a6a50e45ba9f5a41afabc198350038e0" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/b4/03/374bd9e31b58e8a8e5dc65cc3f68ca7cdd716c32b5e5dcb0e1b76bb75b4a/blurb-2.0.0-py3-none-any.whl", upload-time = 2025-01-15T12:48:49Z, size = 18924, hashes = { sha256 = "f6d0e858dbe94765f6a89b8228217ffdb9c19cff08fc8f2c3153954846d31aa1" } }] + +[[packages]] +name = "certifi" +version = "2026.2.25" +sdist = { url = "https://files.pythonhosted.org/packages/af/2d/7bf41579a8986e348fa033a31cdd0e4121114f6bce2457e8876010b092dd/certifi-2026.2.25.tar.gz", upload-time = 2026-02-25T02:54:17Z, size = 155029, hashes = { sha256 = "e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", upload-time = 2026-02-25T02:54:15Z, size = 153684, hashes = { sha256 = "027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa" } }] + +[[packages]] +name = "charset-normalizer" +version = "3.4.7" +sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", upload-time = 2026-04-02T09:28:39Z, size = 144271, hashes = { sha256 = "ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5" } } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/eb/4fc8d0a7110eb5fc9cc161723a34a8a6c200ce3b4fbf681bc86feee22308/charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", upload-time = 2026-04-02T09:26:24Z, size = 311328, hashes = { sha256 = "eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46" } }, + { url = "https://files.pythonhosted.org/packages/f8/e3/0fadc706008ac9d7b9b5be6dc767c05f9d3e5df51744ce4cc9605de7b9f4/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", upload-time = 2026-04-02T09:26:25Z, size = 208061, hashes = { sha256 = "6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2" } }, + { url = "https://files.pythonhosted.org/packages/42/f0/3dd1045c47f4a4604df85ec18ad093912ae1344ac706993aff91d38773a2/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", upload-time = 2026-04-02T09:26:26Z, size = 229031, hashes = { sha256 = "e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b" } }, + { url = "https://files.pythonhosted.org/packages/dc/67/675a46eb016118a2fbde5a277a5d15f4f69d5f3f5f338e5ee2f8948fcf43/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", upload-time = 2026-04-02T09:26:28Z, size = 225239, hashes = { sha256 = "edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a" } }, + { url = "https://files.pythonhosted.org/packages/4b/f8/d0118a2f5f23b02cd166fa385c60f9b0d4f9194f574e2b31cef350ad7223/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", upload-time = 2026-04-02T09:26:29Z, size = 216589, hashes = { sha256 = "5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116" } }, + { url = "https://files.pythonhosted.org/packages/b1/f1/6d2b0b261b6c4ceef0fcb0d17a01cc5bc53586c2d4796fa04b5c540bc13d/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_armv7l.whl", upload-time = 2026-04-02T09:26:30Z, size = 202733, hashes = { sha256 = "203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb" } }, + { url = "https://files.pythonhosted.org/packages/6f/c0/7b1f943f7e87cc3db9626ba17807d042c38645f0a1d4415c7a14afb5591f/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", upload-time = 2026-04-02T09:26:31Z, size = 212652, hashes = { sha256 = "298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1" } }, + { url = "https://files.pythonhosted.org/packages/38/dd/5a9ab159fe45c6e72079398f277b7d2b523e7f716acc489726115a910097/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", upload-time = 2026-04-02T09:26:33Z, size = 211229, hashes = { sha256 = "708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15" } }, + { url = "https://files.pythonhosted.org/packages/d5/ff/531a1cad5ca855d1c1a8b69cb71abfd6d85c0291580146fda7c82857caa1/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_armv7l.whl", upload-time = 2026-04-02T09:26:34Z, size = 203552, hashes = { sha256 = "0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5" } }, + { url = "https://files.pythonhosted.org/packages/c1/4c/a5fb52d528a8ca41f7598cb619409ece30a169fbdf9cdce592e53b46c3a6/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_ppc64le.whl", upload-time = 2026-04-02T09:26:36Z, size = 230806, hashes = { sha256 = "4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d" } }, + { url = "https://files.pythonhosted.org/packages/59/7a/071feed8124111a32b316b33ae4de83d36923039ef8cf48120266844285b/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_riscv64.whl", upload-time = 2026-04-02T09:26:37Z, size = 212316, hashes = { sha256 = "aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7" } }, + { url = "https://files.pythonhosted.org/packages/fd/35/f7dba3994312d7ba508e041eaac39a36b120f32d4c8662b8814dab876431/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_s390x.whl", upload-time = 2026-04-02T09:26:38Z, size = 227274, hashes = { sha256 = "fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464" } }, + { url = "https://files.pythonhosted.org/packages/8a/2d/a572df5c9204ab7688ec1edc895a73ebded3b023bb07364710b05dd1c9be/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", upload-time = 2026-04-02T09:26:40Z, size = 218468, hashes = { sha256 = "bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49" } }, + { url = "https://files.pythonhosted.org/packages/86/eb/890922a8b03a568ca2f336c36585a4713c55d4d67bf0f0c78924be6315ca/charset_normalizer-3.4.7-cp312-cp312-win32.whl", upload-time = 2026-04-02T09:26:41Z, size = 148460, hashes = { sha256 = "2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c" } }, + { url = "https://files.pythonhosted.org/packages/35/d9/0e7dffa06c5ab081f75b1b786f0aefc88365825dfcd0ac544bdb7b2b6853/charset_normalizer-3.4.7-cp312-cp312-win_amd64.whl", upload-time = 2026-04-02T09:26:42Z, size = 159330, hashes = { sha256 = "5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6" } }, + { url = "https://files.pythonhosted.org/packages/9e/5d/481bcc2a7c88ea6b0878c299547843b2521ccbc40980cb406267088bc701/charset_normalizer-3.4.7-cp312-cp312-win_arm64.whl", upload-time = 2026-04-02T09:26:44Z, size = 147828, hashes = { sha256 = "56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d" } }, + { url = "https://files.pythonhosted.org/packages/c1/3b/66777e39d3ae1ddc77ee606be4ec6d8cbd4c801f65e5a1b6f2b11b8346dd/charset_normalizer-3.4.7-cp313-cp313-macosx_10_13_universal2.whl", upload-time = 2026-04-02T09:26:45Z, size = 309627, hashes = { sha256 = "f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063" } }, + { url = "https://files.pythonhosted.org/packages/2e/4e/b7f84e617b4854ade48a1b7915c8ccfadeba444d2a18c291f696e37f0d3b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", upload-time = 2026-04-02T09:26:46Z, size = 207008, hashes = { sha256 = "0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c" } }, + { url = "https://files.pythonhosted.org/packages/c4/bb/ec73c0257c9e11b268f018f068f5d00aa0ef8c8b09f7753ebd5f2880e248/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", upload-time = 2026-04-02T09:26:48Z, size = 228303, hashes = { sha256 = "a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66" } }, + { url = "https://files.pythonhosted.org/packages/85/fb/32d1f5033484494619f701e719429c69b766bfc4dbc61aa9e9c8c166528b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", upload-time = 2026-04-02T09:26:49Z, size = 224282, hashes = { sha256 = "3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18" } }, + { url = "https://files.pythonhosted.org/packages/fa/07/330e3a0dda4c404d6da83b327270906e9654a24f6c546dc886a0eb0ffb23/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", upload-time = 2026-04-02T09:26:50Z, size = 215595, hashes = { sha256 = "e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd" } }, + { url = "https://files.pythonhosted.org/packages/e3/7c/fc890655786e423f02556e0216d4b8c6bcb6bdfa890160dc66bf52dee468/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_armv7l.whl", upload-time = 2026-04-02T09:26:52Z, size = 201986, hashes = { sha256 = "f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215" } }, + { url = "https://files.pythonhosted.org/packages/d8/97/bfb18b3db2aed3b90cf54dc292ad79fdd5ad65c4eae454099475cbeadd0d/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", upload-time = 2026-04-02T09:26:53Z, size = 211711, hashes = { sha256 = "e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859" } }, + { url = "https://files.pythonhosted.org/packages/6f/a5/a581c13798546a7fd557c82614a5c65a13df2157e9ad6373166d2a3e645d/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_aarch64.whl", upload-time = 2026-04-02T09:26:54Z, size = 210036, hashes = { sha256 = "7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8" } }, + { url = "https://files.pythonhosted.org/packages/8c/bf/b3ab5bcb478e4193d517644b0fb2bf5497fbceeaa7a1bc0f4d5b50953861/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_armv7l.whl", upload-time = 2026-04-02T09:26:56Z, size = 202998, hashes = { sha256 = "481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5" } }, + { url = "https://files.pythonhosted.org/packages/e7/4e/23efd79b65d314fa320ec6017b4b5834d5c12a58ba4610aa353af2e2f577/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_ppc64le.whl", upload-time = 2026-04-02T09:26:57Z, size = 230056, hashes = { sha256 = "f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832" } }, + { url = "https://files.pythonhosted.org/packages/b9/9f/1e1941bc3f0e01df116e68dc37a55c4d249df5e6fa77f008841aef68264f/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_riscv64.whl", upload-time = 2026-04-02T09:26:58Z, size = 211537, hashes = { sha256 = "f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6" } }, + { url = "https://files.pythonhosted.org/packages/80/0f/088cbb3020d44428964a6c97fe1edfb1b9550396bf6d278330281e8b709c/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_s390x.whl", upload-time = 2026-04-02T09:27:00Z, size = 226176, hashes = { sha256 = "3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48" } }, + { url = "https://files.pythonhosted.org/packages/6a/9f/130394f9bbe06f4f63e22641d32fc9b202b7e251c9aef4db044324dac493/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_x86_64.whl", upload-time = 2026-04-02T09:27:02Z, size = 217723, hashes = { sha256 = "64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a" } }, + { url = "https://files.pythonhosted.org/packages/73/55/c469897448a06e49f8fa03f6caae97074fde823f432a98f979cc42b90e69/charset_normalizer-3.4.7-cp313-cp313-win32.whl", upload-time = 2026-04-02T09:27:03Z, size = 148085, hashes = { sha256 = "4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e" } }, + { url = "https://files.pythonhosted.org/packages/5d/78/1b74c5bbb3f99b77a1715c91b3e0b5bdb6fe302d95ace4f5b1bec37b0167/charset_normalizer-3.4.7-cp313-cp313-win_amd64.whl", upload-time = 2026-04-02T09:27:04Z, size = 158819, hashes = { sha256 = "3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110" } }, + { url = "https://files.pythonhosted.org/packages/68/86/46bd42279d323deb8687c4a5a811fd548cb7d1de10cf6535d099877a9a9f/charset_normalizer-3.4.7-cp313-cp313-win_arm64.whl", upload-time = 2026-04-02T09:27:05Z, size = 147915, hashes = { sha256 = "80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b" } }, + { url = "https://files.pythonhosted.org/packages/97/c8/c67cb8c70e19ef1960b97b22ed2a1567711de46c4ddf19799923adc836c2/charset_normalizer-3.4.7-cp314-cp314-macosx_10_15_universal2.whl", upload-time = 2026-04-02T09:27:07Z, size = 309234, hashes = { sha256 = "c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0" } }, + { url = "https://files.pythonhosted.org/packages/99/85/c091fdee33f20de70d6c8b522743b6f831a2f1cd3ff86de4c6a827c48a76/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", upload-time = 2026-04-02T09:27:08Z, size = 208042, hashes = { sha256 = "1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a" } }, + { url = "https://files.pythonhosted.org/packages/87/1c/ab2ce611b984d2fd5d86a5a8a19c1ae26acac6bad967da4967562c75114d/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", upload-time = 2026-04-02T09:27:09Z, size = 228706, hashes = { sha256 = "54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b" } }, + { url = "https://files.pythonhosted.org/packages/a8/29/2b1d2cb00bf085f59d29eb773ce58ec2d325430f8c216804a0a5cd83cbca/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", upload-time = 2026-04-02T09:27:11Z, size = 224727, hashes = { sha256 = "715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41" } }, + { url = "https://files.pythonhosted.org/packages/47/5c/032c2d5a07fe4d4855fea851209cca2b6f03ebeb6d4e3afdb3358386a684/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", upload-time = 2026-04-02T09:27:12Z, size = 215882, hashes = { sha256 = "bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e" } }, + { url = "https://files.pythonhosted.org/packages/2c/c2/356065d5a8b78ed04499cae5f339f091946a6a74f91e03476c33f0ab7100/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_armv7l.whl", upload-time = 2026-04-02T09:27:13Z, size = 200860, hashes = { sha256 = "c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae" } }, + { url = "https://files.pythonhosted.org/packages/0c/cd/a32a84217ced5039f53b29f460962abb2d4420def55afabe45b1c3c7483d/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", upload-time = 2026-04-02T09:27:15Z, size = 211564, hashes = { sha256 = "3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18" } }, + { url = "https://files.pythonhosted.org/packages/44/86/58e6f13ce26cc3b8f4a36b94a0f22ae2f00a72534520f4ae6857c4b81f89/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_aarch64.whl", upload-time = 2026-04-02T09:27:16Z, size = 211276, hashes = { sha256 = "e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b" } }, + { url = "https://files.pythonhosted.org/packages/8f/fe/d17c32dc72e17e155e06883efa84514ca375f8a528ba2546bee73fc4df81/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_armv7l.whl", upload-time = 2026-04-02T09:27:18Z, size = 201238, hashes = { sha256 = "a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356" } }, + { url = "https://files.pythonhosted.org/packages/6a/29/f33daa50b06525a237451cdb6c69da366c381a3dadcd833fa5676bc468b3/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_ppc64le.whl", upload-time = 2026-04-02T09:27:19Z, size = 230189, hashes = { sha256 = "2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab" } }, + { url = "https://files.pythonhosted.org/packages/b6/6e/52c84015394a6a0bdcd435210a7e944c5f94ea1055f5cc5d56c5fe368e7b/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_riscv64.whl", upload-time = 2026-04-02T09:27:20Z, size = 211352, hashes = { sha256 = "e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46" } }, + { url = "https://files.pythonhosted.org/packages/8c/d7/4353be581b373033fb9198bf1da3cf8f09c1082561e8e922aa7b39bf9fe8/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_s390x.whl", upload-time = 2026-04-02T09:27:22Z, size = 227024, hashes = { sha256 = "d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44" } }, + { url = "https://files.pythonhosted.org/packages/30/45/99d18aa925bd1740098ccd3060e238e21115fffbfdcb8f3ece837d0ace6c/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_x86_64.whl", upload-time = 2026-04-02T09:27:23Z, size = 217869, hashes = { sha256 = "7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72" } }, + { url = "https://files.pythonhosted.org/packages/5c/05/5ee478aa53f4bb7996482153d4bfe1b89e0f087f0ab6b294fcf92d595873/charset_normalizer-3.4.7-cp314-cp314-win32.whl", upload-time = 2026-04-02T09:27:25Z, size = 148541, hashes = { sha256 = "5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10" } }, + { url = "https://files.pythonhosted.org/packages/48/77/72dcb0921b2ce86420b2d79d454c7022bf5be40202a2a07906b9f2a35c97/charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl", upload-time = 2026-04-02T09:27:26Z, size = 159634, hashes = { sha256 = "92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f" } }, + { url = "https://files.pythonhosted.org/packages/c6/a3/c2369911cd72f02386e4e340770f6e158c7980267da16af8f668217abaa0/charset_normalizer-3.4.7-cp314-cp314-win_arm64.whl", upload-time = 2026-04-02T09:27:28Z, size = 148384, hashes = { sha256 = "67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246" } }, + { url = "https://files.pythonhosted.org/packages/94/09/7e8a7f73d24dba1f0035fbbf014d2c36828fc1bf9c88f84093e57d315935/charset_normalizer-3.4.7-cp314-cp314t-macosx_10_15_universal2.whl", upload-time = 2026-04-02T09:27:29Z, size = 330133, hashes = { sha256 = "effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24" } }, + { url = "https://files.pythonhosted.org/packages/8d/da/96975ddb11f8e977f706f45cddd8540fd8242f71ecdb5d18a80723dcf62c/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", upload-time = 2026-04-02T09:27:30Z, size = 216257, hashes = { sha256 = "fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79" } }, + { url = "https://files.pythonhosted.org/packages/e5/e8/1d63bf8ef2d388e95c64b2098f45f84758f6d102a087552da1485912637b/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", upload-time = 2026-04-02T09:27:32Z, size = 234851, hashes = { sha256 = "733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960" } }, + { url = "https://files.pythonhosted.org/packages/9b/40/e5ff04233e70da2681fa43969ad6f66ca5611d7e669be0246c4c7aaf6dc8/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", upload-time = 2026-04-02T09:27:34Z, size = 233393, hashes = { sha256 = "a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4" } }, + { url = "https://files.pythonhosted.org/packages/be/c1/06c6c49d5a5450f76899992f1ee40b41d076aee9279b49cf9974d2f313d5/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", upload-time = 2026-04-02T09:27:35Z, size = 223251, hashes = { sha256 = "6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e" } }, + { url = "https://files.pythonhosted.org/packages/2b/9f/f2ff16fb050946169e3e1f82134d107e5d4ae72647ec8a1b1446c148480f/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_armv7l.whl", upload-time = 2026-04-02T09:27:36Z, size = 206609, hashes = { sha256 = "a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1" } }, + { url = "https://files.pythonhosted.org/packages/69/d5/a527c0cd8d64d2eab7459784fb4169a0ac76e5a6fc5237337982fd61347e/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", upload-time = 2026-04-02T09:27:38Z, size = 220014, hashes = { sha256 = "3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44" } }, + { url = "https://files.pythonhosted.org/packages/7e/80/8a7b8104a3e203074dc9aa2c613d4b726c0e136bad1cc734594b02867972/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_aarch64.whl", upload-time = 2026-04-02T09:27:39Z, size = 218979, hashes = { sha256 = "8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e" } }, + { url = "https://files.pythonhosted.org/packages/02/9a/b759b503d507f375b2b5c153e4d2ee0a75aa215b7f2489cf314f4541f2c0/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_armv7l.whl", upload-time = 2026-04-02T09:27:40Z, size = 209238, hashes = { sha256 = "cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3" } }, + { url = "https://files.pythonhosted.org/packages/c2/4e/0f3f5d47b86bdb79256e7290b26ac847a2832d9a4033f7eb2cd4bcf4bb5b/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", upload-time = 2026-04-02T09:27:42Z, size = 236110, hashes = { sha256 = "0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0" } }, + { url = "https://files.pythonhosted.org/packages/96/23/bce28734eb3ed2c91dcf93abeb8a5cf393a7b2749725030bb630e554fdd8/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_riscv64.whl", upload-time = 2026-04-02T09:27:43Z, size = 219824, hashes = { sha256 = "752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e" } }, + { url = "https://files.pythonhosted.org/packages/2c/6f/6e897c6984cc4d41af319b077f2f600fc8214eb2fe2d6bcb79141b882400/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_s390x.whl", upload-time = 2026-04-02T09:27:45Z, size = 233103, hashes = { sha256 = "8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb" } }, + { url = "https://files.pythonhosted.org/packages/76/22/ef7bd0fe480a0ae9b656189ec00744b60933f68b4f42a7bb06589f6f576a/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_x86_64.whl", upload-time = 2026-04-02T09:27:46Z, size = 225194, hashes = { sha256 = "ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe" } }, + { url = "https://files.pythonhosted.org/packages/c5/a7/0e0ab3e0b5bc1219bd80a6a0d4d72ca74d9250cb2382b7c699c147e06017/charset_normalizer-3.4.7-cp314-cp314t-win32.whl", upload-time = 2026-04-02T09:27:48Z, size = 159827, hashes = { sha256 = "c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0" } }, + { url = "https://files.pythonhosted.org/packages/7a/1d/29d32e0fb40864b1f878c7f5a0b343ae676c6e2b271a2d55cc3a152391da/charset_normalizer-3.4.7-cp314-cp314t-win_amd64.whl", upload-time = 2026-04-02T09:27:49Z, size = 174168, hashes = { sha256 = "03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c" } }, + { url = "https://files.pythonhosted.org/packages/de/32/d92444ad05c7a6e41fb2036749777c163baf7a0301a040cb672d6b2b1ae9/charset_normalizer-3.4.7-cp314-cp314t-win_arm64.whl", upload-time = 2026-04-02T09:27:51Z, size = 153018, hashes = { sha256 = "c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d" } }, + { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", upload-time = 2026-04-02T09:28:37Z, size = 61958, hashes = { sha256 = "3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d" } }, +] + +[[packages]] +name = "colorama" +version = "0.4.6" +marker = "sys_platform == 'win32'" +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", upload-time = 2022-10-25T02:36:22Z, size = 27697, hashes = { sha256 = "08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", upload-time = 2022-10-25T02:36:20Z, size = 25335, hashes = { sha256 = "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" } }] + +[[packages]] +name = "docutils" +version = "0.21.2" +sdist = { url = "https://files.pythonhosted.org/packages/ae/ed/aefcc8cd0ba62a0560c3c18c33925362d46c6075480bfa4df87b28e169a9/docutils-0.21.2.tar.gz", upload-time = 2024-04-23T18:57:18Z, size = 2204444, hashes = { sha256 = "3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", upload-time = 2024-04-23T18:57:14Z, size = 587408, hashes = { sha256 = "dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2" } }] + +[[packages]] +name = "idna" +version = "3.11" +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", upload-time = 2025-10-12T14:55:20Z, size = 194582, hashes = { sha256 = "795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", upload-time = 2025-10-12T14:55:18Z, size = 71008, hashes = { sha256 = "771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea" } }] + +[[packages]] +name = "imagesize" +version = "1.5.0" +sdist = { url = "https://files.pythonhosted.org/packages/cf/59/4b0dd64676aa6fb4986a755790cb6fc558559cf0084effad516820208ec3/imagesize-1.5.0.tar.gz", upload-time = 2026-03-03T01:59:54Z, size = 1281127, hashes = { sha256 = "8bfc5363a7f2133a89f0098451e0bcb1cd71aba4dc02bbcecb39d99d40e1b94f" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/1e/b1/a0662b03103c66cf77101a187f396ea91167cd9b7d5d3a2e465ad2c7ee9b/imagesize-1.5.0-py2.py3-none-any.whl", upload-time = 2026-03-03T01:59:52Z, size = 5763, hashes = { sha256 = "32677681b3f434c2cb496f00e89c5a291247b35b1f527589909e008057da5899" } }] + +[[packages]] +name = "jinja2" +version = "3.1.6" +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", upload-time = 2025-03-05T20:05:02Z, size = 245115, hashes = { sha256 = "0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", upload-time = 2025-03-05T20:05:00Z, size = 134899, hashes = { sha256 = "85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67" } }] + +[[packages]] +name = "linklint" +version = "0.4.1" +sdist = { url = "https://files.pythonhosted.org/packages/61/bc/9972ace8643a04a74210942717fd20c1c34d96079b59fd7790b4db56df7d/linklint-0.4.1.tar.gz", upload-time = 2026-03-27T10:48:40Z, size = 20588, hashes = { sha256 = "a5d291a0d8a7ab8b1f96f62bb7e1d9d2c79d8eceb934e2efc0235d6b2e77f19b" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/af/88/9c4865cdbd6f73fff668706072c421a329de79c3b69e0aa511679a2ff4f3/linklint-0.4.1-py3-none-any.whl", upload-time = 2026-03-27T10:48:38Z, size = 12186, hashes = { sha256 = "78ff4d23ff3d3c62837fa34f0dcb909593dea52a2a1f426307264f081a8b41b5" } }] + +[[packages]] +name = "markupsafe" +version = "2.1.5" +sdist = { url = "https://files.pythonhosted.org/packages/87/5b/aae44c6655f3801e81aa3eef09dbbf012431987ba564d7231722f68df02d/MarkupSafe-2.1.5.tar.gz", upload-time = 2024-02-02T16:31:22Z, size = 19384, hashes = { sha256 = "d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b" } } +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/bd/583bf3e4c8d6a321938c13f49d44024dbe5ed63e0a7ba127e454a66da974/MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", upload-time = 2024-02-02T16:30:33Z, size = 18215, hashes = { sha256 = "8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1" } }, + { url = "https://files.pythonhosted.org/packages/48/d6/e7cd795fc710292c3af3a06d80868ce4b02bfbbf370b7cee11d282815a2a/MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", upload-time = 2024-02-02T16:30:34Z, size = 14069, hashes = { sha256 = "3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4" } }, + { url = "https://files.pythonhosted.org/packages/51/b5/5d8ec796e2a08fc814a2c7d2584b55f889a55cf17dd1a90f2beb70744e5c/MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", upload-time = 2024-02-02T16:30:35Z, size = 29452, hashes = { sha256 = "ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee" } }, + { url = "https://files.pythonhosted.org/packages/0a/0d/2454f072fae3b5a137c119abf15465d1771319dfe9e4acbb31722a0fff91/MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", upload-time = 2024-02-02T16:30:36Z, size = 28462, hashes = { sha256 = "f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5" } }, + { url = "https://files.pythonhosted.org/packages/2d/75/fd6cb2e68780f72d47e6671840ca517bda5ef663d30ada7616b0462ad1e3/MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", upload-time = 2024-02-02T16:30:37Z, size = 27869, hashes = { sha256 = "ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b" } }, + { url = "https://files.pythonhosted.org/packages/b0/81/147c477391c2750e8fc7705829f7351cf1cd3be64406edcf900dc633feb2/MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", upload-time = 2024-02-02T16:30:39Z, size = 33906, hashes = { sha256 = "d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a" } }, + { url = "https://files.pythonhosted.org/packages/8b/ff/9a52b71839d7a256b563e85d11050e307121000dcebc97df120176b3ad93/MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", upload-time = 2024-02-02T16:30:40Z, size = 32296, hashes = { sha256 = "bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f" } }, + { url = "https://files.pythonhosted.org/packages/88/07/2dc76aa51b481eb96a4c3198894f38b480490e834479611a4053fbf08623/MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", upload-time = 2024-02-02T16:30:42Z, size = 33038, hashes = { sha256 = "58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169" } }, + { url = "https://files.pythonhosted.org/packages/96/0c/620c1fb3661858c0e37eb3cbffd8c6f732a67cd97296f725789679801b31/MarkupSafe-2.1.5-cp312-cp312-win32.whl", upload-time = 2024-02-02T16:30:43Z, size = 16572, hashes = { sha256 = "8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad" } }, + { url = "https://files.pythonhosted.org/packages/3f/14/c3554d512d5f9100a95e737502f4a2323a1959f6d0d01e0d0997b35f7b10/MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", upload-time = 2024-02-02T16:30:44Z, size = 17127, hashes = { sha256 = "823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb" } }, +] + +[[packages]] +name = "packaging" +version = "24.2" +sdist = { url = "https://files.pythonhosted.org/packages/d0/63/68dbb6eb2de9cb10ee4c9c14a0148804425e13c4fb20d61cce69f53106da/packaging-24.2.tar.gz", upload-time = 2024-11-08T09:47:47Z, size = 163950, hashes = { sha256 = "c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/88/ef/eb23f262cca3c0c4eb7ab1933c3b1f03d021f2c48f54763065b6f0e321be/packaging-24.2-py3-none-any.whl", upload-time = 2024-11-08T09:47:44Z, size = 65451, hashes = { sha256 = "09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759" } }] + +[[packages]] +name = "pygments" +version = "2.20.0" +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", upload-time = 2026-03-29T13:29:33Z, size = 4955991, hashes = { sha256 = "6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", upload-time = 2026-03-29T13:29:30Z, size = 1231151, hashes = { sha256 = "81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176" } }] + +[[packages]] +name = "python-docs-theme" +version = "2026.4" +sdist = { url = "https://files.pythonhosted.org/packages/fd/59/dbb07775a15ddf9f7f8d5f6ef4cd4da5e8afd908cc27e6585bb132e6366a/python_docs_theme-2026.4.tar.gz", upload-time = 2026-04-19T18:35:13Z, size = 29782, hashes = { sha256 = "a815f80c5a09f734449eb2498fbcbad05340976a7a543e431f57de92218a9315" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/db/05/b9298eb9330c70a3d1465a6116ab01dad095538c2e574a2d704bb0002f4d/python_docs_theme-2026.4-py3-none-any.whl", upload-time = 2026-04-19T18:35:12Z, size = 73742, hashes = { sha256 = "f755d80ebe8d7aa4fad8ee964ff999635c72eebd24ab10928a0e9726363d65fc" } }] + +[[packages]] +name = "requests" +version = "2.33.1" +sdist = { url = "https://files.pythonhosted.org/packages/5f/a4/98b9c7c6428a668bf7e42ebb7c79d576a1c3c1e3ae2d47e674b468388871/requests-2.33.1.tar.gz", upload-time = 2026-03-30T16:09:15Z, size = 134120, hashes = { sha256 = "18817f8c57c6263968bc123d237e3b8b08ac046f5456bd1e307ee8f4250d3517" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/d7/8e/7540e8a2036f79a125c1d2ebadf69ed7901608859186c856fa0388ef4197/requests-2.33.1-py3-none-any.whl", upload-time = 2026-03-30T16:09:13Z, size = 64947, hashes = { sha256 = "4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a" } }] + +[[packages]] +name = "roman-numerals" +version = "4.1.0" +sdist = { url = "https://files.pythonhosted.org/packages/ae/f9/41dc953bbeb056c17d5f7a519f50fdf010bd0553be2d630bc69d1e022703/roman_numerals-4.1.0.tar.gz", upload-time = 2025-12-17T18:25:34Z, size = 9077, hashes = { sha256 = "1af8b147eb1405d5839e78aeb93131690495fe9da5c91856cb33ad55a7f1e5b2" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl", upload-time = 2025-12-17T18:25:33Z, size = 7676, hashes = { sha256 = "647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7" } }] + +[[packages]] +name = "roman-numerals-py" +version = "4.1.0" +sdist = { url = "https://files.pythonhosted.org/packages/cb/b5/de96fca640f4f656eb79bbee0e79aeec52e3e0e359f8a3e6a0d366378b64/roman_numerals_py-4.1.0.tar.gz", upload-time = 2025-12-17T18:25:41Z, size = 4274, hashes = { sha256 = "f5d7b2b4ca52dd855ef7ab8eb3590f428c0b1ea480736ce32b01fef2a5f8daf9" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/27/2c/daca29684cbe9fd4bc711f8246da3c10adca1ccc4d24436b17572eb2590e/roman_numerals_py-4.1.0-py3-none-any.whl", upload-time = 2025-12-17T18:25:40Z, size = 4547, hashes = { sha256 = "553114c1167141c1283a51743759723ecd05604a1b6b507225e91dc1a6df0780" } }] + +[[packages]] +name = "snowballstemmer" +version = "2.2.0" +sdist = { url = "https://files.pythonhosted.org/packages/44/7b/af302bebf22c749c56c9c3e8ae13190b5b5db37a33d9068652e8f73b7089/snowballstemmer-2.2.0.tar.gz", upload-time = 2021-11-16T18:38:38Z, size = 86699, hashes = { sha256 = "09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/ed/dc/c02e01294f7265e63a7315fe086dd1df7dacb9f840a804da846b96d01b96/snowballstemmer-2.2.0-py2.py3-none-any.whl", upload-time = 2021-11-16T18:38:34Z, size = 93002, hashes = { sha256 = "c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a" } }] + +[[packages]] +name = "sphinx" +version = "8.2.3" +sdist = { url = "https://files.pythonhosted.org/packages/38/ad/4360e50ed56cb483667b8e6dadf2d3fda62359593faabbe749a27c4eaca6/sphinx-8.2.3.tar.gz", upload-time = 2025-03-02T22:31:59Z, size = 8321876, hashes = { sha256 = "398ad29dee7f63a75888314e9424d40f52ce5a6a87ae88e7071e80af296ec348" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/31/53/136e9eca6e0b9dc0e1962e2c908fbea2e5ac000c2a2fbd9a35797958c48b/sphinx-8.2.3-py3-none-any.whl", upload-time = 2025-03-02T22:31:56Z, size = 3589741, hashes = { sha256 = "4405915165f13521d875a8c29c8970800a0141c14cc5416a38feca4ea5d9b9c3" } }] + +[[packages]] +name = "sphinx-notfound-page" +version = "1.0.4" +sdist = { url = "https://files.pythonhosted.org/packages/73/7d/c545883c714319380325a52c9f80d093c97e718d812fd8090e42b1a08508/sphinx_notfound_page-1.0.4.tar.gz", upload-time = 2024-07-31T12:29:21Z, size = 519228, hashes = { sha256 = "2a52f49cd367b5c4e64072de1591cc367714098500abf4ecb9a3ecb4fec25aae" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/87/c4/877a5beffb8dcaf35e919c4c3cad56732c76370d106126394f4ca211ad7f/sphinx_notfound_page-1.0.4-py3-none-any.whl", upload-time = 2024-07-31T12:29:18Z, size = 8170, hashes = { sha256 = "f7c26ae0df3cf3d6f38f56b068762e6203d0ebb7e1c804de1059598d7dd8b9d8" } }] + +[[packages]] +name = "sphinxcontrib-applehelp" +version = "2.0.0" +sdist = { url = "https://files.pythonhosted.org/packages/ba/6e/b837e84a1a704953c62ef8776d45c3e8d759876b4a84fe14eba2859106fe/sphinxcontrib_applehelp-2.0.0.tar.gz", upload-time = 2024-07-29T01:09:00Z, size = 20053, hashes = { sha256 = "2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", upload-time = 2024-07-29T01:08:58Z, size = 119300, hashes = { sha256 = "4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5" } }] + +[[packages]] +name = "sphinxcontrib-devhelp" +version = "2.0.0" +sdist = { url = "https://files.pythonhosted.org/packages/f6/d2/5beee64d3e4e747f316bae86b55943f51e82bb86ecd325883ef65741e7da/sphinxcontrib_devhelp-2.0.0.tar.gz", upload-time = 2024-07-29T01:09:23Z, size = 12967, hashes = { sha256 = "411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", upload-time = 2024-07-29T01:09:21Z, size = 82530, hashes = { sha256 = "aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2" } }] + +[[packages]] +name = "sphinxcontrib-htmlhelp" +version = "2.1.0" +sdist = { url = "https://files.pythonhosted.org/packages/43/93/983afd9aa001e5201eab16b5a444ed5b9b0a7a010541e0ddfbbfd0b2470c/sphinxcontrib_htmlhelp-2.1.0.tar.gz", upload-time = 2024-07-29T01:09:37Z, size = 22617, hashes = { sha256 = "c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", upload-time = 2024-07-29T01:09:36Z, size = 98705, hashes = { sha256 = "166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8" } }] + +[[packages]] +name = "sphinxcontrib-jsmath" +version = "1.0.1" +sdist = { url = "https://files.pythonhosted.org/packages/b2/e8/9ed3830aeed71f17c026a07a5097edcf44b692850ef215b161b8ad875729/sphinxcontrib-jsmath-1.0.1.tar.gz", upload-time = 2019-01-21T16:10:16Z, size = 5787, hashes = { sha256 = "a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", upload-time = 2019-01-21T16:10:14Z, size = 5071, hashes = { sha256 = "2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178" } }] + +[[packages]] +name = "sphinxcontrib-qthelp" +version = "2.0.0" +sdist = { url = "https://files.pythonhosted.org/packages/68/bc/9104308fc285eb3e0b31b67688235db556cd5b0ef31d96f30e45f2e51cae/sphinxcontrib_qthelp-2.0.0.tar.gz", upload-time = 2024-07-29T01:09:56Z, size = 17165, hashes = { sha256 = "4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", upload-time = 2024-07-29T01:09:54Z, size = 88743, hashes = { sha256 = "b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb" } }] + +[[packages]] +name = "sphinxcontrib-serializinghtml" +version = "2.0.0" +sdist = { url = "https://files.pythonhosted.org/packages/3b/44/6716b257b0aa6bfd51a1b31665d1c205fb12cb5ad56de752dfa15657de2f/sphinxcontrib_serializinghtml-2.0.0.tar.gz", upload-time = 2024-07-29T01:10:09Z, size = 16080, hashes = { sha256 = "e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", upload-time = 2024-07-29T01:10:08Z, size = 92072, hashes = { sha256 = "6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331" } }] + +[[packages]] +name = "sphinxext-opengraph" +version = "0.13.0" +sdist = { url = "https://files.pythonhosted.org/packages/f6/c0/eb6838e3bae624ce6c8b90b245d17e84252863150e95efdb88f92c8aa3fb/sphinxext_opengraph-0.13.0.tar.gz", upload-time = 2025-08-29T12:20:31Z, size = 1026875, hashes = { sha256 = "103335d08567ad8468faf1425f575e3b698e9621f9323949a6c8b96d9793e80b" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/bf/a4/66c1fd4f8fab88faf71cee04a945f9806ba0fef753f2cfc8be6353f64508/sphinxext_opengraph-0.13.0-py3-none-any.whl", upload-time = 2025-08-29T12:20:29Z, size = 1004152, hashes = { sha256 = "936c07828edc9ad9a7b07908b29596dc84ed0b3ceaa77acdf51282d232d4d80e" } }] + +[[packages]] +name = "urllib3" +version = "2.6.3" +sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", upload-time = 2026-01-07T16:24:43Z, size = 435556, hashes = { sha256 = "1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", upload-time = 2026-01-07T16:24:42Z, size = 131584, hashes = { sha256 = "bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4" } }] diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 0cf0a41bfb4..72e1cad3bbd 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -858,7 +858,7 @@ A literal pattern corresponds to most : | "None" : | "True" : | "False" - signed_number: ["-"] NUMBER + signed_number: ["+" | "-"] NUMBER The rule ``strings`` and the token ``NUMBER`` are defined in the :doc:`standard Python grammar <./grammar>`. Triple-quoted strings are diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 1e53c0e0e6f..aef5bbe151c 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -926,6 +926,7 @@ Attribute assignment updates the module's namespace dictionary, e.g., single: __doc__ (module attribute) single: __annotations__ (module attribute) single: __annotate__ (module attribute) + single: __lazy_modules__ (module attribute) pair: module; namespace .. _import-mod-attrs: @@ -1121,6 +1122,20 @@ the following writable attributes: .. versionadded:: 3.14 +.. attribute:: module.__lazy_modules__ + + A container (an object implementing :meth:`~object.__contains__`) of fully + qualified module name strings. When defined + at module scope, any regular :keyword:`import` statement in that module whose + target module name appears in this container is treated as a + :ref:`lazy import `, as if the :keyword:`lazy` keyword had + been used. Imports inside functions, class bodies, or + :keyword:`try`/:keyword:`except`/:keyword:`finally` blocks are unaffected. + + See :ref:`lazy-modules-compat` for details and examples. + + .. versionadded:: 3.15 + Module dictionaries ^^^^^^^^^^^^^^^^^^^ @@ -1461,7 +1476,6 @@ indirectly) to mutable objects. single: co_filename (code object attribute) single: co_firstlineno (code object attribute) single: co_flags (code object attribute) - single: co_lnotab (code object attribute) single: co_name (code object attribute) single: co_names (code object attribute) single: co_nlocals (code object attribute) @@ -1534,14 +1548,6 @@ Special read-only attributes * - .. attribute:: codeobject.co_firstlineno - The line number of the first line of the function - * - .. attribute:: codeobject.co_lnotab - - A string encoding the mapping from :term:`bytecode` offsets to line - numbers. For details, see the source code of the interpreter. - - .. deprecated:: 3.12 - This attribute of code objects is deprecated, and may be removed in - Python 3.15. - * - .. attribute:: codeobject.co_stacksize - The required stack size of the code object diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 9b84c2e9ac7..648e3a9bf54 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -920,6 +920,56 @@ See :pep:`810` for the full specification of lazy imports. .. versionadded:: 3.15 +.. _lazy-modules-compat: + +Compatibility via ``__lazy_modules__`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. index:: + single: __lazy_modules__ + +As an alternative to using the :keyword:`lazy` keyword, a module can opt +into lazy loading for specific imports by defining a module-level +:attr:`~module.__lazy_modules__` variable. When present, it must be a +container of fully qualified module name strings. Any regular (non-``lazy``) +:keyword:`import` statement at module scope whose target appears in +:attr:`!__lazy_modules__` is treated as a lazy import, exactly as if the +:keyword:`lazy` keyword had been used. + +This provides a way to enable lazy loading for specific dependencies without +changing individual ``import`` statements. This is useful when supporting +Python versions older than 3.15 while using lazy imports in 3.15+:: + + __lazy_modules__ = ["json", "pathlib"] + + import json # loaded lazily (name is in __lazy_modules__) + import os # loaded eagerly (name not in __lazy_modules__) + + import pathlib # loaded lazily + +Relative imports are resolved to their absolute name before the lookup, so +:attr:`!__lazy_modules__` must always contain fully qualified module names. + +For ``from``-style imports, the relevant name is the module following +``from``, not the names of its members:: + + # In mypackage/mymodule.py + __lazy_modules__ = ["mypackage", "mypackage.sub.utils"] + + from . import helper # loaded lazily: . resolves to mypackage + from .sub.utils import func # loaded lazily: .sub.utils resolves to mypackage.sub.utils + import json # loaded eagerly (not in __lazy_modules__) + +Imports inside functions, class bodies, or +:keyword:`try`/:keyword:`except`/:keyword:`finally` blocks are always eager, +regardless of :attr:`!__lazy_modules__`. + +Setting ``-X lazy_imports=none`` (or the :envvar:`PYTHON_LAZY_IMPORTS` +environment variable to ``none``) overrides :attr:`!__lazy_modules__` and +forces all imports to be eager. + +.. versionadded:: 3.15 + .. _future: Future statements diff --git a/Doc/tools/extensions/changes.py b/Doc/tools/extensions/changes.py index 8de5e7f78c6..02dc51b3a76 100644 --- a/Doc/tools/extensions/changes.py +++ b/Doc/tools/extensions/changes.py @@ -2,8 +2,10 @@ from __future__ import annotations -from typing import TYPE_CHECKING +import re +from docutils import nodes +from sphinx import addnodes from sphinx.domains.changeset import ( VersionChange, versionlabel_classes, @@ -11,6 +13,7 @@ ) from sphinx.locale import _ as sphinx_gettext +TYPE_CHECKING = False if TYPE_CHECKING: from docutils.nodes import Node from sphinx.application import Sphinx @@ -73,6 +76,76 @@ def run(self) -> list[Node]: versionlabel_classes[self.name] = "" +class SoftDeprecated(PyVersionChange): + """Directive for soft deprecations that auto-links to the glossary term. + + Usage:: + + .. soft-deprecated:: 3.15 + + Use :func:`new_thing` instead. + + Renders as: "Soft deprecated since version 3.15: Use new_thing() instead." + with "Soft deprecated" linking to the glossary definition. + """ + + _TERM_RE = re.compile(r":term:`([^`]+)`") + + def run(self) -> list[Node]: + versionlabels[self.name] = sphinx_gettext( + ":term:`Soft deprecated` since version %s" + ) + versionlabel_classes[self.name] = "soft-deprecated" + try: + result = super().run() + finally: + versionlabels[self.name] = "" + versionlabel_classes[self.name] = "" + + for node in result: + # Add "versionchanged" class so existing theme CSS applies + node["classes"] = node.get("classes", []) + ["versionchanged"] + # Replace the plain-text "Soft deprecated" with a glossary reference + for inline in node.findall(nodes.inline): + if "versionmodified" in inline.get("classes", []): + self._add_glossary_link(inline) + + return result + + @classmethod + def _add_glossary_link(cls, inline: nodes.inline) -> None: + """Replace :term:`soft deprecated` text with a cross-reference to the + 'Soft deprecated' glossary entry.""" + for child in inline.children: + if not isinstance(child, nodes.Text): + continue + + text = str(child) + match = cls._TERM_RE.search(text) + if match is None: + continue + + ref = addnodes.pending_xref( + "", + nodes.Text(match.group(1)), + refdomain="std", + reftype="term", + reftarget="soft deprecated", + refwarn=True, + ) + + start, end = match.span() + new_nodes: list[nodes.Node] = [] + if start > 0: + new_nodes.append(nodes.Text(text[:start])) + new_nodes.append(ref) + if end < len(text): + new_nodes.append(nodes.Text(text[end:])) + + child.parent.replace(child, new_nodes) + break + + def setup(app: Sphinx) -> ExtensionMetadata: # Override Sphinx's directives with support for 'next' app.add_directive("versionadded", PyVersionChange, override=True) @@ -83,6 +156,9 @@ def setup(app: Sphinx) -> ExtensionMetadata: # Register the ``.. deprecated-removed::`` directive app.add_directive("deprecated-removed", DeprecatedRemoved) + # Register the ``.. soft-deprecated::`` directive + app.add_directive("soft-deprecated", SoftDeprecated) + return { "version": "1.0", "parallel_read_safe": True, diff --git a/Doc/tools/removed-ids.txt b/Doc/tools/removed-ids.txt index f3cd8bf0ef5..5e3ef2efe27 100644 --- a/Doc/tools/removed-ids.txt +++ b/Doc/tools/removed-ids.txt @@ -1 +1,7 @@ # HTML IDs excluded from the check-html-ids.py check. + +# Remove from here in 3.16 +c-api/allocation.html: deprecated-aliases +c-api/file.html: deprecated-api + +library/asyncio-task.html: terminating-a-task-group diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html index 75f6607d8f3..699e518801c 100644 --- a/Doc/tools/templates/dummy.html +++ b/Doc/tools/templates/dummy.html @@ -29,6 +29,7 @@ {% trans %}Deprecated since version %s, will be removed in version %s{% endtrans %} {% trans %}Deprecated since version %s, removed in version %s{% endtrans %} +{% trans %}:term:`Soft deprecated` since version %s{% endtrans %} In docsbuild-scripts, when rewriting indexsidebar.html with actual versions: diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index 678b71c9274..6c68ba01081 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -1,7 +1,7 @@ .. _tut-brieftourtwo: ********************************************** -Brief Tour of the Standard Library --- Part II +Brief tour of the standard library --- part II ********************************************** This second tour covers more advanced modules that support professional @@ -10,7 +10,7 @@ programming needs. These modules rarely occur in small scripts. .. _tut-output-formatting: -Output Formatting +Output formatting ================= The :mod:`reprlib` module provides a version of :func:`repr` customized for @@ -130,7 +130,7 @@ templates for XML files, plain text reports, and HTML web reports. .. _tut-binary-formats: -Working with Binary Data Record Layouts +Working with binary data record layouts ======================================= The :mod:`struct` module provides :func:`~struct.pack` and @@ -178,14 +178,13 @@ tasks in background while the main program continues to run:: class AsyncZip(threading.Thread): def __init__(self, infile, outfile): - threading.Thread.__init__(self) + super().__init__() self.infile = infile self.outfile = outfile def run(self): - f = zipfile.ZipFile(self.outfile, 'w', zipfile.ZIP_DEFLATED) - f.write(self.infile) - f.close() + with zipfile.ZipFile(self.outfile, 'w', zipfile.ZIP_DEFLATED) as f: + f.write(self.infile) print('Finished background zip of:', self.infile) background = AsyncZip('mydata.txt', 'myarchive.zip') @@ -245,7 +244,7 @@ application. .. _tut-weak-references: -Weak References +Weak references =============== Python does automatic memory management (reference counting for most objects and @@ -286,7 +285,7 @@ applications include caching objects that are expensive to create:: .. _tut-list-tools: -Tools for Working with Lists +Tools for working with lists ============================ Many data structure needs can be met with the built-in list type. However, @@ -352,7 +351,7 @@ not want to run a full list sort:: .. _tut-decimal-fp: -Decimal Floating-Point Arithmetic +Decimal floating-point arithmetic ================================= The :mod:`decimal` module offers a :class:`~decimal.Decimal` datatype for diff --git a/Doc/using/android.rst b/Doc/using/android.rst index 45345d045dd..60a13569318 100644 --- a/Doc/using/android.rst +++ b/Doc/using/android.rst @@ -30,7 +30,7 @@ Adding Python to an Android app Most app developers should use one of the following tools, which will provide a much easier experience: -* `Briefcase `__, from the BeeWare project +* `Briefcase `__, from the BeeWare project * `Buildozer `__, from the Kivy project * `Chaquopy `__ * `pyqtdeploy `__ diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index d5c17560b66..086f6bfa22a 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -780,6 +780,24 @@ also be used to improve performance. .. versionadded:: 3.14 +.. option:: --without-frame-pointers + + Disable frame pointers, which are enabled by default (see :pep:`831`). + + By default, the build appends ``-fno-omit-frame-pointer`` (and + ``-mno-omit-leaf-frame-pointer`` when the compiler supports it) to + ``BASECFLAGS`` so profilers, debuggers, and system tracing tools + (``perf``, ``eBPF``, ``dtrace``, ``gdb``) can walk the C call stack + without DWARF metadata. The flags propagate to third-party C + extensions through :mod:`sysconfig`. On compilers that do not + understand them, the build silently skips them. + + Downstream packagers and authors of native libraries built with + custom build systems should set the same flags so the unwind chain + stays unbroken across all native frames. + + .. versionadded:: 3.15 + .. option:: --without-mimalloc Disable the fast :ref:`mimalloc ` allocator diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 4b092b13959..8a78dbd9038 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -402,7 +402,7 @@ Tracing events, with the correct line number, are generated for all lines of cod The :attr:`~frame.f_lineno` attribute of frame objects will always contain the expected line number. -The :attr:`~codeobject.co_lnotab` attribute of +The :attr:`!codeobject.co_lnotab` attribute of :ref:`code objects ` is deprecated and will be removed in 3.12. Code that needs to convert from offset to line number should use the new diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 221956f3dd3..df6cc98eaf1 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1347,7 +1347,7 @@ Deprecated ``int``, convert to int explicitly: ``~int(x)``. (Contributed by Tim Hoffmann in :gh:`103487`.) -* Accessing :attr:`~codeobject.co_lnotab` on code objects was deprecated in +* Accessing :attr:`!codeobject.co_lnotab` on code objects was deprecated in Python 3.10 via :pep:`626`, but it only got a proper :exc:`DeprecationWarning` in 3.12. May be removed in 3.15. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index dfdfe66be7e..0bb8858aea1 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -953,10 +953,24 @@ when a module is imported) will still emit the syntax warning. (Contributed by Irit Katriel in :gh:`130080`.) +.. _incremental-garbage-collection: .. _whatsnew314-incremental-gc: -Incremental garbage collection ------------------------------- +Garbage collection +------------------ + +**From Python 3.14.5 onwards:** + +The garbage collector (GC) has changed in Python 3.14.5. + +Python 3.14.0-3.14.4 shipped with a new incremental GC. +However, due to a number of `reports +`__ +of significant memory pressure in production environments, +it has been reverted back to the generational GC from 3.13. +This is the GC now used in Python 3.14.5 and later. + +**Previously in Python 3.14.0-3.14.4:** The cycle garbage collector is now incremental. This means that maximum pause times are reduced @@ -2203,7 +2217,18 @@ difflib gc -- -* The new :ref:`incremental garbage collector ` +* **From Python 3.14.5 onwards:** + + Python 3.14.0-3.14.4 shipped with a new incremental garbage collector. + However, due to a number of `reports + `__ + of significant memory pressure in production environments, + it has been reverted back to the generational GC from 3.13. + This is the GC now used in Python 3.14.5 and later. + +* **Previously in Python 3.14.0-3.14.4:** + + The new :ref:`incremental garbage collector ` means that maximum pause times are reduced by an order of magnitude or more for larger heaps. @@ -3447,3 +3472,17 @@ Changes in the C API functions on Python 3.13 and older. .. _pythoncapi-compat project: https://github.com/python/pythoncapi-compat/ + + +Notable changes in 3.14.5 +========================= + +gc +-- + +* The incremental garbage collector shipped in Python 3.14.0-3.14.4 has been + reverted back to the generational garbage collector from 3.13, + due to a number of `reports + `__ + of significant memory pressure in production environments. + See :ref:`whatsnew314-incremental-gc` for details. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 6070e98b459..c65125be8b7 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -69,6 +69,8 @@ Summary -- Release highlights ` * :pep:`814`: :ref:`Add frozendict built-in type ` +* :pep:`661`: :ref:`Add sentinel built-in type + ` * :pep:`799`: :ref:`A dedicated profiling package for organizing Python profiling tools ` * :pep:`799`: :ref:`Tachyon: High frequency statistical sampling profiler @@ -80,9 +82,11 @@ Summary -- Release highlights * :pep:`728`: ``TypedDict`` with typed extra items * :pep:`747`: :ref:`Annotating type forms with TypeForm ` +* :pep:`800`: Disjoint bases in the type system * :pep:`782`: :ref:`A new PyBytesWriter C API to create a Python bytes object ` * :pep:`803`: :ref:`Stable ABI for Free-Threaded Builds ` +* :pep:`831`: :ref:`Frame pointers everywhere ` * :ref:`The JIT compiler has been significantly upgraded ` * :ref:`Improved error messages ` * :ref:`The official Windows 64-bit binaries now use the tail-calling interpreter @@ -141,7 +145,7 @@ In the case where loading a lazily imported module fails (for example, if the module does not exist), Python raises the exception at the point of first use rather than at import time. The associated traceback includes both the location where the name was accessed and the original import statement, -making it straightforward to diagnose & debug the failure. +making it straightforward to diagnose and debug the failure. For cases where you want to enable lazy loading globally without modifying source code, Python provides the :option:`-X lazy_imports <-X>` command-line @@ -183,6 +187,18 @@ function, class body, or ``try``/``except``/``finally`` block raises a (``lazy from module import *`` and ``lazy from __future__ import ...`` both raise :exc:`SyntaxError`). +For code that cannot use the ``lazy`` keyword directly (for example, when +supporting Python versions older than 3.15 while still using lazy +imports on 3.15+), a module can define +:attr:`~module.__lazy_modules__` as a container of fully qualified module +name strings. Regular ``import`` statements for those modules are then treated +as lazy, with the same semantics as the ``lazy`` keyword:: + + __lazy_modules__ = ["json", "pathlib"] + + import json # lazy + import os # still eager + .. seealso:: :pep:`810` for the full specification and rationale. (Contributed by Pablo Galindo Salgado and Dino Viehland in :gh:`142349`.) @@ -234,6 +250,20 @@ to accept also other mapping types such as :class:`~types.MappingProxyType`. (Contributed by Victor Stinner and Donghee Na in :gh:`141510`.) +.. _whatsnew315-sentinel: + +:pep:`661`: Add sentinel built-in type +-------------------------------------- + +A new :class:`sentinel` type is added to the :mod:`builtins` module for +creating unique sentinel values with a concise representation. Sentinel +objects preserve identity when copied, support use in type expressions with +the ``|`` operator, and can be pickled when they are importable by module and +name. + +(PEP by Tal Einat; contributed by Jelle Zijlstra in :gh:`148829`.) + + .. _whatsnew315-profiling-package: :pep:`799`: A dedicated profiling package @@ -259,7 +289,7 @@ The :mod:`profile` module is deprecated and will be removed in Python 3.17. Tachyon: High frequency statistical sampling profiler ----------------------------------------------------- -.. image:: ../library/tachyon-logo.png +.. image:: ../../Lib/profiling/sampling/_assets/tachyon-logo.png :alt: Tachyon profiler logo :align: center :width: 200px @@ -450,14 +480,36 @@ Improved error messages Running this code now produces a clearer suggestion: - .. code-block:: pycon + .. code-block:: pytb Traceback (most recent call last): - File "/home/pablogsal/github/python/main/lel.py", line 42, in - print(container.area) - ^^^^^^^^^^^^^^ + File "/home/pablogsal/github/python/main/lel.py", line 42, in + print(container.area) + ^^^^^^^^^^^^^^ AttributeError: 'Container' object has no attribute 'area'. Did you mean '.inner.area' instead of '.area'? +* The interpreter now tries to provide a suggestion when + :func:`delattr` fails due to a missing attribute. + When an attribute name that closely resembles an existing attribute is used, + the interpreter will suggest the correct attribute name in the error message. + For example: + + .. doctest:: + + >>> class A: + ... pass + >>> a = A() + >>> a.abcde = 1 + >>> del a.abcdf # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'A' object has no attribute 'abcdf'. Did you mean: 'abcde'? + + (Contributed by Nikita Sobolev and Pranjal Prajapati in :gh:`136588`.) + +* Several error messages incorrectly using the term "argument" have been corrected. + (Contributed by Stan Ulbrych in :gh:`133382`.) + Other language changes ====================== @@ -489,28 +541,6 @@ Other language changes (Contributed by Adam Turner in :gh:`133711`; PEP 686 written by Inada Naoki.) -* Several error messages incorrectly using the term "argument" have been corrected. - (Contributed by Stan Ulbrych in :gh:`133382`.) - -* The interpreter now tries to provide a suggestion when - :func:`delattr` fails due to a missing attribute. - When an attribute name that closely resembles an existing attribute is used, - the interpreter will suggest the correct attribute name in the error message. - For example: - - .. doctest:: - - >>> class A: - ... pass - >>> a = A() - >>> a.abcde = 1 - >>> del a.abcdf # doctest: +ELLIPSIS - Traceback (most recent call last): - ... - AttributeError: 'A' object has no attribute 'abcdf'. Did you mean: 'abcde'? - - (Contributed by Nikita Sobolev and Pranjal Prajapati in :gh:`136588`.) - * Unraisable exceptions are now highlighted with color by default. This can be controlled by :ref:`environment variables `. (Contributed by Peter Bierma in :gh:`134170`.) @@ -644,6 +674,17 @@ Other language changes * Allow the *count* argument of :meth:`bytes.replace` to be a keyword. (Contributed by Stan Ulbrych in :gh:`147856`.) +* Unary plus is now accepted in :keyword:`match` literal patterns, mirroring + the existing support for unary minus. + (Contributed by Bartosz Sławecki in :gh:`145239`.) + +* The import system now acquires per-module locks in hierarchical order + (parent packages before their submodules). This fixes a long-standing + deadlock where one thread importing ``pkg.sub`` and another importing + ``pkg.sub.mod`` could each block the other when ``pkg/sub/__init__.py`` + imports ``pkg.sub.mod``. + (Contributed by Gregory P. Smith in :gh:`83065`.) + New modules =========== @@ -692,6 +733,31 @@ array (Contributed by Victor Stinner in :gh:`148675`.) +ast +--- + +* Add *color* parameter to :func:`~ast.dump`. + If ``True``, the returned string is syntax highlighted using ANSI escape + sequences. + If ``False`` (the default), colored output is always disabled. + (Contributed by Stan Ulbrych in :gh:`148981`.) + +* The :ref:`command-line ` output is now syntax highlighted by default. + This can be :ref:`controlled using environment variables `. + (Contributed by Stan Ulbrych in :gh:`148981`.) + + +asyncio +------- + +* Added :meth:`TaskGroup.cancel ` to allow early + termination of a task group, for instance, when the goal of the tasks has + been achieved or their services are no longer needed. + Previously this would involve unintuitive boilerplate such as an extra task + raising a custom exception which is then suppressed as it exits the task group. + (Contributed by John Belmonte in :gh:`127214`.) + + base64 ------ @@ -712,11 +778,20 @@ base64 (Contributed by Serhiy Storchaka in :gh:`143214` and :gh:`146431`.) * Added the *ignorechars* parameter in :func:`~base64.b16decode`, - :func:`~base64.b32decode`, :func:`~base64.b32hexdecode`, + :func:`~base64.b32decode`, :func:`~base64.b32hexdecode`, :func:`~base64.b64decode`, :func:`~base64.b85decode`, and :func:`~base64.z85decode`. (Contributed by Serhiy Storchaka in :gh:`144001` and :gh:`146431`.) +* Added the *canonical* parameter in + :func:`~base64.b32decode`, :func:`~base64.b32hexdecode`, + :func:`~base64.b64decode`, :func:`~base64.urlsafe_b64decode`, + :func:`~base64.a85decode`, :func:`~base64.b85decode`, and + :func:`~base64.z85decode`, + to reject encodings with non-zero padding bits or other non-canonical + forms. + (Contributed by Gregory P. Smith in :gh:`146311`.) + binascii -------- @@ -750,6 +825,10 @@ binascii :func:`~binascii.unhexlify`, and :func:`~binascii.a2b_base64`. (Contributed by Serhiy Storchaka in :gh:`144001` and :gh:`146431`.) +* Added the *canonical* parameter in :func:`~binascii.a2b_base64`, + to reject encodings with non-zero padding bits. + (Contributed by Gregory P. Smith in :gh:`146311`.) + calendar -------- @@ -791,6 +870,15 @@ contextlib consistency with the :keyword:`with` and :keyword:`async with` statements. (Contributed by Serhiy Storchaka in :gh:`144386`.) +* :class:`~contextlib.ContextDecorator` and + :class:`~contextlib.AsyncContextDecorator` (and therefore + :func:`~contextlib.contextmanager` and :func:`~contextlib.asynccontextmanager` + used as decorators) now detect generator functions, coroutine functions, and + asynchronous generator functions and keep the context manager open across + iteration or await. Previously the context manager exited as soon as the + generator or coroutine object was created. + (Contributed by Alex Grönholm & Gregory P. Smith in :gh:`125862`.) + dataclasses ----------- @@ -823,6 +911,25 @@ difflib (Contributed by Jiahao Li in :gh:`134580`.) +faulthandler +------------ + +* Added the *max_threads* parameter in :func:`faulthandler.enable`, + :func:`faulthandler.dump_traceback`, :func:`faulthandler.dump_traceback_later`, + and :func:`faulthandler.register`. + (Contributed by Eric Froemling in :gh:`149085`.) + + +email +----- + +* Email generators now raise an error when an :class:`.EmailMessage` cannot be + accurately flattened due to a non-ASCII email address (mailbox) in an address + header. Options for supporting Email Address Internationalization (EAI) are + discussed in :attr:`.EmailPolicy.utf8`. + (Contributed by R David Murray and Mike Edmunds in :gh:`122540`.) + + functools --------- @@ -856,13 +963,6 @@ http.client (Contributed by Alexander Enrique Urieles Nieto in :gh:`131724`.) -http.cookies ------------- - -* Allow '``"``' double quotes in cookie values. - (Contributed by Nick Burns and Senthil Kumaran in :gh:`92936`.) - - http.server ----------- @@ -873,6 +973,12 @@ http.server `. (Contributed by Hugo van Kemenade in :gh:`146292`.) +* Added :attr:`~http.server.SimpleHTTPRequestHandler.default_content_type` + and the :option:`--content-type ` command-line + option to allow customizing the default ``Content-Type`` header + for files with unknown extensions. + (Contributed by John Comeau and Hugo van Kemenade in :gh:`113471`.) + inspect ------- @@ -884,13 +990,13 @@ inspect json ---- -* Add the *array_hook* parameter to :func:`~json.load` and +* Add the *array_hook* parameter to :func:`~json.load` and :func:`~json.loads` functions: allow a callback for JSON literal array types to customize Python lists in the resulting decoded object. Passing combined :class:`frozendict` to *object_pairs_hook* param and :class:`tuple` to ``array_hook`` will yield a deeply nested immutable Python structure representing the JSON data. - (Contributed by Joao S. O. Bueno in :gh:`146440`) + (Contributed by Joao S. O. Bueno in :gh:`146440`.) locale @@ -918,23 +1024,11 @@ math mimetypes --------- -* Add ``application/dicom`` MIME type for ``.dcm`` extension. - (Contributed by Benedikt Johannes in :gh:`144217`.) -* Add ``application/efi``. (Contributed by Charlie Lin in :gh:`145720`.) -* Add ``application/node`` MIME type for ``.cjs`` extension. - (Contributed by John Franey in :gh:`140937`.) -* Add ``application/toml``. (Contributed by Gil Forcada in :gh:`139959`.) -* Add ``application/sql`` and ``application/vnd.sqlite3``. - (Contributed by Charlie Lin in :gh:`145698`.) -* Add the following MIME types: - - - ``application/vnd.ms-cab-compressed`` for ``.cab`` extension - - ``application/vnd.ms-htmlhelp`` for ``.chm`` extension - - ``application/vnd.ms-officetheme`` for ``.thmx`` extension - - (Contributed by Charlie Lin in :gh:`145718`.) - -* Add ``image/jxl``. (Contributed by Foolbar in :gh:`144213`.) +* Add more MIME types. + (Contributed by Benedikt Johannes, Charlie Lin, Foolbar, Gil Forcada and + John Franey + in :gh:`144217`, :gh:`145720`, :gh:`140937`, :gh:`139959`, :gh:`145698`, + :gh:`145718` and :gh:`144213`.) * Rename ``application/x-texinfo`` to ``application/texinfo``. (Contributed by Charlie Lin in :gh:`140165`.) * Changed the MIME type for ``.ai`` files to ``application/pdf``. @@ -976,6 +1070,13 @@ os.path (Contributed by Petr Viktorin for :cve:`2025-4517`.) +pdb +--- + +* Use the new interactive shell as the default input shell for :mod:`pdb`. + (Contributed by Tian Gao in :gh:`145379`.) + + pickle ------ @@ -983,6 +1084,29 @@ pickle (Contributed by Zackery Spytz and Serhiy Storchaka in :gh:`77188`.) +pickletools +----------- + +* The output of the :mod:`pickletools` command-line interface is colored by + default. This can be controlled with + :ref:`environment variables `. + (Contributed by Hugo van Kemenade in :gh:`149026`.) + + +pprint +------ + +* Add an *expand* keyword argument for :func:`pprint.pprint`, + :func:`pprint.pformat`, :func:`pprint.pp`. If true, the output will be + formatted similar to pretty-printed :func:`json.dumps` when + *indent* is supplied. + (Contributed by Stefan Todoran, Semyon Moroz and Hugo van Kemenade in + :gh:`112632`.) + +* Add t-string support to :mod:`pprint`. + (Contributed by Loïc Simon and Hugo van Kemenade in :gh:`134551`.) + + re -- @@ -1104,7 +1228,7 @@ subprocess If none of these mechanisms are available, the function falls back to the traditional busy loop (non-blocking call and short sleeps). - (Contributed by Giampaolo Rodola in :gh:`83069`). + (Contributed by Giampaolo Rodola in :gh:`83069`.) symtable @@ -1121,6 +1245,19 @@ sys (Contributed by Klaus Zimmermann in :gh:`137476`.) +sys.monitoring +-------------- + +* The :ref:`other events ` + (:monitoring-event:`PY_THROW`, :monitoring-event:`PY_UNWIND`, + :monitoring-event:`RAISE`, :monitoring-event:`EXCEPTION_HANDLED`, and + :monitoring-event:`RERAISE`) can now be turned on and disabled on a per code + object basis. Returning :data:`~sys.monitoring.DISABLE` from a callback for + one of these events disables the event for the entire code object (for the + current tool), rather than raising :exc:`ValueError` as in prior versions. + (Contributed by Gabriele N. Tornetta in :gh:`146182`.) + + tarfile ------- @@ -1147,6 +1284,16 @@ tarfile (Contributed by Christoph Walcher in :gh:`57911`.) +threading +--------- + +* Added :class:`~threading.serialize_iterator`, + :func:`~threading.synchronized_iterator`, + and :func:`~threading.concurrent_tee` to support concurrent access to + generators and iterators. + (Contributed by Raymond Hettinger in :gh:`124397`.) + + timeit ------ @@ -1161,7 +1308,7 @@ timeit * Make the target time of :meth:`timeit.Timer.autorange` configurable and add ``--target-time`` option to the command-line interface. - (Contributed by Alessandro Cucci and Miikka Koskinen in :gh:`140283`.) + (Contributed by Alessandro Cucci and Miikka Koskinen in :gh:`80642`.) tkinter @@ -1190,6 +1337,15 @@ tkinter (Contributed by Matthias Kievernagel and Serhiy Storchaka in :gh:`47655`.) +tokenize +-------- + +* The output of the :mod:`tokenize` :ref:`command-line interface + ` is colored by default. This can be controlled with + :ref:`environment variables `. + (Contributed by Hugo van Kemenade in :gh:`148991`.) + + .. _whatsnew315-tomllib-1-1-0: tomllib @@ -1206,10 +1362,7 @@ tomllib Previously an inline table had to be on a single line and couldn't end with a trailing comma. This is now relaxed so that the following is valid: - .. syntax highlighting needs TOML 1.1.0 support in Pygments, - see https://github.com/pygments/pygments/issues/3026 - - .. code-block:: text + .. code-block:: toml tbl = { key = "a string", @@ -1221,7 +1374,7 @@ tomllib - Add ``\xHH`` notation to basic strings for codepoints under 255, and the ``\e`` escape for the escape character: - .. code-block:: text + .. code-block:: toml null = "null byte: \x00; letter a: \x61" csi = "\e[" @@ -1229,7 +1382,7 @@ tomllib - Seconds in datetime and time values are now optional. The following are now valid: - .. code-block:: text + .. code-block:: toml dt = 2010-02-03 14:15 t = 14:15 @@ -1281,6 +1434,18 @@ typing as it was incorrectly inferred in runtime before. (Contributed by Nikita Sobolev in :gh:`137191`.) +* :pep:`800`: Add :deco:`typing.disjoint_base`, a new decorator marking a class + as a disjoint base. This is an advanced feature primarily intended to allow + type checkers to faithfully reflect the runtime semantics of types defined + as builtins or in compiled extensions. If a class ``C`` is a disjoint base, then + child classes of that class cannot inherit from other disjoint bases that are + not parent or child classes of ``C``. (Contributed by Jelle Zijlstra in :gh:`148639`.) + +* :class:`~typing.TypeVarTuple` now accepts ``bound``, ``covariant``, + ``contravariant``, and ``infer_variance`` keyword arguments, matching the + interface of :class:`~typing.TypeVar` and :class:`~typing.ParamSpec`. + ``bound`` semantics remain undefined in the specification. + unicodedata ----------- @@ -1404,7 +1569,7 @@ Optimizations ============= * ``mimalloc`` is now used as the default allocator for - for raw memory allocations such as via :c:func:`PyMem_RawMalloc` + raw memory allocations such as via :c:func:`PyMem_RawMalloc` for better performance on :term:`free-threaded builds `. (Contributed by Kumar Aditya in :gh:`144914`.) @@ -1423,7 +1588,7 @@ base64 & binascii * Implementation for Base32 has been rewritten in C. Encoding and decoding is now two orders of magnitude faster. - (Contributed by James Seo in :gh:`146192`) + (Contributed by James Seo in :gh:`146192`.) csv @@ -1440,11 +1605,11 @@ Upgraded JIT compiler Results from the `pyperformance `__ benchmark suite report -`6-7% `__ +`8-9% `__ geometric mean performance improvement for the JIT over the standard CPython interpreter built with all optimizations enabled on x86-64 Linux. On AArch64 macOS, the JIT has a -`12-13% `__ +`12-13% `__ speedup over the :ref:`tail calling interpreter ` with all optimizations enabled. The speedups for JIT builds versus no JIT builds range from roughly 15% slowdown to over @@ -1517,7 +1682,7 @@ The JIT compiler's machine code generator now produces better machine code for x86-64 and AArch64 macOS and Linux targets. In general, users should experience lower memory usage for generated machine code and more efficient machine code versus 3.14. -(Contributed by Brandt Bucher in :gh:`136528` and :gh:`136528`. +(Contributed by Brandt Bucher in :gh:`136528` and :gh:`135905`. Implementation for AArch64 contributed by Mark Shannon in :gh:`139855`. Additional optimizations for AArch64 contributed by Mark Shannon and Diego Russo in :gh:`140683` and :gh:`142305`.) @@ -1532,6 +1697,16 @@ This was made possible by a refactoring of JIT data structures. Removed ======== +ast +--- + +* The constructors of :ref:`AST nodes ` now raise a :exc:`TypeError` + when a required argument is omitted or when a keyword argument that does not + map to a field on the AST node is passed. These cases had previously raised a + :exc:`DeprecationWarning` since Python 3.13. + (Contributed by Brian Schubert and Jelle Zijlstra in :gh:`137600` and :gh:`105858`.) + + collections.abc --------------- @@ -1540,15 +1715,6 @@ collections.abc deprecated since Python 3.12, and is scheduled for removal in Python 3.17. -datetime --------- - -* :meth:`~datetime.datetime.strptime` now raises :exc:`ValueError` when the - format string contains ``%d`` (day of month) without a year directive. - This has been deprecated since Python 3.13. - (Contributed by Stan Ulbrych and Gregory P. Smith in :gh:`70647`.) - - ctypes ------ @@ -1563,6 +1729,15 @@ ctypes (Contributed by Victor Stinner in :gh:`148675`.) +datetime +-------- + +* :meth:`~datetime.datetime.strptime` now raises :exc:`ValueError` when the + format string contains ``%d`` (day of month) without a year directive. + This has been deprecated since Python 3.13. + (Contributed by Stan Ulbrych and Gregory P. Smith in :gh:`70647`.) + + glob ---- @@ -1605,20 +1780,6 @@ platform (Contributed by Alexey Makridenko in :gh:`133604`.) -pprint ------- - -* Add an *expand* keyword argument for :func:`pprint.pprint`, - :func:`pprint.pformat`, :func:`pprint.pp`. If true, the output will be - formatted similar to pretty-printed :func:`json.dumps` when - *indent* is supplied. - (Contributed by Stefan Todoran, Semyon Moroz and Hugo van Kemenade in - :gh:`112632`.) - -* Add t-string support to :mod:`pprint`. - (Contributed by Loïc Simon and Hugo van Kemenade in :gh:`134551`.) - - sre_* ----- @@ -1642,6 +1803,14 @@ threading (Contributed by Bénédikt Tran in :gh:`134087`.) +types +----- + +* Removed deprecated in :pep:`626` since Python 3.12 + :attr:`!codeobject.co_lnotab` from :class:`types.CodeType`. + (Contributed by Nikita Sobolev in :gh:`134690`.) + + typing ------ @@ -1661,6 +1830,9 @@ typing or ``TD = TypedDict("TD", {})`` instead. (Contributed by Bénédikt Tran in :gh:`133823`.) +* Deprecated :func:`!typing.no_type_check_decorator` has been removed. + (Contributed by Nikita Sobolev in :gh:`133601`.) + wave ---- @@ -1776,8 +1948,6 @@ New deprecations :func:`issubclass`, but warnings were not previously emitted if it was merely imported or accessed from the :mod:`!typing` module. - * Deprecated :func:`!typing.no_type_check_decorator` has been removed. - (Contributed by Nikita Sobolev in :gh:`133601`.) * ``__version__`` @@ -2064,8 +2234,8 @@ Deprecated C APIs - :c:macro:`Py_ALIGNED`: Prefer ``alignas`` instead. - :c:macro:`PY_FORMAT_SIZE_T`: Use ``"z"`` directly. - - :c:macro:`Py_LL` & :c:macro:`Py_ULL`: - Use standard suffixes, ``LL`` & ``ULL``. + - :c:macro:`Py_LL` and :c:macro:`Py_ULL`: + Use standard suffixes, ``LL`` and ``ULL``. - :c:macro:`PY_LONG_LONG`, :c:macro:`PY_LLONG_MIN`, :c:macro:`PY_LLONG_MAX`, :c:macro:`PY_ULLONG_MAX`, :c:macro:`PY_INT32_T`, :c:macro:`PY_UINT32_T`, :c:macro:`PY_INT64_T`, :c:macro:`PY_UINT64_T`, :c:macro:`PY_SIZE_MAX`: @@ -2114,6 +2284,16 @@ Build changes and :option:`-X dev <-X>` is passed to the Python or Python is built in :ref:`debug mode `. (Contributed by Donghee Na in :gh:`141770`.) +.. _whatsnew315-frame-pointers: + +* CPython is now built with frame pointers enabled by default + (:pep:`831`). Pass :option:`--without-frame-pointers` to opt out. + Authors of C extensions and native libraries built with custom build + systems should add ``-fno-omit-frame-pointer`` and + ``-mno-omit-leaf-frame-pointer`` to their own ``CFLAGS`` to keep the + unwind chain intact. + (Contributed by Pablo Galindo Salgado and Savannah Ostrowski in :gh:`149201`.) + .. _whatsnew315-windows-tail-calling-interpreter: * 64-bit builds using Visual Studio 2026 (MSVC 18) may now use the new diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 9eafc09dbee..bdd35d39e36 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -2173,7 +2173,7 @@ Changes in the Python API * :c:func:`PyErr_SetImportError` now sets :exc:`TypeError` when its **msg** argument is not set. Previously only ``NULL`` was returned. -* The format of the :attr:`~codeobject.co_lnotab` attribute of code objects +* The format of the :attr:`!codeobject.co_lnotab` attribute of code objects changed to support a negative line number delta. By default, Python does not emit bytecode with a negative line number delta. Functions using :attr:`frame.f_lineno`, diff --git a/Grammar/python.gram b/Grammar/python.gram index 3a91d426c36..9bf3a67939f 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -554,10 +554,12 @@ complex_number[expr_ty]: signed_number[expr_ty]: | NUMBER + | '+' number=NUMBER { number } | '-' number=NUMBER { _PyAST_UnaryOp(USub, number, EXTRA) } signed_real_number[expr_ty]: | real_number + | '+' real=real_number { real } | '-' real=real_number { _PyAST_UnaryOp(USub, real, EXTRA) } real_number[expr_ty]: @@ -565,6 +567,7 @@ real_number[expr_ty]: imaginary_number[expr_ty]: | imag=NUMBER { _PyPegen_ensure_imaginary(p, imag) } + | '+' imag=NUMBER { _PyPegen_ensure_imaginary(p, imag) } capture_pattern[pattern_ty]: | target=pattern_capture_target { _PyAST_MatchAs(NULL, target->v.Name.id, EXTRA) } diff --git a/Include/Python.h b/Include/Python.h index e6e5cab67e2..d5e38b8b020 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -9,10 +9,11 @@ // is not needed. -// Include Python header files -#include "patchlevel.h" -#include "pyconfig.h" -#include "pymacconfig.h" +// Include Python configuration headers +#include "patchlevel.h" // the Python version +#include "pyconfig.h" // information from configure +#include "pymacconfig.h" // overrides for pyconfig +#include "pyabi.h" // feature/ABI selection // Include standard header files @@ -46,13 +47,11 @@ # endif #endif -#if defined(Py_GIL_DISABLED) -# if defined(_MSC_VER) -# include // __readgsqword() -# endif - -# if defined(__MINGW32__) -# include // __readgsqword() +#if !defined(Py_LIMITED_API) +# if defined(Py_GIL_DISABLED) +# if defined(_MSC_VER) || defined(__MINGW32__) +# include // __readgsqword() +# endif # endif #endif // Py_GIL_DISABLED @@ -67,6 +66,7 @@ __pragma(warning(disable: 4201)) // Include Python header files #include "pyport.h" +#include "exports.h" #include "pymacro.h" #include "pymath.h" #include "pymem.h" @@ -117,6 +117,7 @@ __pragma(warning(disable: 4201)) #include "cpython/genobject.h" #include "descrobject.h" #include "genericaliasobject.h" +#include "cpython/sentinelobject.h" #include "warnings.h" #include "weakrefobject.h" #include "structseq.h" diff --git a/Include/cpython/ceval.h b/Include/cpython/ceval.h index bbab8d35b75..5b66fa1040d 100644 --- a/Include/cpython/ceval.h +++ b/Include/cpython/ceval.h @@ -38,7 +38,7 @@ typedef struct { PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void); PyAPI_FUNC(int) PyUnstable_WritePerfMapEntry( const void *code_addr, - unsigned int code_size, + size_t code_size, const char *entry_name); PyAPI_FUNC(void) PyUnstable_PerfMapState_Fini(void); PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename); diff --git a/Include/cpython/monitoring.h b/Include/cpython/monitoring.h index 5094c8c23ae..c93271f6ca9 100644 --- a/Include/cpython/monitoring.h +++ b/Include/cpython/monitoring.h @@ -24,9 +24,10 @@ extern "C" { #define PY_MONITORING_EVENT_STOP_ITERATION 10 #define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \ - ((ev) < _PY_MONITORING_LOCAL_EVENTS) +((ev) <= PY_MONITORING_EVENT_STOP_ITERATION) -/* Other events, mainly exceptions */ +/* Other events, mainly exceptions. + * These can now be turned on and disabled on a per code object basis. */ #define PY_MONITORING_EVENT_RAISE 11 #define PY_MONITORING_EVENT_EXCEPTION_HANDLED 12 @@ -34,6 +35,9 @@ extern "C" { #define PY_MONITORING_EVENT_PY_THROW 14 #define PY_MONITORING_EVENT_RERAISE 15 +#define _PY_MONITORING_IS_UNGROUPED_EVENT(ev) \ +((ev) < _PY_MONITORING_UNGROUPED_EVENTS) + /* Ancillary events */ diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index e473110eca7..5d1f44988a6 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -144,6 +144,7 @@ typedef struct _optimization_stats { uint64_t unknown_callee; uint64_t trace_immediately_deopts; uint64_t executors_invalidated; + uint64_t fitness_terminated_traces; UOpStats opcode[PYSTATS_MAX_UOP_ID + 1]; uint64_t unsupported_opcode[256]; uint64_t trace_length_hist[_Py_UOP_HIST_SIZE]; diff --git a/Include/cpython/sentinelobject.h b/Include/cpython/sentinelobject.h new file mode 100644 index 00000000000..0b6ff0f17e6 --- /dev/null +++ b/Include/cpython/sentinelobject.h @@ -0,0 +1,22 @@ +/* Sentinel object interface */ + +#ifndef Py_LIMITED_API +#ifndef Py_SENTINELOBJECT_H +#define Py_SENTINELOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +PyAPI_DATA(PyTypeObject) PySentinel_Type; + +#define PySentinel_Check(op) Py_IS_TYPE((op), &PySentinel_Type) + +PyAPI_FUNC(PyObject *) PySentinel_New( + const char *name, + const char *module_name); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_SENTINELOBJECT_H */ +#endif /* !Py_LIMITED_API */ diff --git a/Include/exports.h b/Include/exports.h index 97a674ec240..a863ecb3307 100644 --- a/Include/exports.h +++ b/Include/exports.h @@ -36,7 +36,7 @@ #define Py_LOCAL_SYMBOL #endif /* module init functions outside the core must be exported */ - #if defined(Py_BUILD_CORE) + #if defined(_PyEXPORTS_CORE) #define _PyINIT_EXPORTED_SYMBOL Py_EXPORTED_SYMBOL #else #define _PyINIT_EXPORTED_SYMBOL __declspec(dllexport) @@ -64,13 +64,13 @@ /* only get special linkage if built as shared or platform is Cygwin */ #if defined(Py_ENABLE_SHARED) || defined(__CYGWIN__) # if defined(HAVE_DECLSPEC_DLL) -# if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# if defined(_PyEXPORTS_CORE) && !defined(_PyEXPORTS_CORE_MODULE) /* module init functions inside the core need no external linkage */ /* except for Cygwin to handle embedding */ # if !defined(__CYGWIN__) # define _PyINIT_FUNC_DECLSPEC # endif /* __CYGWIN__ */ -# else /* Py_BUILD_CORE */ +# else /* _PyEXPORTS_CORE */ /* Building an extension module, or an embedded situation */ /* public Python functions and data are imported */ /* Under Cygwin, auto-import functions to prevent compilation */ @@ -80,7 +80,7 @@ # define PyAPI_FUNC(RTYPE) Py_IMPORTED_SYMBOL RTYPE # endif /* !__CYGWIN__ */ # define PyAPI_DATA(RTYPE) extern Py_IMPORTED_SYMBOL RTYPE -# endif /* Py_BUILD_CORE */ +# endif /* _PyEXPORTS_CORE */ # endif /* HAVE_DECLSPEC_DLL */ #endif /* Py_ENABLE_SHARED */ diff --git a/Include/internal/pycore_blocks_output_buffer.h b/Include/internal/pycore_blocks_output_buffer.h index 016e7a18665..322c1e93344 100644 --- a/Include/internal/pycore_blocks_output_buffer.h +++ b/Include/internal/pycore_blocks_output_buffer.h @@ -242,9 +242,12 @@ static inline PyObject * _BlocksOutputBuffer_Finish(_BlocksOutputBuffer *buffer, const Py_ssize_t avail_out) { + PyObject *obj; assert(buffer->writer != NULL); - return PyBytesWriter_FinishWithSize(buffer->writer, - buffer->allocated - avail_out); + obj = PyBytesWriter_FinishWithSize(buffer->writer, + buffer->allocated - avail_out); + buffer->writer = NULL; + return obj; } /* Clean up the buffer when an error occurred. */ diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 94a1f687b7b..fd4221f0816 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -94,7 +94,7 @@ typedef struct { void* (*init_state)(void); // Callback to register every trampoline being created void (*write_state)(void* state, const void *code_addr, - unsigned int code_size, PyCodeObject* code); + size_t code_size, PyCodeObject* code); // Callback to free the trampoline state int (*free_state)(void* state); } _PyPerf_Callbacks; @@ -108,6 +108,10 @@ extern PyStatus _PyPerfTrampoline_AfterFork_Child(void); #ifdef PY_HAVE_PERF_TRAMPOLINE extern _PyPerf_Callbacks _Py_perfmap_callbacks; extern _PyPerf_Callbacks _Py_perfmap_jit_callbacks; +extern void _PyPerfJit_WriteNamedCode(const void *code_addr, + size_t code_size, + const char *entry, + const char *filename); #endif static inline PyObject* @@ -121,18 +125,11 @@ _PyEval_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwfl } #ifdef _Py_TIER2 -#ifdef _Py_JIT -_Py_CODEUNIT *_Py_LazyJitShim( - struct _PyExecutorObject *current_executor, _PyInterpreterFrame *frame, - _PyStackRef *stack_pointer, PyThreadState *tstate -); -#else _Py_CODEUNIT *_PyTier2Interpreter( struct _PyExecutorObject *current_executor, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate ); #endif -#endif extern _PyJitEntryFuncPtr _Py_jit_entry; @@ -320,7 +317,7 @@ PyObject * _PyEval_ImportNameWithImport( PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); PyAPI_FUNC(void) _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); -PyAPI_FUNC(bool) _PyEval_NoToolsForUnwind(PyThreadState *tstate); +PyAPI_FUNC(bool) _PyEval_NoToolsForUnwind(PyThreadState *tstate, _PyInterpreterFrame *frame); PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, _PyStackRef *sp); PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch); diff --git a/Include/internal/pycore_faulthandler.h b/Include/internal/pycore_faulthandler.h index 78cd657e6ae..9ddd70d39ed 100644 --- a/Include/internal/pycore_faulthandler.h +++ b/Include/internal/pycore_faulthandler.h @@ -42,6 +42,7 @@ struct faulthandler_user_signal { int chain; _Py_sighandler_t previous; PyInterpreterState *interp; + Py_ssize_t max_threads; }; #endif /* FAULTHANDLER_USER */ @@ -57,6 +58,7 @@ struct _faulthandler_runtime_state { void *exc_handler; #endif int c_stack; + Py_ssize_t max_threads; } fatal_error; struct { @@ -68,6 +70,7 @@ struct _faulthandler_runtime_state { int exit; char *header; size_t header_len; + Py_ssize_t max_threads; /* The main thread always holds this lock. It is only released when faulthandler_thread() is interrupted before this thread exits, or at Python exit. */ diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index fd284d0e4ec..e105677cd2e 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -118,21 +118,6 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) { /* Bit 1 is set when the object is in generation which is GCed currently. */ #define _PyGC_PREV_MASK_COLLECTING ((uintptr_t)2) -/* Bit 0 in _gc_next is the old space bit. - * It is set as follows: - * Young: gcstate->visited_space - * old[0]: 0 - * old[1]: 1 - * permanent: 0 - * - * During a collection all objects handled should have the bit set to - * gcstate->visited_space, as objects are moved from the young gen - * and the increment into old[gcstate->visited_space]. - * When object are moved from the pending space, old[gcstate->visited_space^1] - * into the increment, the old space bit is flipped. -*/ -#define _PyGC_NEXT_MASK_OLD_SPACE_1 1 - #define _PyGC_PREV_SHIFT 2 #define _PyGC_PREV_MASK (((uintptr_t) -1) << _PyGC_PREV_SHIFT) @@ -159,13 +144,11 @@ typedef enum { // Lowest bit of _gc_next is used for flags only in GC. // But it is always 0 for normal code. static inline PyGC_Head* _PyGCHead_NEXT(PyGC_Head *gc) { - uintptr_t next = gc->_gc_next & _PyGC_PREV_MASK; + uintptr_t next = gc->_gc_next; return (PyGC_Head*)next; } static inline void _PyGCHead_SET_NEXT(PyGC_Head *gc, PyGC_Head *next) { - uintptr_t unext = (uintptr_t)next; - assert((unext & ~_PyGC_PREV_MASK) == 0); - gc->_gc_next = (gc->_gc_next & ~_PyGC_PREV_MASK) | unext; + gc->_gc_next = (uintptr_t)next; } // Lowest two bits of _gc_prev is used for _PyGC_PREV_MASK_* flags. @@ -207,10 +190,6 @@ static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) { extern void _Py_ScheduleGC(PyThreadState *tstate); -#ifndef Py_GIL_DISABLED -extern void _Py_TriggerGC(struct _gc_runtime_state *gcstate); -#endif - /* Tell the GC to track this object. * @@ -220,7 +199,7 @@ extern void _Py_TriggerGC(struct _gc_runtime_state *gcstate); * ob_traverse method. * * Internal note: interp->gc.generation0->_gc_prev doesn't have any bit flags - * because it's not object header. So we don't use _PyGCHead_PREV() and + * because it's not an object header. So we don't use _PyGCHead_PREV() and * _PyGCHead_SET_PREV() for it to avoid unnecessary bitwise operations. * * See also the public PyObject_GC_Track() function. @@ -244,19 +223,12 @@ static inline void _PyObject_GC_TRACK( "object is in generation which is garbage collected", filename, lineno, __func__); - struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc; - PyGC_Head *generation0 = &gcstate->young.head; + PyGC_Head *generation0 = _PyInterpreterState_GET()->gc.generation0; PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev); _PyGCHead_SET_NEXT(last, gc); _PyGCHead_SET_PREV(gc, last); - uintptr_t not_visited = 1 ^ gcstate->visited_space; - gc->_gc_next = ((uintptr_t)generation0) | not_visited; + _PyGCHead_SET_NEXT(gc, generation0); generation0->_gc_prev = (uintptr_t)gc; - gcstate->young.count++; /* number of tracked GC objects */ - gcstate->heap_size++; - if (gcstate->young.count > gcstate->young.threshold) { - _Py_TriggerGC(gcstate); - } #endif } @@ -291,11 +263,6 @@ static inline void _PyObject_GC_UNTRACK( _PyGCHead_SET_PREV(next, prev); gc->_gc_next = 0; gc->_gc_prev &= _PyGC_PREV_MASK_FINALIZED; - struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc; - if (gcstate->young.count > 0) { - gcstate->young.count--; - } - gcstate->heap_size--; #endif } diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index a3badb59cb7..2c264c39ae9 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -33,6 +33,9 @@ PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); PyAPI_FUNC(PyObject *)_PyCoro_GetAwaitableIter(PyObject *o); PyAPI_FUNC(PyObject *)_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *); +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyCoro_ComputeOrigin(int origin_depth, _PyInterpreterFrame *current_frame); + extern PyTypeObject _PyCoroWrapper_Type; extern PyTypeObject _PyAsyncGenWrappedValue_Type; extern PyTypeObject _PyAsyncGenAThrow_Type; diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index beae65213a2..4d6d5ce9c5e 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1636,6 +1636,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(callable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cancel)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(canonical)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(capath)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(capitals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(category)); @@ -1896,6 +1897,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mask)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(match)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(max_length)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(max_threads)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxdigits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxevents)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxlen)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index bb1c6dbaf03..20dcf81ccf1 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -359,6 +359,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(callable) STRUCT_FOR_ID(callback) STRUCT_FOR_ID(cancel) + STRUCT_FOR_ID(canonical) STRUCT_FOR_ID(capath) STRUCT_FOR_ID(capitals) STRUCT_FOR_ID(category) @@ -619,6 +620,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(mask) STRUCT_FOR_ID(match) STRUCT_FOR_ID(max_length) + STRUCT_FOR_ID(max_threads) STRUCT_FOR_ID(maxdigits) STRUCT_FOR_ID(maxevents) STRUCT_FOR_ID(maxlen) diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 1da8237e93f..56b55e93a01 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -70,16 +70,15 @@ PyAPI_DATA(PyObject) _PyInstrumentation_DISABLE; /* Total tool ids available */ #define PY_MONITORING_TOOL_IDS 8 -/* Count of all local monitoring events */ -#define _PY_MONITORING_LOCAL_EVENTS 11 -/* Count of all "real" monitoring events (not derived from other events) */ +/* Count of all "real" monitoring events (not derived from other events). + * "Other" events can now be turned on/disabled per code object. */ #define _PY_MONITORING_UNGROUPED_EVENTS 16 /* Count of all monitoring events */ #define _PY_MONITORING_EVENTS 19 /* Tables of which tools are active for each monitored event. */ typedef struct _Py_LocalMonitors { - uint8_t tools[_PY_MONITORING_LOCAL_EVENTS]; + uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS]; } _Py_LocalMonitors; typedef struct _Py_GlobalMonitors { @@ -122,6 +121,15 @@ typedef struct _PyCoMonitoringData { extern int _Py_Instrumentation_GetLine(PyCodeObject *code, _PyCoLineInstrumentationData *line_data, int index); +static inline uint8_t +_PyCode_GetOriginalOpcode(_PyCoLineInstrumentationData *line_data, int index) +{ + return line_data->data[index*line_data->bytes_per_entry]; +} + +// Exported for external JIT support +PyAPI_FUNC(uint8_t) _PyCode_Deinstrument(uint8_t opcode); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 2bfb84da36c..86f018e3286 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -69,7 +69,7 @@ struct code_arena_st; struct trampoline_api_st { void* (*init_state)(void); void (*write_state)(void* state, const void *code_addr, - unsigned int code_size, PyCodeObject* code); + size_t code_size, PyCodeObject* code); int (*free_state)(void* state); void *state; Py_ssize_t code_padding; @@ -181,29 +181,14 @@ struct gc_generation { struct gc_generation_stats { PyTime_t ts_start; PyTime_t ts_stop; - - /* heap_size on the start of the collection */ - Py_ssize_t heap_size; - - /* work_to_do on the start of the collection */ - Py_ssize_t work_to_do; - /* total number of collections */ Py_ssize_t collections; - - /* total number of visited objects */ - Py_ssize_t object_visits; - /* total number of collected objects */ Py_ssize_t collected; /* total number of uncollectable objects (put into gc.garbage) */ Py_ssize_t uncollectable; // Total number of objects considered for collection and traversed: Py_ssize_t candidates; - - Py_ssize_t objects_transitively_reachable; - Py_ssize_t objects_not_transitively_reachable; - // Total duration of the collection in seconds: double duration; }; @@ -225,11 +210,6 @@ struct gc_old_stats_buffer { int8_t index; }; -enum _GCPhase { - GC_PHASE_MARK = 0, - GC_PHASE_COLLECT = 1 -}; - /* If we change this, we need to change the default value in the signature of gc.collect and change the size of PyStats.gc_stats */ #define NUM_GENERATIONS 3 @@ -244,8 +224,13 @@ struct _gc_runtime_state { int enabled; int debug; /* linked lists of container objects */ +#ifndef Py_GIL_DISABLED + struct gc_generation generations[NUM_GENERATIONS]; + PyGC_Head *generation0; +#else struct gc_generation young; struct gc_generation old[2]; +#endif /* a permanent generation which won't be collected */ struct gc_generation permanent_generation; struct gc_stats *generation_stats; @@ -259,13 +244,6 @@ struct _gc_runtime_state { /* a list of callbacks to be invoked when collection is performed */ PyObject *callbacks; - Py_ssize_t heap_size; - Py_ssize_t work_to_do; - /* Which of the old spaces is the visited space */ - int visited_space; - int phase; - -#ifdef Py_GIL_DISABLED /* This is the number of objects that survived the last full collection. It approximates the number of long lived objects tracked by the GC. @@ -278,6 +256,7 @@ struct _gc_runtime_state { the first time. */ Py_ssize_t long_lived_pending; +#ifdef Py_GIL_DISABLED /* True if gc.freeze() has been used. */ int freeze_active; @@ -293,6 +272,22 @@ struct _gc_runtime_state { #endif }; +#ifndef Py_GIL_DISABLED +#define GC_GENERATION_INIT \ + .generations = { \ + { .threshold = 2000, }, \ + { .threshold = 10, }, \ + { .threshold = 10, }, \ + }, +#else +#define GC_GENERATION_INIT \ + .young = { .threshold = 2000, }, \ + .old = { \ + { .threshold = 10, }, \ + { .threshold = 10, }, \ + }, +#endif + #include "pycore_gil.h" // struct _gil_runtime_state /**** Import ********/ @@ -449,6 +444,9 @@ typedef struct _PyOptimizationConfig { uint16_t side_exit_initial_value; uint16_t side_exit_initial_backoff; + // Trace fitness thresholds + uint16_t fitness_initial; + // Optimization flags bool specialization_enabled; bool uops_optimize_enabled; @@ -527,8 +525,13 @@ struct _py_func_state { /****** type state *********/ /* For now we hard-code this to a value for which we are confident - all the static builtin types will fit (for all builds). */ -#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES 202 + all the static builtin types will fit (for all builds). + If you add a new static type to the standard library, you may have to + update one of these numbers. + */ +#define _Py_NUM_MANAGED_PREINITIALIZED_TYPES 120 +#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES \ + (_Py_NUM_MANAGED_PREINITIALIZED_TYPES + 83) #define _Py_MAX_MANAGED_STATIC_EXT_TYPES 10 #define _Py_MAX_MANAGED_STATIC_TYPES \ (_Py_MAX_MANAGED_STATIC_BUILTIN_TYPES + _Py_MAX_MANAGED_STATIC_EXT_TYPES) diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h index 70bccce4166..2f97cc26eaf 100644 --- a/Include/internal/pycore_jit.h +++ b/Include/internal/pycore_jit.h @@ -23,9 +23,13 @@ typedef _Py_CODEUNIT *(*jit_func)( _PyStackRef _tos_cache0, _PyStackRef _tos_cache1, _PyStackRef _tos_cache2 ); +_Py_CODEUNIT *_PyJIT_Entry( + _PyExecutorObject *executor, _PyInterpreterFrame *frame, + _PyStackRef *stack_pointer, PyThreadState *tstate +); + int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length); void _PyJIT_Free(_PyExecutorObject *executor); -void _PyJIT_Fini(void); PyAPI_FUNC(int) _PyJIT_AddressInJitCode(PyInterpreterState *interp, uintptr_t addr); #endif // _Py_JIT diff --git a/Include/internal/pycore_jit_unwind.h b/Include/internal/pycore_jit_unwind.h new file mode 100644 index 00000000000..2d325ad9562 --- /dev/null +++ b/Include/internal/pycore_jit_unwind.h @@ -0,0 +1,68 @@ +#ifndef Py_INTERNAL_JIT_UNWIND_H +#define Py_INTERNAL_JIT_UNWIND_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include +#include + +#if defined(_Py_JIT) && defined(__linux__) && defined(__ELF__) +# define PY_HAVE_JIT_GDB_UNWIND +#endif + +#if defined(PY_HAVE_PERF_TRAMPOLINE) || defined(PY_HAVE_JIT_GDB_UNWIND) + +#if defined(PY_HAVE_JIT_GDB_UNWIND) +extern PyMutex _Py_jit_debug_mutex; +#endif + +/* DWARF exception-handling pointer encodings shared by JIT unwind users. */ +enum { + DWRF_EH_PE_absptr = 0x00, + DWRF_EH_PE_omit = 0xff, + + /* Data type encodings */ + DWRF_EH_PE_uleb128 = 0x01, + DWRF_EH_PE_udata2 = 0x02, + DWRF_EH_PE_udata4 = 0x03, + DWRF_EH_PE_udata8 = 0x04, + DWRF_EH_PE_sleb128 = 0x09, + DWRF_EH_PE_sdata2 = 0x0a, + DWRF_EH_PE_sdata4 = 0x0b, + DWRF_EH_PE_sdata8 = 0x0c, + DWRF_EH_PE_signed = 0x08, + + /* Reference type encodings */ + DWRF_EH_PE_pcrel = 0x10, + DWRF_EH_PE_textrel = 0x20, + DWRF_EH_PE_datarel = 0x30, + DWRF_EH_PE_funcrel = 0x40, + DWRF_EH_PE_aligned = 0x50, + DWRF_EH_PE_indirect = 0x80 +}; + +/* Return the size of the generated .eh_frame data for the given encoding. */ +size_t _PyJitUnwind_EhFrameSize(int absolute_addr); + +/* + * Build DWARF .eh_frame data for JIT code; returns size written or 0 on error. + * absolute_addr selects the FDE address encoding: + * - 0: PC-relative offsets (perf jitdump synthesized DSO). + * - nonzero: absolute addresses (GDB JIT in-memory ELF). + */ +size_t _PyJitUnwind_BuildEhFrame(uint8_t *buffer, size_t buffer_size, + const void *code_addr, size_t code_size, + int absolute_addr); + +void *_PyJitUnwind_GdbRegisterCode(const void *code_addr, + size_t code_size, + const char *entry, + const char *filename); + +void _PyJitUnwind_GdbUnregisterCode(void *handle); + +#endif // defined(PY_HAVE_PERF_TRAMPOLINE) || defined(PY_HAVE_JIT_GDB_UNWIND) + +#endif // Py_INTERNAL_JIT_UNWIND_H diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index a809a7e2552..a0727c045e5 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -15,6 +15,50 @@ extern "C" { #include "pycore_optimizer_types.h" #include +/* Fitness controls how long a trace can grow. + * Starts at FITNESS_INITIAL, then decreases from per-bytecode buffer usage + * plus branch/frame heuristics. The trace stops when fitness drops below the + * current exit_quality. + * + * Design targets for the constants below: + * 1. Reaching the abstract frame-depth limit should drop fitness below + * EXIT_QUALITY_SPECIALIZABLE. + * 2. A backward edge should leave budget for roughly N_BACKWARD_SLACK more + * bytecodes, assuming AVG_SLOTS_PER_INSTRUCTION. + * 3. Roughly seven balanced branches should reduce fitness to + * EXIT_QUALITY_DEFAULT after per-slot costs. + * 4. A push followed by a matching return is net-zero on frame-specific + * fitness, excluding per-slot costs. + */ +#define MAX_TARGET_LENGTH (UOP_MAX_TRACE_LENGTH / 2) +#define OPTIMIZER_EFFECTIVENESS 2 +#define FITNESS_INITIAL (MAX_TARGET_LENGTH * OPTIMIZER_EFFECTIVENESS) + +/* Exit quality thresholds: trace stops when fitness < exit_quality. + * Higher = trace is more willing to stop here. */ +#define EXIT_QUALITY_CLOSE_LOOP (FITNESS_INITIAL - AVG_SLOTS_PER_INSTRUCTION*4) +#define EXIT_QUALITY_ENTER_EXECUTOR (FITNESS_INITIAL * 1 / 8) +#define EXIT_QUALITY_DEFAULT (FITNESS_INITIAL / 40) +#define EXIT_QUALITY_SPECIALIZABLE (FITNESS_INITIAL / 80) + +/* Estimated buffer slots per bytecode, used only to derive heuristics. + * Runtime charging uses trace-buffer capacity consumed for each bytecode. */ +#define AVG_SLOTS_PER_INSTRUCTION 6 + +/* Heuristic backward-edge exit quality: leave room for about 1 unroll and + * N_BACKWARD_SLACK more bytecodes before reaching EXIT_QUALITY_CLOSE_LOOP, + * based on AVG_SLOTS_PER_INSTRUCTION. */ +#define N_BACKWARD_SLACK 10 +#define EXIT_QUALITY_BACKWARD_EDGE (EXIT_QUALITY_CLOSE_LOOP / 2 - N_BACKWARD_SLACK * AVG_SLOTS_PER_INSTRUCTION) + +/* Penalty for a balanced branch. + * It is sized so repeated balanced branches can drive a trace toward + * EXIT_QUALITY_DEFAULT, while compute_branch_penalty() keeps any single branch + * from dominating the budget. + */ +#define FITNESS_BRANCH_BALANCED ((FITNESS_INITIAL - EXIT_QUALITY_DEFAULT - \ + (MAX_TARGET_LENGTH / 14 * AVG_SLOTS_PER_INSTRUCTION)) / (14)) + typedef struct _PyJitUopBuffer { _PyUOpInstruction *start; @@ -103,7 +147,8 @@ typedef struct _PyJitTracerPreviousState { } _PyJitTracerPreviousState; typedef struct _PyJitTracerTranslatorState { - int jump_backward_seen; + int32_t fitness; // Current trace fitness, starts high, decrements + int frame_depth; // Current inline depth (0 = root frame) } _PyJitTracerTranslatorState; typedef struct _PyJitTracerState { @@ -153,6 +198,7 @@ typedef struct _PyExecutorObject { uint32_t code_size; size_t jit_size; void *jit_code; + void *jit_gdb_handle; _PyExitData exits[1]; } _PyExecutorObject; @@ -489,7 +535,21 @@ typedef struct { uint8_t count; uint8_t indices[MAX_RECORDED_VALUES]; } _PyOpcodeRecordEntry; + +typedef struct { + uint8_t count; + uint8_t transform_mask; + uint8_t slots[MAX_RECORDED_VALUES]; +} _PyOpcodeRecordSlotMap; + PyAPI_DATA(const _PyOpcodeRecordEntry) _PyOpcode_RecordEntries[256]; +PyAPI_DATA(const _PyOpcodeRecordSlotMap) _PyOpcode_RecordSlotMaps[256]; + +/* Convert a family-recorded value to the form a recorder uop expects. + * If no transform is needed, return the input value unchanged. + * Takes ownership of `value` and returns a new strong reference or NULL. + */ +PyAPI_FUNC(PyObject *) _PyOpcode_RecordTransformValue(int uop, PyObject *value); #endif #ifdef __cplusplus diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index e8d1098c207..6c48ac0dccf 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -130,13 +130,7 @@ extern PyTypeObject _PyExc_MemoryError; }, \ .gc = { \ .enabled = 1, \ - .young = { .threshold = 2000, }, \ - .old = { \ - { .threshold = 10, }, \ - { .threshold = 0, }, \ - }, \ - .work_to_do = -5000, \ - .phase = GC_PHASE_MARK, \ + GC_GENERATION_INIT \ }, \ .qsbr = { \ .wr_seq = QSBR_INITIAL, \ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 64b029797ab..1ce91dc51ea 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1634,6 +1634,7 @@ extern "C" { INIT_ID(callable), \ INIT_ID(callback), \ INIT_ID(cancel), \ + INIT_ID(canonical), \ INIT_ID(capath), \ INIT_ID(capitals), \ INIT_ID(category), \ @@ -1894,6 +1895,7 @@ extern "C" { INIT_ID(mask), \ INIT_ID(match), \ INIT_ID(max_length), \ + INIT_ID(max_threads), \ INIT_ID(maxdigits), \ INIT_ID(maxevents), \ INIT_ID(maxlen), \ diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 329045b5faa..ca4a7c216ed 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -71,8 +71,10 @@ static const _PyStackRef PyStackRef_NULL = { .index = 0 }; static const _PyStackRef PyStackRef_ERROR = { .index = (1 << Py_TAGGED_SHIFT) }; #define PyStackRef_None ((_PyStackRef){ .index = (2 << Py_TAGGED_SHIFT) } ) -#define PyStackRef_False ((_PyStackRef){ .index = (3 << Py_TAGGED_SHIFT) }) -#define PyStackRef_True ((_PyStackRef){ .index = (4 << Py_TAGGED_SHIFT) }) +#define _Py_STACKREF_FALSE_INDEX (3 << Py_TAGGED_SHIFT) +#define _Py_STACKREF_TRUE_INDEX (4 << Py_TAGGED_SHIFT) +#define PyStackRef_False ((_PyStackRef){ .index = _Py_STACKREF_FALSE_INDEX }) +#define PyStackRef_True ((_PyStackRef){ .index = _Py_STACKREF_TRUE_INDEX }) #define INITIAL_STACKREF_INDEX (5 << Py_TAGGED_SHIFT) diff --git a/Include/internal/pycore_traceback.h b/Include/internal/pycore_traceback.h index 6b5e24979d5..fbf6bc2c41f 100644 --- a/Include/internal/pycore_traceback.h +++ b/Include/internal/pycore_traceback.h @@ -61,7 +61,8 @@ extern void _Py_DumpTraceback( extern const char* _Py_DumpTracebackThreads( int fd, PyInterpreterState *interp, - PyThreadState *current_tstate); + PyThreadState *current_tstate, + Py_ssize_t max_threads); /* Write a Unicode object into the file descriptor fd. Encode the string to ASCII using the backslashreplace error handler. diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 461ee36dceb..c7c23494845 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1216,6 +1216,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(canonical); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(capath); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2256,6 +2260,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(max_threads); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxdigits); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Include/internal/pycore_uop.h b/Include/internal/pycore_uop.h index 7bc8947cfa9..320508e8b7a 100644 --- a/Include/internal/pycore_uop.h +++ b/Include/internal/pycore_uop.h @@ -31,6 +31,7 @@ typedef struct _PyUOpInstruction{ uint64_t operand0; // A cache entry uint64_t operand1; #ifdef Py_STATS + int32_t fitness; uint64_t execution_count; #endif } _PyUOpInstruction; diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 6b96f9bc78e..bd1440a89bd 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -375,14 +375,14 @@ extern "C" { #define _RESUME_CHECK 601 #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE 602 -#define _SAVE_RETURN_OFFSET 603 -#define _SEND 604 -#define _SEND_GEN_FRAME 605 +#define _RROT_3 603 +#define _SAVE_RETURN_OFFSET 604 +#define _SEND 605 +#define _SEND_GEN_FRAME 606 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE -#define _SET_UPDATE 606 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW 607 +#define _SET_UPDATE 607 #define _SPILL_OR_RELOAD 608 #define _START_EXECUTOR 609 #define _STORE_ATTR 610 @@ -1275,27 +1275,27 @@ extern "C" { #define _RESUME_CHECK_r33 1487 #define _RETURN_GENERATOR_r01 1488 #define _RETURN_VALUE_r11 1489 -#define _SAVE_RETURN_OFFSET_r00 1490 -#define _SAVE_RETURN_OFFSET_r11 1491 -#define _SAVE_RETURN_OFFSET_r22 1492 -#define _SAVE_RETURN_OFFSET_r33 1493 -#define _SEND_r33 1494 -#define _SEND_GEN_FRAME_r33 1495 -#define _SETUP_ANNOTATIONS_r00 1496 -#define _SET_ADD_r10 1497 -#define _SET_FUNCTION_ATTRIBUTE_r01 1498 -#define _SET_FUNCTION_ATTRIBUTE_r11 1499 -#define _SET_FUNCTION_ATTRIBUTE_r21 1500 -#define _SET_FUNCTION_ATTRIBUTE_r32 1501 -#define _SET_IP_r00 1502 -#define _SET_IP_r11 1503 -#define _SET_IP_r22 1504 -#define _SET_IP_r33 1505 -#define _SET_UPDATE_r11 1506 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03 1507 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13 1508 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23 1509 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 1510 +#define _RROT_3_r03 1490 +#define _RROT_3_r13 1491 +#define _RROT_3_r23 1492 +#define _RROT_3_r33 1493 +#define _SAVE_RETURN_OFFSET_r00 1494 +#define _SAVE_RETURN_OFFSET_r11 1495 +#define _SAVE_RETURN_OFFSET_r22 1496 +#define _SAVE_RETURN_OFFSET_r33 1497 +#define _SEND_r33 1498 +#define _SEND_GEN_FRAME_r33 1499 +#define _SETUP_ANNOTATIONS_r00 1500 +#define _SET_ADD_r10 1501 +#define _SET_FUNCTION_ATTRIBUTE_r01 1502 +#define _SET_FUNCTION_ATTRIBUTE_r11 1503 +#define _SET_FUNCTION_ATTRIBUTE_r21 1504 +#define _SET_FUNCTION_ATTRIBUTE_r32 1505 +#define _SET_IP_r00 1506 +#define _SET_IP_r11 1507 +#define _SET_IP_r22 1508 +#define _SET_IP_r33 1509 +#define _SET_UPDATE_r11 1510 #define _SPILL_OR_RELOAD_r01 1511 #define _SPILL_OR_RELOAD_r02 1512 #define _SPILL_OR_RELOAD_r03 1513 diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 8465fd4345e..8f543dbeeb8 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -397,7 +397,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW] = 0, + [_RROT_3] = HAS_PURE_FLAG, [_START_EXECUTOR] = HAS_DEOPT_FLAG, [_MAKE_WARM] = 0, [_FATAL_ERROR] = 0, @@ -3700,13 +3700,13 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW] = { + [_RROT_3] = { .best = { 0, 1, 2, 3 }, .entries = { - { 3, 0, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03 }, - { 3, 1, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13 }, - { 3, 2, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23 }, - { 3, 3, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 }, + { 3, 0, _RROT_3_r03 }, + { 3, 1, _RROT_3_r13 }, + { 3, 2, _RROT_3_r23 }, + { 3, 3, _RROT_3_r33 }, }, }, [_START_EXECUTOR] = { @@ -4705,10 +4705,10 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_LOAD_CONST_INLINE_BORROW_r01] = _LOAD_CONST_INLINE_BORROW, [_LOAD_CONST_INLINE_BORROW_r12] = _LOAD_CONST_INLINE_BORROW, [_LOAD_CONST_INLINE_BORROW_r23] = _LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03] = _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13] = _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23] = _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33] = _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, + [_RROT_3_r03] = _RROT_3, + [_RROT_3_r13] = _RROT_3, + [_RROT_3_r23] = _RROT_3, + [_RROT_3_r33] = _RROT_3, [_START_EXECUTOR_r00] = _START_EXECUTOR, [_MAKE_WARM_r00] = _MAKE_WARM, [_MAKE_WARM_r11] = _MAKE_WARM, @@ -5910,6 +5910,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_RETURN_GENERATOR_r01] = "_RETURN_GENERATOR_r01", [_RETURN_VALUE] = "_RETURN_VALUE", [_RETURN_VALUE_r11] = "_RETURN_VALUE_r11", + [_RROT_3] = "_RROT_3", + [_RROT_3_r03] = "_RROT_3_r03", + [_RROT_3_r13] = "_RROT_3_r13", + [_RROT_3_r23] = "_RROT_3_r23", + [_RROT_3_r33] = "_RROT_3_r33", [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET", [_SAVE_RETURN_OFFSET_r00] = "_SAVE_RETURN_OFFSET_r00", [_SAVE_RETURN_OFFSET_r11] = "_SAVE_RETURN_OFFSET_r11", @@ -5933,11 +5938,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_SET_IP_r33] = "_SET_IP_r33", [_SET_UPDATE] = "_SET_UPDATE", [_SET_UPDATE_r11] = "_SET_UPDATE_r11", - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW", - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03", - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13", - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23", - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33", [_SPILL_OR_RELOAD] = "_SPILL_OR_RELOAD", [_SPILL_OR_RELOAD_r01] = "_SPILL_OR_RELOAD_r01", [_SPILL_OR_RELOAD_r02] = "_SPILL_OR_RELOAD_r02", @@ -6827,8 +6827,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _LOAD_CONST_INLINE_BORROW: return 0; - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW: - return 3; + case _RROT_3: + return 0; case _START_EXECUTOR: return 0; case _MAKE_WARM: diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 9f5c36230a7..974246f896e 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -61,32 +61,4 @@ #define PYTHON_ABI_VERSION 3 #define PYTHON_ABI_STRING "3" - -/* Stable ABI for free-threaded builds (introduced in PEP 803) - is enabled by one of: - - Py_TARGET_ABI3T, or - - Py_LIMITED_API and Py_GIL_DISABLED. - "Output" macros to be used internally: - - Py_LIMITED_API (defines the subset of API we expose) - - _Py_OPAQUE_PYOBJECT (additionally hides what's ABI-incompatible between - free-threaded & GIL) - (Don't use Py_TARGET_ABI3T directly: it's currently only used to set these - 2 macros. It's also available for users' convenience.) - */ -#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) \ - && !defined(Py_TARGET_ABI3T) -# define Py_TARGET_ABI3T Py_LIMITED_API -#endif -#if defined(Py_TARGET_ABI3T) -# define _Py_OPAQUE_PYOBJECT -# if !defined(Py_LIMITED_API) -# define Py_LIMITED_API Py_TARGET_ABI3T -# elif Py_LIMITED_API > Py_TARGET_ABI3T - // if both are defined, use the *lower* version, - // i.e. maximum compatibility -# undef Py_LIMITED_API -# define Py_LIMITED_API Py_TARGET_ABI3T -# endif -#endif - #endif //_Py_PATCHLEVEL_H diff --git a/Include/pyabi.h b/Include/pyabi.h new file mode 100644 index 00000000000..8c4ae281a43 --- /dev/null +++ b/Include/pyabi.h @@ -0,0 +1,121 @@ +/* Macros that restrict available definitions and select implementations + * to match an ABI stability promise: + * + * - internal API/ABI (may change at any time) -- Py_BUILD_CORE* + * - general CPython API/ABI (may change in 3.x.0) -- default + * - Stable ABI: abi3, abi3t (long-term stable) -- Py_LIMITED_API, + * Py_TARGET_ABI3T, _Py_OPAQUE_PYOBJECT + * - Free-threading (incompatible with non-free-threading builds) + * -- Py_GIL_DISABLED + */ + +#ifndef _Py_PYABI_H +#define _Py_PYABI_H + +/* Defines to build Python and its standard library: + * + * - Py_BUILD_CORE: Build Python core. Gives access to Python internals; should + * not be used by third-party modules. + * - Py_BUILD_CORE_BUILTIN: Build a Python stdlib module as a built-in module. + * - Py_BUILD_CORE_MODULE: Build a Python stdlib module as a dynamic library. + * + * Py_BUILD_CORE_BUILTIN and Py_BUILD_CORE_MODULE imply Py_BUILD_CORE. + * + * On Windows, Py_BUILD_CORE_MODULE exports "PyInit_xxx" symbol, whereas + * Py_BUILD_CORE_BUILTIN does not. + */ +#if defined(Py_BUILD_CORE_BUILTIN) && !defined(Py_BUILD_CORE) +# define Py_BUILD_CORE +#endif +#if defined(Py_BUILD_CORE_MODULE) && !defined(Py_BUILD_CORE) +# define Py_BUILD_CORE +#endif + +/* Check valid values for target ABI macros. + */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 3 + // Empty Py_LIMITED_API used to work; redefine to + // Python 3.2 to be explicit. +# undef Py_LIMITED_API +# define Py_LIMITED_API 0x03020000 +#endif +#if defined(Py_TARGET_ABI3T) && Py_TARGET_ABI3T+0 < 0x030f0000 +# error "Py_TARGET_ABI3T must be 0x030f0000 (3.15) or above" +#endif + +/* Stable ABI for free-threaded builds (abi3t, introduced in PEP 803) + * is enabled by one of: + * - Py_TARGET_ABI3T, or + * - Py_LIMITED_API and Py_GIL_DISABLED. + * + * These affect set the following, which Python.h should use internally: + * - Py_LIMITED_API (defines the subset of API we expose) + * - _Py_OPAQUE_PYOBJECT (additionally hides what's ABI-incompatible between + * free-threaded & GIL) + * + * (Don't use Py_TARGET_ABI3T directly. It's currently only used to set these + * 2 macros, and defined for users' convenience.) + */ +#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) \ + && !defined(Py_TARGET_ABI3T) +# define Py_TARGET_ABI3T Py_LIMITED_API +#endif +#if defined(Py_TARGET_ABI3T) +# define _Py_OPAQUE_PYOBJECT +# if !defined(Py_LIMITED_API) +# define Py_LIMITED_API Py_TARGET_ABI3T +# elif Py_LIMITED_API > Py_TARGET_ABI3T + // if both are defined, use the *lower* version, + // i.e. maximum compatibility +# undef Py_LIMITED_API +# define Py_LIMITED_API Py_TARGET_ABI3T +# endif +#else +# ifdef _Py_OPAQUE_PYOBJECT + // _Py_OPAQUE_PYOBJECT is a private macro; do not define it directly. +# error "Define Py_TARGET_ABI3T to target abi3t." +# endif +#endif + +#if defined(Py_TARGET_ABI3T) +# if !defined(Py_GIL_DISABLED) + // Define Py_GIL_DISABLED for users' needs. Users check this macro to see + // whether they need extra synchronization. +# define Py_GIL_DISABLED +# endif +# if defined(_Py_IS_TESTCEXT) + // When compiling for abi3t, contents of Python.h should not depend + // on Py_GIL_DISABLED. + // We ask GCC to error if it sees the macro from this point on. + // Since users are free to the macro, and there's no way to undo the + // poisoning at the end of Python.h, we only do this in a test module + // (test_cext). + // + // Clang's poisoning is stricter than GCC's: it looks in `#elif` + // expressions after matching `#if`s. We disable it for now. + // We also provide an undocumented, unsupported opt-out macro to help + // porting to other compilers. Consider reaching out if you use it. +# if defined(__GNUC__) && !defined(__clang__) && !defined(_Py_NO_GCC_POISON) +# undef Py_GIL_DISABLED +# pragma GCC poison Py_GIL_DISABLED +# endif +# endif +#endif + +/* The internal C API must not be used with the limited C API: make sure + * that Py_BUILD_CORE* macros are not defined in this case. + * But, keep the "original" values, under different names, for "exports.h" + */ +#ifdef Py_BUILD_CORE +# define _PyEXPORTS_CORE +#endif +#ifdef Py_BUILD_CORE_MODULE +# define _PyEXPORTS_CORE_MODULE +#endif +#ifdef Py_LIMITED_API +# undef Py_BUILD_CORE +# undef Py_BUILD_CORE_BUILTIN +# undef Py_BUILD_CORE_MODULE +#endif + +#endif // _Py_PYABI_H diff --git a/Include/pyport.h b/Include/pyport.h index 62cba4c1421..73a3e6cdaf0 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -58,34 +58,6 @@ #endif -/* Defines to build Python and its standard library: - * - * - Py_BUILD_CORE: Build Python core. Give access to Python internals, but - * should not be used by third-party modules. - * - Py_BUILD_CORE_BUILTIN: Build a Python stdlib module as a built-in module. - * - Py_BUILD_CORE_MODULE: Build a Python stdlib module as a dynamic library. - * - * Py_BUILD_CORE_BUILTIN and Py_BUILD_CORE_MODULE imply Py_BUILD_CORE. - * - * On Windows, Py_BUILD_CORE_MODULE exports "PyInit_xxx" symbol, whereas - * Py_BUILD_CORE_BUILTIN does not. - */ -#if defined(Py_BUILD_CORE_BUILTIN) && !defined(Py_BUILD_CORE) -# define Py_BUILD_CORE -#endif -#if defined(Py_BUILD_CORE_MODULE) && !defined(Py_BUILD_CORE) -# define Py_BUILD_CORE -#endif - -#if defined(Py_TARGET_ABI3T) -# if !defined(Py_GIL_DISABLED) -// Define Py_GIL_DISABLED for users' needs. This macro is used to enable -// locking needed in for free-threaded interpreters builds. -# define Py_GIL_DISABLED -# endif -#endif - - /************************************************************************** Symbols and macros to supply platform-independent interfaces to basic C language & library operations whose spellings vary across platforms. @@ -393,17 +365,6 @@ extern "C" { # define Py_NO_INLINE #endif -#include "exports.h" - -#ifdef Py_LIMITED_API - // The internal C API must not be used with the limited C API: make sure - // that Py_BUILD_CORE macro is not defined in this case. These 3 macros are - // used by exports.h, so only undefine them afterwards. -# undef Py_BUILD_CORE -# undef Py_BUILD_CORE_BUILTIN -# undef Py_BUILD_CORE_MODULE -#endif - /* limits.h constants that may be missing */ #ifndef INT_MAX @@ -592,6 +553,7 @@ extern "C" { # if !defined(_Py_MEMORY_SANITIZER) # define _Py_MEMORY_SANITIZER # define _Py_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory)) +# define _Py_MSAN_UNPOISON(PTR, SIZE) (__msan_unpoison(PTR, SIZE)) # endif # endif # if __has_feature(address_sanitizer) @@ -630,6 +592,9 @@ extern "C" { #ifndef _Py_NO_SANITIZE_MEMORY # define _Py_NO_SANITIZE_MEMORY #endif +#ifndef _Py_MSAN_UNPOISON +# define _Py_MSAN_UNPOISON(PTR, SIZE) +#endif /* AIX has __bool__ redefined in it's system header file. */ #if defined(_AIX) && defined(__bool__) diff --git a/InternalDocs/code_objects.md b/InternalDocs/code_objects.md index a91a7043c1b..cccbe715886 100644 --- a/InternalDocs/code_objects.md +++ b/InternalDocs/code_objects.md @@ -70,14 +70,6 @@ ### Format of the locations table representation of the source code positions of instructions, which are returned by the `co_positions()` iterator. -> [!NOTE] -> `co_linetable` is not to be confused with `co_lnotab`. -> For backwards compatibility, `co_lnotab` exposes the format -> as it existed in Python 3.10 and lower: this older format -> stores only the start line for each instruction. -> It is lazily created from `co_linetable` when accessed. -> See [`Objects/lnotab_notes.txt`](../Objects/lnotab_notes.txt) for more details. - `co_linetable` consists of a sequence of location entries. Each entry starts with a byte with the most significant bit set, followed by zero or more bytes with the most significant bit unset. diff --git a/InternalDocs/garbage_collector.md b/InternalDocs/garbage_collector.md index 94e6fb05b68..0ef45ff8e02 100644 --- a/InternalDocs/garbage_collector.md +++ b/InternalDocs/garbage_collector.md @@ -107,7 +107,7 @@ [Optimization: reusing fields to save memory](#optimization-reusing-fields-to-save-memory) section, these two extra fields are normally used to keep doubly linked lists of all the objects tracked by the garbage collector (these lists are the GC generations, more on -that in the [Optimization: incremental collection](#Optimization-incremental-collection) section), but +that in the [Optimization: generations](#Optimization-generations) section), but they are also reused to fulfill other purposes when the full doubly linked list structure is not needed as a memory optimization. @@ -203,22 +203,22 @@ ```pycon >>> import gc ->>> +>>> >>> class Link: ... def __init__(self, next_link=None): ... self.next_link = next_link -... +... >>> link_3 = Link() >>> link_2 = Link(link_3) >>> link_1 = Link(link_2) >>> link_3.next_link = link_1 >>> A = link_1 >>> del link_1, link_2, link_3 ->>> +>>> >>> link_4 = Link() >>> link_4.next_link = link_4 >>> del link_4 ->>> +>>> >>> # Collect the unreachable Link object (and its .__dict__ dict). >>> gc.collect() 2 @@ -360,11 +360,12 @@ the reference counts fall to 0, triggering the destruction of all unreachable objects. -Optimization: incremental collection -==================================== +Optimization: generations +========================= -In order to bound the length of each garbage collection pause, the GC implementation -for the default build uses incremental collection with two generations. +In order to limit the time each garbage collection takes, the GC +implementation for the default build uses a popular optimization: +generations. Generational garbage collection takes advantage of what is known as the weak generational hypothesis: Most objects die young. @@ -372,76 +373,29 @@ programs as many temporary objects are created and destroyed very quickly. To take advantage of this fact, all container objects are segregated into -two generations: young and old. Every new object starts in the young generation. -Each garbage collection scans the entire young generation and part of the old generation. - -The time taken to scan the young generation can be controlled by controlling its -size, but the size of the old generation cannot be controlled. -In order to keep pause times down, scanning of the old generation of the heap -occurs in increments. - -To keep track of what has been scanned, the old generation contains two lists: - -* Those objects that have not yet been scanned, referred to as the `pending` list. -* Those objects that have been scanned, referred to as the `visited` list. - -To detect and collect all unreachable objects in the heap, the garbage collector -must scan the whole heap. This whole heap scan is called a full scavenge. - -Increments ----------- - -Each full scavenge is performed in a series of increments. -For each full scavenge, the combined increments will cover the whole heap. - -Each increment is made up of: - -* The young generation -* The old generation's least recently scanned objects -* All objects reachable from those objects that have not yet been scanned this full scavenge - -The surviving objects (those that are not collected) are moved to the back of the -`visited` list in the old generation. - -When a full scavenge starts, no objects in the heap are considered to have been scanned, -so all objects in the old generation must be in the `pending` space. -When all objects in the heap have been scanned a cycle ends, and all objects are moved -to the `pending` list again. To avoid having to traverse the entire list, which list is -`pending` and which is `visited` is determined by a field in the `GCState` struct. -The `visited` and `pending` lists can be swapped by toggling this bit. - -Correctness ------------ - -The [algorithm for identifying cycles](#Identifying-reference-cycles) will find all -unreachable cycles in a list of objects, but will not find any cycles that are -even partly outside of that list. -Therefore, to be guaranteed that a full scavenge will find all unreachable cycles, -each cycle must be fully contained within a single increment. - -To make sure that no partial cycles are included in the increment we perform a -[transitive closure](https://en.wikipedia.org/wiki/Transitive_closure) -over reachable, unscanned objects from the initial increment. -Since the transitive closure of objects reachable from an object must be a (non-strict) -superset of any unreachable cycle including that object, we are guaranteed that a -transitive closure cannot contain any partial cycles. -We can exclude scanned objects, as they must have been reachable when scanned. -If a scanned object becomes part of an unreachable cycle after being scanned, it will -not be collected at this time, but it will be collected in the next full scavenge. +three spaces/generations. Every new +object starts in the first generation (generation 0). The previous algorithm is +executed only over the objects of a particular generation and if an object +survives a collection of its generation it will be moved to the next one +(generation 1), where it will be surveyed for collection less often. If +the same object survives another GC round in this new generation (generation 1) +it will be moved to the last generation (generation 2) where it will be +surveyed the least often. > [!NOTE] -> The GC implementation for the free-threaded build does not use incremental collection. -> Every collection operates on the entire heap. +> The GC implementation for the free-threaded build does not use generational +> collection. Every collection operates on the entire heap. + In order to decide when to run, the collector keeps track of the number of object allocations and deallocations since the last collection. When the number of allocations minus the number of deallocations exceeds `threshold0`, -collection starts. `threshold1` determines the fraction of the old -collection that is included in the increment. -The fraction is inversely proportional to `threshold1`, -as historically a larger `threshold1` meant that old generation -collections were performed less frequently. -`threshold2` is ignored. +collection starts. Initially only generation 0 is examined. If generation 0 has +been examined more than `threshold_1` times since generation 1 has been +examined, then generation 1 is examined as well. With generation 2, +things are a bit more complicated; see +[Collecting the oldest generation](#Collecting-the-oldest-generation) for +more information. These thresholds can be examined using the [`gc.get_threshold()`](https://docs.python.org/3/library/gc.html#gc.get_threshold) @@ -450,7 +404,7 @@ ```pycon >>> import gc >>> gc.get_threshold() -(700, 10, 10) +(2000, 10, 10) ``` The content of these generations can be examined using the @@ -463,84 +417,61 @@ ... pass ... >>> # Move everything to the old generation so it's easier to inspect ->>> # the young generation. +>>> # the younger generation. >>> gc.collect() 0 >>> # Create a reference cycle. >>> x = MyObj() >>> x.self = x ->>> ->>> # Initially the object is in the young generation. +>>> +>>> # Initially the object is in the youngest generation. >>> gc.get_objects(generation=0) [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] ->>> +>>> >>> # After a collection of the youngest generation the object ->>> # moves to the old generation. +>>> # moves to the next generation. >>> gc.collect(generation=0) 0 >>> gc.get_objects(generation=0) [] >>> gc.get_objects(generation=1) -[] ->>> gc.get_objects(generation=2) [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] ``` +Collecting the oldest generation +-------------------------------- + +In addition to the various configurable thresholds, the GC only triggers a full +collection of the oldest generation if the ratio `long_lived_pending / long_lived_total` +is above a given value (hardwired to 25%). The reason is that, while "non-full" +collections (that is, collections of the young and middle generations) will always +examine roughly the same number of objects (determined by the aforementioned +thresholds) the cost of a full collection is proportional to the total +number of long-lived objects, which is virtually unbounded. Indeed, it has +been remarked that doing a full collection every of object +creations entails a dramatic performance degradation in workloads which consist +of creating and storing lots of long-lived objects (for example, building a large list +of GC-tracked objects would show quadratic performance, instead of linear as +expected). Using the above ratio, instead, yields amortized linear performance +in the total number of objects (the effect of which can be summarized thusly: +"each full garbage collection is more and more costly as the number of objects +grows, but we do fewer and fewer of them"). + Optimization: excluding reachable objects ========================================= An object cannot be garbage if it can be reached. To avoid having to identify -reference cycles across the whole heap, we can reduce the amount of work done -considerably by first identifying objects reachable from objects known to be -alive. These objects are excluded from the normal cyclic detection process. - -The default and free-threaded build both implement this optimization but in -slightly different ways. - -Finding reachable objects for the default build GC --------------------------------------------------- - -This works by first moving most reachable objects to the `visited` space. -Empirically, most reachable objects can be reached from a small set of global -objects and local variables. This step does much less work per object, so -reduces the time spent performing garbage collection by at least half. - -> [!NOTE] -> Objects that are not determined to be reachable by this pass are not necessarily -> unreachable. We still need to perform the main algorithm to determine which objects -> are actually unreachable. -We use the same technique of forming a transitive closure as the incremental -collector does to find reachable objects, seeding the list with some global -objects and the currently executing frames. - -This phase moves objects to the `visited` space, as follows: - -1. All objects directly referred to by any builtin class, the `sys` module, the `builtins` -module and all objects directly referred to from stack frames are added to a working -set of reachable objects. -2. Until this working set is empty: - 1. Pop an object from the set and move it to the `visited` space - 2. For each object directly reachable from that object: - * If it is not already in `visited` space and it is a GC object, - add it to the working set - - -Before each increment of collection is performed, the stacks are scanned -to check for any new stack frames that have been created since the last -increment. All objects directly referred to from those stack frames are -added to the working set. -Then the above algorithm is repeated, starting from step 2. - +reference cycles across the whole heap, the free-threaded build first identifies +objects reachable from objects known to be alive. These objects are excluded +from the normal cyclic detection process. Finding reachable objects for the free-threaded GC -------------------------------------------------- Within the `gc_free_threading.c` implementation, this is known as the "mark -alive" pass or phase. It is similar in concept to what is done for the default -build GC. Rather than moving objects between double-linked lists, the -free-threaded GC uses a flag in `ob_gc_bits` to track if an object is -found to be definitely alive (not garbage). +alive" pass or phase. The free-threaded GC uses a flag in `ob_gc_bits` to track +if an object is found to be definitely alive (not garbage). To find objects reachable from known alive objects, known as the "roots", the `gc_mark_alive_from_roots()` function is used. Root objects include @@ -771,6 +702,14 @@ already not tracked. Tuples are examined for untracking in all garbage collection cycles. +Dictionaries are always tracked from creation and are not untracked by the +garbage collector. Earlier versions (up to 3.13) used lazy tracking: empty or +atomic-only dicts were untracked on creation and re-tracked when a trackable +value was inserted (via `MAINTAIN_TRACKING`), and full collections called +`_PyDict_MaybeUntrack` to prune dicts whose values had become atomic. That +machinery was removed in 3.14 (GH-127010) because the per-set-item cost of +checking the tracking invariant outweighed the savings on full collections. + The garbage collector module provides the Python function `is_tracked(obj)`, which returns the current tracking status of the object. Subsequent garbage collections may change the tracking status of the object. diff --git a/InternalDocs/parser.md b/InternalDocs/parser.md index 1d0ffe6d40d..1bb4cdea543 100644 --- a/InternalDocs/parser.md +++ b/InternalDocs/parser.md @@ -819,6 +819,13 @@ $ python -m pegen python ``` +> [!CAUTION] +> Python's grammar (the `Grammar/python.gram` file) is written for the +> C backend. To experiment, you will need to write a grammar +> without C-specific parts like actions and the trailer. +> See [#133560](https://github.com/python/cpython/issues/133560) +> and [#96424](https://github.com/python/cpython/issues/96424) for more information. + This will generate a file called `parse.py` in the same directory that you can use to parse some input: diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 478f8189491..62806b1d8d7 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -1,3 +1,4 @@ +import builtins import os import sys @@ -188,6 +189,17 @@ class Argparse(ThemeSection): message: str = ANSIColors.MAGENTA +@dataclass(frozen=True, kw_only=True) +class Ast(ThemeSection): + node: str = ANSIColors.CYAN + field: str = ANSIColors.BLUE + attribute: str = ANSIColors.GREY + string: str = ANSIColors.GREEN + number: str = ANSIColors.YELLOW + keyword: str = ANSIColors.BOLD_BLUE + reset: str = ANSIColors.RESET + + @dataclass(frozen=True, kw_only=True) class Difflib(ThemeSection): """A 'git diff'-like theme for `difflib.unified_diff`.""" @@ -202,25 +214,25 @@ class Difflib(ThemeSection): @dataclass(frozen=True, kw_only=True) class FancyCompleter(ThemeSection): # functions and methods - function: str = ANSIColors.BOLD_BLUE - builtin_function_or_method: str = ANSIColors.BOLD_BLUE - method: str = ANSIColors.BOLD_CYAN - method_wrapper: str = ANSIColors.BOLD_CYAN - wrapper_descriptor: str = ANSIColors.BOLD_CYAN - method_descriptor: str = ANSIColors.BOLD_CYAN + function: builtins.str = ANSIColors.BOLD_BLUE + builtin_function_or_method: builtins.str = ANSIColors.BOLD_BLUE + method: builtins.str = ANSIColors.BOLD_CYAN + method_wrapper: builtins.str = ANSIColors.BOLD_CYAN + wrapper_descriptor: builtins.str = ANSIColors.BOLD_CYAN + method_descriptor: builtins.str = ANSIColors.BOLD_CYAN # numbers - int: str = ANSIColors.BOLD_YELLOW - float: str = ANSIColors.BOLD_YELLOW - complex: str = ANSIColors.BOLD_YELLOW - bool: str = ANSIColors.BOLD_YELLOW + int: builtins.str = ANSIColors.BOLD_YELLOW + float: builtins.str = ANSIColors.BOLD_YELLOW + complex: builtins.str = ANSIColors.BOLD_YELLOW + bool: builtins.str = ANSIColors.BOLD_YELLOW # others - type: str = ANSIColors.BOLD_MAGENTA - module: str = ANSIColors.CYAN - NoneType: str = ANSIColors.GREY - bytes: str = ANSIColors.BOLD_GREEN - str: str = ANSIColors.BOLD_GREEN + type: builtins.str = ANSIColors.BOLD_MAGENTA + module: builtins.str = ANSIColors.CYAN + NoneType: builtins.str = ANSIColors.GREY + bytes: builtins.str = ANSIColors.BOLD_GREEN + str: builtins.str = ANSIColors.BOLD_GREEN @dataclass(frozen=True, kw_only=True) @@ -347,6 +359,23 @@ class LiveProfiler(ThemeSection): ) +@dataclass(frozen=True, kw_only=True) +class Pickletools(ThemeSection): + annotation: str = ANSIColors.GREY + arg_number: str = ANSIColors.YELLOW + arg_string: str = ANSIColors.GREEN + mark: str = ANSIColors.GREY + op_call: str = ANSIColors.GREEN + op_container: str = ANSIColors.INTENSE_BLUE + op_memo: str = ANSIColors.MAGENTA + op_meta: str = ANSIColors.GREY + op_stack: str = ANSIColors.BOLD_RED + opcode_code: str = ANSIColors.CYAN + position: str = ANSIColors.GREY + proto: str = ANSIColors.YELLOW + reset: str = ANSIColors.RESET + + @dataclass(frozen=True, kw_only=True) class Syntax(ThemeSection): prompt: str = ANSIColors.BOLD_MAGENTA @@ -374,6 +403,14 @@ class Timeit(ThemeSection): reset: str = ANSIColors.RESET +@dataclass(frozen=True, kw_only=True) +class Tokenize(ThemeSection): + whitespace: str = ANSIColors.GREY + error: str = ANSIColors.BOLD_RED + position: str = ANSIColors.GREY + delimiter: str = ANSIColors.RESET + + @dataclass(frozen=True, kw_only=True) class Traceback(ThemeSection): type: str = ANSIColors.BOLD_MAGENTA @@ -404,12 +441,15 @@ class Theme: below. """ argparse: Argparse = field(default_factory=Argparse) + ast: Ast = field(default_factory=Ast) difflib: Difflib = field(default_factory=Difflib) fancycompleter: FancyCompleter = field(default_factory=FancyCompleter) http_server: HttpServer = field(default_factory=HttpServer) live_profiler: LiveProfiler = field(default_factory=LiveProfiler) + pickletools: Pickletools = field(default_factory=Pickletools) syntax: Syntax = field(default_factory=Syntax) timeit: Timeit = field(default_factory=Timeit) + tokenize: Tokenize = field(default_factory=Tokenize) traceback: Traceback = field(default_factory=Traceback) unittest: Unittest = field(default_factory=Unittest) @@ -417,12 +457,15 @@ def copy_with( self, *, argparse: Argparse | None = None, + ast: Ast | None = None, difflib: Difflib | None = None, fancycompleter: FancyCompleter | None = None, http_server: HttpServer | None = None, live_profiler: LiveProfiler | None = None, + pickletools: Pickletools | None = None, syntax: Syntax | None = None, timeit: Timeit | None = None, + tokenize: Tokenize | None = None, traceback: Traceback | None = None, unittest: Unittest | None = None, ) -> Self: @@ -433,12 +476,15 @@ def copy_with( """ return type(self)( argparse=argparse or self.argparse, + ast=ast or self.ast, difflib=difflib or self.difflib, fancycompleter=fancycompleter or self.fancycompleter, http_server=http_server or self.http_server, live_profiler=live_profiler or self.live_profiler, + pickletools=pickletools or self.pickletools, syntax=syntax or self.syntax, timeit=timeit or self.timeit, + tokenize=tokenize or self.tokenize, traceback=traceback or self.traceback, unittest=unittest or self.unittest, ) @@ -453,12 +499,15 @@ def no_colors(cls) -> Self: """ return cls( argparse=Argparse.no_colors(), + ast=Ast.no_colors(), difflib=Difflib.no_colors(), fancycompleter=FancyCompleter.no_colors(), http_server=HttpServer.no_colors(), live_profiler=LiveProfiler.no_colors(), + pickletools=Pickletools.no_colors(), syntax=Syntax.no_colors(), timeit=Timeit.no_colors(), + tokenize=Tokenize.no_colors(), traceback=Traceback.no_colors(), unittest=Unittest.no_colors(), ) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 9fee2564114..5c9a0812646 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -47,6 +47,7 @@ class Format(enum.IntEnum): "__cell__", "__owner__", "__stringifier_dict__", + "__resolved_str_cache__", ) @@ -94,6 +95,7 @@ def __init__( # value later. self.__code__ = None self.__ast_node__ = None + self.__resolved_str_cache__ = None def __init_subclass__(cls, /, *args, **kwds): raise TypeError("Cannot subclass ForwardRef") @@ -113,7 +115,7 @@ def evaluate( """ match format: case Format.STRING: - return self.__forward_arg__ + return self.__resolved_str__ case Format.VALUE: is_forwardref_format = False case Format.FORWARDREF: @@ -258,6 +260,24 @@ def __forward_arg__(self): "Attempted to access '__forward_arg__' on an uninitialized ForwardRef" ) + @property + def __resolved_str__(self): + # __forward_arg__ with any names from __extra_names__ replaced + # with the type_repr of the value they represent + if self.__resolved_str_cache__ is None: + resolved_str = self.__forward_arg__ + names = self.__extra_names__ + + if names: + visitor = _ExtraNameFixer(names) + ast_expr = ast.parse(resolved_str, mode="eval").body + node = visitor.visit(ast_expr) + resolved_str = ast.unparse(node) + + self.__resolved_str_cache__ = resolved_str + + return self.__resolved_str_cache__ + @property def __forward_code__(self): if self.__code__ is not None: @@ -321,7 +341,7 @@ def __repr__(self): extra.append(", is_class=True") if self.__owner__ is not None: extra.append(f", owner={self.__owner__!r}") - return f"ForwardRef({self.__forward_arg__!r}{''.join(extra)})" + return f"ForwardRef({self.__resolved_str__!r}{''.join(extra)})" _Template = type(t"") @@ -357,6 +377,7 @@ def __init__( self.__cell__ = cell self.__owner__ = owner self.__stringifier_dict__ = stringifier_dict + self.__resolved_str_cache__ = None # Needed for ForwardRef def __convert_to_ast(self, other): if isinstance(other, _Stringifier): @@ -1163,3 +1184,14 @@ def _get_dunder_annotations(obj): if not isinstance(ann, dict): raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") return ann + + +class _ExtraNameFixer(ast.NodeTransformer): + """Fixer for __extra_names__ items in ForwardRef __repr__ and string evaluation""" + def __init__(self, extra_names): + self.extra_names = extra_names + + def visit_Name(self, node: ast.Name): + if (new_name := self.extra_names.get(node.id, _sentinel)) is not _sentinel: + node = ast.Name(id=type_repr(new_name)) + return node diff --git a/Lib/ast.py b/Lib/ast.py index d9743ba7ab4..ba4ee0197b8 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -21,6 +21,7 @@ :license: Python License. """ from _ast import * +lazy from _colorize import can_colorize, get_theme def parse(source, filename='', mode='exec', *, @@ -117,21 +118,32 @@ def _convert_literal(node): def dump( node, annotate_fields=True, include_attributes=False, *, - indent=None, show_empty=False, + color=False, indent=None, show_empty=False, ): """ Return a formatted dump of the tree in node. This is mainly useful for - debugging purposes. If annotate_fields is true (by default), - the returned string will show the names and the values for fields. - If annotate_fields is false, the result string will be more compact by - omitting unambiguous field names. Attributes such as line - numbers and column offsets are not dumped by default. If this is wanted, - include_attributes can be set to true. If indent is a non-negative - integer or string, then the tree will be pretty-printed with that indent - level. None (the default) selects the single line representation. + debugging purposes. + + If annotate_fields is true (by default), the returned string will show the + names and the values for fields. If annotate_fields is false, the result + string will be more compact by omitting unambiguous field names. + + Attributes such as line numbers and column offsets are not dumped by default. + If this is wanted, include_attributes can be set to true. + + If color is true, the returned string is syntax highlighted using ANSI + escape sequences. If color is false (the default), colored output is always + disabled. + + If indent is a non-negative integer or string, then the tree will be + pretty-printed with that indent level. If indent is None (the default), + the tree is dumped on a single line. + If show_empty is False, then empty lists and fields that are None will be omitted from the output for better readability. """ + t = get_theme(force_color=color, force_no_color=not color).ast + def _format(node, level=0): if indent is not None: level += 1 @@ -166,7 +178,9 @@ def _format(node, level=0): field_type = cls._field_types.get(name, object) if field_type is expr_context: if not keywords: - args_buffer.append(repr(value)) + args_buffer.append( + f'{t.node}{type(value).__name__}' + f'{t.reset}()') continue if not keywords: args.extend(args_buffer) @@ -174,7 +188,7 @@ def _format(node, level=0): value, simple = _format(value, level) allsimple = allsimple and simple if keywords: - args.append('%s=%s' % (name, value)) + args.append(f'{t.field}{name}{t.reset}={value}') else: args.append(value) if include_attributes and node._attributes: @@ -187,14 +201,21 @@ def _format(node, level=0): continue value, simple = _format(value, level) allsimple = allsimple and simple - args.append('%s=%s' % (name, value)) + args.append(f'{t.attribute}{name}{t.reset}={value}') + cls_name = f'{t.node}{cls.__name__}{t.reset}' if allsimple and len(args) <= 3: - return '%s(%s)' % (node.__class__.__name__, ', '.join(args)), not args - return '%s(%s%s)' % (node.__class__.__name__, prefix, sep.join(args)), False + return f'{cls_name}({", ".join(args)})', not args + return f'{cls_name}({prefix}{sep.join(args)})', False elif isinstance(node, list): if not node: return '[]', True return '[%s%s]' % (prefix, sep.join(_format(x, level)[0] for x in node)), False + if isinstance(node, bool) or node is None or node is Ellipsis: + return f'{t.keyword}{node!r}{t.reset}', True + if isinstance(node, (int, float, complex)): + return f'{t.number}{node!r}{t.reset}', True + if isinstance(node, (str, bytes)): + return f'{t.string}{node!r}{t.reset}', True return repr(node), True if not isinstance(node, AST): @@ -642,7 +663,7 @@ def main(args=None): import argparse import sys - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('infile', nargs='?', default='-', help='the file to parse; defaults to stdin') parser.add_argument('-m', '--mode', default='exec', @@ -661,7 +682,7 @@ def main(args=None): '(for example, 3.10)') parser.add_argument('-O', '--optimize', type=int, default=-1, metavar='LEVEL', - help='optimization level for parser (default -1)') + help='optimization level for parser') parser.add_argument('--show-empty', default=False, action='store_true', help='show empty lists and fields in dump output') args = parser.parse_args(args) @@ -688,6 +709,7 @@ def main(args=None): tree = parse(source, name, args.mode, type_comments=args.no_type_comments, feature_version=feature_version, optimize=args.optimize) print(dump(tree, include_attributes=args.include_attributes, + color=can_colorize(file=sys.stdout), indent=args.indent, show_empty=args.show_empty)) if __name__ == '__main__': diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 8ee09b38469..37eba9657ac 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -101,11 +101,15 @@ def run(self): if not sys.flags.isolated and (startup_path := os.getenv("PYTHONSTARTUP")): sys.audit("cpython.run_startup", startup_path) - - import tokenize - with tokenize.open(startup_path) as f: - startup_code = compile(f.read(), startup_path, "exec") + try: + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") exec(startup_code, console.locals) + except SystemExit: + raise + except BaseException: + console.showtraceback() ps1 = getattr(sys, "ps1", ">>> ") if CAN_USE_PYREPL: diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 00e8f6d5d1a..45dfebc6590 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -37,6 +37,7 @@ def __init__(self): self._errors = [] self._base_error = None self._on_completed_fut = None + self._cancel_on_enter = False def __repr__(self): info = [''] @@ -63,6 +64,8 @@ async def __aenter__(self): raise RuntimeError( f'TaskGroup {self!r} cannot determine the parent task') self._entered = True + if self._cancel_on_enter: + self.cancel() return self @@ -178,6 +181,9 @@ async def _aexit(self, et, exc): finally: exc = None + # Suppress any remaining exception (exceptions deserving to be raised + # were raised above). + return True def create_task(self, coro, **kwargs): """Create a new task in this group and return it. @@ -278,3 +284,30 @@ def _on_task_done(self, task): self._abort() self._parent_cancel_requested = True self._parent_task.cancel() + + def cancel(self): + """Cancel the task group + + `cancel()` will be called on any tasks in the group that aren't yet + done, as well as the parent (body) of the group. This will cause the + task group context manager to exit *without* `asyncio.CancelledError` + being raised. + + If `cancel()` is called before entering the task group, the group will be + cancelled upon entry. This is useful for patterns where one piece of + code passes an unused TaskGroup instance to another in order to have + the ability to cancel anything run within the group. + + `cancel()` is idempotent and may be called after the task group has + already exited. + """ + if not self._entered: + self._cancel_on_enter = True + return + if self._exiting and not self._tasks: + return + if not self._aborting: + self._abort() + if self._parent_task and not self._parent_cancel_requested: + self._parent_cancel_requested = True + self._parent_task.cancel() diff --git a/Lib/base64.py b/Lib/base64.py index 7f39c68070b..4b810e08569 100644 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -68,7 +68,7 @@ def b64encode(s, altchars=None, *, padded=True, wrapcol=0): def b64decode(s, altchars=None, validate=_NOT_SPECIFIED, - *, padded=True, ignorechars=_NOT_SPECIFIED): + *, padded=True, ignorechars=_NOT_SPECIFIED, canonical=False): """Decode the Base64 encoded bytes-like object or ASCII string s. Optional altchars must be a bytes-like object or ASCII string of length 2 @@ -110,11 +110,13 @@ def b64decode(s, altchars=None, validate=_NOT_SPECIFIED, alphabet = binascii.BASE64_ALPHABET[:-2] + altchars return binascii.a2b_base64(s, strict_mode=validate, alphabet=alphabet, - padded=padded, ignorechars=ignorechars) + padded=padded, ignorechars=ignorechars, + canonical=canonical) if ignorechars is _NOT_SPECIFIED: ignorechars = b'' result = binascii.a2b_base64(s, strict_mode=validate, - padded=padded, ignorechars=ignorechars) + padded=padded, ignorechars=ignorechars, + canonical=canonical) if badchar is not None: import warnings if validate: @@ -230,7 +232,8 @@ def b32encode(s, *, padded=True, wrapcol=0): return binascii.b2a_base32(s, padded=padded, wrapcol=wrapcol) b32encode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32') -def b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b''): +def b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b'', + canonical=False): s = _bytes_from_decode_data(s) # Handle section 2.4 zero and one mapping. The flag map01 will be either # False, or the character to map the digit 1 (one) to. It should be @@ -240,7 +243,8 @@ def b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b''): s = s.translate(bytes.maketrans(b'01', b'O' + map01)) if casefold: s = s.upper() - return binascii.a2b_base32(s, padded=padded, ignorechars=ignorechars) + return binascii.a2b_base32(s, padded=padded, ignorechars=ignorechars, + canonical=canonical) b32decode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32', extra_args=_B32_DECODE_MAP01_DOCSTRING) @@ -249,13 +253,15 @@ def b32hexencode(s, *, padded=True, wrapcol=0): alphabet=binascii.BASE32HEX_ALPHABET) b32hexencode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32hex') -def b32hexdecode(s, casefold=False, *, padded=True, ignorechars=b''): +def b32hexdecode(s, casefold=False, *, padded=True, ignorechars=b'', + canonical=False): s = _bytes_from_decode_data(s) # base32hex does not have the 01 mapping if casefold: s = s.upper() return binascii.a2b_base32(s, alphabet=binascii.BASE32HEX_ALPHABET, - padded=padded, ignorechars=ignorechars) + padded=padded, ignorechars=ignorechars, + canonical=canonical) b32hexdecode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32hex', extra_args='') @@ -323,7 +329,8 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): return binascii.b2a_ascii85(b, foldspaces=foldspaces, adobe=adobe, wrapcol=wrapcol, pad=pad) -def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'): +def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v', + canonical=False): """Decode the Ascii85 encoded bytes-like object or ASCII string b. foldspaces is a flag that specifies whether the 'y' short sequence should be @@ -337,10 +344,13 @@ def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'): input. This should only contain whitespace characters, and by default contains all whitespace characters in ASCII. + If canonical is true, non-canonical encodings are rejected. + The result is returned as a bytes object. """ return binascii.a2b_ascii85(b, foldspaces=foldspaces, - adobe=adobe, ignorechars=ignorechars) + adobe=adobe, ignorechars=ignorechars, + canonical=canonical) def b85encode(b, pad=False, *, wrapcol=0): """Encode bytes-like object b in base85 format and return a bytes object. @@ -353,12 +363,15 @@ def b85encode(b, pad=False, *, wrapcol=0): """ return binascii.b2a_base85(b, wrapcol=wrapcol, pad=pad) -def b85decode(b, *, ignorechars=b''): +def b85decode(b, *, ignorechars=b'', canonical=False): """Decode the base85-encoded bytes-like object or ASCII string b + If canonical is true, non-canonical encodings are rejected. + The result is returned as a bytes object. """ - return binascii.a2b_base85(b, ignorechars=ignorechars) + return binascii.a2b_base85(b, ignorechars=ignorechars, + canonical=canonical) def z85encode(s, pad=False, *, wrapcol=0): """Encode bytes-like object b in z85 format and return a bytes object. @@ -372,12 +385,15 @@ def z85encode(s, pad=False, *, wrapcol=0): return binascii.b2a_base85(s, wrapcol=wrapcol, pad=pad, alphabet=binascii.Z85_ALPHABET) -def z85decode(s, *, ignorechars=b''): +def z85decode(s, *, ignorechars=b'', canonical=False): """Decode the z85-encoded bytes-like object or ASCII string b + If canonical is true, non-canonical encodings are rejected. + The result is returned as a bytes object. """ - return binascii.a2b_base85(s, alphabet=binascii.Z85_ALPHABET, ignorechars=ignorechars) + return binascii.a2b_base85(s, alphabet=binascii.Z85_ALPHABET, + ignorechars=ignorechars, canonical=canonical) # Legacy interface. This code could be cleaned up since I don't believe # binascii has any line length limitations. It just doesn't seem worth it diff --git a/Lib/contextlib.py b/Lib/contextlib.py index cac3e39eba8..efc02bfa924 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -1,10 +1,16 @@ """Utilities for with-statement contexts. See PEP 343.""" + import abc import os import sys import _collections_abc from collections import deque from functools import wraps +lazy from inspect import ( + isasyncgenfunction as _isasyncgenfunction, + iscoroutinefunction as _iscoroutinefunction, + isgeneratorfunction as _isgeneratorfunction, +) from types import GenericAlias __all__ = ["asynccontextmanager", "contextmanager", "closing", "nullcontext", @@ -79,11 +85,37 @@ def _recreate_cm(self): return self def __call__(self, func): - @wraps(func) - def inner(*args, **kwds): - with self._recreate_cm(): - return func(*args, **kwds) - return inner + wrapper = wraps(func) + if _isasyncgenfunction(func): + + async def asyncgen_inner(*args, **kwds): + with self._recreate_cm(): + async with aclosing(func(*args, **kwds)) as gen: + async for value in gen: + yield value + + return wrapper(asyncgen_inner) + elif _iscoroutinefunction(func): + + async def async_inner(*args, **kwds): + with self._recreate_cm(): + return await func(*args, **kwds) + + return wrapper(async_inner) + elif _isgeneratorfunction(func): + + def gen_inner(*args, **kwds): + with self._recreate_cm(), closing(func(*args, **kwds)) as gen: + return (yield from gen) + + return wrapper(gen_inner) + else: + + def inner(*args, **kwds): + with self._recreate_cm(): + return func(*args, **kwds) + + return wrapper(inner) class AsyncContextDecorator(object): @@ -95,11 +127,41 @@ def _recreate_cm(self): return self def __call__(self, func): - @wraps(func) - async def inner(*args, **kwds): - async with self._recreate_cm(): - return await func(*args, **kwds) - return inner + wrapper = wraps(func) + if _isasyncgenfunction(func): + + async def asyncgen_inner(*args, **kwds): + async with ( + self._recreate_cm(), + aclosing(func(*args, **kwds)) as gen + ): + async for value in gen: + yield value + + return wrapper(asyncgen_inner) + elif _iscoroutinefunction(func): + + async def async_inner(*args, **kwds): + async with self._recreate_cm(): + return await func(*args, **kwds) + + return wrapper(async_inner) + elif _isgeneratorfunction(func): + + async def gen_inner(*args, **kwds): + async with self._recreate_cm(): + with closing(func(*args, **kwds)) as gen: + for value in gen: + yield value + + return wrapper(gen_inner) + else: + + async def inner(*args, **kwds): + async with self._recreate_cm(): + return func(*args, **kwds) + + return wrapper(inner) class _GeneratorContextManagerBase: diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 0c7e01cb16b..e9810d6bd5d 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1,12 +1,12 @@ import sys import types -import inspect import keyword import itertools import annotationlib import abc from reprlib import recursive_repr lazy import copy +lazy import inspect lazy import re @@ -178,17 +178,12 @@ def __repr__(self): return '' _HAS_DEFAULT_FACTORY = _HAS_DEFAULT_FACTORY_CLASS() -# A sentinel object to detect if a parameter is supplied or not. Use -# a class to give it a better repr. -class _MISSING_TYPE: - pass -MISSING = _MISSING_TYPE() +# A sentinel object to detect if a parameter is supplied or not. +MISSING = sentinel("MISSING") # A sentinel object to indicate that following fields are keyword-only by -# default. Use a class to give it a better repr. -class _KW_ONLY_TYPE: - pass -KW_ONLY = _KW_ONLY_TYPE() +# default. +KW_ONLY = sentinel("KW_ONLY") # Since most per-field metadata will be unused, create an empty # read-only dictionary that can be shared among all fields. @@ -988,6 +983,28 @@ def _hash_exception(cls, fields, func_builder): # See https://bugs.python.org/issue32929#msg312829 for an if-statement # version of this table. +# A non-data descriptor to autogenerate class docstring +# from the signature of its __init__ method on demand. +# The primary reason is to be able to lazy import `inspect` module. +class _AutoDocstring: + + def __get__(self, _obj, cls): + try: + # In some cases fetching a signature is not possible. + # But, we surely should not fail in this case. + text_sig = str(inspect.signature( + cls, + annotation_format=annotationlib.Format.FORWARDREF, + )).replace(' -> None', '') + except TypeError, ValueError: + text_sig = '' + + doc = cls.__name__ + text_sig + setattr(cls, '__doc__', doc) + return doc + +_auto_docstring = _AutoDocstring() + def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, match_args, kw_only, slots, weakref_slot): @@ -1215,23 +1232,13 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, if hash_action: cls.__hash__ = hash_action(cls, field_list, func_builder) - # Generate the methods and add them to the class. This needs to be done - # before the __doc__ logic below, since inspect will look at the __init__ - # signature. + # Generate the methods and add them to the class. func_builder.add_fns_to_class(cls) if not getattr(cls, '__doc__'): - # Create a class doc-string. - try: - # In some cases fetching a signature is not possible. - # But, we surely should not fail in this case. - text_sig = str(inspect.signature( - cls, - annotation_format=annotationlib.Format.FORWARDREF, - )).replace(' -> None', '') - except (TypeError, ValueError): - text_sig = '' - cls.__doc__ = (cls.__name__ + text_sig) + # Create a class doc-string lazily via descriptor protocol + # to avoid importing `inspect` module. + cls.__doc__ = _auto_docstring if match_args: # I could probably compute this once. @@ -1298,10 +1305,18 @@ def _update_func_cell_for__class__(f, oldcls, newcls): # This function doesn't reference __class__, so nothing to do. return False # Fix the cell to point to the new class, if it's already pointing - # at the old class. I'm not convinced that the "is oldcls" test - # is needed, but other than performance can't hurt. + # at the old class. closure = f.__closure__[idx] - if closure.cell_contents is oldcls: + + try: + contents = closure.cell_contents + except ValueError: + # Cell is empty + return False + + # This check makes it so we avoid updating an incorrect cell if the + # class body contains a function that was defined in a different class. + if contents is oldcls: closure.cell_contents = newcls return True return False @@ -1383,8 +1398,10 @@ def _add_slots(cls, is_frozen, weakref_slot, defined_fields): # make an update, since all closures for a class will share a # given cell. for member in newcls.__dict__.values(): + # If this is a wrapped function, unwrap it. - member = inspect.unwrap(member) + if not isinstance(member, type) and hasattr(member, '__wrapped__'): + member = inspect.unwrap(member) if isinstance(member, types.FunctionType): if _update_func_cell_for__class__(member, cls, newcls): @@ -1496,7 +1513,8 @@ class C: If given, 'dict_factory' will be used instead of built-in dict. The function applies recursively to field values that are dataclass instances. This will also look into built-in containers: - tuples, lists, and dicts. Other objects are copied with 'copy.deepcopy()'. + tuples, lists, dicts, and frozendicts. Other objects are copied + with 'copy.deepcopy()'. """ if not _is_dataclass_instance(obj): raise TypeError("asdict() should be called on dataclass instances") @@ -1552,7 +1570,7 @@ def _asdict_inner(obj, dict_factory): return obj_type(*[_asdict_inner(v, dict_factory) for v in obj]) else: return obj_type(_asdict_inner(v, dict_factory) for v in obj) - elif issubclass(obj_type, dict): + elif issubclass(obj_type, (dict, frozendict)): if hasattr(obj_type, 'default_factory'): # obj is a defaultdict, which has a different constructor from # dict as it requires the default_factory as its first arg. @@ -1587,7 +1605,8 @@ class C: If given, 'tuple_factory' will be used instead of built-in tuple. The function applies recursively to field values that are dataclass instances. This will also look into built-in containers: - tuples, lists, and dicts. Other objects are copied with 'copy.deepcopy()'. + tuples, lists, dicts, and frozendicts. Other objects are copied + with 'copy.deepcopy()'. """ if not _is_dataclass_instance(obj): @@ -1616,7 +1635,7 @@ def _astuple_inner(obj, tuple_factory): # generator (which is not true for namedtuples, handled # above). return type(obj)(_astuple_inner(v, tuple_factory) for v in obj) - elif isinstance(obj, dict): + elif isinstance(obj, (dict, frozendict)): obj_type = type(obj) if hasattr(obj_type, 'default_factory'): # obj is a defaultdict, which has a different constructor from diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 4c5394ab635..9873958f5c2 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -157,10 +157,7 @@ def all_defects(self): def startswith_fws(self): return self[0].startswith_fws() - @property - def as_ew_allowed(self): - """True if all top level tokens of this part may be RFC2047 encoded.""" - return all(part.as_ew_allowed for part in self) + as_ew_allowed = True @property def comments(self): @@ -429,6 +426,7 @@ def addr_spec(self): class AngleAddr(TokenList): token_type = 'angle-addr' + as_ew_allowed = False @property def local_part(self): @@ -639,11 +637,11 @@ def local_part(self): for tok in self[0] + [DOT]: if tok.token_type == 'cfws': continue - if (last_is_tl and tok.token_type == 'dot' and + if (last_is_tl and tok.token_type == 'dot' and last and last[-1].token_type == 'cfws'): res[-1] = TokenList(last[:-1]) is_tl = isinstance(tok, TokenList) - if (is_tl and last.token_type == 'dot' and + if (is_tl and last.token_type == 'dot' and tok and tok[0].token_type == 'cfws'): res.append(TokenList(tok[1:])) else: @@ -847,26 +845,22 @@ def params(self): class ContentType(ParameterizedHeaderValue): token_type = 'content-type' - as_ew_allowed = False maintype = 'text' subtype = 'plain' class ContentDisposition(ParameterizedHeaderValue): token_type = 'content-disposition' - as_ew_allowed = False content_disposition = None class ContentTransferEncoding(TokenList): token_type = 'content-transfer-encoding' - as_ew_allowed = False cte = '7bit' class HeaderLabel(TokenList): token_type = 'header-label' - as_ew_allowed = False class MsgID(TokenList): @@ -1249,8 +1243,7 @@ def get_bare_quoted_string(value): bare_quoted_string = BareQuotedString() value = value[1:] if value and value[0] == '"': - token, value = get_qcontent(value) - bare_quoted_string.append(token) + return bare_quoted_string, value[1:] while value and value[0] != '"': if value[0] in WSP: token, value = get_fws(value) @@ -1510,11 +1503,6 @@ def get_local_part(value): local_part.defects.append(errors.ObsoleteHeaderDefect( "local-part is not a dot-atom (contains CFWS)")) local_part[0] = obs_local_part - try: - local_part.value.encode('ascii') - except UnicodeEncodeError: - local_part.defects.append(errors.NonASCIILocalPartDefect( - "local-part contains non-ASCII characters)")) return local_part, value def get_obs_local_part(value): @@ -2063,12 +2051,10 @@ def get_address_list(value): address_list.defects.append(errors.InvalidHeaderDefect( "invalid address in address-list")) if value and value[0] != ',': - # Crap after address; treat it as an invalid mailbox. - # The mailbox info will still be available. - mailbox = address_list[-1][0] - mailbox.token_type = 'invalid-mailbox' + # Crap after address: add it to the address list + # as an invalid mailbox token, value = get_invalid_mailbox(value, ',') - mailbox.extend(token) + address_list.append(Address([token])) address_list.defects.append(errors.InvalidHeaderDefect( "invalid address in address-list")) if value: # Must be a , at this point. @@ -2838,13 +2824,68 @@ def _steal_trailing_WSP_if_exists(lines): def _refold_parse_tree(parse_tree, *, policy): - """Return string of contents of parse_tree folded according to RFC rules. - - """ # max_line_length 0/None means no limit, ie: infinitely long. maxlen = policy.max_line_length or sys.maxsize encoding = 'utf-8' if policy.utf8 else 'us-ascii' lines = [''] # Folded lines to be output + if parse_tree.as_ew_allowed: + _refold_with_ew(parse_tree, lines, maxlen, encoding, policy=policy) + else: + _refold_without_ew(parse_tree, lines, maxlen, encoding, policy=policy) + return policy.linesep.join(lines) + policy.linesep + +def _refold_without_ew(parse_tree, lines, maxlen, encoding, *, policy): + parts = list(parse_tree) + while parts: + part = parts.pop(0) + tstr = str(part) + try: + tstr.encode(encoding) + except UnicodeEncodeError: + if any(isinstance(x, errors.UndecodableBytesDefect) + for x in part.all_defects): + # There is garbage data from parsing a message in binary mode, + # just pass it through. Not good, but the best we can do. + pass + elif policy.utf8: + # If this happens, it's a programmer error. + raise + else: + raise errors.HeaderWriteError( + f"Non-ASCII {part.token_type} '{part}' is invalid" + " under current policy setting (utf8=False)" + ) + if len(tstr) <= maxlen - len(lines[-1]): + lines[-1] += tstr + continue + # This part is too long to fit. The RFC wants us to break at + # "major syntactic breaks", so unless we don't consider this + # to be one, check if it will fit on the next line by itself. + if (part.syntactic_break and + len(tstr) + 1 <= maxlen): + newline = _steal_trailing_WSP_if_exists(lines) + if newline or part.startswith_fws(): + lines.append(newline + tstr) + continue + if not hasattr(part, 'encode'): + # It's not a terminal, try folding the subparts. + newparts = list(part) + parts = newparts + parts + continue + # We can't figure out how to wrap, it, so give up. + newline = _steal_trailing_WSP_if_exists(lines) + if newline or part.startswith_fws(): + lines.append(newline + tstr) + else: + # We can't fold it onto the next line either... + lines[-1] += tstr + return + + +def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy): + """Return string of contents of parse_tree folded according to RFC rules. + + """ last_word_is_ew = False last_ew = None # if there is an encoded word in the last line of lines, # points to the encoded word's first character @@ -2858,6 +2899,11 @@ def _refold_parse_tree(parse_tree, *, policy): if part is end_ew_not_allowed: wrap_as_ew_blocked -= 1 continue + if part.token_type == 'mime-parameters': + # Mime parameter folding (using RFC2231) is extra special. + _fold_mime_parameters(part, lines, maxlen, encoding) + last_word_is_ew = False + continue tstr = str(part) if not want_encoding: if part.token_type in ('ptext', 'vtext'): @@ -2879,14 +2925,11 @@ def _refold_parse_tree(parse_tree, *, policy): charset = 'utf-8' want_encoding = True - if part.token_type == 'mime-parameters': - # Mime parameter folding (using RFC2231) is extra special. - _fold_mime_parameters(part, lines, maxlen, encoding) - last_word_is_ew = False - continue - if want_encoding and not wrap_as_ew_blocked: - if not part.as_ew_allowed: + if any( + not x.as_ew_allowed for x in part + if hasattr(x, 'as_ew_allowed') + ): want_encoding = False last_ew = None if part.syntactic_break: @@ -2967,6 +3010,8 @@ def _refold_parse_tree(parse_tree, *, policy): [ValueTerminal(make_quoted_pairs(p), 'ptext') for p in newparts] + [ValueTerminal('"', 'ptext')]) + _refold_without_ew(newparts, lines, maxlen, encoding, policy=policy) + continue if part.token_type == 'comment': newparts = ( [ValueTerminal('(', 'ptext')] + @@ -2994,7 +3039,7 @@ def _refold_parse_tree(parse_tree, *, policy): lines[-1] += tstr last_word_is_ew = last_word_is_ew and not bool(tstr.strip(_WSP)) - return policy.linesep.join(lines) + policy.linesep + return def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset, last_word_is_ew): """Fold string to_encode into lines as encoded word, combining if allowed. diff --git a/Lib/email/errors.py b/Lib/email/errors.py index 6bc744bd59c..859307dd85b 100644 --- a/Lib/email/errors.py +++ b/Lib/email/errors.py @@ -109,9 +109,9 @@ class ObsoleteHeaderDefect(HeaderDefect): """Header uses syntax declared obsolete by RFC 5322""" class NonASCIILocalPartDefect(HeaderDefect): - """local_part contains non-ASCII characters""" - # This defect only occurs during unicode parsing, not when - # parsing messages decoded from binary. + """Unused. Note: this error is deprecated and may be removed in the future.""" + # RFC 6532 permits a non-ASCII local-part. _header_value_parser previously + # treated this as a parse-time defect (when parsing Unicode, but not bytes). class InvalidDateDefect(HeaderDefect): """Header has unparsable or invalid date""" diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 93b4e7a820f..5a55525d6bd 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -10,7 +10,7 @@ __all__ = ["version", "bootstrap"] -_PIP_VERSION = "26.0.1" +_PIP_VERSION = "26.1" # Directory of system wheel packages. Some Linux distribution packaging # policies recommend against bundling dependencies. For example, Fedora diff --git a/Lib/ensurepip/_bundled/pip-26.0.1-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-26.1-py3-none-any.whl similarity index 73% rename from Lib/ensurepip/_bundled/pip-26.0.1-py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-26.1-py3-none-any.whl index 580d09a9204..b51afa14f7c 100644 Binary files a/Lib/ensurepip/_bundled/pip-26.0.1-py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-26.1-py3-none-any.whl differ diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index 68cf16c93cc..13e5b104a81 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -1032,10 +1032,13 @@ def set_ok_domain(self, cookie, request): if j == 0: # domain like .foo.bar tld = domain[i+1:] sld = domain[j+1:i] - if sld.lower() in ("co", "ac", "com", "edu", "org", "net", - "gov", "mil", "int", "aero", "biz", "cat", "coop", - "info", "jobs", "mobi", "museum", "name", "pro", - "travel", "eu") and len(tld) == 2: + known_slds = ( + "co", "ac", "com", "edu", "org", "net", + "gov", "mil", "int", "aero", "biz", "cat", "coop", + "info", "jobs", "mobi", "museum", "name", "pro", + "travel", "eu", "tv", "or", "nom", "sch", "web", + ) + if sld.lower() in known_slds and len(tld) == 2: # domain like .co.uk _debug(" country-code second level domain %s", domain) return False diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index 76954111699..5c5b14788dc 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -391,17 +391,21 @@ def __repr__(self): return '<%s: %s>' % (self.__class__.__name__, self.OutputString()) def js_output(self, attrs=None): + import base64 # Print javascript output_string = self.OutputString(attrs) if _has_control_character(output_string): raise CookieError("Control characters are not allowed in cookies") + # Base64-encode value to avoid template + # injection in cookie values. + output_encoded = base64.b64encode(output_string.encode('utf-8')).decode("ascii") return """ - """ % (output_string.replace('"', r'\"')) + """ % (output_encoded,) def OutputString(self, attrs=None): # Build up our result @@ -458,7 +462,7 @@ def OutputString(self, attrs=None): ( # Optional group: there may not be a value. \s*=\s* # Equal Sign (?P # Start of group 'val' - "(?:\\"|.)*?" # Any double-quoted string + "(?:[^\\"]|\\.)*" # Any double-quoted string | # or # Special case for "expires" attr (\w{3,6}day|\w{3}),\s # Day of the week or abbreviated day diff --git a/Lib/http/server.py b/Lib/http/server.py index 568d3bb38de..27ab37303a0 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -727,6 +727,7 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): """ server_version = "SimpleHTTP" + default_content_type = "application/octet-stream" index_pages = ("index.html", "index.htm") extensions_map = _encodings_map_default = { '.gz': 'application/gzip', @@ -974,7 +975,7 @@ def guess_type(self, path): guess, _ = mimetypes.guess_file_type(path) if guess: return guess - return 'application/octet-stream' + return self.default_content_type nobody = None @@ -1010,9 +1011,10 @@ def _get_best_family(*address): return family, sockaddr -def test(HandlerClass=BaseHTTPRequestHandler, +def test(HandlerClass=SimpleHTTPRequestHandler, ServerClass=ThreadingHTTPServer, protocol="HTTP/1.0", port=8000, bind=None, + content_type=SimpleHTTPRequestHandler.default_content_type, tls_cert=None, tls_key=None, tls_password=None): """Test the HTTP request handler class. @@ -1021,6 +1023,7 @@ def test(HandlerClass=BaseHTTPRequestHandler, """ ServerClass.address_family, addr = _get_best_family(bind, port) HandlerClass.protocol_version = protocol + HandlerClass.default_content_type = content_type if tls_cert: server = ServerClass(addr, HandlerClass, certfile=tls_cert, @@ -1060,6 +1063,10 @@ def _main(args=None): default='HTTP/1.0', help='conform to this HTTP version ' '(default: %(default)s)') + parser.add_argument('--content-type', + default=SimpleHTTPRequestHandler.default_content_type, + help='default content type for unknown extensions ' + '(default: %(default)s)') parser.add_argument('--tls-cert', metavar='PATH', help='path to the TLS certificate chain file') parser.add_argument('--tls-key', metavar='PATH', @@ -1112,6 +1119,7 @@ class HTTPSDualStackServer(DualStackServerMixin, ThreadingHTTPSServer): port=args.port, bind=args.bind, protocol=args.protocol, + content_type=args.content_type, tls_cert=args.tls_cert, tls_key=args.tls_key, tls_password=tls_key_password, diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 45beb51659f..06dc45d71d7 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -424,6 +424,64 @@ def __exit__(self, *args, **kwargs): self._lock.release() +def _get_module_chain(name): + """Return the chain of dotted-name prefixes from root to leaf. + + For example: 'a.b.c' -> ['a', 'a.b', 'a.b.c'] + """ + parts = name.split('.') + return ['.'.join(parts[:i+1]) for i in range(len(parts))] + + +class _HierarchicalLockManager: + """Manages acquisition of multiple module locks in hierarchical order. + + This prevents deadlocks by ensuring all threads acquire locks in the + same order (parent modules before child modules). + """ + + def __init__(self, name): + self._name = name + self._module_chain = _get_module_chain(name) + self._locks = [] + + def __enter__(self): + try: + for module_name in self._module_chain: + # Only acquire lock if module is not already fully loaded + module = sys.modules.get(module_name) + if (module is None or + getattr(getattr(module, "__spec__", None), + "_initializing", False)): + lock = _get_module_lock(module_name) + try: + lock.acquire() + except _DeadlockError: + if module_name == self._name: + raise + # The parent is being initialised by a thread that + # is (transitively) waiting on a lock we hold. + # Apply the same policy as _lock_unlock_module(): + # accept a partially-initialised parent for circular + # imports rather than failing the whole chain. + continue + self._locks.append((module_name, lock)) + except: + # __exit__ is not called when __enter__ raises (e.g. _DeadlockError + # on the leaf lock, or KeyboardInterrupt), so release whatever we + # already hold to avoid permanently leaking held module locks. + for module_name, lock in reversed(self._locks): + lock.release() + self._locks.clear() + raise + return self + + def __exit__(self, *args, **kwargs): + for module_name, lock in reversed(self._locks): + lock.release() + self._locks.clear() + + # The following two functions are for consumption by Python/import.c. def _get_module_lock(name): @@ -1276,7 +1334,13 @@ def _find_and_load(name, import_): module = sys.modules.get(name, _NEEDS_LOADING) if (module is _NEEDS_LOADING or getattr(getattr(module, "__spec__", None), "_initializing", False)): - with _ModuleLockManager(name): + + if '.' in name: + lock_manager = _HierarchicalLockManager(name) + else: + lock_manager = _ModuleLockManager(name) + + with lock_manager: module = sys.modules.get(name, _NEEDS_LOADING) if module is _NEEDS_LOADING: return _find_and_load_unlocked(name, import_) diff --git a/Lib/inspect.py b/Lib/inspect.py index dfc5503dee5..d3af61b26e2 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -416,7 +416,6 @@ def iscode(object): co_freevars tuple of names of free variables co_posonlyargcount number of positional only arguments co_kwonlyargcount number of keyword only arguments (not including ** arg) - co_lnotab encoded mapping of line numbers to bytecode indices co_name name with which this code object was defined co_names tuple of names other than arguments and function locals co_nlocals number of local variables @@ -1634,7 +1633,6 @@ def getframeinfo(frame, context=1): def getlineno(frame): """Get the line number from a frame object, allowing for optimization.""" - # FrameType.f_lineno is now a descriptor that grovels co_lnotab return frame.f_lineno _FrameInfo = namedtuple('_FrameInfo', ('frame',) + Traceback._fields) diff --git a/Lib/json/tool.py b/Lib/json/tool.py index e0b944b197d..e56a601c581 100644 --- a/Lib/json/tool.py +++ b/Lib/json/tool.py @@ -89,7 +89,8 @@ def main(): infile = open(options.infile, encoding='utf-8') try: if options.json_lines: - objs = (json.loads(line) for line in infile) + lines = infile.readlines() + objs = (json.loads(line) for line in lines) else: objs = (json.load(infile),) finally: diff --git a/Lib/locale.py b/Lib/locale.py index e7382796905..4ff6f8c0f0a 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -1505,8 +1505,8 @@ def getpreferredencoding(do_setlocale=True): # This maps Windows language identifiers to locale strings. # # This list has been updated from -# http://msdn.microsoft.com/library/default.asp?url=/library/en-us/intl/nls_238z.asp -# to include every locale up to Windows Vista. +# https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-lcid/70feba9f-294e-491e-b6eb-56532684c37f +# to include every locale up to protocol revision 16.0 (2024-04-23). # # NOTE: this mapping is incomplete. If your language is missing, please # submit a bug report as detailed in the Python devguide at: @@ -1516,10 +1516,15 @@ def getpreferredencoding(do_setlocale=True): # windows_locale = { - 0x0436: "af_ZA", # Afrikaans - 0x041c: "sq_AL", # Albanian - 0x0484: "gsw_FR",# Alsatian - France + 0x0036: "af", # Afrikaans + 0x0436: "af_ZA", # Afrikaans - South Africa + 0x001c: "sq", # Albanian + 0x041c: "sq_AL", # Albanian - Albania + 0x0084: "gsw", # Alsatian + 0x0484: "gsw_FR", # Alsatian - France + 0x005e: "am", # Amharic 0x045e: "am_ET", # Amharic - Ethiopia + 0x0001: "ar", # Arabic 0x0401: "ar_SA", # Arabic - Saudi Arabia 0x0801: "ar_IQ", # Arabic - Iraq 0x0c01: "ar_EG", # Arabic - Egypt @@ -1533,39 +1538,72 @@ def getpreferredencoding(do_setlocale=True): 0x2c01: "ar_JO", # Arabic - Jordan 0x3001: "ar_LB", # Arabic - Lebanon 0x3401: "ar_KW", # Arabic - Kuwait - 0x3801: "ar_AE", # Arabic - United Arab Emirates + 0x3801: "ar_AE", # Arabic - U.A.E. 0x3c01: "ar_BH", # Arabic - Bahrain 0x4001: "ar_QA", # Arabic - Qatar - 0x042b: "hy_AM", # Armenian + 0x002b: "hy", # Armenian + 0x042b: "hy_AM", # Armenian - Armenia + 0x004d: "as", # Assamese 0x044d: "as_IN", # Assamese - India - 0x042c: "az_AZ", # Azeri - Latin - 0x082c: "az_AZ", # Azeri - Cyrillic - 0x046d: "ba_RU", # Bashkir - 0x042d: "eu_ES", # Basque - Russia - 0x0423: "be_BY", # Belarusian - 0x0445: "bn_IN", # Begali - 0x201a: "bs_BA", # Bosnian - Cyrillic - 0x141a: "bs_BA", # Bosnian - Latin + 0x002c: "az", # Azerbaijani (Latin) + 0x742c: "az", # Azerbaijani (Cyrillic) + 0x782c: "az", # Azerbaijani (Latin) + 0x042c: "az_AZ", # Azerbaijani (Latin) - Azerbaijan + 0x0045: "bn", # Bangla + 0x0445: "bn_IN", # Bangla - India + 0x0845: "bn_BD", # Bangla - Bangladesh + 0x006d: "ba", # Bashkir + 0x046d: "ba_RU", # Bashkir - Russia + 0x002d: "eu", # Basque + 0x042d: "eu_ES", # Basque - Spain + 0x0023: "be", # Belarusian + 0x0423: "be_BY", # Belarusian - Belarus + 0x641a: "bs", # Bosnian (Cyrillic) + 0x681a: "bs", # Bosnian (Latin) + 0x141a: "bs_BA", # Bosnian (Latin) - Bosnia and Herzegovina + 0x201a: "bs_BA", # Bosnian (Cyrillic) - Bosnia and Herzegovina + 0x781a: "bs", # Bosnian (Latin) + 0x007e: "br", # Breton 0x047e: "br_FR", # Breton - France - 0x0402: "bg_BG", # Bulgarian -# 0x0455: "my_MM", # Burmese - Not supported - 0x0403: "ca_ES", # Catalan - 0x0004: "zh_CHS",# Chinese - Simplified - 0x0404: "zh_TW", # Chinese - Taiwan - 0x0804: "zh_CN", # Chinese - PRC - 0x0c04: "zh_HK", # Chinese - Hong Kong S.A.R. - 0x1004: "zh_SG", # Chinese - Singapore - 0x1404: "zh_MO", # Chinese - Macao S.A.R. - 0x7c04: "zh_CHT",# Chinese - Traditional + 0x0002: "bg", # Bulgarian + 0x0402: "bg_BG", # Bulgarian - Bulgaria + 0x0055: "my", # Burmese + 0x0455: "my_MM", # Burmese - Myanmar + 0x0003: "ca", # Catalan + 0x0403: "ca_ES", # Catalan - Spain + 0x0803: "ca_ES", # Valencian - Spain + 0x0092: "ku", # Central Kurdish + 0x7c92: "ku", # Central Kurdish + 0x0492: "ku_IQ", # Central Kurdish - Iraq + 0x005c: "chr", # Cherokee + 0x7c5c: "chr", # Cherokee + 0x045c: "chr_US", # Cherokee - United States + 0x0004: "zh", # Chinese (Simplified) + 0x7804: "zh", # Chinese (Simplified) + 0x7c04: "zh", # Chinese (Traditional) + 0x0404: "zh_TW", # Chinese (Traditional) - Taiwan + 0x0804: "zh_CN", # Chinese (Simplified) - People's Republic of China + 0x0c04: "zh_HK", # Chinese (Traditional) - Hong Kong S.A.R. + 0x1004: "zh_SG", # Chinese (Simplified) - Singapore + 0x1404: "zh_MO", # Chinese (Traditional) - Macao S.A.R. + 0x0083: "co", # Corsican 0x0483: "co_FR", # Corsican - France - 0x041a: "hr_HR", # Croatian - 0x101a: "hr_BA", # Croatian - Bosnia - 0x0405: "cs_CZ", # Czech - 0x0406: "da_DK", # Danish - 0x048c: "gbz_AF",# Dari - Afghanistan - 0x0465: "div_MV",# Divehi - Maldives - 0x0413: "nl_NL", # Dutch - The Netherlands + 0x001a: "hr", # Croatian + 0x041a: "hr_HR", # Croatian - Croatia + 0x101a: "hr_BA", # Croatian (Latin) - Bosnia and Herzegovina + 0x0005: "cs", # Czech + 0x0405: "cs_CZ", # Czech - Czech Republic + 0x0006: "da", # Danish + 0x0406: "da_DK", # Danish - Denmark + 0x008c: "prs", # Dari + 0x048c: "prs_AF", # Dari - Afghanistan + 0x0065: "dv", # Divehi + 0x0465: "dv_MV", # Divehi - Maldives + 0x0013: "nl", # Dutch + 0x0413: "nl_NL", # Dutch - Netherlands 0x0813: "nl_BE", # Dutch - Belgium + 0x0c51: "dz_BT", # Dzongkha - Bhutan + 0x0009: "en", # English 0x0409: "en_US", # English - United States 0x0809: "en_GB", # English - United Kingdom 0x0c09: "en_AU", # English - Australia @@ -1573,122 +1611,248 @@ def getpreferredencoding(do_setlocale=True): 0x1409: "en_NZ", # English - New Zealand 0x1809: "en_IE", # English - Ireland 0x1c09: "en_ZA", # English - South Africa - 0x2009: "en_JA", # English - Jamaica - 0x2409: "en_CB", # English - Caribbean + 0x2009: "en_JM", # English - Jamaica 0x2809: "en_BZ", # English - Belize - 0x2c09: "en_TT", # English - Trinidad + 0x2c09: "en_TT", # English - Trinidad and Tobago 0x3009: "en_ZW", # English - Zimbabwe - 0x3409: "en_PH", # English - Philippines + 0x3409: "en_PH", # English - Republic of the Philippines + 0x3c09: "en_HK", # English - Hong Kong 0x4009: "en_IN", # English - India 0x4409: "en_MY", # English - Malaysia - 0x4809: "en_IN", # English - Singapore - 0x0425: "et_EE", # Estonian - 0x0438: "fo_FO", # Faroese - 0x0464: "fil_PH",# Filipino - 0x040b: "fi_FI", # Finnish + 0x4809: "en_SG", # English - Singapore + 0x4c09: "en_AE", # English - United Arab Emirates + 0x0025: "et", # Estonian + 0x0425: "et_EE", # Estonian - Estonia + 0x0038: "fo", # Faroese + 0x0438: "fo_FO", # Faroese - Faroe Islands + 0x0064: "fil", # Filipino + 0x0464: "fil_PH", # Filipino - Philippines + 0x000b: "fi", # Finnish + 0x040b: "fi_FI", # Finnish - Finland + 0x000c: "fr", # French 0x040c: "fr_FR", # French - France 0x080c: "fr_BE", # French - Belgium 0x0c0c: "fr_CA", # French - Canada 0x100c: "fr_CH", # French - Switzerland 0x140c: "fr_LU", # French - Luxembourg - 0x180c: "fr_MC", # French - Monaco + 0x180c: "fr_MC", # French - Principality of Monaco + 0x1c0c: "fr_029", # French - Caribbean + 0x200c: "fr_RE", # French - Reunion + 0x240c: "fr_CD", # French - Congo, DRC + 0x280c: "fr_SN", # French - Senegal + 0x2c0c: "fr_CM", # French - Cameroon + 0x300c: "fr_CI", # French - Côte d'Ivoire + 0x340c: "fr_ML", # French - Mali + 0x380c: "fr_MA", # French - Morocco + 0x3c0c: "fr_HT", # French - Haiti + 0x0062: "fy", # Frisian 0x0462: "fy_NL", # Frisian - Netherlands - 0x0456: "gl_ES", # Galician - 0x0437: "ka_GE", # Georgian + 0x0067: "ff", # Fulah + 0x7c67: "ff", # Fulah (Latin) + 0x0467: "ff_NG", + 0x0867: "ff_SN", # Fulah - Senegal + 0x0056: "gl", # Galician + 0x0456: "gl_ES", # Galician - Spain + 0x0037: "ka", # Georgian + 0x0437: "ka_GE", # Georgian - Georgia + 0x0007: "de", # German 0x0407: "de_DE", # German - Germany 0x0807: "de_CH", # German - Switzerland 0x0c07: "de_AT", # German - Austria 0x1007: "de_LU", # German - Luxembourg 0x1407: "de_LI", # German - Liechtenstein - 0x0408: "el_GR", # Greek + 0x0008: "el", # Greek + 0x0408: "el_GR", # Greek - Greece + 0x006f: "kl", # Greenlandic 0x046f: "kl_GL", # Greenlandic - Greenland - 0x0447: "gu_IN", # Gujarati - 0x0468: "ha_NG", # Hausa - Latin - 0x040d: "he_IL", # Hebrew - 0x0439: "hi_IN", # Hindi - 0x040e: "hu_HU", # Hungarian - 0x040f: "is_IS", # Icelandic - 0x0421: "id_ID", # Indonesian - 0x045d: "iu_CA", # Inuktitut - Syllabics - 0x085d: "iu_CA", # Inuktitut - Latin + 0x0074: "gn", # Guarani + 0x0474: "gn_PY", # Guarani - Paraguay + 0x0047: "gu", # Gujarati + 0x0447: "gu_IN", # Gujarati - India + 0x0068: "ha", # Hausa (Latin) + 0x7c68: "ha", # Hausa (Latin) + 0x0468: "ha_NG", # Hausa (Latin) - Nigeria + 0x0075: "haw", # Hawaiian + 0x0475: "haw_US", # Hawaiian - United States + 0x000d: "he", # Hebrew + 0x040d: "he_IL", # Hebrew - Israel + 0x0039: "hi", # Hindi + 0x0439: "hi_IN", # Hindi - India + 0x000e: "hu", # Hungarian + 0x040e: "hu_HU", # Hungarian - Hungary + 0x000f: "is", # Icelandic + 0x040f: "is_IS", # Icelandic - Iceland + 0x0070: "ig", # Igbo + 0x0470: "ig_NG", # Igbo - Nigeria + 0x0021: "id", # Indonesian + 0x0421: "id_ID", # Indonesian - Indonesia + 0x005d: "iu", # Inuktitut (Latin) + 0x785d: "iu", # Inuktitut (Syllabics) + 0x7c5d: "iu", # Inuktitut (Latin) + 0x045d: "iu_CA", # Inuktitut (Syllabics) - Canada + 0x085d: "iu_CA", # Inuktitut (Latin) - Canada + 0x003c: "ga", # Irish 0x083c: "ga_IE", # Irish - Ireland + 0x0010: "it", # Italian 0x0410: "it_IT", # Italian - Italy 0x0810: "it_CH", # Italian - Switzerland - 0x0411: "ja_JP", # Japanese + 0x0011: "ja", # Japanese + 0x0411: "ja_JP", # Japanese - Japan + 0x004b: "kn", # Kannada 0x044b: "kn_IN", # Kannada - India - 0x043f: "kk_KZ", # Kazakh - 0x0453: "kh_KH", # Khmer - Cambodia - 0x0486: "qut_GT",# K'iche - Guatemala + 0x0471: "kr_NG", # Kanuri (Latin) - Nigeria + 0x0060: "ks", # Kashmiri + 0x0460: "ks", # Kashmiri - Perso_Arabic + 0x0860: "ks_IN", # Kashmiri (Devanagari) - India + 0x003f: "kk", # Kazakh + 0x043f: "kk_KZ", # Kazakh - Kazakhstan + 0x0053: "km", # Khmer + 0x0453: "km_KH", # Khmer - Cambodia + 0x0087: "rw", # Kinyarwanda 0x0487: "rw_RW", # Kinyarwanda - Rwanda - 0x0457: "kok_IN",# Konkani - 0x0412: "ko_KR", # Korean - 0x0440: "ky_KG", # Kyrgyz - 0x0454: "lo_LA", # Lao - Lao PDR - 0x0426: "lv_LV", # Latvian - 0x0427: "lt_LT", # Lithuanian - 0x082e: "dsb_DE",# Lower Sorbian - Germany - 0x046e: "lb_LU", # Luxembourgish - 0x042f: "mk_MK", # FYROM Macedonian + 0x0041: "sw", # Kiswahili + 0x0441: "sw_KE", # Kiswahili - Kenya + 0x0057: "kok", # Konkani + 0x0457: "kok_IN", # Konkani - India + 0x0012: "ko", # Korean + 0x0412: "ko_KR", # Korean - Korea + 0x0040: "ky", # Kyrgyz + 0x0440: "ky_KG", # Kyrgyz - Kyrgyzstan + 0x0054: "lo", # Lao + 0x0454: "lo_LA", # Lao - Lao P.D.R. + 0x0476: "la_VA", # Latin - Vatican City + 0x0026: "lv", # Latvian + 0x0426: "lv_LV", # Latvian - Latvia + 0x0027: "lt", # Lithuanian + 0x0427: "lt_LT", # Lithuanian - Lithuania + 0x7c2e: "dsb", # Lower Sorbian + 0x082e: "dsb_DE", # Lower Sorbian - Germany + 0x006e: "lb", # Luxembourgish + 0x046e: "lb_LU", # Luxembourgish - Luxembourg + 0x002f: "mk", # Macedonian + 0x042f: "mk_MK", # Macedonian - North Macedonia + 0x003e: "ms", # Malay 0x043e: "ms_MY", # Malay - Malaysia 0x083e: "ms_BN", # Malay - Brunei Darussalam + 0x004c: "ml", # Malayalam 0x044c: "ml_IN", # Malayalam - India - 0x043a: "mt_MT", # Maltese - 0x0481: "mi_NZ", # Maori - 0x047a: "arn_CL",# Mapudungun - 0x044e: "mr_IN", # Marathi - 0x047c: "moh_CA",# Mohawk - Canada - 0x0450: "mn_MN", # Mongolian - Cyrillic - 0x0850: "mn_CN", # Mongolian - PRC - 0x0461: "ne_NP", # Nepali - 0x0414: "nb_NO", # Norwegian - Bokmal - 0x0814: "nn_NO", # Norwegian - Nynorsk + 0x003a: "mt", # Maltese + 0x043a: "mt_MT", # Maltese - Malta + 0x0081: "mi", # Maori + 0x0481: "mi_NZ", # Maori - New Zealand + 0x007a: "arn", # Mapudungun + 0x047a: "arn_CL", # Mapudungun - Chile + 0x004e: "mr", # Marathi + 0x044e: "mr_IN", # Marathi - India + 0x007c: "moh", # Mohawk + 0x047c: "moh_CA", # Mohawk - Canada + 0x0050: "mn", # Mongolian (Cyrillic) + 0x7850: "mn", # Mongolian (Cyrillic) + 0x7c50: "mn", # Mongolian (Traditional Mongolian) + 0x0450: "mn_MN", # Mongolian (Cyrillic) - Mongolia + 0x0c50: "mn_MN", # Mongolian (Traditional Mongolian) - Mongolia + 0x0061: "ne", # Nepali + 0x0461: "ne_NP", # Nepali - Nepal + 0x0861: "ne_IN", # Nepali - India + 0x0014: "no", # Norwegian (Bokmal) + 0x0414: "nb_NO", # Norwegian (Bokmal) - Norway + 0x0814: "nn_NO", # Norwegian (Nynorsk) - Norway + 0x7814: "nn", # Norwegian (Nynorsk) + 0x7c14: "nb", # Norwegian (Bokmal) + 0x0082: "oc", # Occitan 0x0482: "oc_FR", # Occitan - France - 0x0448: "or_IN", # Oriya - India + 0x0048: "or", # Odia + 0x0448: "or_IN", # Odia - India + 0x0072: "om", # Oromo + 0x0472: "om_ET", # Oromo - Ethiopia + 0x0063: "ps", # Pashto 0x0463: "ps_AF", # Pashto - Afghanistan - 0x0429: "fa_IR", # Persian - 0x0415: "pl_PL", # Polish + 0x0029: "fa", # Persian + 0x0429: "fa_IR", # Persian - Iran + 0x0015: "pl", # Polish + 0x0415: "pl_PL", # Polish - Poland + 0x0016: "pt", # Portuguese 0x0416: "pt_BR", # Portuguese - Brazil 0x0816: "pt_PT", # Portuguese - Portugal - 0x0446: "pa_IN", # Punjabi - 0x046b: "quz_BO",# Quechua (Bolivia) - 0x086b: "quz_EC",# Quechua (Ecuador) - 0x0c6b: "quz_PE",# Quechua (Peru) + 0x0046: "pa", # Punjabi + 0x7c46: "pa", # Punjabi + 0x0446: "pa_IN", # Punjabi - India + 0x0846: "pa_PK", # Punjabi - Islamic Republic of Pakistan + 0x006b: "quz", # Quechua + 0x046b: "quz_BO", # Quechua - Bolivia + 0x086b: "quz_EC", # Quechua - Ecuador + 0x0c6b: "quz_PE", # Quechua - Peru + 0x0018: "ro", # Romanian 0x0418: "ro_RO", # Romanian - Romania - 0x0417: "rm_CH", # Romansh - 0x0419: "ru_RU", # Russian - 0x243b: "smn_FI",# Sami Finland - 0x103b: "smj_NO",# Sami Norway - 0x143b: "smj_SE",# Sami Sweden - 0x043b: "se_NO", # Sami Northern Norway - 0x083b: "se_SE", # Sami Northern Sweden - 0x0c3b: "se_FI", # Sami Northern Finland - 0x203b: "sms_FI",# Sami Skolt - 0x183b: "sma_NO",# Sami Southern Norway - 0x1c3b: "sma_SE",# Sami Southern Sweden - 0x044f: "sa_IN", # Sanskrit - 0x0c1a: "sr_SP", # Serbian - Cyrillic - 0x1c1a: "sr_BA", # Serbian - Bosnia Cyrillic - 0x081a: "sr_SP", # Serbian - Latin - 0x181a: "sr_BA", # Serbian - Bosnia Latin + 0x0818: "ro_MD", # Romanian - Moldova + 0x0017: "rm", # Romansh + 0x0417: "rm_CH", # Romansh - Switzerland + 0x0019: "ru", # Russian + 0x0419: "ru_RU", # Russian - Russia + 0x0819: "ru_MD", # Russian - Moldova + 0x0085: "sah", # Sakha + 0x0485: "sah_RU", # Sakha - Russia + 0x003b: "se", # Sami (Northern) + 0x043b: "se_NO", # Sami (Northern) - Norway + 0x083b: "se_SE", # Sami (Northern) - Sweden + 0x0c3b: "se_FI", # Sami (Northern) - Finland + 0x7c3b: "smj", # Sami (Lule) + 0x103b: "smj_NO", # Sami (Lule) - Norway + 0x143b: "smj_SE", # Sami (Lule) - Sweden + 0x783b: "sma", # Sami (Southern) + 0x183b: "sma_NO", # Sami (Southern) - Norway + 0x1c3b: "sma_SE", # Sami (Southern) - Sweden + 0x743b: "sms", # Sami (Skolt) + 0x203b: "sms_FI", # Sami (Skolt) - Finland + 0x703b: "smn", # Sami (Inari) + 0x243b: "smn_FI", # Sami (Inari) - Finland + 0x004f: "sa", # Sanskrit + 0x044f: "sa_IN", # Sanskrit - India + 0x0091: "gd", # Scottish Gaelic + 0x0491: "gd_GB", # Scottish Gaelic - United Kingdom + 0x6c1a: "sr", # Serbian (Cyrillic) + 0x701a: "sr", # Serbian (Latin) + 0x7c1a: "sr", # Serbian (Latin) + 0x081a: "sr_CS", # Serbian (Latin) - Serbia and Montenegro (Former) + 0x0c1a: "sr_CS", # Serbian (Cyrillic) - Serbia and Montenegro (Former) + 0x181a: "sr_BA", # Serbian (Latin) - Bosnia and Herzegovina + 0x1c1a: "sr_BA", # Serbian (Cyrillic) - Bosnia and Herzegovina + 0x241a: "sr_RS", # Serbian (Latin) - Serbia + 0x281a: "sr_RS", # Serbian (Cyrillic) - Serbia + 0x2c1a: "sr_ME", # Serbian (Latin) - Montenegro + 0x301a: "sr_ME", # Serbian (Cyrillic) - Montenegro + 0x006c: "nso", # Sesotho sa Leboa + 0x046c: "nso_ZA", # Sesotho sa Leboa - South Africa + 0x0032: "tn", # Setswana + 0x0432: "tn_ZA", # Setswana - South Africa + 0x0832: "tn_BW", # Setswana - Botswana + 0x0059: "sd", # Sindhi + 0x7c59: "sd", # Sindhi + 0x0859: "sd_PK", # Sindhi - Islamic Republic of Pakistan + 0x005b: "si", # Sinhala 0x045b: "si_LK", # Sinhala - Sri Lanka - 0x046c: "ns_ZA", # Northern Sotho - 0x0432: "tn_ZA", # Setswana - Southern Africa - 0x041b: "sk_SK", # Slovak - 0x0424: "sl_SI", # Slovenian + 0x001b: "sk", # Slovak + 0x041b: "sk_SK", # Slovak - Slovakia + 0x0024: "sl", # Slovenian + 0x0424: "sl_SI", # Slovenian - Slovenia + 0x0477: "so_SO", # Somali - Somalia + 0x0030: "st", # Sotho + 0x0430: "st_ZA", # Sotho - South Africa + 0x000a: "es", # Spanish 0x040a: "es_ES", # Spanish - Spain 0x080a: "es_MX", # Spanish - Mexico - 0x0c0a: "es_ES", # Spanish - Spain (Modern) + 0x0c0a: "es_ES", # Spanish - Spain 0x100a: "es_GT", # Spanish - Guatemala 0x140a: "es_CR", # Spanish - Costa Rica 0x180a: "es_PA", # Spanish - Panama 0x1c0a: "es_DO", # Spanish - Dominican Republic - 0x200a: "es_VE", # Spanish - Venezuela + 0x200a: "es_VE", # Spanish - Bolivarian Republic of Venezuela 0x240a: "es_CO", # Spanish - Colombia 0x280a: "es_PE", # Spanish - Peru 0x2c0a: "es_AR", # Spanish - Argentina 0x300a: "es_EC", # Spanish - Ecuador 0x340a: "es_CL", # Spanish - Chile - 0x380a: "es_UR", # Spanish - Uruguay + 0x380a: "es_UY", # Spanish - Uruguay 0x3c0a: "es_PY", # Spanish - Paraguay 0x400a: "es_BO", # Spanish - Bolivia 0x440a: "es_SV", # Spanish - El Salvador @@ -1696,36 +1860,87 @@ def getpreferredencoding(do_setlocale=True): 0x4c0a: "es_NI", # Spanish - Nicaragua 0x500a: "es_PR", # Spanish - Puerto Rico 0x540a: "es_US", # Spanish - United States -# 0x0430: "", # Sutu - Not supported - 0x0441: "sw_KE", # Swahili + 0x5c0a: "es_CU", # Spanish - Cuba + 0x001d: "sv", # Swedish 0x041d: "sv_SE", # Swedish - Sweden 0x081d: "sv_FI", # Swedish - Finland - 0x045a: "syr_SY",# Syriac - 0x0428: "tg_TJ", # Tajik - Cyrillic - 0x085f: "tmz_DZ",# Tamazight - Latin - 0x0449: "ta_IN", # Tamil - 0x0444: "tt_RU", # Tatar - 0x044a: "te_IN", # Telugu - 0x041e: "th_TH", # Thai - 0x0851: "bo_BT", # Tibetan - Bhutan - 0x0451: "bo_CN", # Tibetan - PRC - 0x041f: "tr_TR", # Turkish - 0x0442: "tk_TM", # Turkmen - Cyrillic - 0x0480: "ug_CN", # Uighur - Arabic - 0x0422: "uk_UA", # Ukrainian - 0x042e: "wen_DE",# Upper Sorbian - Germany - 0x0420: "ur_PK", # Urdu + 0x005a: "syr", # Syriac + 0x045a: "syr_SY", # Syriac - Syria + 0x0028: "tg", # Tajik (Cyrillic) + 0x7c28: "tg", # Tajik (Cyrillic) + 0x0428: "tg_TJ", # Tajik (Cyrillic) - Tajikistan + 0x005f: "tzm", # Tamazight (Latin) + 0x785f: "tzm", + 0x7c5f: "tzm", # Tamazight (Latin) + 0x085f: "tzm_DZ", # Tamazight (Latin) - Algeria + 0x045f: "tzm_MA", # Central Atlas Tamazight (Arabic) - Morocco + 0x105f: "tzm_MA", + 0x0049: "ta", # Tamil + 0x0449: "ta_IN", # Tamil - India + 0x0849: "ta_LK", # Tamil - Sri Lanka + 0x0044: "tt", # Tatar + 0x0444: "tt_RU", # Tatar - Russia + 0x004a: "te", # Telugu + 0x044a: "te_IN", # Telugu - India + 0x001e: "th", # Thai + 0x041e: "th_TH", # Thai - Thailand + 0x0051: "bo", # Tibetan + 0x0451: "bo_CN", # Tibetan - People's Republic of China + 0x0073: "ti", # Tigrinya + 0x0473: "ti_ET", # Tigrinya - Ethiopia + 0x0873: "ti_ER", # Tigrinya - Eritrea + 0x0031: "ts", # Tsonga + 0x0431: "ts_ZA", # Tsonga - South Africa + 0x001f: "tr", # Turkish + 0x041f: "tr_TR", # Turkish - Turkey + 0x0042: "tk", # Turkmen + 0x0442: "tk_TM", # Turkmen - Turkmenistan + 0x0022: "uk", # Ukrainian + 0x0422: "uk_UA", # Ukrainian - Ukraine + 0x002e: "hsb", # Upper Sorbian + 0x042e: "hsb_DE", # Upper Sorbian - Germany + 0x0020: "ur", # Urdu + 0x0420: "ur_PK", # Urdu - Islamic Republic of Pakistan 0x0820: "ur_IN", # Urdu - India - 0x0443: "uz_UZ", # Uzbek - Latin - 0x0843: "uz_UZ", # Uzbek - Cyrillic - 0x042a: "vi_VN", # Vietnamese - 0x0452: "cy_GB", # Welsh + 0x0080: "ug", # Uyghur + 0x0480: "ug_CN", # Uyghur - People's Republic of China + 0x0043: "uz", # Uzbek (Latin) + 0x7843: "uz", # Uzbek (Cyrillic) + 0x7c43: "uz", # Uzbek (Latin) + 0x0443: "uz_UZ", # Uzbek (Latin) - Uzbekistan + 0x0033: "ve", # Venda + 0x0433: "ve_ZA", # Venda - South Africa + 0x002a: "vi", # Vietnamese + 0x042a: "vi_VN", # Vietnamese - Vietnam + 0x0052: "cy", # Welsh + 0x0452: "cy_GB", # Welsh - United Kingdom + 0x0088: "wo", # Wolof 0x0488: "wo_SN", # Wolof - Senegal + 0x0034: "xh", # Xhosa 0x0434: "xh_ZA", # Xhosa - South Africa - 0x0485: "sah_RU",# Yakut - Cyrillic - 0x0478: "ii_CN", # Yi - PRC + 0x0078: "ii", # Yi + 0x0478: "ii_CN", # Yi - People's Republic of China + 0x043d: "yi_001", # Yiddish - World + 0x006a: "yo", # Yoruba 0x046a: "yo_NG", # Yoruba - Nigeria - 0x0435: "zu_ZA", # Zulu + 0x0035: "zu", # Zulu + 0x0435: "zu_ZA", # Zulu - South Africa + 0x0086: "qut", + +# 0x0001007f: "x-IV-mathan", # math alphanumeric sorting + 0x00010407: "de_DE", + 0x0001040e: "hu_HU", + 0x00010437: "ka_GE", + 0x00020804: "zh_CN", + 0x00021004: "zh_SG", + 0x00021404: "zh_MO", + 0x00030404: "zh_TW", + 0x00040404: "zh_TW", + 0x00040411: "ja_JP", + 0x00040c04: "zh_HK", + 0x00041404: "zh_MO", + 0x00050804: "zh_CN", + 0x00051004: "zh_SG", } def _print_locale(): diff --git a/Lib/pdb.py b/Lib/pdb.py index 7b08d2bb701..4dd974b375c 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -318,12 +318,34 @@ def namespace(self): class _PdbInteractiveConsole(code.InteractiveConsole): - def __init__(self, ns, message): + def __init__(self, ns=None, message=None): self._message = message super().__init__(locals=ns, local_exit=True) def write(self, data): - self._message(data, end='') + if self._message is not None: + self._message(data, end='') + else: + super().write(data) + + def more_lines(self, text): + # Generic Python multi-line completeness heuristic. + # Strips pyrepl's trailing auto-indent before compiling. + # This should be functionally identical to simple_interact._more_lines + src = text.rstrip(" \t") + n = len(src) + if n > 0 and text[n-1] == '\n': + text = src + try: + code_obj = self.compile(text, "", "single") + except (OverflowError, SyntaxError, ValueError): + lines = text.splitlines(keepends=True) + if len(lines) == 1: + return False + last = lines[-1] + return ((last.startswith((" ", "\t")) or last.strip() != "") + and not last.endswith("\n")) + return code_obj is None # Interaction prompt line will separate file and call info from code @@ -352,6 +374,96 @@ def get_default_backend(): return _default_backend +def _pyrepl_available(): + """return whether pdb should use _pyrepl for input""" + if os.getenv("PYTHON_BASIC_REPL"): + CAN_USE_PYREPL = False + else: + try: + from _pyrepl.main import CAN_USE_PYREPL + except ModuleNotFoundError: + CAN_USE_PYREPL = False + return CAN_USE_PYREPL + + +class PdbPyReplInput: + def __init__(self, pdb_instance, stdin, stdout, prompt): + import _pyrepl.readline + + self.pdb_instance = pdb_instance + self.prompt = prompt + self.console = _PdbInteractiveConsole() + if not (os.isatty(stdin.fileno())): + raise ValueError("stdin is not a TTY") + self.readline_wrapper = _pyrepl.readline._ReadlineWrapper( + f_in=stdin.fileno(), + f_out=stdout.fileno(), + config=_pyrepl.readline.ReadlineConfig( + completer_delims=frozenset(' \t\n`@#%^&*()=+[{]}\\|;:\'",<>?') + ) + ) + + def readline(self): + + def more_lines(text): + if text.strip() == "\x1a": + # Ctrl + Z raises EOFError to quit pdb + # This is similarly handled in simple_interact.py + raise EOFError + cmd, _, line = self.pdb_instance.parseline(text) + if not line or not cmd: + return False + func = getattr(self.pdb_instance, 'do_' + cmd, None) + if func is not None: + return False + return self.console.more_lines(text) + + try: + pyrepl_completer = self.readline_wrapper.get_completer() + self.readline_wrapper.set_completer(self.complete) + multiline = ( + self.readline_wrapper.multiline_input( + more_lines, + self.prompt, + '... ' + ' ' * (len(self.prompt) - 4) + ) + '\n' + ) + return multiline + except EOFError: + return 'EOF' + finally: + self.readline_wrapper.set_completer(pyrepl_completer) + + def complete(self, text, state): + """ + This function is very similar to cmd.Cmd.complete. + However, cmd.Cmd.complete assumes that we use readline module, but + pyrepl does not use it. + """ + if state == 0: + origline = self.readline_wrapper.get_line_buffer() + line = origline.lstrip() + stripped = len(origline) - len(line) + begidx = self.readline_wrapper.get_begidx() - stripped + endidx = self.readline_wrapper.get_endidx() - stripped + if begidx > 0: + cmd, args, foo = self.pdb_instance.parseline(line) + if not cmd: + compfunc = self.pdb_instance.completedefault + else: + try: + compfunc = getattr(self.pdb_instance, 'complete_' + cmd) + except AttributeError: + compfunc = self.pdb_instance.completedefault + else: + compfunc = self.pdb_instance.completenames + self.completion_matches = compfunc(text, line, begidx, endidx) + try: + return self.completion_matches[state] + except IndexError: + return None + + class Pdb(bdb.Bdb, cmd.Cmd): _previous_sigint_handler = None @@ -386,6 +498,12 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, except ImportError: pass + self.pyrepl_input = None + if _pyrepl_available(): + try: + self.pyrepl_input = PdbPyReplInput(self, self.stdin, self.stdout, self.prompt) + except Exception: + pass self.allow_kbdint = False self.nosigint = nosigint # Consider these characters as part of the command so when the users type @@ -624,6 +742,31 @@ def user_exception(self, frame, exc_info): self.message('%s%s' % (prefix, self._format_exc(exc_value))) self.interaction(frame, exc_traceback) + @contextmanager + def _replace_attribute(self, attrs): + original_attrs = {} + for attr, value in attrs.items(): + original_attrs[attr] = getattr(self, attr) + setattr(self, attr, value) + try: + yield + finally: + for attr, value in original_attrs.items(): + setattr(self, attr, value) + + @contextmanager + def _maybe_use_pyrepl_as_stdin(self): + if self.pyrepl_input is None: + yield + return + + with self._replace_attribute({ + 'stdin': self.pyrepl_input, + 'use_rawinput': False, + 'prompt': '', + }): + yield + # General interaction function def _cmdloop(self): while True: @@ -631,7 +774,8 @@ def _cmdloop(self): # keyboard interrupts allow for an easy way to cancel # the current command, so allow them during interactive input self.allow_kbdint = True - self.cmdloop() + with self._maybe_use_pyrepl_as_stdin(): + self.cmdloop() self.allow_kbdint = False break except KeyboardInterrupt: @@ -2364,10 +2508,21 @@ def do_interact(self, arg): contains all the (global and local) names found in the current scope. """ ns = {**self.curframe.f_globals, **self.curframe.f_locals} - with self._enable_rlcompleter(ns): - console = _PdbInteractiveConsole(ns, message=self.message) - console.interact(banner="*pdb interact start*", - exitmsg="*exit from pdb interact command*") + console = _PdbInteractiveConsole(ns, message=self.message) + banner = "*pdb interact start*" + exitmsg = "*exit from pdb interact command*" + if self.pyrepl_input is not None: + from _pyrepl.simple_interact import run_multiline_interactive_console + self.message(banner) + try: + run_multiline_interactive_console(console) + except SystemExit: + pass + self.message(exitmsg) + else: + with self._enable_rlcompleter(ns): + console.interact(banner=banner, + exitmsg=exitmsg) def do_alias(self, arg): """alias [name [command]] diff --git a/Lib/pickle.py b/Lib/pickle.py index 3e7cf25cb05..95836afdc2b 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -920,17 +920,11 @@ def save_picklebuffer(self, obj): # Write data in-band # XXX The C implementation avoids a copy here buf = m.tobytes() - in_memo = id(buf) in self.memo if m.readonly: - if in_memo: - self._save_bytes_no_memo(buf) - else: - self.save_bytes(buf) + self._save_bytes_no_memo(buf) else: - if in_memo: - self._save_bytearray_no_memo(buf) - else: - self.save_bytearray(buf) + self._save_bytearray_no_memo(buf) + self.memoize(obj) else: # Write data out-of-band self.write(NEXT_BUFFER) diff --git a/Lib/pickletools.py b/Lib/pickletools.py index 29baf3be7eb..976e218db19 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -16,6 +16,8 @@ import re import sys +lazy from _colorize import decolor, get_theme + __all__ = ['dis', 'genops', 'optimize'] bytes_types = pickle.bytes_types @@ -2209,6 +2211,32 @@ def __init__(self, name, code, arg, name2i[d.name] = i code2i[d.code] = i +# Group opcode names into categories for colourised CLI output. +_opcode_categories = frozendict( + op_call=frozenset({ + "BUILD", "EXT1", "EXT2", "EXT4", "GLOBAL", "INST", "NEWOBJ", + "NEWOBJ_EX", "OBJ", "REDUCE", "STACK_GLOBAL", + }), + op_container=frozenset({ + "ADDITEMS", "APPEND", "APPENDS", "DICT", "EMPTY_DICT", "EMPTY_LIST", + "EMPTY_SET", "EMPTY_TUPLE", "FROZENSET", "LIST", "SETITEM", + "SETITEMS", "TUPLE", "TUPLE1", "TUPLE2", "TUPLE3", + }), + op_memo=frozenset({ + "BINGET", "BINPUT", "GET", "LONG_BINGET", "LONG_BINPUT", "MEMOIZE", + "PUT", + }), + op_meta=frozenset({"BINPERSID", "FRAME", "MARK", "PERSID", "PROTO"}), + op_stack=frozenset({"DUP", "POP", "POP_MARK", "STOP"}), +) +_opcode_color_attr = frozendict({ + name: attr + for attr, names in _opcode_categories.items() + for name in names +}) +assert _opcode_color_attr.keys() <= name2i.keys(), ( + f"unknown opcodes: {_opcode_color_attr.keys() - name2i.keys()}" +) del name2i, code2i, i, d ############################################################################## @@ -2443,13 +2471,19 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): indentchunk = ' ' * indentlevel errormsg = None annocol = annotate # column hint for annotations + t = get_theme(tty_file=out).pickletools for opcode, arg, pos in genops(pickle): if pos is not None: - print("%5d:" % pos, end=' ', file=out) + print(f"{t.position}{pos:5d}:{t.reset}", end=' ', file=out) - line = "%-4s %s%s" % (repr(opcode.code)[1:-1], - indentchunk * len(markstack), - opcode.name) + attr = _opcode_color_attr.get(opcode.name) + opcode_color = getattr(t, attr) if attr else "" + opcode_reset = t.reset if attr else "" + line = ( + f"{t.opcode_code}{repr(opcode.code)[1:-1]:<4}{t.reset} " + f"{indentchunk * len(markstack)}" + f"{opcode_color}{opcode.name}{opcode_reset}" + ) maxproto = max(maxproto, opcode.proto) before = opcode.stack_before # don't mutate @@ -2510,18 +2544,26 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): line += ' ' * (10 - len(opcode.name)) if arg is not None: if opcode.name in ("STRING", "BINSTRING", "SHORT_BINSTRING"): - line += ' ' + ascii(arg) + arg_text = ascii(arg) else: - line += ' ' + repr(arg) + arg_text = repr(arg) + arg_color = ( + t.arg_number + if isinstance(arg, (int, float)) + else t.arg_string + ) + line += f" {arg_color}{arg_text}{t.reset}" if markmsg: - line += ' ' + markmsg + line += f" {t.mark}{markmsg}{t.reset}" if annotate: - line += ' ' * (annocol - len(line)) + visible_len = len(decolor(line)) + line += ' ' * (annocol - visible_len) # make a mild effort to align annotations - annocol = len(line) + annocol = max(visible_len, annocol) if annocol > 50: annocol = annotate - line += ' ' + opcode.doc.split('\n', 1)[0] + doc = opcode.doc.split('\n', 1)[0] + line += f" {t.annotation}{doc}{t.reset}" print(line, file=out) if errormsg: @@ -2541,7 +2583,11 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): stack.extend(after) - print("highest protocol among opcodes =", maxproto, file=out) + print( + "highest protocol among opcodes =", + f"{t.proto}{maxproto}{t.reset}", + file=out, + ) if stack: raise ValueError("stack not empty after STOP: %r" % stack) @@ -2841,10 +2887,7 @@ def __init__(self, value): def _main(args=None): import argparse - parser = argparse.ArgumentParser( - description='disassemble one or more pickle files', - color=True, - ) + parser = argparse.ArgumentParser(description='disassemble one or more pickle files') parser.add_argument( 'pickle_file', nargs='+', help='the pickle file') diff --git a/Lib/profiling/sampling/_assets/tachyon-logo.png b/Lib/profiling/sampling/_assets/tachyon-logo.png index f87e006b14f..03e67823d5a 100644 Binary files a/Lib/profiling/sampling/_assets/tachyon-logo.png and b/Lib/profiling/sampling/_assets/tachyon-logo.png differ diff --git a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css index c4da169d15d..c93ee1e9dd4 100644 --- a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css @@ -315,6 +315,12 @@ .section-title { } /* View Mode Section */ +.view-mode-section { + display: flex; + flex-direction: column; + gap: 8px; +} + .view-mode-section .section-content { display: flex; flex-direction: column; @@ -1067,7 +1073,8 @@ .d3-flame-graph g:first-of-type .d3-flame-graph-label { -------------------------------------------------------------------------- */ #toggle-invert .toggle-track.on, -#toggle-elided .toggle-track.on { +#toggle-elided .toggle-track.on, +#toggle-path-display .toggle-track.on { background: #8e44ad; border-color: #8e44ad; box-shadow: 0 0 8px rgba(142, 68, 173, 0.3); diff --git a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js index d7a8890d4a1..1611bf75442 100644 --- a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js @@ -6,6 +6,7 @@ let normalData = null; let invertedData = null; let currentThreadFilter = 'all'; let isInverted = false; +let useModuleNames = true; // Heat colors are now defined in CSS variables (--heat-1 through --heat-8) // and automatically switch with theme changes - no JS color arrays needed! @@ -64,6 +65,12 @@ function resolveStringIndices(node, table) { if (typeof resolved.funcname === 'number') { resolved.funcname = resolveString(resolved.funcname, table); } + if (typeof resolved.module === 'number') { + resolved.module = resolveString(resolved.module, table); + } + if (typeof resolved.label === 'number') { + resolved.label = resolveString(resolved.label, table); + } if (Array.isArray(resolved.source)) { resolved.source = resolved.source.map(index => @@ -78,6 +85,19 @@ function resolveStringIndices(node, table) { return resolved; } +// Escape HTML special characters +function escapeHtml(str) { + return str.replace(/&/g, "&").replace(//g, ">"); +} + +// Get display path based on user preference (module or full path) +function getDisplayName(moduleName, filename) { + if (useModuleNames) { + return moduleName || filename; + } + return filename; +} + function selectFlamegraphData() { const baseData = isShowingElided ? elidedFlamegraphData : normalData; @@ -228,6 +248,7 @@ function setupLogos() { function updateStatusBar(nodeData, rootValue) { const funcname = resolveString(nodeData.funcname) || resolveString(nodeData.name) || "--"; const filename = resolveString(nodeData.filename) || ""; + const moduleName = resolveString(nodeData.module) || ""; const lineno = nodeData.lineno; const timeMs = (nodeData.value / 1000).toFixed(2); const percent = rootValue > 0 ? ((nodeData.value / rootValue) * 100).toFixed(1) : "0.0"; @@ -249,8 +270,8 @@ function updateStatusBar(nodeData, rootValue) { const fileEl = document.getElementById('status-file'); if (fileEl && filename && filename !== "~") { - const basename = filename.split('/').pop(); - fileEl.textContent = lineno ? `${basename}:${lineno}` : basename; + const displayName = getDisplayName(moduleName, filename); + fileEl.textContent = lineno ? `${displayName}:${lineno}` : displayName; } const funcEl = document.getElementById('status-func'); @@ -301,6 +322,8 @@ function createPythonTooltip(data) { const funcname = resolveString(d.data.funcname) || resolveString(d.data.name); const filename = resolveString(d.data.filename) || ""; + const moduleName = resolveString(d.data.module) || ""; + const displayName = escapeHtml(useModuleNames ? (moduleName || filename) : filename); const isSpecialFrame = filename === "~"; // Build source section @@ -309,7 +332,7 @@ function createPythonTooltip(data) { const sourceLines = source .map((line) => { const isCurrent = line.startsWith("→"); - const escaped = line.replace(/&/g, "&").replace(//g, ">"); + const escaped = escapeHtml(line); return `
${escaped}
`; }) .join(""); @@ -369,7 +392,7 @@ function createPythonTooltip(data) { } const fileLocationHTML = isSpecialFrame ? "" : ` -
${filename}${d.data.lineno ? ":" + d.data.lineno : ""}
`; +
${displayName}${d.data.lineno ? ":" + d.data.lineno : ""}
`; // Differential stats section let diffSection = ""; @@ -586,6 +609,7 @@ function createFlamegraph(tooltip, rootValue, data) { .minFrameSize(1) .tooltip(tooltip) .inverted(true) + .getName(d => resolveString(useModuleNames ? d.data.label : d.data.name) || resolveString(d.data.name) || '') .setColorMapper(function (d) { if (d.depth === 0) return 'transparent'; @@ -628,25 +652,25 @@ function updateSearchHighlight(searchTerm, searchInput) { const name = resolveString(d.data.name) || ""; const funcname = resolveString(d.data.funcname) || ""; const filename = resolveString(d.data.filename) || ""; + const moduleName = resolveString(d.data.module) || ""; + const displayName = getDisplayName(moduleName, filename); const lineno = d.data.lineno; const term = searchTerm.toLowerCase(); - // Check if search term looks like file:line pattern + // Check if search term looks like path:line pattern const fileLineMatch = term.match(/^(.+):(\d+)$/); let matches = false; if (fileLineMatch) { - // Exact file:line matching const searchFile = fileLineMatch[1]; const searchLine = parseInt(fileLineMatch[2], 10); - const basename = filename.split('/').pop().toLowerCase(); - matches = basename.includes(searchFile) && lineno === searchLine; + matches = displayName.toLowerCase().includes(searchFile) && lineno === searchLine; } else { // Regular substring search matches = name.toLowerCase().includes(term) || funcname.toLowerCase().includes(term) || - filename.toLowerCase().includes(term); + displayName.toLowerCase().includes(term); } if (matches) { @@ -1047,6 +1071,7 @@ function populateStats(data) { let filename = resolveString(node.filename); let funcname = resolveString(node.funcname); + let moduleName = resolveString(node.module); if (!filename || !funcname) { const nameStr = resolveString(node.name); @@ -1061,13 +1086,10 @@ function populateStats(data) { filename = filename || 'unknown'; funcname = funcname || 'unknown'; + moduleName = moduleName || 'unknown'; if (filename !== 'unknown' && funcname !== 'unknown' && node.value > 0) { - let childrenValue = 0; - if (node.children) { - childrenValue = node.children.reduce((sum, child) => sum + child.value, 0); - } - const directSamples = Math.max(0, node.value - childrenValue); + const directSamples = node.self || 0; const funcKey = `${filename}:${node.lineno || '?'}:${funcname}`; @@ -1077,12 +1099,14 @@ function populateStats(data) { existing.directPercent = (existing.directSamples / totalSamples) * 100; if (directSamples > existing.maxSingleSamples) { existing.filename = filename; + existing.module = moduleName; existing.lineno = node.lineno || '?'; existing.maxSingleSamples = directSamples; } } else { functionMap.set(funcKey, { filename: filename, + module: moduleName, lineno: node.lineno || '?', funcname: funcname, directSamples, @@ -1117,6 +1141,7 @@ function populateStats(data) { const h = hotSpots[i]; const filename = h.filename || 'unknown'; const lineno = h.lineno ?? '?'; + const moduleName = h.module || 'unknown'; const isSpecialFrame = filename === '~' && (lineno === 0 || lineno === '?'); let funcDisplay = h.funcname || 'unknown'; @@ -1127,8 +1152,8 @@ function populateStats(data) { if (isSpecialFrame) { fileEl.textContent = '--'; } else { - const basename = filename !== 'unknown' ? filename.split('/').pop() : 'unknown'; - fileEl.textContent = `${basename}:${lineno}`; + const displayName = getDisplayName(moduleName, filename); + fileEl.textContent = `${displayName}:${lineno}`; } } if (percentEl) percentEl.textContent = `${h.directPercent.toFixed(1)}%`; @@ -1144,8 +1169,11 @@ function populateStats(data) { if (card) { if (i < hotSpots.length && hotSpots[i]) { const h = hotSpots[i]; - const basename = h.filename !== 'unknown' ? h.filename.split('/').pop() : ''; - const searchTerm = basename && h.lineno !== '?' ? `${basename}:${h.lineno}` : h.funcname; + const moduleName = h.module || 'unknown'; + const filename = h.filename || 'unknown'; + const displayName = getDisplayName(moduleName, filename); + const hasValidLocation = displayName !== 'unknown' && h.lineno !== '?'; + const searchTerm = hasValidLocation ? `${displayName}:${h.lineno}` : h.funcname; card.dataset.searchterm = searchTerm; card.onclick = () => searchForHotspot(searchTerm); card.style.cursor = 'pointer'; @@ -1277,10 +1305,12 @@ function accumulateInvertedNode(parent, stackFrame, leaf, isDifferential) { if (!parent.children[key]) { const newNode = { name: stackFrame.name, + label: stackFrame.label, value: 0, self: 0, children: {}, filename: stackFrame.filename, + module: stackFrame.module, lineno: stackFrame.lineno, funcname: stackFrame.funcname, source: stackFrame.source, @@ -1345,14 +1375,13 @@ function processLeaf(invertedRoot, path, leafNode, isDifferential) { } function traverseInvert(path, currentNode, invertedRoot, isDifferential) { - const children = currentNode.children || []; - const childThreads = new Set(children.flatMap(c => c.threads || [])); - const selfThreads = (currentNode.threads || []).filter(t => !childThreads.has(t)); + const selfValue = currentNode.self || 0; - if (selfThreads.length > 0) { - processLeaf(invertedRoot, path, { ...currentNode, threads: selfThreads }, isDifferential); + if (selfValue > 0) { + processLeaf(invertedRoot, path, { ...currentNode, value: selfValue }, isDifferential); } + const children = currentNode.children || []; children.forEach(child => traverseInvert(path.concat([child]), child, invertedRoot, isDifferential)); } @@ -1375,6 +1404,7 @@ function generateInvertedFlamegraph(data) { const invertedRoot = { name: data.name, + label: data.label, value: data.value, children: {}, stats: data.stats, @@ -1399,6 +1429,12 @@ function toggleInvert() { updateFlamegraphView(); } +function togglePathDisplay() { + useModuleNames = !useModuleNames; + updateToggleUI('toggle-path-display', useModuleNames); + updateFlamegraphView(); +} + // ============================================================================ // Initialization // ============================================================================ @@ -1446,6 +1482,11 @@ function initFlamegraph() { if (toggleInvertBtn) { toggleInvertBtn.addEventListener('click', toggleInvert); } + + const togglePathDisplayBtn = document.getElementById('toggle-path-display'); + if (togglePathDisplayBtn) { + togglePathDisplayBtn.addEventListener('click', togglePathDisplay); + } } // Keyboard shortcut: Enter/Space activates toggle switches diff --git a/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html b/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html index 9a77178aeff..f1c5bb03006 100644 --- a/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html @@ -117,6 +117,12 @@

View Mode

Elided +
+ File Paths +
+ Module Names +
+
Flamegraph
diff --git a/Lib/profiling/sampling/heatmap_collector.py b/Lib/profiling/sampling/heatmap_collector.py index ea1beec70d3..5c36d78f553 100644 --- a/Lib/profiling/sampling/heatmap_collector.py +++ b/Lib/profiling/sampling/heatmap_collector.py @@ -20,6 +20,7 @@ from .collector import normalize_location, extract_lineno from .opcode_utils import get_opcode_info, format_opcode from .stack_collector import StackTraceCollector +from .module_utils import extract_module_name, get_python_path_info # ============================================================================ @@ -49,126 +50,6 @@ class TreeNode: children: Dict[str, 'TreeNode'] = field(default_factory=dict) -# ============================================================================ -# Module Path Analysis -# ============================================================================ - -def get_python_path_info(): - """Get information about Python installation paths for module extraction. - - Returns: - dict: Dictionary containing stdlib path, site-packages paths, and sys.path entries. - """ - info = { - 'stdlib': None, - 'site_packages': [], - 'sys_path': [] - } - - # Get standard library path from os module location - try: - if hasattr(os, '__file__') and os.__file__: - info['stdlib'] = Path(os.__file__).parent - except (AttributeError, OSError): - pass # Silently continue if we can't determine stdlib path - - # Get site-packages directories - site_packages = [] - try: - site_packages.extend(Path(p) for p in site.getsitepackages()) - except (AttributeError, OSError): - pass # Continue without site packages if unavailable - - # Get user site-packages - try: - user_site = site.getusersitepackages() - if user_site and Path(user_site).exists(): - site_packages.append(Path(user_site)) - except (AttributeError, OSError): - pass # Continue without user site packages - - info['site_packages'] = site_packages - info['sys_path'] = [Path(p) for p in sys.path if p] - - return info - - -def extract_module_name(filename, path_info): - """Extract Python module name and type from file path. - - Args: - filename: Path to the Python file - path_info: Dictionary from get_python_path_info() - - Returns: - tuple: (module_name, module_type) where module_type is one of: - 'stdlib', 'site-packages', 'project', or 'other' - """ - if not filename: - return ('unknown', 'other') - - try: - file_path = Path(filename) - except (ValueError, OSError): - return (str(filename), 'other') - - # Check if it's in stdlib - if path_info['stdlib'] and _is_subpath(file_path, path_info['stdlib']): - try: - rel_path = file_path.relative_to(path_info['stdlib']) - return (_path_to_module(rel_path), 'stdlib') - except ValueError: - pass - - # Check site-packages - for site_pkg in path_info['site_packages']: - if _is_subpath(file_path, site_pkg): - try: - rel_path = file_path.relative_to(site_pkg) - return (_path_to_module(rel_path), 'site-packages') - except ValueError: - continue - - # Check other sys.path entries (project files) - if not str(file_path).startswith(('<', '[')): # Skip special files - for path_entry in path_info['sys_path']: - if _is_subpath(file_path, path_entry): - try: - rel_path = file_path.relative_to(path_entry) - return (_path_to_module(rel_path), 'project') - except ValueError: - continue - - # Fallback: just use the filename - return (_path_to_module(file_path), 'other') - - -def _is_subpath(file_path, parent_path): - try: - file_path.relative_to(parent_path) - return True - except (ValueError, OSError): - return False - - -def _path_to_module(path): - if isinstance(path, str): - path = Path(path) - - # Remove .py extension - if path.suffix == '.py': - path = path.with_suffix('') - - # Convert path separators to dots - parts = path.parts - - # Handle __init__ files - they represent the package itself - if parts and parts[-1] == '__init__': - parts = parts[:-1] - - return '.'.join(parts) if parts else path.stem - - # ============================================================================ # Helper Classes # ============================================================================ diff --git a/Lib/profiling/sampling/module_utils.py b/Lib/profiling/sampling/module_utils.py new file mode 100644 index 00000000000..dfde2b28ab2 --- /dev/null +++ b/Lib/profiling/sampling/module_utils.py @@ -0,0 +1,102 @@ +"""Utilities for extracting module names from file paths.""" + +import os +import site +import sys +from pathlib import Path + + +def get_python_path_info(): + """Get information about Python's search paths. + + Returns: + dict: Dictionary containing stdlib path, site-packages paths, and sys.path entries. + """ + info = { + 'stdlib': None, + 'site_packages': [], + 'sys_path': [] + } + + # Get standard library path from os module location + try: + if hasattr(os, '__file__') and os.__file__: + info['stdlib'] = Path(os.__file__).parent + except (AttributeError, OSError): + pass # Silently continue if we can't determine stdlib path + + # Get site-packages directories + site_packages = [] + try: + site_packages.extend(Path(p) for p in site.getsitepackages()) + except (AttributeError, OSError): + pass # Continue without site packages if unavailable + + # Get user site-packages + try: + user_site = site.getusersitepackages() + if user_site and Path(user_site).exists(): + site_packages.append(Path(user_site)) + except (AttributeError, OSError): + pass # Continue without user site packages + + info['site_packages'] = site_packages + info['sys_path'] = [Path(p) for p in sys.path if p] + + return info + + +def extract_module_name(filename, path_info): + """Extract Python module name and type from file path. + + Args: + filename: Path to the Python file + path_info: Dictionary from get_python_path_info() + + Returns: + tuple: (module_name, module_type) where module_type is one of: + 'stdlib', 'site-packages', 'project', or 'other' + """ + if not filename: + return ('unknown', 'other') + + try: + file_path = Path(filename) + except (ValueError, OSError): + return (str(filename), 'other') + + # Check if it's in stdlib + if path_info['stdlib'] and file_path.is_relative_to(path_info['stdlib']): + return (_path_to_module(file_path.relative_to(path_info['stdlib'])), 'stdlib') + + # Check site-packages + for site_pkg in path_info['site_packages']: + if file_path.is_relative_to(site_pkg): + return (_path_to_module(file_path.relative_to(site_pkg)), 'site-packages') + + # Check other sys.path entries (project files) + if not str(file_path).startswith(('<', '[')): # Skip special files + for path_entry in path_info['sys_path']: + if file_path.is_relative_to(path_entry): + return (_path_to_module(file_path.relative_to(path_entry)), 'project') + + # Fallback: just use the filename + return (_path_to_module(file_path), 'other') + + +def _path_to_module(path): + if isinstance(path, str): + path = Path(path) + + # Remove .py extension + if path.suffix == '.py': + path = path.with_suffix('') + + # Convert path separators to dots, stripping root/drive (e.g. "/" or "C:\") + parts = [p for p in path.parts if p != path.root and p != path.drive] + + # Handle __init__ files - they represent the package itself + if parts and parts[-1] == '__init__': + parts = parts[:-1] + + return '.'.join(parts) if parts else path.stem diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 6a76bbeeb24..9195f5ee6dd 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -58,6 +58,10 @@ def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MOD try: self.unwinder = self._new_unwinder(native, gc, opcodes, skip_non_matching_threads) except RuntimeError as err: + if os.name == "nt" and sys.executable.endswith("python.exe"): + raise SystemExit( + "Running profiling.sampling from virtualenv on Windows platform is not supported" + ) from err raise SystemExit(err) from err # Track sample intervals and total sample count self.sample_intervals = deque(maxlen=100) diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index 461ce95a258..04622a8c1e8 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -12,6 +12,7 @@ from .collector import Collector, extract_lineno from .opcode_utils import get_opcode_mapping from .string_table import StringTable +from .module_utils import extract_module_name, get_python_path_info class StackTraceCollector(Collector): @@ -72,6 +73,7 @@ def __init__(self, *args, **kwargs): self._sample_count = 0 # Track actual number of samples (not thread traces) self._func_intern = {} self._string_table = StringTable() + self._module_cache = {} self._all_threads = set() # Thread status statistics (similar to LiveStatsCollector) @@ -182,6 +184,24 @@ def _format_function_name(func): return f"{funcname} ({filename}:{lineno})" + @staticmethod + @functools.lru_cache(maxsize=None) + def _format_module_name(func, module_name): + filename, lineno, funcname = func + + # Special frames like and should not show file:line + if filename == "~" and lineno == 0: + return funcname + + return f"{funcname} ({module_name}:{lineno})" + + def _get_module_name(self, filename, path_info): + module_name = self._module_cache.get(filename) + if module_name is None: + module_name, _ = extract_module_name(filename, path_info) + self._module_cache[filename] = module_name + return module_name + def _convert_to_flamegraph_format(self): if self._total_samples == 0: return { @@ -192,7 +212,7 @@ def _convert_to_flamegraph_format(self): "strings": self._string_table.get_strings() } - def convert_children(children, min_samples): + def convert_children(children, min_samples, path_info): out = [] for func, node in children.items(): samples = node["samples"] @@ -202,14 +222,20 @@ def convert_children(children, min_samples): # Intern all string components for maximum efficiency filename_idx = self._string_table.intern(func[0]) funcname_idx = self._string_table.intern(func[2]) + module_name = self._get_module_name(func[0], path_info) + + module_idx = self._string_table.intern(module_name) name_idx = self._string_table.intern(self._format_function_name(func)) + label_idx = self._string_table.intern(self._format_module_name(func, module_name)) child_entry = { "name": name_idx, + "label": label_idx, "value": samples, "self": node.get("self", 0), "children": [], "filename": filename_idx, + "module": module_idx, "lineno": func[1], "funcname": funcname_idx, "threads": sorted(list(node.get("threads", set()))), @@ -228,7 +254,7 @@ def convert_children(children, min_samples): # Recurse child_entry["children"] = convert_children( - node["children"], min_samples + node["children"], min_samples, path_info ) out.append(child_entry) @@ -239,8 +265,9 @@ def convert_children(children, min_samples): # Filter out very small functions (less than 0.1% of total samples) total_samples = self._total_samples min_samples = max(1, int(total_samples * 0.001)) + path_info = get_python_path_info() - root_children = convert_children(self._root["children"], min_samples) + root_children = convert_children(self._root["children"], min_samples, path_info) if not root_children: return { "name": self._string_table.intern("No significant data"), @@ -282,10 +309,11 @@ def convert_children(children, min_samples): # If we only have one root child, make it the root to avoid redundant level if len(root_children) == 1: main_child = root_children[0] - # Update the name to indicate it's the program root + # Update name and label to indicate it's the program root old_name = self._string_table.get_string(main_child["name"]) - new_name = f"Program Root: {old_name}" - main_child["name"] = self._string_table.intern(new_name) + main_child["name"] = self._string_table.intern(f"Program Root: {old_name}") + old_label = self._string_table.get_string(main_child["label"]) + main_child["label"] = self._string_table.intern(f"Program Root: {old_label}") main_child["stats"] = { **self.stats, "thread_stats": thread_stats, @@ -296,8 +324,10 @@ def convert_children(children, min_samples): main_child["opcode_mapping"] = opcode_mapping return main_child + program_root_idx = self._string_table.intern("Program Root") return { - "name": self._string_table.intern("Program Root"), + "name": program_root_idx, + "label": program_root_idx, "value": total_samples, "children": root_children, "stats": { diff --git a/Lib/random.py b/Lib/random.py index c89cbb755ab..726a71e7828 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -836,7 +836,11 @@ def binomialvariate(self, n=1, p=0.5): if not c: return x while True: - y += _floor(_log2(random()) / c) + 1 + try: + y += _floor(_log2(random()) / c) + 1 + except ValueError: + # Reject case where random() returned 0.0 + continue if y > n: return x x += 1 diff --git a/Lib/runpy.py b/Lib/runpy.py index 9f62d20e9a2..a535b4f651a 100644 --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -102,8 +102,10 @@ def _run_module_code(code, init_globals=None, # Helper to get the full name, spec and code for a module def _get_module_details(mod_name, error=ImportError): + # name= is only accepted by ImportError and its subclasses. + kwargs = {"name": mod_name} if issubclass(error, ImportError) else {} if mod_name.startswith("."): - raise error("Relative module names not supported") + raise error("Relative module names not supported", **kwargs) pkg_name, _, _ = mod_name.rpartition(".") if pkg_name: # Try importing the parent to avoid catching initialization errors @@ -136,12 +138,13 @@ def _get_module_details(mod_name, error=ImportError): if mod_name.endswith(".py"): msg += (f". Try using '{mod_name[:-3]}' instead of " f"'{mod_name}' as the module name.") - raise error(msg.format(mod_name, type(ex).__name__, ex)) from ex + raise error(msg.format(mod_name, type(ex).__name__, ex), + **kwargs) from ex if spec is None: - raise error("No module named %s" % mod_name) + raise error("No module named %s" % mod_name, **kwargs) if spec.submodule_search_locations is not None: if mod_name == "__main__" or mod_name.endswith(".__main__"): - raise error("Cannot use package as __main__ module") + raise error("Cannot use package as __main__ module", **kwargs) try: pkg_main_name = mod_name + ".__main__" return _get_module_details(pkg_main_name, error) @@ -149,17 +152,19 @@ def _get_module_details(mod_name, error=ImportError): if mod_name not in sys.modules: raise # No module loaded; being a package is irrelevant raise error(("%s; %r is a package and cannot " + - "be directly executed") %(e, mod_name)) + "be directly executed") %(e, mod_name), + **kwargs) loader = spec.loader if loader is None: raise error("%r is a namespace package and cannot be executed" - % mod_name) + % mod_name, + **kwargs) try: code = loader.get_code(mod_name) except ImportError as e: - raise error(format(e)) from e + raise error(format(e), **kwargs) from e if code is None: - raise error("No code object available for %s" % mod_name) + raise error("No code object available for %s" % mod_name, **kwargs) return mod_name, spec, code class _Error(Exception): @@ -232,6 +237,7 @@ def _get_main_module_details(error=ImportError): # Also moves the standard __main__ out of the way so that the # preexisting __loader__ entry doesn't cause issues main_name = "__main__" + kwargs = {"name": main_name} if issubclass(error, ImportError) else {} saved_main = sys.modules[main_name] del sys.modules[main_name] try: @@ -239,7 +245,8 @@ def _get_main_module_details(error=ImportError): except ImportError as exc: if main_name in str(exc): raise error("can't find %r module in %r" % - (main_name, sys.path[0])) from exc + (main_name, sys.path[0]), + **kwargs) from exc raise finally: sys.modules[main_name] = saved_main diff --git a/Lib/shutil.py b/Lib/shutil.py index 44ccdbb503d..45cbe4c855b 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -1317,27 +1317,9 @@ def _unpack_zipfile(filename, extract_dir): if not zipfile.is_zipfile(filename): raise ReadError("%s is not a zip file" % filename) - zip = zipfile.ZipFile(filename) - try: - for info in zip.infolist(): - name = info.filename - - # don't extract absolute paths or ones with .. in them - if name.startswith('/') or '..' in name: - continue - - targetpath = os.path.join(extract_dir, *name.split('/')) - if not targetpath: - continue - - _ensure_directory(targetpath) - if not name.endswith('/'): - # file - with zip.open(name, 'r') as source, \ - open(targetpath, 'wb') as target: - copyfileobj(source, target) - finally: - zip.close() + with zipfile.ZipFile(filename) as zip: + zip._ignore_invalid_names = True + zip.extractall(extract_dir) def _unpack_tarfile(filename, extract_dir, *, filter=None): """Unpack tar/tar.gz/tar.bz2/tar.xz/tar.zst `filename` to `extract_dir` diff --git a/Lib/statistics.py b/Lib/statistics.py index e635b99f958..01ca6c51daf 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -136,7 +136,7 @@ from fractions import Fraction from decimal import Decimal -from itertools import count, groupby, repeat +from itertools import compress, count, groupby, repeat from bisect import bisect_left, bisect_right from math import hypot, sqrt, fabs, exp, erfc, tau, log, fsum, sumprod from math import isfinite, isinf, pi, cos, sin, tan, cosh, asin, atan, acos @@ -195,9 +195,9 @@ def fmean(data, weights=None): n = len(data) except TypeError: # Handle iterators that do not define __len__(). - counter = count() - total = fsum(map(itemgetter(0), zip(data, counter))) - n = next(counter) + counter = count(1) + total = fsum(compress(data, counter)) + n = next(counter) - 1 else: total = fsum(data) @@ -248,7 +248,7 @@ def count_positive(iterable): elif x == 0.0: found_zero = True else: - raise StatisticsError('No negative inputs allowed', x) + raise StatisticsError(f'No negative inputs allowed: {x!r}') total = fsum(map(log, count_positive(data))) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 7ac2289f535..38b655f2f7b 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -250,6 +250,82 @@ def __repr__(self): else: _PopenSelector = selectors.SelectSelector + def _communicate_io_posix(selector, stdin, input_view, input_offset, + output_buffers, endtime, *, close_on_eof=False): + """ + Low-level POSIX I/O multiplexing loop used by Popen._communicate. + + Handles the select loop for reading/writing but does not manage + stream lifecycle or raise timeout exceptions. + + Args: + selector: A _PopenSelector with streams already registered + stdin: Writable file object for input, or None + input_view: memoryview of input bytes, or None + input_offset: Starting offset into input_view (for resume support) + output_buffers: Dict {file_object: list} to append read chunks to + endtime: Deadline timestamp, or None for no timeout + close_on_eof: If True, close output streams immediately when they + EOF rather than leaving them open for the caller to close. + Used by Popen._communicate() to match its historical behavior + of releasing fds as soon as the child closes the corresponding + pipe. + + Returns: + (new_input_offset, completed) + - new_input_offset: How many bytes of input were written + - completed: True if all I/O finished, False if timed out + + Note: + - Closes output streams on EOF only if close_on_eof=True + - Does NOT raise TimeoutExpired (caller handles) + - Appends to output_buffers lists in place + """ + stdin_fd = stdin.fileno() if stdin else None + + while selector.get_map(): + remaining = _deadline_remaining(endtime) + if remaining is not None and remaining <= 0: + return (input_offset, False) # Timed out + + ready = selector.select(remaining) + + # Check timeout after select (may have woken spuriously) + if endtime is not None and _time() > endtime: + return (input_offset, False) # Timed out + + for key, events in ready: + if key.fd == stdin_fd: + chunk = input_view[input_offset:input_offset + _PIPE_BUF] + try: + input_offset += os.write(key.fd, chunk) + except BrokenPipeError: + selector.unregister(key.fd) + try: + stdin.close() + except BrokenPipeError: + pass + else: + if input_offset >= len(input_view): + selector.unregister(key.fd) + try: + stdin.close() + except BrokenPipeError: + pass + elif key.fileobj in output_buffers: + data = os.read(key.fd, 32768) + if not data: + selector.unregister(key.fileobj) + if close_on_eof: + try: + key.fileobj.close() + except OSError: + pass + else: + output_buffers[key.fileobj].append(data) + + return (input_offset, True) # Completed + if _mswindows: # On Windows we just need to close `Popen._handle` when we no longer need @@ -289,6 +365,45 @@ def _cleanup(): DEVNULL = -3 +def _deadline_remaining(endtime): + """Calculate remaining time until deadline.""" + if endtime is None: + return None + return endtime - _time() + + +def _flush_stdin(stdin): + """Flush stdin, ignoring BrokenPipeError and closed file ValueError.""" + try: + stdin.flush() + except BrokenPipeError: + pass # communicate() must ignore BrokenPipeError. + except ValueError: + # Ignore ValueError: I/O operation on closed file. + if not stdin.closed: + raise + + +def _make_input_view(input_data): + """Convert input data to a byte memoryview for writing. + + Handles the case where input_data is already a memoryview with + non-byte elements (e.g., int32 array) by casting to a byte view. + This ensures len(view) returns the byte count, not element count. + """ + if not input_data: + return None + if isinstance(input_data, memoryview): + return input_data.cast("b") # ensure byte view for correct len() + return memoryview(input_data) + + +def _translate_newlines(data, encoding, errors): + """Decode bytes to str and translate newlines to \n.""" + data = data.decode(encoding, errors) + return data.replace("\r\n", "\n").replace("\r", "\n") + + # XXX This function is only used by multiprocessing and the test suite, # but it's here so that it can be imported when Python is compiled without # threads. @@ -1149,8 +1264,8 @@ def universal_newlines(self, universal_newlines): self.text_mode = bool(universal_newlines) def _translate_newlines(self, data, encoding, errors): - data = data.decode(encoding, errors) - return data.replace("\r\n", "\n").replace("\r", "\n") + # Subclass-overridable hook; defers to the module-level helper. + return _translate_newlines(data, encoding, errors) def __enter__(self): return self @@ -1277,7 +1392,7 @@ def communicate(self, input=None, timeout=None): # See the detailed comment in .wait(). if timeout is not None: sigint_timeout = min(self._sigint_wait_secs, - self._remaining_time(endtime)) + _deadline_remaining(endtime)) else: sigint_timeout = self._sigint_wait_secs self._sigint_wait_secs = 0 # nothing else should wait. @@ -1290,7 +1405,7 @@ def communicate(self, input=None, timeout=None): finally: self._communication_started = True try: - self.wait(timeout=self._remaining_time(endtime)) + self.wait(timeout=_deadline_remaining(endtime)) except TimeoutExpired as exc: exc.timeout = timeout raise @@ -1304,14 +1419,6 @@ def poll(self): return self._internal_poll() - def _remaining_time(self, endtime): - """Convenience for _communicate when computing timeouts.""" - if endtime is None: - return None - else: - return endtime - _time() - - def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq, skip_check_and_raise=False): """Convenience for checking if a timeout has expired.""" @@ -1337,7 +1444,7 @@ def wait(self, timeout=None): # generated SIGINT and will exit rapidly. if timeout is not None: sigint_timeout = min(self._sigint_wait_secs, - self._remaining_time(endtime)) + _deadline_remaining(endtime)) else: sigint_timeout = self._sigint_wait_secs self._sigint_wait_secs = 0 # nothing else should wait. @@ -1704,7 +1811,7 @@ def _communicate(self, input, endtime, orig_timeout): # thread remains writing and the fd left open in case the user # calls communicate again. if hasattr(self, "_stdin_thread"): - self._stdin_thread.join(self._remaining_time(endtime)) + self._stdin_thread.join(_deadline_remaining(endtime)) if self._stdin_thread.is_alive(): raise TimeoutExpired(self.args, orig_timeout) @@ -1712,11 +1819,11 @@ def _communicate(self, input, endtime, orig_timeout): # threads remain reading and the fds left open in case the user # calls communicate again. if self.stdout is not None: - self.stdout_thread.join(self._remaining_time(endtime)) + self.stdout_thread.join(_deadline_remaining(endtime)) if self.stdout_thread.is_alive(): raise TimeoutExpired(self.args, orig_timeout) if self.stderr is not None: - self.stderr_thread.join(self._remaining_time(endtime)) + self.stderr_thread.join(_deadline_remaining(endtime)) if self.stderr_thread.is_alive(): raise TimeoutExpired(self.args, orig_timeout) @@ -2210,7 +2317,7 @@ def _wait(self, timeout): break finally: self._waitpid_lock.release() - remaining = self._remaining_time(endtime) + remaining = _deadline_remaining(endtime) if remaining <= 0: raise TimeoutExpired(self.args, timeout) delay = min(delay * 2, remaining, .05) @@ -2234,14 +2341,7 @@ def _communicate(self, input, endtime, orig_timeout): if self.stdin and not self._communication_started: # Flush stdio buffer. This might block, if the user has # been writing to .stdin in an uncontrolled fashion. - try: - self.stdin.flush() - except BrokenPipeError: - pass # communicate() must ignore BrokenPipeError. - except ValueError: - # ignore ValueError: I/O operation on closed file. - if not self.stdin.closed: - raise + _flush_stdin(self.stdin) if not input: try: self.stdin.close() @@ -2266,11 +2366,8 @@ def _communicate(self, input, endtime, orig_timeout): self._save_input(input) - if self._input: - if not isinstance(self._input, memoryview): - input_view = memoryview(self._input) - else: - input_view = self._input.cast("b") # byte input required + input_view = _make_input_view(self._input) + input_offset = self._input_offset if self._input else 0 with _PopenSelector() as selector: if self.stdin and not self.stdin.closed and self._input: @@ -2280,43 +2377,31 @@ def _communicate(self, input, endtime, orig_timeout): if self.stderr and not self.stderr.closed: selector.register(self.stderr, selectors.EVENT_READ) - while selector.get_map(): - timeout = self._remaining_time(endtime) - if timeout is not None and timeout <= 0: - self._check_timeout(endtime, orig_timeout, - stdout, stderr, - skip_check_and_raise=True) - raise RuntimeError( # Impossible :) - '_check_timeout(..., skip_check_and_raise=True) ' - 'failed to raise TimeoutExpired.') + stdin_to_write = (self.stdin if self.stdin and self._input + and not self.stdin.closed else None) + # Persist the returned offset on self so a subsequent + # communicate() after a TimeoutExpired resumes mid-input + # rather than re-sending bytes the child already consumed. + new_offset, completed = _communicate_io_posix( + selector, + stdin_to_write, + input_view, + input_offset, + self._fileobj2output, + endtime, + close_on_eof=True) + if self._input: + self._input_offset = new_offset - ready = selector.select(timeout) - self._check_timeout(endtime, orig_timeout, stdout, stderr) + if not completed: + self._check_timeout(endtime, orig_timeout, stdout, stderr, + skip_check_and_raise=True) + raise RuntimeError( # Impossible :) + '_check_timeout(..., skip_check_and_raise=True) ' + 'failed to raise TimeoutExpired.') - # XXX Rewrite these to use non-blocking I/O on the file - # objects; they are no longer using C stdio! - - for key, events in ready: - if key.fileobj is self.stdin: - chunk = input_view[self._input_offset : - self._input_offset + _PIPE_BUF] - try: - self._input_offset += os.write(key.fd, chunk) - except BrokenPipeError: - selector.unregister(key.fileobj) - key.fileobj.close() - else: - if self._input_offset >= len(input_view): - selector.unregister(key.fileobj) - key.fileobj.close() - elif key.fileobj in (self.stdout, self.stderr): - data = os.read(key.fd, 32768) - if not data: - selector.unregister(key.fileobj) - key.fileobj.close() - self._fileobj2output[key.fileobj].append(data) try: - self.wait(timeout=self._remaining_time(endtime)) + self.wait(timeout=_deadline_remaining(endtime)) except TimeoutExpired as exc: exc.timeout = orig_timeout raise diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 7f0b0b3c632..4f47aaab902 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -337,7 +337,7 @@ class _Stream: """ def __init__(self, name, mode, comptype, fileobj, bufsize, - compresslevel, preset): + compresslevel, preset, mtime): """Construct a _Stream object. """ self._extfileobj = True @@ -372,7 +372,7 @@ def __init__(self, name, mode, comptype, fileobj, bufsize, self.exception = zlib.error self._init_read_gz() else: - self._init_write_gz(compresslevel) + self._init_write_gz(compresslevel, mtime) elif comptype == "bz2": try: @@ -421,7 +421,7 @@ def __del__(self): if hasattr(self, "closed") and not self.closed: self.close() - def _init_write_gz(self, compresslevel): + def _init_write_gz(self, compresslevel, mtime): """Initialize for writing with gzip compression. """ self.cmp = self.zlib.compressobj(compresslevel, @@ -429,7 +429,9 @@ def _init_write_gz(self, compresslevel): -self.zlib.MAX_WBITS, self.zlib.DEF_MEM_LEVEL, 0) - timestamp = struct.pack(" leftover 0001 (non-zero) + with self.assertRaises(binascii.Error): + binascii.a2b_base64(self.type2test(b'AB=='), canonical=True) + # 'A' = 0, 'P' = 15 -> leftover 1111 (non-zero) + with self.assertRaises(binascii.Error): + binascii.a2b_base64(self.type2test(b'AP=='), canonical=True) + + # 3 data chars + "=": last char has 2 padding bits + # 'A' = 0, 'A' = 0, 'B' = 1 -> leftover 01 (non-zero) + with self.assertRaises(binascii.Error): + binascii.a2b_base64(self.type2test(b'AAB='), canonical=True) + # 'A' = 0, 'A' = 0, 'D' = 3 -> leftover 11 (non-zero) + with self.assertRaises(binascii.Error): + binascii.a2b_base64(self.type2test(b'AAD='), canonical=True) + + # Verify that zero padding bits are accepted + binascii.a2b_base64(self.type2test(b'AA=='), canonical=True) + binascii.a2b_base64(self.type2test(b'AAA='), canonical=True) + + # Full quads with no padding have no leftover bits -- always valid + binascii.a2b_base64(self.type2test(b'AAAA'), canonical=True) + + @hypothesis.given(payload=hypothesis.strategies.binary()) + @hypothesis.example(b'') + @hypothesis.example(b'\x00') + @hypothesis.example(b'\xff\xff') + @hypothesis.example(b'abc') + def test_base64_canonical_roundtrip(self, payload): + # The encoder must always produce canonical output. + encoded = binascii.b2a_base64(payload, newline=False) + decoded = binascii.a2b_base64(encoded, canonical=True) + self.assertEqual(decoded, payload) + def test_base64_alphabet(self): alphabet = (b'!"#$%&\'()*+,-012345689@' b'ABCDEFGHIJKLMNPQRSTUVXYZ[`abcdefhijklmpqr') @@ -443,20 +486,22 @@ def test_ascii85_valid(self): res += b self.assertEqual(res, rawdata) - # Test decoding inputs with length 1 mod 5 - params = [ - (b"a", False, False, b"", b""), - (b"xbw", False, False, b"wx", b""), - (b"<~c~>", False, True, b"", b""), - (b"{d ~>", False, True, b" {", b""), - (b"ye", True, False, b"", b" "), - (b"z\x01y\x00f", True, False, b"\x00\x01", b"\x00\x00\x00\x00 "), - (b"<~FCfN8yg~>", True, True, b"", b"test "), - (b"FE;\x03#8zFCf\x02N8yh~>", True, True, b"\x02\x03", b"tset\x00\x00\x00\x00test "), + # Inputs with length 1 mod 5 end with a 1-char group, which is + # an encoding violation per the PLRM spec. + error_params = [ + (b"a", False, False, b""), + (b"xbw", False, False, b"wx"), + (b"<~c~>", False, True, b""), + (b"{d ~>", False, True, b" {"), + (b"ye", True, False, b""), + (b"z\x01y\x00f", True, False, b"\x00\x01"), + (b"<~FCfN8yg~>", True, True, b""), + (b"FE;\x03#8zFCf\x02N8yh~>", True, True, b"\x02\x03"), ] - for a, foldspaces, adobe, ignorechars, b in params: + for a, foldspaces, adobe, ignorechars in error_params: kwargs = {"foldspaces": foldspaces, "adobe": adobe, "ignorechars": ignorechars} - self.assertEqual(binascii.a2b_ascii85(self.type2test(a), **kwargs), b) + with self.assertRaises(binascii.Error): + binascii.a2b_ascii85(self.type2test(a), **kwargs) def test_ascii85_invalid(self): # Test Ascii85 with invalid characters interleaved @@ -670,16 +715,18 @@ def test_base85_valid(self): self.assertEqual(res, self.rawdata) # Test decoding inputs with different length - self.assertEqual(binascii.a2b_base85(self.type2test(b'a')), b'') - self.assertEqual(binascii.a2b_base85(self.type2test(b'a')), b'') + # 1-char groups are rejected (encoding violation) + with self.assertRaises(binascii.Error): + binascii.a2b_base85(self.type2test(b'a')) self.assertEqual(binascii.a2b_base85(self.type2test(b'ab')), b'q') self.assertEqual(binascii.a2b_base85(self.type2test(b'abc')), b'qa') self.assertEqual(binascii.a2b_base85(self.type2test(b'abcd')), b'qa\x9e') self.assertEqual(binascii.a2b_base85(self.type2test(b'abcde')), b'qa\x9e\xb6') - self.assertEqual(binascii.a2b_base85(self.type2test(b'abcdef')), - b'qa\x9e\xb6') + # 6-char input = full 5-char group + trailing 1-char group (rejected) + with self.assertRaises(binascii.Error): + binascii.a2b_base85(self.type2test(b'abcdef')) self.assertEqual(binascii.a2b_base85(self.type2test(b'abcdefg')), b'qa\x9e\xb6\x81') @@ -767,6 +814,169 @@ def test_base85_alphabet(self): with self.assertRaises(TypeError): binascii.a2b_base64(data, alphabet=bytearray(alphabet)) + def test_base85_canonical(self): + # Non-canonical encodings are accepted without canonical=True + self.assertEqual(binascii.a2b_base85(b'VF'), b'a') + + # 1-char partial groups are always rejected (encoding violation: + # no conforming encoder produces them) + with self.assertRaises(binascii.Error): + binascii.a2b_base85(b'V') + with self.assertRaises(binascii.Error): + binascii.a2b_base85(b'0') + + # Verify round-trip: encode then decode with canonical=True works + for data in [b'a', b'ab', b'abc', b'abcd', b'abcde', + b'\x00', b'\xff', b'\x00\x00', b'\xff\xff\xff']: + encoded = binascii.b2a_base85(data) + decoded = binascii.a2b_base85(encoded, canonical=True) + self.assertEqual(decoded, data) + + # Test non-canonical rejection for each partial group size + # (2-char/1-byte, 3-char/2-byte, 4-char/3-byte). + # Incrementing the last digit by 1 produces a non-canonical + # encoding. For 4-char groups (n_pad=1) a +1 can change the + # output byte, so we use b'ab\x00' whose canonical form allows + # a +1 that still decodes to the same 3 bytes. + for data in [b'a', b'ab', b'ab\x00']: + canonical_enc = binascii.b2a_base85(data) + non_canonical = (canonical_enc[:-1] + + bytes([canonical_enc[-1] + 1])) + # Same decoded output without canonical check + self.assertEqual(binascii.a2b_base85(non_canonical), data) + # Rejected with canonical=True + with self.assertRaises(binascii.Error): + binascii.a2b_base85(non_canonical, canonical=True) + + # Boundary bytes: \x00 and \xff for each partial group size + for data in [b'\x00', b'\x00\x00', b'\x00\x00\x00', + b'\xff', b'\xff\xff', b'\xff\xff\xff']: + canonical_enc = binascii.b2a_base85(data) + binascii.a2b_base85(canonical_enc, canonical=True) + + # Full 5-char groups are always canonical (no padding bits) + self.assertEqual( + binascii.a2b_base85(b'VPa!s', canonical=True), b'abcd') + + # Empty input is valid + self.assertEqual(binascii.a2b_base85(b'', canonical=True), b'') + + @hypothesis.given(payload=hypothesis.strategies.binary()) + @hypothesis.example(b'') + @hypothesis.example(b'\x00') + @hypothesis.example(b'\xff\xff') + @hypothesis.example(b'abc') + def test_base85_canonical_roundtrip(self, payload): + encoded = binascii.b2a_base85(payload) + decoded = binascii.a2b_base85(encoded, canonical=True) + self.assertEqual(decoded, payload) + + @hypothesis.given(payload=hypothesis.strategies.binary(min_size=1, max_size=3)) + @hypothesis.example(b'\x00') + @hypothesis.example(b'\xff') + @hypothesis.example(b'ab\x00') + def test_base85_canonical_unique(self, payload): + # For a partial group, sweeping all 85 last-digit values should + # yield exactly one encoding that both decodes to the original + # payload AND passes canonical=True. + hypothesis.assume(len(payload) % 4 != 0) + canonical_enc = binascii.b2a_base85(payload) + table = binascii.BASE85_ALPHABET + accepted = [] + for digit in table: + candidate = canonical_enc[:-1] + bytes([digit]) + try: + result = binascii.a2b_base85(candidate, canonical=True) + if result == payload: + accepted.append(candidate) + except binascii.Error: + pass + self.assertEqual(accepted, [canonical_enc]) + + def test_ascii85_canonical(self): + # Non-canonical encodings are accepted without canonical=True + self.assertEqual(binascii.a2b_ascii85(b'@0'), b'a') + + # 1-char partial groups are always rejected (PLRM encoding violation) + with self.assertRaises(binascii.Error): + binascii.a2b_ascii85(b'@') + with self.assertRaises(binascii.Error): + binascii.a2b_ascii85(b'!') + + # Verify round-trip: encode then decode with canonical=True works + for data in [b'a', b'ab', b'abc', b'abcd', b'abcde', + b'\x00', b'\xff', b'\x00\x00', b'\xff\xff\xff']: + encoded = binascii.b2a_ascii85(data) + decoded = binascii.a2b_ascii85(encoded, canonical=True) + self.assertEqual(decoded, data) + + # Test non-canonical rejection for each partial group size. + # See test_base85_canonical for why b'ab\x00' is used for 3 bytes. + for data in [b'a', b'ab', b'ab\x00']: + canonical_enc = binascii.b2a_ascii85(data) + non_canonical = (canonical_enc[:-1] + + bytes([canonical_enc[-1] + 1])) + self.assertEqual(binascii.a2b_ascii85(non_canonical), data) + with self.assertRaises(binascii.Error): + binascii.a2b_ascii85(non_canonical, canonical=True) + + # Full 5-char groups are always canonical + self.assertEqual( + binascii.a2b_ascii85(b'@:E_W', canonical=True), b'abcd') + + # 'z' is the canonical form for all-zero groups per the PLRM. + # '!!!!!' decodes identically but is non-canonical. + self.assertEqual(binascii.a2b_ascii85(b'!!!!!'), b'\x00' * 4) + self.assertEqual(binascii.a2b_ascii85(b'z'), b'\x00' * 4) + self.assertEqual( + binascii.a2b_ascii85(b'z', canonical=True), b'\x00' * 4) + with self.assertRaises(binascii.Error): + binascii.a2b_ascii85(b'!!!!!', canonical=True) + # Multiple groups: z + !!!!! should fail + with self.assertRaises(binascii.Error): + binascii.a2b_ascii85(b'z!!!!!', canonical=True) + # Multiple z groups are fine + self.assertEqual( + binascii.a2b_ascii85(b'zz', canonical=True), b'\x00' * 8) + + # Empty input is valid + self.assertEqual(binascii.a2b_ascii85(b'', canonical=True), b'') + + # Adobe-wrapped with canonical + self.assertEqual( + binascii.a2b_ascii85(b'<~@:E_W~>', canonical=True, adobe=True), + b'abcd') + + @hypothesis.given(payload=hypothesis.strategies.binary()) + @hypothesis.example(b'') + @hypothesis.example(b'\x00') + @hypothesis.example(b'\x00\x00\x00\x00') # triggers z abbreviation + @hypothesis.example(b'\xff\xff') + @hypothesis.example(b'abc') + def test_ascii85_canonical_roundtrip(self, payload): + encoded = binascii.b2a_ascii85(payload) + decoded = binascii.a2b_ascii85(encoded, canonical=True) + self.assertEqual(decoded, payload) + + @hypothesis.given(payload=hypothesis.strategies.binary(min_size=1, max_size=3)) + @hypothesis.example(b'\x00') + @hypothesis.example(b'\xff') + @hypothesis.example(b'ab\x00') + def test_ascii85_canonical_unique(self, payload): + hypothesis.assume(len(payload) % 4 != 0) + canonical_enc = binascii.b2a_ascii85(payload) + # Ascii85 alphabet: '!' (33) through 'u' (117) + accepted = [] + for digit in range(33, 118): + candidate = canonical_enc[:-1] + bytes([digit]) + try: + result = binascii.a2b_ascii85(candidate, canonical=True) + if result == payload: + accepted.append(candidate) + except binascii.Error: + pass + self.assertEqual(accepted, [canonical_enc]) + def test_base32_valid(self): # Test base32 with valid data lines = [] @@ -935,6 +1145,55 @@ def assertInvalidLength(data, *args, length=None, **kwargs): assertInvalidLength(b" ABC=====", ignorechars=b' ') assertInvalidLength(b" ABCDEF==", ignorechars=b' ') + def test_base32_canonical(self): + # https://datatracker.ietf.org/doc/html/rfc4648.html#section-3.5 + # Decoders MAY reject encoded data if the pad bits are not zero. + + # Without canonical=True, non-zero padding bits are accepted + self.assertEqual(binascii.a2b_base32(self.type2test(b'AB======')), + b'\x00') + + # 2 data chars + "======": last char has 2 padding bits + with self.assertRaises(binascii.Error): + binascii.a2b_base32(self.type2test(b'AB======'), canonical=True) + with self.assertRaises(binascii.Error): + binascii.a2b_base32(self.type2test(b'AD======'), canonical=True) + + # 4 data chars + "====": last char has 4 padding bits + with self.assertRaises(binascii.Error): + binascii.a2b_base32(self.type2test(b'AAAB===='), canonical=True) + with self.assertRaises(binascii.Error): + binascii.a2b_base32(self.type2test(b'AAAP===='), canonical=True) + + # 5 data chars + "===": last char has 1 padding bit + with self.assertRaises(binascii.Error): + binascii.a2b_base32(self.type2test(b'AAAAB==='), canonical=True) + + # 7 data chars + "=": last char has 3 padding bits + with self.assertRaises(binascii.Error): + binascii.a2b_base32(self.type2test(b'AAAAAAB='), canonical=True) + with self.assertRaises(binascii.Error): + binascii.a2b_base32(self.type2test(b'AAAAAAH='), canonical=True) + + # Verify that zero padding bits are accepted + binascii.a2b_base32(self.type2test(b'AA======'), canonical=True) + binascii.a2b_base32(self.type2test(b'AAAA===='), canonical=True) + binascii.a2b_base32(self.type2test(b'AAAAA==='), canonical=True) + binascii.a2b_base32(self.type2test(b'AAAAAAA='), canonical=True) + + # Full octet with no padding -- always valid + binascii.a2b_base32(self.type2test(b'AAAAAAAA'), canonical=True) + + @hypothesis.given(payload=hypothesis.strategies.binary()) + @hypothesis.example(b'') + @hypothesis.example(b'\x00') + @hypothesis.example(b'\xff\xff') + @hypothesis.example(b'abc') + def test_base32_canonical_roundtrip(self, payload): + encoded = binascii.b2a_base32(payload) + decoded = binascii.a2b_base32(encoded, canonical=True) + self.assertEqual(decoded, payload) + def test_a2b_base32_padded(self): a2b_base32 = binascii.a2b_base32 t = self.type2test diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 844656eb0e2..81967fb8a83 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -4,6 +4,7 @@ import builtins import collections import contextlib +import copy import decimal import fractions import gc @@ -21,6 +22,7 @@ import typing import unittest import warnings +import weakref from contextlib import ExitStack from functools import partial from inspect import CO_COROUTINE @@ -52,6 +54,10 @@ # used as proof of globals being used A_GLOBAL_VALUE = 123 +A_SENTINEL = sentinel("A_SENTINEL") + +class SentinelContainer: + CLASS_SENTINEL = sentinel("SentinelContainer.CLASS_SENTINEL") class Squares: @@ -302,6 +308,27 @@ def f_set(): self.assertEqual(overridden_outputs, ['all', 'any', 'tuple', 'list', 'set']) + def test_builtin_call_async_genexpr_no_crash(self): + async def f_all(): + return all(await 2 for _ in []) + + async def f_any(): + return any(await 2 for _ in []) + + async def f_tuple(): + return tuple(await 2 for _ in []) + + async def f_list(): + return list(await 2 for _ in []) + + async def f_set(): + return set(await 2 for _ in []) + + for f in (f_all, f_any, f_tuple, f_list, f_set): + with self.subTest(func=f.__name__): + with self.assertRaises(TypeError): + run_yielding_async_fn(f) + def test_ascii(self): self.assertEqual(ascii(''), '\'\'') self.assertEqual(ascii(0), '0') @@ -1903,6 +1930,98 @@ class C: __repr__ = None self.assertRaises(TypeError, repr, C()) + def test_sentinel(self): + missing = sentinel("MISSING") + other = sentinel("MISSING") + + self.assertIsInstance(missing, sentinel) + self.assertIs(type(missing), sentinel) + self.assertEqual(missing.__name__, "MISSING") + self.assertEqual(missing.__module__, __name__) + self.assertIsNot(missing, other) + self.assertEqual(repr(missing), "MISSING") + self.assertTrue(missing) + self.assertIs(copy.copy(missing), missing) + self.assertIs(copy.deepcopy(missing), missing) + self.assertEqual(missing, missing) + self.assertNotEqual(missing, other) + self.assertRaises(TypeError, sentinel) + self.assertRaises(TypeError, sentinel, "MISSING", "EXTRA") + self.assertRaises(TypeError, sentinel, name="MISSING") + with self.assertRaisesRegex(TypeError, "must be str"): + sentinel(1) + self.assertTrue(sentinel.__flags__ & support._TPFLAGS_IMMUTABLETYPE) + self.assertTrue(sentinel.__flags__ & support._TPFLAGS_HAVE_GC) + self.assertFalse(sentinel.__flags__ & support._TPFLAGS_BASETYPE) + with self.assertRaises(TypeError): + class SubSentinel(sentinel): + pass + with self.assertRaises(TypeError): + sentinel.attribute = "value" + with self.assertRaises(AttributeError): + missing.__name__ = "CHANGED" + with self.assertRaises(AttributeError): + missing.__module__ = "changed" + with self.assertRaises(AttributeError): + del missing.__name__ + with self.assertRaises(AttributeError): + del missing.__module__ + + def test_sentinel_pickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + self.assertIs( + pickle.loads(pickle.dumps(A_SENTINEL, protocol=proto)), + A_SENTINEL) + self.assertIs( + pickle.loads(pickle.dumps( + SentinelContainer.CLASS_SENTINEL, protocol=proto)), + SentinelContainer.CLASS_SENTINEL) + + missing = sentinel("MISSING") + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + with self.assertRaises(pickle.PicklingError): + pickle.dumps(missing, protocol=proto) + + def test_sentinel_str_subclass_name_cycle(self): + class Name(str): + pass + + name = Name("MISSING") + missing = sentinel(name) + self.assertIs(missing.__name__, name) + self.assertTrue(gc.is_tracked(missing)) + + name.missing = missing + ref = weakref.ref(name) + del name, missing + support.gc_collect() + self.assertIsNone(ref()) + + def test_sentinel_union(self): + missing = sentinel("MISSING") + + self.assertIsInstance(missing | int, typing.Union) + self.assertEqual((missing | int).__args__, (missing, int)) + self.assertIsInstance(int | missing, typing.Union) + self.assertEqual((int | missing).__args__, (int, missing)) + self.assertIs(missing | missing, missing) + self.assertEqual(repr(int | missing), "int | MISSING") + self.assertIsInstance(missing | None, typing.Union) + self.assertEqual((missing | None).__args__, (missing, type(None))) + self.assertIsInstance(None | missing, typing.Union) + self.assertEqual((None | missing).__args__, (type(None), missing)) + self.assertIsInstance(missing | list[int], typing.Union) + self.assertEqual((missing | list[int]).__args__, (missing, list[int])) + self.assertIsInstance(missing | (int | str), typing.Union) + self.assertEqual((missing | (int | str)).__args__, (missing, int, str)) + + with self.assertRaises(TypeError): + missing | 1 + with self.assertRaises(TypeError): + 1 | missing + def test_round(self): self.assertEqual(round(0.0), 0.0) self.assertEqual(type(round(0.0)), int) diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index fe9a59d335b..79f0ebb78ff 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -495,12 +495,17 @@ def test_formatmonth(self): calendar.TextCalendar().formatmonth(0, 2), result_0_02_text ) + def test_formatmonth_with_invalid_month(self): with self.assertRaises(calendar.IllegalMonthError): calendar.TextCalendar().formatmonth(2017, 13) with self.assertRaises(calendar.IllegalMonthError): calendar.TextCalendar().formatmonth(2017, -1) + def test_illegal_month_error_bases(self): + self.assertIsSubclass(calendar.IllegalMonthError, ValueError) + self.assertIsSubclass(calendar.IllegalMonthError, IndexError) + def test_formatmonthname_with_year(self): self.assertEqual( calendar.HTMLCalendar().formatmonthname(2004, 1, withyear=True), diff --git a/Lib/test/test_capi/test_object.py b/Lib/test/test_capi/test_object.py index 67572ab1ba2..635deaa73f7 100644 --- a/Lib/test/test_capi/test_object.py +++ b/Lib/test/test_capi/test_object.py @@ -1,5 +1,6 @@ import enum import os +import pickle import sys import textwrap import unittest @@ -63,6 +64,27 @@ def test_get_constant_borrowed(self): self.check_get_constant(_testlimitedcapi.get_constant_borrowed) +class SentinelTest(unittest.TestCase): + + def test_pysentinel_new(self): + marker = _testcapi.pysentinel_new("CAPI_SENTINEL", __name__) + self.assertIs(type(marker), sentinel) + self.assertTrue(_testcapi.pysentinel_check(marker)) + self.assertFalse(_testcapi.pysentinel_check(object())) + self.assertEqual(marker.__name__, "CAPI_SENTINEL") + self.assertEqual(marker.__module__, __name__) + self.assertEqual(repr(marker), "CAPI_SENTINEL") + + no_module = _testcapi.pysentinel_new("NO_MODULE") + self.assertIs(type(no_module), sentinel) + self.assertEqual(no_module.__name__, "NO_MODULE") + self.assertIs(no_module.__module__, None) + + globals()["CAPI_SENTINEL"] = marker + self.addCleanup(globals().pop, "CAPI_SENTINEL", None) + self.assertIs(pickle.loads(pickle.dumps(marker)), marker) + + class PrintTest(unittest.TestCase): def testPyObjectPrintObject(self): diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 59266b000ed..7118dfeed9f 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1427,9 +1427,13 @@ def testfunc(n): for _ in gen(n): pass testfunc(TIER2_THRESHOLD * 2) + # The generator may be inlined into testfunc's trace, + # so check whichever executor contains _YIELD_VALUE. gen_ex = get_first_executor(gen) - self.assertIsNotNone(gen_ex) - uops = get_opnames(gen_ex) + testfunc_ex = get_first_executor(testfunc) + ex = gen_ex or testfunc_ex + self.assertIsNotNone(ex) + uops = get_opnames(ex) self.assertNotIn("_MAKE_HEAP_SAFE", uops) self.assertIn("_YIELD_VALUE", uops) @@ -2959,7 +2963,7 @@ def testfunc(n): self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertNotIn("_CALL_LEN", uops) - self.assertEqual(count_ops(ex, "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW"), 4) + self.assertGreaterEqual(count_ops(ex, "_LOAD_CONST_INLINE_BORROW"), 8) def test_call_len_known_length_small_int(self): # Make sure that len(t) is optimized for a tuple of length 5. @@ -3915,6 +3919,38 @@ def testfunc(args): expected = TIER2_THRESHOLD * (5.0 / Fraction(4)) self.assertAlmostEqual(res, float(expected)) + def test_float_truediv_partial_float_no_stack_underflow(self): + # gh-149049: a speculative _GUARD_*_FLOAT for a partially-float + # truediv/remainder must not drop the original _BINARY_OP. + def truediv(args): + n, = args + nan = float("nan") + def victim(a=0, b=nan, c=2): + return (a + b) / c + for _ in range(n): + victim() + + def remainder(args): + n, = args + nan = float("nan") + def victim(a=0, b=nan, c=2): + return (a + b) % c + for _ in range(n): + victim() + + for testfunc in (truediv, remainder): + with self.subTest(op=testfunc.__name__): + # Iterations must be high enough that the buggy trace + # is not only built but executed (where it underflows). + _, ex = self._run_with_optimizer( + testfunc, (TIER2_THRESHOLD * 10,)) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertTrue( + "_GUARD_TOS_FLOAT" in uops or "_GUARD_NOS_FLOAT" in uops, + uops, + ) + def test_int_add_inplace_unique_lhs(self): # a * b produces a unique compact int; adding c reuses it in place def testfunc(args): @@ -5845,6 +5881,19 @@ def testfunc(n): self.assertNotIn("_LOAD_SUPER_ATTR_METHOD", uops) self.assertEqual(uops.count("_GUARD_NOS_TYPE_VERSION"), 2) + def test_settrace_then_polymorphic_call_does_not_crash(self): + script_helper.assert_python_ok("-c", textwrap.dedent(""" + import sys + sys.settrace(lambda *_: None) + sys.settrace(None) + + class C: + def __init__(self, x): + pass + + for i in 0, 1, 0, 1: + C(0) if i else str(0) + """)) def global_identity(x): return x diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index 7262a110d83..25fe50df603 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -18,6 +18,11 @@ # The purpose of test_cext extension is to check that building a C # extension using the Python C API does not emit C compiler warnings. '-Werror', + # Enable extra checks for header files, which: + # - need to be enabled somewhere inside Python headers (rather than + # before including Python.h) + # - should not be checked for user code + '-D_Py_IS_TESTCEXT', ] # C compiler flags for GCC and clang diff --git a/Lib/test/test_cmath.py b/Lib/test/test_cmath.py index 389a3fa0e0a..a986fd6b892 100644 --- a/Lib/test/test_cmath.py +++ b/Lib/test/test_cmath.py @@ -516,6 +516,7 @@ def test_isinf(self): self.assertFalse(cmath.isinf(1j)) self.assertFalse(cmath.isinf(NAN)) self.assertTrue(cmath.isinf(INF)) + self.assertTrue(cmath.isinf(-INF)) self.assertTrue(cmath.isinf(complex(INF, 0))) self.assertTrue(cmath.isinf(complex(0, INF))) self.assertTrue(cmath.isinf(complex(INF, INF))) diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index fac7e9148f1..5e802a929b1 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -424,13 +424,6 @@ def func(): new_code = code = func.__code__.replace(co_linetable=b'') self.assertEqual(list(new_code.co_lines()), []) - def test_co_lnotab_is_deprecated(self): # TODO: remove in 3.14 - def func(): - pass - - with self.assertWarns(DeprecationWarning): - func.__code__.co_lnotab - @unittest.skipIf(_testinternalcapi is None, '_testinternalcapi is missing') def test_returns_only_none(self): value = True diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 1fd8b3cb18c..e291f814edb 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -680,6 +680,154 @@ def test(x): self.assertEqual(state, [1, 'something else', 999]) + def test_contextmanager_decorate_generator_function(self): + @contextmanager + def woohoo(y): + state.append(y) + yield + state.append(999) + + state = [] + @woohoo(1) + def test(x): + self.assertEqual(state, [1]) + state.append(x) + yield + state.append("second item") + return "result" + + gen = test("something") + for _ in gen: + self.assertEqual(state, [1, "something"]) + self.assertEqual(state, [1, "something", "second item", 999]) + + # The wrapped generator's return value is preserved. + state = [] + gen = test("something") + with self.assertRaises(StopIteration) as cm: + while True: + next(gen) + self.assertEqual(cm.exception.value, "result") + + + def test_contextmanager_decorate_generator_function_exception(self): + @contextmanager + def woohoo(): + state.append("enter") + try: + yield + finally: + state.append("exit") + + state = [] + @woohoo() + def test(): + state.append("body") + yield + raise ZeroDivisionError + + with self.assertRaises(ZeroDivisionError): + for _ in test(): + pass + self.assertEqual(state, ["enter", "body", "exit"]) + + + def test_contextmanager_decorate_generator_function_early_stop(self): + @contextmanager + def woohoo(): + state.append("enter") + try: + yield + finally: + state.append("exit") + + state = [] + @woohoo() + def test(): + try: + yield 1 + yield 2 + finally: + state.append("inner closed") + + gen = test() + self.assertEqual(next(gen), 1) + gen.close() + # The inner generator is closed before the context manager exits. + self.assertEqual(state, ["enter", "inner closed", "exit"]) + + + def test_contextmanager_decorate_generator_function_send_throw(self): + @contextmanager + def woohoo(): + yield + + @woohoo() + def test(): + received = yield "first" + state.append(("received", received)) + try: + yield "second" + except ValueError as exc: + state.append(("caught", type(exc))) + yield "after throw" + + # .send() and .throw() are forwarded to the wrapped generator. + state = [] + gen = test() + self.assertEqual(next(gen), "first") + self.assertEqual(gen.send("VALUE"), "second") + self.assertEqual(gen.throw(ValueError), "after throw") + gen.close() + self.assertEqual( + state, [("received", "VALUE"), ("caught", ValueError)] + ) + + + def test_contextmanager_decorate_coroutine_function(self): + @contextmanager + def woohoo(y): + state.append(y) + yield + state.append(999) + + state = [] + @woohoo(1) + async def test(x): + self.assertEqual(state, [1]) + state.append(x) + + coro = test("something") + with self.assertRaises(StopIteration): + coro.send(None) + + self.assertEqual(state, [1, "something", 999]) + + + def test_contextmanager_decorate_asyncgen_function(self): + @contextmanager + def woohoo(y): + state.append(y) + yield + state.append(999) + + state = [] + @woohoo(1) + async def test(x): + self.assertEqual(state, [1]) + state.append(x) + yield + state.append("second item") + + agen = test("something") + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + with self.assertRaises(StopAsyncIteration): + agen.asend(None).send(None) + + self.assertEqual(state, [1, "something", "second item", 999]) + + class TestBaseExitStack: exit_stack = None diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py index 248d32d6152..95bdfdb3d9d 100644 --- a/Lib/test/test_contextlib_async.py +++ b/Lib/test/test_contextlib_async.py @@ -402,6 +402,144 @@ async def test(): await test() self.assertFalse(entered) + @_async_test + async def test_decorator_decorate_sync_function(self): + @asynccontextmanager + async def context(): + state.append(1) + yield + state.append(999) + + state = [] + @context() + def test(x): + self.assertEqual(state, [1]) + state.append(x) + + await test("something") + self.assertEqual(state, [1, "something", 999]) + + @_async_test + async def test_decorator_decorate_generator_function(self): + @asynccontextmanager + async def context(): + state.append(1) + yield + state.append(999) + + state = [] + @context() + def test(x): + self.assertEqual(state, [1]) + state.append(x) + yield + state.append("second item") + + async for _ in test("something"): + self.assertEqual(state, [1, "something"]) + self.assertEqual(state, [1, "something", "second item", 999]) + + @_async_test + async def test_decorator_decorate_asyncgen_function(self): + @asynccontextmanager + async def context(): + state.append(1) + yield + state.append(999) + + state = [] + @context() + async def test(x): + self.assertEqual(state, [1]) + state.append(x) + yield + state.append("second item") + + async for _ in test("something"): + self.assertEqual(state, [1, "something"]) + self.assertEqual(state, [1, "something", "second item", 999]) + + @_async_test + async def test_decorator_decorate_asyncgen_function_exception(self): + @asynccontextmanager + async def context(): + state.append("enter") + try: + yield + finally: + state.append("exit") + + state = [] + @context() + async def test(): + state.append("body") + yield + raise ZeroDivisionError + + with self.assertRaises(ZeroDivisionError): + async for _ in test(): + pass + self.assertEqual(state, ["enter", "body", "exit"]) + + @_async_test + async def test_decorator_decorate_asyncgen_function_early_stop(self): + @asynccontextmanager + async def context(): + state.append("enter") + try: + yield + finally: + state.append("exit") + + state = [] + @context() + async def test(): + try: + yield 1 + yield 2 + finally: + state.append("inner closed") + + agen = test() + async for value in agen: + self.assertEqual(value, 1) + break + await agen.aclose() + # The inner async generator is closed before the context + # manager exits. + self.assertEqual(state, ["enter", "inner closed", "exit"]) + + @_async_test + async def test_decorator_decorate_asyncgen_function_asend_athrow(self): + @asynccontextmanager + async def context(): + yield + + @context() + async def test(): + try: + received = yield "first" + state.append(("received", received)) + yield "second" + except ValueError: + state.append("inner saw ValueError") + raise + finally: + state.append("inner closed") + + # asend() values and athrow() exceptions are not forwarded to the + # wrapped generator (a documented limitation). + state = [] + agen = test() + self.assertEqual(await agen.__anext__(), "first") + self.assertEqual(await agen.asend("VALUE"), "second") + # The inner generator received None, not "VALUE". + self.assertEqual(state, [("received", None)]) + with self.assertRaises(ValueError): + await agen.athrow(ValueError) + # The inner generator was closed, not thrown into. + self.assertEqual(state, [("received", None), "inner closed"]) + @_async_test async def test_decorator_with_exception(self): entered = False diff --git a/Lib/test/test_ctypes/test_byteswap.py b/Lib/test/test_ctypes/test_byteswap.py index f14e1aa32e1..6a1bae14773 100644 --- a/Lib/test/test_ctypes/test_byteswap.py +++ b/Lib/test/test_ctypes/test_byteswap.py @@ -1,4 +1,5 @@ import binascii +import ctypes import math import struct import sys @@ -165,6 +166,48 @@ def test_endian_double(self): self.assertEqual(s.value, math.pi) self.assertEqual(bin(struct.pack(">d", math.pi)), bin(s)) + @unittest.skipUnless(hasattr(ctypes, 'c_float_complex'), "No complex types") + def test_endian_float_complex(self): + c_float_complex = ctypes.c_float_complex + if sys.byteorder == "little": + self.assertIs(c_float_complex.__ctype_le__, c_float_complex) + self.assertIs(c_float_complex.__ctype_be__.__ctype_le__, + c_float_complex) + else: + self.assertIs(c_float_complex.__ctype_be__, c_float_complex) + self.assertIs(c_float_complex.__ctype_le__.__ctype_be__, + c_float_complex) + s = c_float_complex(math.pi+1j) + self.assertEqual(bin(struct.pack("F", math.pi+1j)), bin(s)) + self.assertAlmostEqual(s.value, math.pi+1j, places=6) + s = c_float_complex.__ctype_le__(math.pi+1j) + self.assertAlmostEqual(s.value, math.pi+1j, places=6) + self.assertEqual(bin(struct.pack("F", math.pi+1j)), bin(s)) + + @unittest.skipUnless(hasattr(ctypes, 'c_double_complex'), "No complex types") + def test_endian_double_complex(self): + c_double_complex = ctypes.c_double_complex + if sys.byteorder == "little": + self.assertIs(c_double_complex.__ctype_le__, c_double_complex) + self.assertIs(c_double_complex.__ctype_be__.__ctype_le__, + c_double_complex) + else: + self.assertIs(c_double_complex.__ctype_be__, c_double_complex) + self.assertIs(c_double_complex.__ctype_le__.__ctype_be__, + c_double_complex) + s = c_double_complex(math.pi+1j) + self.assertEqual(bin(struct.pack("D", math.pi+1j)), bin(s)) + self.assertAlmostEqual(s.value, math.pi+1j, places=6) + s = c_double_complex.__ctype_le__(math.pi+1j) + self.assertAlmostEqual(s.value, math.pi+1j, places=6) + self.assertEqual(bin(struct.pack("D", math.pi+1j)), bin(s)) + def test_endian_other(self): self.assertIs(c_byte.__ctype_le__, c_byte) self.assertIs(c_byte.__ctype_be__, c_byte) diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index b44b1da0336..8a0a7d12c04 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -27,11 +27,21 @@ import dataclasses # Needed for the string "dataclasses.InitVar[int]" to work as an annotation. from test import support -from test.support import import_helper +from test.support import cpython_only, import_helper # Just any custom exception we can catch. class CustomError(Exception): pass + +class TestImportTime(unittest.TestCase): + + @cpython_only + def test_lazy_import(self): + import_helper.ensure_lazy_imports( + "dataclasses", {"inspect", "re", "copy"} + ) + + class TestCase(unittest.TestCase): def test_no_fields(self): @dataclass @@ -985,7 +995,7 @@ class D: self.assertNotIn('x', D.__dict__) def test_missing_repr(self): - self.assertIn('MISSING_TYPE object', repr(MISSING)) + self.assertEqual(repr(MISSING), 'MISSING') def test_dont_include_other_annotations(self): @dataclass @@ -1693,17 +1703,24 @@ class GroupTuple: class GroupDict: id: int users: Dict[str, User] + @dataclass + class GroupFrozenDict: + id: int + users: frozendict[str, User] a = User('Alice', 1) b = User('Bob', 2) gl = GroupList(0, [a, b]) gt = GroupTuple(0, (a, b)) gd = GroupDict(0, {'first': a, 'second': b}) + gfd = GroupFrozenDict(0, frozendict({'first': a, 'second': b})) self.assertEqual(asdict(gl), {'id': 0, 'users': [{'name': 'Alice', 'id': 1}, {'name': 'Bob', 'id': 2}]}) self.assertEqual(asdict(gt), {'id': 0, 'users': ({'name': 'Alice', 'id': 1}, {'name': 'Bob', 'id': 2})}) - self.assertEqual(asdict(gd), {'id': 0, 'users': {'first': {'name': 'Alice', 'id': 1}, - 'second': {'name': 'Bob', 'id': 2}}}) + expected_dict = {'id': 0, 'users': {'first': {'name': 'Alice', 'id': 1}, + 'second': {'name': 'Bob', 'id': 2}}} + self.assertEqual(asdict(gd), expected_dict) + self.assertEqual(asdict(gfd), expected_dict) def test_helper_asdict_builtin_object_containers(self): @dataclass @@ -1884,14 +1901,21 @@ class GroupTuple: class GroupDict: id: int users: Dict[str, User] + @dataclass + class GroupFrozenDict: + id: int + users: frozendict[str, User] a = User('Alice', 1) b = User('Bob', 2) gl = GroupList(0, [a, b]) gt = GroupTuple(0, (a, b)) gd = GroupDict(0, {'first': a, 'second': b}) + gfd = GroupFrozenDict(0, frozendict({'first': a, 'second': b})) self.assertEqual(astuple(gl), (0, [('Alice', 1), ('Bob', 2)])) self.assertEqual(astuple(gt), (0, (('Alice', 1), ('Bob', 2)))) - self.assertEqual(astuple(gd), (0, {'first': ('Alice', 1), 'second': ('Bob', 2)})) + d = {'first': ('Alice', 1), 'second': ('Bob', 2)} + self.assertEqual(astuple(gd), (0, d)) + self.assertEqual(astuple(gfd), (0, frozendict(d))) def test_helper_astuple_builtin_object_containers(self): @dataclass @@ -2295,6 +2319,20 @@ class C: self.assertDocStrEqual(C.__doc__, "C()") + def test_docstring_slotted(self): + @dataclass(slots=True) + class C: + x: int + + self.assertDocStrEqual(C.__doc__, "C(x:int)") + + def test_docstring_recursive(self): + @dataclass() + class C: + x: list[C] + + self.assertDocStrEqual(C.__doc__, "C(x:list[test.test_dataclasses.TestDocString.test_docstring_recursive..C])") + def test_docstring_one_field(self): @dataclass class C: @@ -5361,5 +5399,51 @@ def cls(self): # one will be keeping a reference to the underlying class A. self.assertIs(A().cls(), B) + def test_empty_class_cell(self): + # gh-148947: Make sure that we explicitly handle the empty class cell. + def maker(): + if False: + __class__ = 42 + + def method(self): + return __class__ + return method + + from dataclasses import dataclass + + @dataclass(slots=True) + class X: + a: int + + meth = maker() + + with self.assertRaisesRegex(NameError, '__class__'): + X(1).meth() + + def test_class_cell_from_other_class(self): + # This test fails without the "is oldcls" check in + # _update_func_cell_for__class__. + class Base: + def meth(self): + return "Base" + + class Child(Base): + def meth(self): + return super().meth() + " Child" + + @dataclass(slots=True) + class DC(Child): + a: int + + meth = Child.meth + + closure = DC.meth.__closure__ + self.assertEqual(len(closure), 1) + self.assertIs(closure[0].cell_contents, Child) + + self.assertEqual(DC(1).meth(), "Base Child") + + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index e28fe389201..aded44e85ee 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -1235,17 +1235,6 @@ def test_get_local_part_valid_and_invalid_qp_in_atom_list(self): '@example.com') self.assertEqual(local_part.local_part, r'\example\\ example') - def test_get_local_part_unicode_defect(self): - # Currently this only happens when parsing unicode, not when parsing - # stuff that was originally binary. - local_part = self._test_get_x(parser.get_local_part, - 'exámple@example.com', - 'exámple', - 'exámple', - [errors.NonASCIILocalPartDefect], - '@example.com') - self.assertEqual(local_part.local_part, 'exámple') - # get_dtext def test_get_dtext_only(self): @@ -2617,7 +2606,7 @@ def test_get_address_list_mailboxes_invalid_addresses(self): '') self.assertEqual(address_list.token_type, 'address-list') self.assertEqual(len(address_list.mailboxes), 1) - self.assertEqual(len(address_list.all_mailboxes), 3) + self.assertEqual(len(address_list.all_mailboxes), 4) self.assertEqual([str(x) for x in address_list.all_mailboxes], [str(x) for x in address_list.addresses]) self.assertEqual(address_list.mailboxes[0].domain, 'example.com') @@ -2626,11 +2615,13 @@ def test_get_address_list_mailboxes_invalid_addresses(self): self.assertEqual(address_list.addresses[1].token_type, 'address') self.assertEqual(len(address_list.addresses[0].mailboxes), 1) self.assertEqual(len(address_list.addresses[1].mailboxes), 0) - self.assertEqual(len(address_list.addresses[1].mailboxes), 0) + self.assertEqual(len(address_list.addresses[2].mailboxes), 0) + self.assertEqual(len(address_list.addresses[3].mailboxes), 0) self.assertEqual( address_list.addresses[1].all_mailboxes[0].local_part, 'Foo x') + self.assertEqual(address_list.addresses[2].all_mailboxes[0].value, '[]') self.assertEqual( - address_list.addresses[2].all_mailboxes[0].display_name, + address_list.addresses[3].all_mailboxes[0].display_name, "Nobody Is. Special") def test_get_address_list_group_empty(self): @@ -2695,6 +2686,14 @@ def test_get_address_list_group_and_mailboxes(self): self.assertEqual(str(address_list.addresses[1]), str(address_list.mailboxes[2])) + def test_get_address_list_trailing_garbage(self): + address_list = self._test_get_x(parser.get_address_list, + 'unlisted-recipients:; (no To-header on input)', + 'unlisted-recipients:; (no To-header on input)', + 'unlisted-recipients:; ', + [errors.InvalidHeaderDefect]*2 + [errors.ObsoleteHeaderDefect], + '') + def test_invalid_content_disposition(self): content_disp = self._test_parse_x( parser.parse_content_disposition_header, @@ -3364,10 +3363,12 @@ def test_fold_unfoldable_element_stealing_whitespace(self): self._test(token, expected, policy=policy) def test_encoded_word_with_undecodable_bytes(self): - self._test(parser.get_address_list( - ' =?utf-8?Q?=E5=AE=A2=E6=88=B6=E6=AD=A3=E8=A6=8F=E4=BA=A4=E7?=' + self._test( + parser.get_address_list( + ' =?utf-8?Q?=E5=AE=A2=E6=88=B6=E6=AD=A3=E8=A6=8F=E4=BA=A4=E7?=' + ' ' )[0], - ' =?unknown-8bit?b?5a6i5oi25q2j6KaP5Lqk5w==?=\n', + ' =?unknown-8bit?b?5a6i5oi25q2j6KaP5Lqk5w==?= \n', ) diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py index 3c9a86f3e8c..8d912738029 100644 --- a/Lib/test/test_email/test_generator.py +++ b/Lib/test/test_email/test_generator.py @@ -1,4 +1,5 @@ import io +import re import textwrap import unittest import random @@ -295,6 +296,69 @@ def test_keep_long_encoded_newlines(self): g.flatten(msg) self.assertEqual(s.getvalue(), self.typ(expected)) + def test_non_ascii_addr_spec_raises(self): + # non-ascii is not permitted in any part of an addr-spec. If the + # programmer generated it, it's an error. (See also + # test_non_ascii_addr_spec_preserved below.) + p = self.policy.clone(utf8=False, max_line_length=20) + g = self.genclass(self.ioclass(), policy=p) + # XXX The particular part detected here isn't part of a behavioral + # spec and may change in the future. + cases = [ + ('wők@example.com', 'wők', 'local-part'), + ('wok@exàmple.com', 'exàmple.com', 'domain'), + ('wők@exàmple.com', 'wők', 'local-part'), + ( + '"Name, for display" ', + 'wők@example.com', + 'addr-spec', + ), + ( + 'Näyttönimi ', + 'wők@example.com', + 'addr-spec', + ), + ( + '"a lőng quoted string as the local part"@example.com', + 'a lőng quoted string as the local part', + 'local-part', + ), + + ] + for address, badtoken, partname in cases: + with self.subTest(address=address): + msg = EmailMessage() + msg['To'] = address + expected_error = ( + fr"(?i)(?=.*non-ascii)" + fr"(?=.*{re.escape(badtoken)})" + fr"(?=.*{partname})" + fr"(?=.*policy.*utf8)" + ) + with self.assertRaisesRegex( + email.errors.HeaderWriteError, expected_error + ): + g.flatten(msg) + + def test_local_part_quoted_string_wrapped_correctly(self): + msg = self.msgmaker(self.typ(textwrap.dedent("""\ + To: <"a long local part in a quoted string"@example.com> + Subject: test + + None + """)), policy=self.policy.clone(max_line_length=20)) + expected = textwrap.dedent("""\ + To: <"a long local part in a + quoted string"@example.com> + Subject: test + + None + """) + s = self.ioclass() + g = self.genclass(s, policy=self.policy.clone(max_line_length=30)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(expected)) + def _test_boundary_detection(self, linesep): # Generate a boundary token in the same way as _make_boundary token = random.randrange(sys.maxsize) @@ -515,12 +579,12 @@ def test_cte_type_7bit_transforms_8bit_cte(self): def test_smtputf8_policy(self): msg = EmailMessage() - msg['From'] = "Páolo " + msg['From'] = "Páolo " msg['To'] = 'Dinsdale' msg['Subject'] = 'Nudge nudge, wink, wink \u1F609' msg.set_content("oh là là, know what I mean, know what I mean?") expected = textwrap.dedent("""\ - From: Páolo + From: Páolo To: Dinsdale Subject: Nudge nudge, wink, wink \u1F609 Content-Type: text/plain; charset="utf-8" @@ -555,6 +619,37 @@ def test_smtp_policy(self): g.flatten(msg) self.assertEqual(s.getvalue(), expected) + def test_non_ascii_addr_spec_preserved(self): + # A defective non-ASCII addr-spec parsed from the original + # message is left unchanged when flattening. + # (See also test_non_ascii_addr_spec_raises above.) + source = ( + 'To: jörg@example.com, "But a long name still works with refold_source" ' + ).encode() + expected = ( + b'To: j\xc3\xb6rg@example.com,\n' + b' "But a long name still works with refold_source" \n' + b'\n' + ) + msg = message_from_bytes(source, policy=policy.default) + s = io.BytesIO() + g = BytesGenerator(s, policy=policy.default) + g.flatten(msg) + self.assertEqual(s.getvalue(), expected) + + def test_idna_encoding_preserved(self): + # Nothing tries to decode a pre-encoded IDNA domain. + msg = EmailMessage() + msg["To"] = Address( + username='jörg', + domain='☕.example'.encode('idna').decode() # IDNA 2003 + ) + expected = 'To: jörg@xn--53h.example\n\n'.encode() + s = io.BytesIO() + g = BytesGenerator(s, policy=policy.default.clone(utf8=True)) + g.flatten(msg) + self.assertEqual(s.getvalue(), expected) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index c9c63951597..aa918255d15 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -1271,12 +1271,12 @@ class TestAddressHeader(TestHeaderBase): 'example.com', None), - } - # XXX: Need many more examples, and in particular some with names in # trailing comments, which aren't currently handled. comments in # general are not handled yet. + } + def example_as_address(self, source, defects, decoded, display_name, addr_spec, username, domain, comment): h = self.make_header('sender', source) @@ -1294,6 +1294,43 @@ def example_as_address(self, source, defects, decoded, display_name, # XXX: we have no comment support yet. #self.assertEqual(a.comment, comment) + example_broken_header_params = { + + 'just_dquote': + ('"', + [errors.InvalidHeaderDefect]*2, + '<>', + '', + '<>', + '', + '', + ), + + } + + def example_broken_header_as_address( + self, + source, + defects, + decoded, + display_name, + addr_spec, + username, + domain, + ): + h = self.make_header('sender', source) + self.assertEqual(h, decoded) + self.assertDefectsEqual(h.defects, defects) + a = h.address + self.assertEqual(str(a), decoded) + self.assertEqual(len(h.groups), 1) + self.assertEqual([a], list(h.groups[0].addresses)) + self.assertEqual([a], list(h.addresses)) + self.assertEqual(a.display_name, display_name) + self.assertEqual(a.addr_spec, addr_spec) + self.assertEqual(a.username, username) + self.assertEqual(a.domain, domain) + def example_as_group(self, source, defects, decoded, display_name, addr_spec, username, domain, comment): source = 'foo: {};'.format(source) @@ -1506,17 +1543,19 @@ def test_quoting(self): self.assertEqual(str(a), '"Sara J." <"bad name"@example.com>') def test_il8n(self): - a = Address('Éric', 'wok', 'exàmple.com') + a = Address('Éric', 'wők', 'exàmple.com') self.assertEqual(a.display_name, 'Éric') - self.assertEqual(a.username, 'wok') + self.assertEqual(a.username, 'wők') self.assertEqual(a.domain, 'exàmple.com') - self.assertEqual(a.addr_spec, 'wok@exàmple.com') - self.assertEqual(str(a), 'Éric ') + self.assertEqual(a.addr_spec, 'wők@exàmple.com') + self.assertEqual(str(a), 'Éric ') - # XXX: there is an API design issue that needs to be solved here. - #def test_non_ascii_username_raises(self): - # with self.assertRaises(ValueError): - # Address('foo', 'wők', 'example.com') + def test_i18n_in_addr_spec(self): + a = Address(addr_spec='wők@exàmple.com') + self.assertEqual(a.username, 'wők') + self.assertEqual(a.domain, 'exàmple.com') + self.assertEqual(a.addr_spec, 'wők@exàmple.com') + self.assertEqual(str(a), 'wők@exàmple.com') def test_crlf_in_constructor_args_raises(self): cases = ( @@ -1537,10 +1576,6 @@ def test_crlf_in_constructor_args_raises(self): with self.subTest(kwargs=kwargs), self.assertRaisesRegex(ValueError, "invalid arguments"): Address(**kwargs) - def test_non_ascii_username_in_addr_spec_raises(self): - with self.assertRaises(ValueError): - Address('foo', addr_spec='wők@example.com') - def test_address_addr_spec_and_username_raises(self): with self.assertRaises(TypeError): Address('foo', username='bing', addr_spec='bar@baz') diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 5e88ae47775..11df59f2346 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -719,6 +719,76 @@ def test_dump_traceback_later_fd(self): def test_dump_traceback_later_twice(self): self.check_dump_traceback_later(loops=2) + def test_dump_traceback_max_threads(self): + # max_threads caps the dump and writes "...\n" when truncated. + # Spawn N worker threads, dump with cap < N, and verify the + # marker is present and exactly CAP thread headers are written. + code = dedent(""" + import faulthandler + import sys + import threading + + NTHREADS = 6 + CAP = 3 + + ready = threading.Barrier(NTHREADS + 1) + stop = threading.Event() + + def worker(): + ready.wait() + stop.wait() + + threads = [threading.Thread(target=worker) for _ in range(NTHREADS)] + for t in threads: + t.start() + ready.wait() + try: + faulthandler.dump_traceback(file=sys.stderr, max_threads=CAP) + finally: + stop.set() + for t in threads: + t.join() + """).strip() + proc = script_helper.assert_python_ok('-c', code) + output = proc.err + # Truncation marker is written on its own line when the cap is hit. + self.assertIn(b"\n...\n", output) + # Cap of 3 means exactly 3 thread headers in the dump. + self.assertEqual(output.count(b"Thread 0x"), 3) + + @skip_segfault_on_android + @unittest.skipIf(support.Py_GIL_DISABLED, + "fatal-signal handler only dumps the current thread " + "when the GIL is disabled") + def test_enable_max_threads(self): + # enable(max_threads=N) caps the thread dump produced when a + # fatal signal fires. + code = dedent(""" + import faulthandler + import threading + + NTHREADS = 6 + CAP = 3 + + ready = threading.Barrier(NTHREADS + 1) + stop = threading.Event() + + def worker(): + ready.wait() + stop.wait() + + for _ in range(NTHREADS): + threading.Thread(target=worker, daemon=True).start() + ready.wait() + faulthandler.enable(max_threads=CAP) + faulthandler._sigsegv() + """).strip() + output, exitcode = self.get_output(code) + output = '\n'.join(output) + # Cap of 3 means the dump is truncated with "..." on its own line. + self.assertIn("\n...\n", output) + self.assertNotEqual(exitcode, 0) + @unittest.skipIf(not hasattr(faulthandler, "register"), "need faulthandler.register") def check_register(self, filename=False, all_threads=False, @@ -825,6 +895,46 @@ def test_register_threads(self): def test_register_chain(self): self.check_register(chain=True) + @unittest.skipIf(not hasattr(faulthandler, "register"), + "need faulthandler.register") + def test_register_max_threads(self): + # register(max_threads=N) caps the thread dump produced when + # the registered signal fires. + code = dedent(""" + import faulthandler + import signal + import threading + + NTHREADS = 6 + CAP = 3 + + ready = threading.Barrier(NTHREADS + 1) + stop = threading.Event() + + def worker(): + ready.wait() + stop.wait() + + threads = [threading.Thread(target=worker) for _ in range(NTHREADS)] + for t in threads: + t.start() + ready.wait() + try: + faulthandler.register(signal.SIGUSR1, all_threads=True, + max_threads=CAP) + signal.raise_signal(signal.SIGUSR1) + finally: + stop.set() + for t in threads: + t.join() + """).strip() + proc = script_helper.assert_python_ok('-c', code) + output = proc.err + # Cap of 3 means the dump is truncated with "..." on its own line. + self.assertIn(b"\n...\n", output) + # Cap of 3 means exactly 3 thread headers in the dump. + self.assertEqual(output.count(b"Thread 0x"), 3) + @contextmanager def check_stderr_none(self): stderr = sys.stderr diff --git a/Lib/test/test_frame_pointer_unwind.py b/Lib/test/test_frame_pointer_unwind.py index c70ec281686..2f9ce2bf049 100644 --- a/Lib/test/test_frame_pointer_unwind.py +++ b/Lib/test/test_frame_pointer_unwind.py @@ -27,9 +27,8 @@ def _frame_pointers_expected(machine): ) if "no-omit-frame-pointer" in cflags: - # For example, configure adds -fno-omit-frame-pointer if Python - # has perf trampoline (PY_HAVE_PERF_TRAMPOLINE) and Python is built - # in debug mode. + # For example, configure adds -fno-omit-frame-pointer by default on + # supported GCC-compatible builds. return True if "omit-frame-pointer" in cflags: return False diff --git a/Lib/test/test_free_threading/test_str.py b/Lib/test/test_free_threading/test_str.py index 9a1ce3620ac..11e04009956 100644 --- a/Lib/test/test_free_threading/test_str.py +++ b/Lib/test/test_free_threading/test_str.py @@ -1,7 +1,9 @@ +import sys +import threading import unittest from itertools import cycle -from threading import Event, Thread +from threading import Barrier, Event, Thread from unittest import TestCase from test.support import threading_helper @@ -69,6 +71,24 @@ def reader_func(): for reader in readers: reader.join() + def test_intern_unowned_string(self): + # Test interning strings owned by various threads. + strings = [f"intern_race_owner_{i}" for i in range(50)] + + NUM_THREADS = 5 + b = Barrier(NUM_THREADS) + + def interner(): + tid = threading.get_ident() + for i in range(20): + strings.append(f"intern_{tid}_{i}") + b.wait() + for s in strings: + r = sys.intern(s) + self.assertTrue(sys._is_interned(r)) + + threading_helper.run_concurrently(interner, nthreads=NUM_THREADS) + def test_maketrans_dict_concurrent_modification(self): for _ in range(5): d = {2000: 'a'} diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 6aa6361d5d0..88d265cbc21 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -7,7 +7,7 @@ Py_GIL_DISABLED) from test.support.import_helper import import_module from test.support.os_helper import temp_dir, TESTFN, unlink -from test.support.script_helper import assert_python_ok, make_script, run_test_script +from test.support.script_helper import assert_python_ok, make_script from test.support import threading_helper, gc_threshold import gc @@ -419,11 +419,19 @@ def test_collect_generations(self): # each call to collect(N) x = [] gc.collect(0) - # x is now in the old gen + # x is now in gen 1 a, b, c = gc.get_count() - # We don't check a since its exact values depends on + gc.collect(1) + # x is now in gen 2 + d, e, f = gc.get_count() + gc.collect(2) + # x is now in gen 3 + g, h, i = gc.get_count() + # We don't check a, d, g since their exact values depends on # internal implementation details of the interpreter. self.assertEqual((b, c), (1, 0)) + self.assertEqual((e, f), (0, 1)) + self.assertEqual((h, i), (0, 0)) def test_trashcan(self): class Ouch: @@ -907,10 +915,42 @@ def test_get_objects_generations(self): self.assertTrue( any(l is element for element in gc.get_objects(generation=0)) ) - gc.collect() + self.assertFalse( + any(l is element for element in gc.get_objects(generation=1)) + ) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=2)) + ) + gc.collect(generation=0) self.assertFalse( any(l is element for element in gc.get_objects(generation=0)) ) + self.assertTrue( + any(l is element for element in gc.get_objects(generation=1)) + ) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=2)) + ) + gc.collect(generation=1) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=0)) + ) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=1)) + ) + self.assertTrue( + any(l is element for element in gc.get_objects(generation=2)) + ) + gc.collect(generation=2) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=0)) + ) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=1)) + ) + self.assertTrue( + any(l is element for element in gc.get_objects(generation=2)) + ) del l gc.collect() @@ -1249,17 +1289,6 @@ def test_tuple_untrack_counts(self): self.assertTrue(new_count - count > (n // 2)) -class IncrementalGCTests(unittest.TestCase): - @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") - @requires_gil_enabled("Free threading does not support incremental GC") - def test_incremental_gc_handles_fast_cycle_creation(self): - # Run this test in a fresh process. The number of alive objects (which can - # be from unit tests run before this one) can influence how quickly cyclic - # garbage is found. - script = support.findfile("_test_gc_fast_cycles.py") - run_test_script(script) - - class GCCallbackTests(unittest.TestCase): def setUp(self): # Save gc state and disable it. @@ -1514,8 +1543,8 @@ def callback(ignored): assert not detector.gc_happened while not detector.gc_happened: i += 1 - if i > 100000: - self.fail("gc didn't happen after 100000 iterations") + if i > 10000: + self.fail("gc didn't happen after 10000 iterations") self.assertEqual(len(ouch), 0) junk.append([]) # this will eventually trigger gc @@ -1587,8 +1616,8 @@ def __del__(self): gc.collect() while not detector.gc_happened: i += 1 - if i > 50000: - self.fail("gc didn't happen after 50000 iterations") + if i > 10000: + self.fail("gc didn't happen after 10000 iterations") self.assertEqual(len(ouch), 0) junk.append([]) # this will eventually trigger gc @@ -1605,8 +1634,8 @@ def test_indirect_calls_with_gc_disabled(self): detector = GC_Detector() while not detector.gc_happened: i += 1 - if i > 100000: - self.fail("gc didn't happen after 100000 iterations") + if i > 10000: + self.fail("gc didn't happen after 10000 iterations") junk.append([]) # this will eventually trigger gc try: @@ -1616,11 +1645,11 @@ def test_indirect_calls_with_gc_disabled(self): detector = GC_Detector() while not detector.gc_happened: i += 1 - if i > 100000: + if i > 10000: break junk.append([]) # this may eventually trigger gc (if it is enabled) - self.assertEqual(i, 100001) + self.assertEqual(i, 10001) finally: gc.enable() diff --git a/Lib/test/test_gdb/gdb_jit_sample.py b/Lib/test/test_gdb/gdb_jit_sample.py new file mode 100644 index 00000000000..b439e82e8b3 --- /dev/null +++ b/Lib/test/test_gdb/gdb_jit_sample.py @@ -0,0 +1,27 @@ +# Sample script for use by test_gdb.test_jit + +import _testinternalcapi +import operator + + +WARMUP_ITERATIONS = _testinternalcapi.TIER2_THRESHOLD + 10 + + +def jit_bt_hot(depth, warming_up_caller=False): + if depth == 0: + if not warming_up_caller: + id(42) + return + + for iteration in range(WARMUP_ITERATIONS): + operator.call( + jit_bt_hot, + depth - 1, + warming_up_caller or iteration + 1 != WARMUP_ITERATIONS, + ) + + +# Warm the shared shim once without hitting builtin_id so the real run uses +# the steady-state shim path when GDB breaks inside id(42). +jit_bt_hot(1, warming_up_caller=True) +jit_bt_hot(1) diff --git a/Lib/test/test_gdb/test_jit.py b/Lib/test/test_gdb/test_jit.py new file mode 100644 index 00000000000..ea88d7b0a1f --- /dev/null +++ b/Lib/test/test_gdb/test_jit.py @@ -0,0 +1,201 @@ +import os +import platform +import re +import sys +import unittest + +from .util import setup_module, DebuggerTests + + +JIT_SAMPLE_SCRIPT = os.path.join(os.path.dirname(__file__), "gdb_jit_sample.py") +# In batch GDB, break in builtin_id() while it is running under JIT, +# then repeatedly "finish" until the selected frame is the JIT executor. +# That gives a deterministic backtrace starting with py::jit:executor. +# +# builtin_id() sits only a few helper frames above the JIT entry on this path. +# This bound is just a generous upper limit so the test fails clearly if the +# expected stack shape changes. +MAX_FINISH_STEPS = 20 +# After landing on the JIT entry frame, single-step a bounded number of +# instructions further into the blob so the backtrace is taken from JIT code +# itself rather than the immediate helper-return site. The exact number of +# steps is not significant: each step is cross-checked against the selected +# frame's symbol so the test fails loudly if stepping escapes the registered +# JIT region, instead of asserting against a misleading backtrace. +MAX_JIT_ENTRY_STEPS = 4 +EVAL_FRAME_RE = r"(_PyEval_EvalFrameDefault|_PyEval_Vector)" +JIT_EXECUTOR_FRAME = "py::jit:executor" +JIT_ENTRY_SYMBOL = "_PyJIT_Entry" +BACKTRACE_FRAME_RE = re.compile(r"^#\d+\s+.*$", re.MULTILINE) + +FINISH_TO_JIT_EXECUTOR = ( + "python exec(\"import gdb\\n" + f"target = {JIT_EXECUTOR_FRAME!r}\\n" + f"for _ in range({MAX_FINISH_STEPS}):\\n" + " frame = gdb.selected_frame()\\n" + " if frame is not None and frame.name() == target:\\n" + " break\\n" + " gdb.execute('finish')\\n" + "else:\\n" + " raise RuntimeError('did not reach %s' % target)\\n\")" +) +STEP_INSIDE_JIT_EXECUTOR = ( + "python exec(\"import gdb\\n" + f"target = {JIT_EXECUTOR_FRAME!r}\\n" + f"for _ in range({MAX_JIT_ENTRY_STEPS}):\\n" + " frame = gdb.selected_frame()\\n" + " if frame is None or frame.name() != target:\\n" + " raise RuntimeError('left JIT region during stepping: '\\n" + " + repr(frame and frame.name()))\\n" + " gdb.execute('si')\\n" + "frame = gdb.selected_frame()\\n" + "if frame is None or frame.name() != target:\\n" + " raise RuntimeError('stepped out of JIT region after si')\\n\")" +) + + +def setUpModule(): + setup_module() + + +# The GDB JIT interface registration is gated on __linux__ && __ELF__ in +# Python/jit_unwind.c, and the synthetic EH-frame is only implemented for +# x86_64 and AArch64 (a #error fires otherwise). Skip cleanly on other +# platforms or architectures instead of producing timeouts / empty backtraces. +# is_enabled() implies is_available() and also implies that the runtime has +# JIT execution active; interpreter-only tier 2 builds don't hit this path. +@unittest.skipUnless(sys.platform == "linux", + "GDB JIT interface is only implemented for Linux + ELF") +@unittest.skipUnless(platform.machine() in ("x86_64", "aarch64"), + "GDB JIT CFI emitter only supports x86_64 and AArch64") +@unittest.skipUnless(hasattr(sys, "_jit") and sys._jit.is_enabled(), + "requires a JIT-enabled build with JIT execution active") +class JitBacktraceTests(DebuggerTests): + def get_stack_trace(self, **kwargs): + # These tests validate the JIT-relevant part of the backtrace via + # _assert_jit_backtrace_shape, so an unrelated "?? ()" frame below + # the JIT/eval segment (e.g. libc without debug info) is tolerable. + kwargs.setdefault("skip_on_truncation", False) + return super().get_stack_trace(**kwargs) + + def _extract_backtrace_frames(self, gdb_output): + frames = BACKTRACE_FRAME_RE.findall(gdb_output) + self.assertGreater( + len(frames), 0, + f"expected at least one GDB backtrace frame in output:\n{gdb_output}", + ) + return frames + + def _assert_jit_backtrace_shape(self, gdb_output, *, anchor_at_top): + # Shape assertions applied to every JIT backtrace we produce: + # 1. The synthetic JIT symbol appears exactly once. A second + # py::jit:executor frame would mean the unwinder is + # materializing two native frames for a single logical JIT + # region, or failing to unwind out of the region entirely. + # 2. The unwinder must climb directly back out of the JIT region + # into the eval loop. _PyJIT_Entry only exists to establish the + # physical frame; the synthetic executor FDE collapses it away. + # 3. For tests that assert a specific entry PC, the JIT frame + # is also at #0. + frames = self._extract_backtrace_frames(gdb_output) + backtrace = "\n".join(frames) + + jit_frames = [frame for frame in frames if JIT_EXECUTOR_FRAME in frame] + jit_count = len(jit_frames) + self.assertEqual( + jit_count, 1, + f"expected exactly 1 {JIT_EXECUTOR_FRAME} frame, got {jit_count}\n" + f"backtrace:\n{backtrace}", + ) + eval_frames = [frame for frame in frames if re.search(EVAL_FRAME_RE, frame)] + eval_count = len(eval_frames) + self.assertGreaterEqual( + eval_count, 1, + f"expected at least one _PyEval_* frame, got {eval_count}\n" + f"backtrace:\n{backtrace}", + ) + jit_frame_index = next( + i for i, frame in enumerate(frames) if JIT_EXECUTOR_FRAME in frame + ) + frames_after_jit = frames[jit_frame_index + 1:] + first_eval_offset = next( + ( + i for i, frame in enumerate(frames_after_jit) + if re.search(EVAL_FRAME_RE, frame) + ), + None, + ) + self.assertIsNotNone( + first_eval_offset, + f"expected an eval frame after the JIT frame\n" + f"backtrace:\n{backtrace}", + ) + unexpected_between = frames_after_jit[:first_eval_offset] + self.assertFalse( + unexpected_between, + "expected the executor frame to unwind directly into eval\n" + f"backtrace:\n{backtrace}", + ) + relevant_end = max( + i + for i, frame in enumerate(frames) + if ( + JIT_EXECUTOR_FRAME in frame + or re.search(EVAL_FRAME_RE, frame) + ) + ) + truncated_frames = [ + frame for frame in frames[: relevant_end + 1] + if " ?? ()" in frame + ] + self.assertFalse( + truncated_frames, + "unexpected truncated frame before the validated JIT/eval segment\n" + f"backtrace:\n{backtrace}", + ) + if anchor_at_top: + self.assertRegex( + frames[0], + re.compile(rf"^#0\s+{re.escape(JIT_EXECUTOR_FRAME)}"), + ) + + def test_bt_unwinds_through_jit_frames(self): + gdb_output = self.get_stack_trace( + script=JIT_SAMPLE_SCRIPT, + cmds_after_breakpoint=["bt"], + PYTHON_JIT="1", + ) + # The executor should appear as a named JIT frame and unwind back into + # the eval loop. + self._assert_jit_backtrace_shape(gdb_output, anchor_at_top=False) + + def test_bt_handoff_from_jit_entry_to_executor(self): + gdb_output = self.get_stack_trace( + script=JIT_SAMPLE_SCRIPT, + breakpoint=JIT_ENTRY_SYMBOL, + cmds_after_breakpoint=[ + "delete 1", + "tbreak builtin_id", + "continue", + "bt", + ], + PYTHON_JIT="1", + ) + # If we stop first in the shim and then continue into the real JIT + # workload, the final backtrace should match the architecture's + # executor unwind contract. + self._assert_jit_backtrace_shape(gdb_output, anchor_at_top=False) + + def test_bt_unwinds_from_inside_jit_executor(self): + gdb_output = self.get_stack_trace( + script=JIT_SAMPLE_SCRIPT, + cmds_after_breakpoint=[ + FINISH_TO_JIT_EXECUTOR, + STEP_INSIDE_JIT_EXECUTOR, + "bt", + ], + PYTHON_JIT="1", + ) + # Once the selected PC is inside the JIT executor, we require that GDB + # identifies the JIT frame at #0 and keeps unwinding into _PyEval_*. + self._assert_jit_backtrace_shape(gdb_output, anchor_at_top=True) diff --git a/Lib/test/test_gdb/util.py b/Lib/test/test_gdb/util.py index 8097fd52aba..d903adcf290 100644 --- a/Lib/test/test_gdb/util.py +++ b/Lib/test/test_gdb/util.py @@ -20,6 +20,27 @@ PYTHONHASHSEED = '123' +# gh-91960, bpo-40019: gdb reports these when the optimizer has dropped +# python-frame debug info; the test can't read what's not there. +_OPTIMIZED_OUT_PATTERNS = ( + '(frame information optimized out)', + 'Unable to read information on python frame', + '(unable to read python frame information)', +) +# gdb prints this when the unwinder genuinely failed to walk a frame — +# i.e. the CFI (ours or a library's) is wrong. Treat as a hard failure, +# not a skip, so regressions in our own unwind info don't hide. +_UNWIND_FAILURE_PATTERNS = ( + 'Backtrace stopped: frame did not save the PC', +) +# gh-104736: " ?? ()" in the bt usually means the unwinder bailed early, +# but can also be unrelated frames without debug info (e.g. libc). Tests +# that validate the JIT-relevant part of the backtrace themselves can +# opt out via skip_on_truncation=False. +_TRUNCATED_BACKTRACE_PATTERNS = ( + ' ?? ()', +) + def clean_environment(): # Remove PYTHON* environment variables such as PYTHONHOME @@ -160,7 +181,9 @@ def get_stack_trace(self, source=None, script=None, breakpoint=BREAKPOINT_FN, cmds_after_breakpoint=None, import_site=False, - ignore_stderr=False): + ignore_stderr=False, + skip_on_truncation=True, + **env_vars): ''' Run 'python -c SOURCE' under gdb with a breakpoint. @@ -239,7 +262,7 @@ def get_stack_trace(self, source=None, script=None, args += [script] # Use "args" to invoke gdb, capturing stdout, stderr: - out, err = run_gdb(*args, PYTHONHASHSEED=PYTHONHASHSEED) + out, err = run_gdb(*args, PYTHONHASHSEED=PYTHONHASHSEED, **env_vars) if not ignore_stderr: for line in err.splitlines(): @@ -255,26 +278,20 @@ def get_stack_trace(self, source=None, script=None, " because the Program Counter is" " not present") + for pattern in _UNWIND_FAILURE_PATTERNS: + if pattern in out: + raise AssertionError( + f"gdb unwinder failed ({pattern!r}) — CFI bug in our " + f"generated code or in a linked library.\n" + f"Full gdb output:\n{out}" + ) + # bpo-40019: Skip the test if gdb failed to read debug information # because the Python binary is optimized. - for pattern in ( - '(frame information optimized out)', - 'Unable to read information on python frame', - - # gh-91960: On Python built with "clang -Og", gdb gets - # "frame=" for _PyEval_EvalFrameDefault() parameter - '(unable to read python frame information)', - - # gh-104736: On Python built with "clang -Og" on ppc64le, - # "py-bt" displays a truncated or not traceback, but "where" - # logs this error message: - 'Backtrace stopped: frame did not save the PC', - - # gh-104736: When "bt" command displays something like: - # "#1 0x0000000000000000 in ?? ()", the traceback is likely - # truncated or wrong. - ' ?? ()', - ): + patterns = _OPTIMIZED_OUT_PATTERNS + if skip_on_truncation: + patterns = patterns + _TRUNCATED_BACKTRACE_PATTERNS + for pattern in patterns: if pattern in out: raise unittest.SkipTest(f"{pattern!r} found in gdb output") diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 62cf0c0c6af..748309b5459 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -2074,19 +2074,33 @@ def tearDown(self) -> None: pass super().tearDown() - def generate_tables(self, input: str) -> str: - import io + def analyze_input(self, input: str): with open(self.temp_input_filename, "w+") as f: f.write(parser.BEGIN_MARKER) f.write(input) f.write(parser.END_MARKER) with handle_stderr(): - analysis = analyze_files([self.temp_input_filename]) + return analyze_files([self.temp_input_filename]) + + def generate_tables(self, input: str) -> str: + import io + analysis = self.analyze_input(input) buf = io.StringIO() out = CWriter(buf, 0, False) record_function_generator.generate_recorder_tables(analysis, out) return buf.getvalue() + def get_slot_map_section(self, output: str) -> str: + return output.split( + "const _PyOpcodeRecordSlotMap _PyOpcode_RecordSlotMaps[256] = {\n", + 1, + )[1].split("};\n\n", 1)[0] + + def assert_slot_map_lines(self, output: str, *lines: str) -> None: + slot_map_section = self.get_slot_map_section(output) + for line in lines: + self.assertIn(line, slot_map_section) + def test_single_recording_uop_generates_count(self): input = """ tier2 op(_RECORD_TOS, (value -- value)) { @@ -2145,6 +2159,173 @@ def test_four_recording_uops_rejected(self): with self.assertRaisesRegex(ValueError, "exceeds MAX_RECORDED_VALUES"): self.generate_tables(input) + def test_family_member_needs_transform_only_when_shape_changes(self): + input = """ + tier2 op(_RECORD_TOS, (value -- value)) { + RECORD_VALUE(value); + } + tier2 op(_RECORD_TOS_TYPE, (value -- value)) { + RECORD_VALUE(Py_TYPE(value)); + } + op(_DO_STUFF, (value -- res)) { + res = value; + } + macro(OP_RAW) = _RECORD_TOS + _DO_STUFF; + macro(OP_RAW_SPECIALIZED) = _RECORD_TOS_TYPE + _DO_STUFF; + family(OP_RAW, INLINE_CACHE_ENTRIES_OP_RAW) = { OP_RAW_SPECIALIZED }; + + macro(OP_TYPED) = _RECORD_TOS_TYPE + _DO_STUFF; + macro(OP_TYPED_SPECIALIZED) = _RECORD_TOS_TYPE + _DO_STUFF; + family(OP_TYPED, INLINE_CACHE_ENTRIES_OP_TYPED) = { OP_TYPED_SPECIALIZED }; + """ + output = self.generate_tables(input) + self.assert_slot_map_lines( + output, + "[OP_RAW] = {1, 1, {0}}", + "[OP_RAW_SPECIALIZED] = {1, 0, {0}}", + "[OP_TYPED] = {1, 0, {0}}", + "[OP_TYPED_SPECIALIZED] = {1, 0, {0}}", + ) + + def test_family_member_maps_positional_recorders_to_family_slots(self): + input = """ + tier2 op(_RECORD_TOS, (sub -- sub)) { + RECORD_VALUE(sub); + } + tier2 op(_RECORD_NOS, (container, sub -- container, sub)) { + RECORD_VALUE(container); + } + op(_DO_STUFF, (container, sub -- res)) { + res = container; + } + macro(OP) = _RECORD_TOS + _RECORD_NOS + _DO_STUFF; + macro(OP_SPECIALIZED) = _RECORD_NOS + _DO_STUFF; + family(OP, INLINE_CACHE_ENTRIES_OP) = { OP_SPECIALIZED }; + """ + output = self.generate_tables(input) + self.assert_slot_map_lines( + output, + "[OP] = {2, 0, {1, 0}}", + "[OP_SPECIALIZED] = {1, 0, {0}}", + ) + + def test_family_member_maps_non_positional_recorders_by_stack_shape(self): + input = """ + tier2 op(_RECORD_CALLABLE, (callable, self, args[oparg] -- callable, self, args[oparg])) { + RECORD_VALUE(callable); + } + tier2 op(_RECORD_BOUND_METHOD, (callable, self, args[oparg] -- callable, self, args[oparg])) { + RECORD_VALUE(callable); + } + op(_DO_STUFF, (callable, self, args[oparg] -- res)) { + res = callable; + } + macro(OP) = _RECORD_CALLABLE + _DO_STUFF; + macro(OP_SPECIALIZED) = _RECORD_BOUND_METHOD + _DO_STUFF; + family(OP, INLINE_CACHE_ENTRIES_OP) = { OP_SPECIALIZED }; + """ + output = self.generate_tables(input) + self.assert_slot_map_lines( + output, + "[OP] = {1, 1, {0}}", + "[OP_SPECIALIZED] = {1, 0, {0}}", + ) + + def test_family_head_records_union_of_member_recorders(self): + input = """ + tier2 op(_RECORD_TOS, (value -- value)) { + RECORD_VALUE(value); + } + op(_DO_STUFF, (value -- res)) { + res = value; + } + macro(OP) = _DO_STUFF; + macro(OP_SPECIALIZED) = _RECORD_TOS + _DO_STUFF; + family(OP, INLINE_CACHE_ENTRIES_OP) = { OP_SPECIALIZED }; + """ + output = self.generate_tables(input) + self.assertIn("[OP] = {1, {_RECORD_TOS_INDEX}}", output) + self.assertIn("[OP_SPECIALIZED] = {1, {_RECORD_TOS_INDEX}}", output) + self.assert_slot_map_lines(output, "[OP_SPECIALIZED] = {1, 0, {0}}") + + def test_family_detects_base_and_specialized_recording_difference(self): + input = """ + tier2 op(_RECORD_TOS, (value -- value)) { + RECORD_VALUE(value); + } + tier2 op(_RECORD_TOS_TYPE, (value -- value)) { + RECORD_VALUE(Py_TYPE(value)); + } + op(_DO_STUFF, (value -- res)) { + res = value; + } + macro(OP) = _RECORD_TOS + _DO_STUFF; + macro(OP_SPECIALIZED) = _RECORD_TOS_TYPE + _DO_STUFF; + family(OP, INLINE_CACHE_ENTRIES_OP) = { OP_SPECIALIZED }; + """ + analysis = self.analyze_input(input) + output = self.generate_tables(input) + self.assertEqual( + record_function_generator.get_instruction_record_names( + analysis.instructions["OP"] + ), + ["_RECORD_TOS"], + ) + self.assertEqual( + record_function_generator.get_instruction_record_names( + analysis.instructions["OP_SPECIALIZED"] + ), + ["_RECORD_TOS_TYPE"], + ) + self.assertIn("[OP] = {1, {_RECORD_TOS_TYPE_INDEX}}", output) + self.assertIn("[OP_SPECIALIZED] = {1, {_RECORD_TOS_TYPE_INDEX}}", output) + self.assert_slot_map_lines( + output, + "[OP] = {1, 1, {0}}", + "[OP_SPECIALIZED] = {1, 0, {0}}", + ) + + def test_family_head_falls_back_for_missing_member_slots(self): + input = """ + tier2 op(_RECORD_TOS, (value -- value)) { + RECORD_VALUE(value); + } + op(_DO_STUFF, (value -- res)) { + res = value; + } + macro(OP) = _RECORD_TOS + _DO_STUFF; + macro(OP_SPECIALIZED) = _DO_STUFF; + family(OP, INLINE_CACHE_ENTRIES_OP) = { OP_SPECIALIZED }; + """ + output = self.generate_tables(input) + self.assertIn("[OP] = {1, {_RECORD_TOS_INDEX}}", output) + self.assertIn("[OP_SPECIALIZED] = {1, {_RECORD_TOS_INDEX}}", output) + + def test_family_mixed_slots_only_transform_changed_recorders(self): + input = """ + tier2 op(_RECORD_TOS_TYPE, (left, right -- left, right)) { + RECORD_VALUE(Py_TYPE(right)); + } + tier2 op(_RECORD_NOS_TYPE, (left, right -- left, right)) { + RECORD_VALUE(Py_TYPE(left)); + } + tier2 op(_RECORD_NOS, (left, right -- left, right)) { + RECORD_VALUE(left); + } + op(_DO_STUFF, (left, right -- res)) { + res = left; + } + macro(OP) = _RECORD_TOS_TYPE + _RECORD_NOS_TYPE + _DO_STUFF; + macro(OP_SPECIALIZED) = _RECORD_NOS + _DO_STUFF; + family(OP, INLINE_CACHE_ENTRIES_OP) = { OP_SPECIALIZED }; + """ + output = self.generate_tables(input) + self.assertIn("[OP] = {2, {_RECORD_NOS_INDEX, _RECORD_TOS_TYPE_INDEX}}", output) + self.assert_slot_map_lines( + output, + "[OP] = {2, 2, {1, 0}}", + "[OP_SPECIALIZED] = {1, 0, {0}}", + ) class TestGeneratedAbstractCases(unittest.TestCase): def setUp(self) -> None: diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index e2c7551c0b3..4884b07c95b 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -1,5 +1,5 @@ # Simple test suite for http/cookies.py - +import base64 import copy import unittest import doctest @@ -48,29 +48,6 @@ def test_basic(self): 'Set-Cookie: d=r', 'Set-Cookie: f=h' )) - }, - - # gh-92936: allow double quote in cookie values - { - 'data': 'cookie="{"key": "value"}"', - 'dict': {'cookie': '{"key": "value"}'}, - 'repr': "", - 'output': 'Set-Cookie: cookie="{"key": "value"}"', - }, - { - 'data': 'key="some value; surrounded by quotes"', - 'dict': {'key': 'some value; surrounded by quotes'}, - 'repr': "", - 'output': 'Set-Cookie: key="some value; surrounded by quotes"', - }, - { - 'data': 'session="user123"; preferences="{"theme": "dark"}"', - 'dict': {'session': 'user123', 'preferences': '{"theme": "dark"}'}, - 'repr': "", - 'output': '\n'.join(( - 'Set-Cookie: preferences="{"theme": "dark"}"', - 'Set-Cookie: session="user123"', - )) } ] @@ -175,17 +152,19 @@ def test_load(self): self.assertEqual(C.output(['path']), 'Set-Cookie: Customer="WILE_E_COYOTE"; Path=/acme') - self.assertEqual(C.js_output(), r""" + cookie_encoded = base64.b64encode(b'Customer="WILE_E_COYOTE"; Path=/acme; Version=1').decode('ascii') + self.assertEqual(C.js_output(), fr""" """) - self.assertEqual(C.js_output(['path']), r""" + cookie_encoded = base64.b64encode(b'Customer="WILE_E_COYOTE"; Path=/acme').decode('ascii') + self.assertEqual(C.js_output(['path']), fr""" """) @@ -290,17 +269,19 @@ def test_quoted_meta(self): self.assertEqual(C.output(['path']), 'Set-Cookie: Customer="WILE_E_COYOTE"; Path=/acme') - self.assertEqual(C.js_output(), r""" + expected_encoded_cookie = base64.b64encode(b'Customer=\"WILE_E_COYOTE\"; Path=/acme; Version=1').decode('ascii') + self.assertEqual(C.js_output(), fr""" """) - self.assertEqual(C.js_output(['path']), r""" + expected_encoded_cookie = base64.b64encode(b'Customer=\"WILE_E_COYOTE\"; Path=/acme').decode('ascii') + self.assertEqual(C.js_output(['path']), fr""" """) @@ -391,13 +372,16 @@ def test_setter(self): self.assertEqual( M.output(), "Set-Cookie: %s=%s; Path=/foo" % (i, "%s_coded_val" % i)) + expected_encoded_cookie = base64.b64encode( + ("%s=%s; Path=/foo" % (i, "%s_coded_val" % i)).encode("ascii") + ).decode('ascii') expected_js_output = """ - """ % (i, "%s_coded_val" % i) + """ % (expected_encoded_cookie,) self.assertEqual(M.js_output(), expected_js_output) for i in ["foo bar", "foo@bar"]: # Try some illegal characters diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index d78b94e3a37..1f7a5a42fda 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -1379,6 +1379,7 @@ class CommandLineTestCase(unittest.TestCase): 'protocol': default_protocol, 'port': default_port, 'bind': default_bind, + 'content_type': 'application/octet-stream', 'tls_cert': None, 'tls_key': None, 'tls_password': None, @@ -1447,6 +1448,16 @@ def test_protocol_flag(self, mock_func): mock_func.assert_called_once_with(**call_args) mock_func.reset_mock() + @mock.patch('http.server.test') + def test_content_type_flag(self, mock_func): + content_types = ['text/html', 'text/plain', 'application/json'] + for content_type in content_types: + with self.subTest(content_type=content_type): + self.invoke_httpd('--content-type', content_type) + call_args = self.args | dict(content_type=content_type) + mock_func.assert_called_once_with(**call_args) + mock_func.reset_mock() + @unittest.skipIf(ssl is None, "requires ssl") @mock.patch('http.server.test') def test_tls_cert_and_key_flags(self, mock_func): diff --git a/Lib/test/test_importlib/test_threaded_import.py b/Lib/test/test_importlib/test_threaded_import.py index 8b793ebf29b..6875fdca9c8 100644 --- a/Lib/test/test_importlib/test_threaded_import.py +++ b/Lib/test/test_importlib/test_threaded_import.py @@ -324,6 +324,143 @@ def do_import(delay=0): # Neither thread should have errors about stale modules self.assertEqual(errors, [], f"Race condition detected: {errors}") + def test_hierarchical_import_deadlock(self): + # Regression test for bpo-38884 / gh-83065 + # Tests that concurrent imports at different hierarchy levels + # don't deadlock when parent imports child in __init__.py + + # Create package structure: + # package/__init__.py: from package import subpackage + # package/subpackage/__init__.py: from package.subpackage.module import * + # package/subpackage/module.py: class SomeClass: pass + + pkg_dir = os.path.join(TESTFN, 'hier_deadlock_pkg') + os.makedirs(pkg_dir) + self.addCleanup(shutil.rmtree, TESTFN) + + subpkg_dir = os.path.join(pkg_dir, 'subpackage') + os.makedirs(subpkg_dir) + + with open(os.path.join(pkg_dir, "__init__.py"), "w") as f: + f.write("from hier_deadlock_pkg import subpackage\n") + + with open(os.path.join(subpkg_dir, "__init__.py"), "w") as f: + f.write("from hier_deadlock_pkg.subpackage.module import *\n") + + with open(os.path.join(subpkg_dir, "module.py"), "w") as f: + f.write("class SomeClass:\n pass\n") + + sys.path.insert(0, TESTFN) + self.addCleanup(sys.path.remove, TESTFN) + self.addCleanup(forget, 'hier_deadlock_pkg') + self.addCleanup(forget, 'hier_deadlock_pkg.subpackage') + self.addCleanup(forget, 'hier_deadlock_pkg.subpackage.module') + + importlib.invalidate_caches() + + errors = [] + results = [] + barrier = threading.Barrier(2) + + def t1(): + barrier.wait() + try: + import hier_deadlock_pkg.subpackage + results.append('t1_success') + except Exception as e: + errors.append(('t1', type(e).__name__, str(e))) + + def t2(): + barrier.wait() + try: + import hier_deadlock_pkg.subpackage.module + results.append('t2_success') + except Exception as e: + errors.append(('t2', type(e).__name__, str(e))) + + # Run multiple times to increase chance of hitting race condition + for i in range(10): + for mod in ['hier_deadlock_pkg', 'hier_deadlock_pkg.subpackage', + 'hier_deadlock_pkg.subpackage.module']: + sys.modules.pop(mod, None) + + errors.clear() + results.clear() + barrier.reset() + + thread1 = threading.Thread(target=t1) + thread2 = threading.Thread(target=t2) + + thread1.start() + thread2.start() + + thread1.join(timeout=5) + thread2.join(timeout=5) + + if thread1.is_alive() or thread2.is_alive(): + self.fail(f"Threads deadlocked on iteration {i}") + + self.assertEqual( + errors, [], + f"Import(s) failed on iteration {i}: {errors}") + self.assertEqual( + sorted(results), ['t1_success', 't2_success'], + f"Not all imports succeeded on iteration {i}: {results}") + + def test_cross_package_circular_import(self): + # Two packages whose __init__.py each import a submodule of the + # other. Concurrent imports of submodules of each must not raise + # _DeadlockError; the import system accepts a partially-initialised + # parent in this case (see _lock_unlock_module). + os.makedirs(os.path.join(TESTFN, "circ_a")) + os.makedirs(os.path.join(TESTFN, "circ_b")) + self.addCleanup(shutil.rmtree, TESTFN) + with open(os.path.join(TESTFN, "circ_a", "__init__.py"), "w") as f: + f.write("import time; time.sleep(0.03)\nimport circ_b.other\n") + with open(os.path.join(TESTFN, "circ_b", "__init__.py"), "w") as f: + f.write("import time; time.sleep(0.03)\nimport circ_a.other\n") + for pkg in ("circ_a", "circ_b"): + for mod in ("sub.py", "other.py"): + with open(os.path.join(TESTFN, pkg, mod), "w") as f: + f.write("X = 1\n") + + sys.path.insert(0, TESTFN) + self.addCleanup(sys.path.remove, TESTFN) + for mod in ("circ_a", "circ_a.sub", "circ_a.other", + "circ_b", "circ_b.sub", "circ_b.other"): + self.addCleanup(forget, mod) + importlib.invalidate_caches() + + errors = [] + barrier = threading.Barrier(2) + + def do_import(name): + barrier.wait() + try: + importlib.import_module(name) + except Exception as e: + errors.append((name, type(e).__name__, str(e))) + + for i in range(10): + for mod in ("circ_a", "circ_a.sub", "circ_a.other", + "circ_b", "circ_b.sub", "circ_b.other"): + sys.modules.pop(mod, None) + errors.clear() + barrier.reset() + + thread1 = threading.Thread(target=do_import, args=("circ_a.sub",)) + thread2 = threading.Thread(target=do_import, args=("circ_b.sub",)) + thread1.start() + thread2.start() + thread1.join(timeout=5) + thread2.join(timeout=5) + + if thread1.is_alive() or thread2.is_alive(): + self.fail(f"Threads deadlocked on iteration {i}") + self.assertEqual( + errors, [], + f"Import(s) failed on iteration {i}: {errors}") + def setUpModule(): thread_info = threading_helper.threading_setup() diff --git a/Lib/test/test_json/json_lines.jsonl b/Lib/test/test_json/json_lines.jsonl new file mode 100644 index 00000000000..d2f29211195 --- /dev/null +++ b/Lib/test/test_json/json_lines.jsonl @@ -0,0 +1,2 @@ +{"ingredients":["frog", "water", "chocolate", "glucose"]} +{"ingredients":["chocolate","steel bolts"]} diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 7b5d217a215..0a96b318b15 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -1,4 +1,5 @@ import errno +import pathlib import os import sys import textwrap @@ -157,6 +158,14 @@ def test_jsonlines(self): self.assertEqual(process.stdout, self.jsonlines_expect) self.assertEqual(process.stderr, '') + @force_not_colorized + def test_jsonlines_from_file(self): + jsonl = pathlib.Path(__file__).parent / 'json_lines.jsonl' + args = sys.executable, '-m', self.module, '--json-lines', jsonl + process = subprocess.run(args, capture_output=True, text=True, check=True) + self.assertEqual(process.stdout, self.jsonlines_expect) + self.assertEqual(process.stderr, '') + def test_help_flag(self): rc, out, err = assert_python_ok('-m', self.module, '-h', PYTHON_COLORS='0') diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 78db4219e29..9c4d91c456d 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -317,6 +317,138 @@ def test_recursion_limit(self): last.append([0]) self.assertRaises(ValueError, marshal.dumps, head) + def test_reference_loop_list(self): + a = [] + a.append(a) + for v in range(3): + self.assertRaises(ValueError, marshal.dumps, a, v) + for v in range(3, marshal.version + 1): + d = marshal.dumps(a, v) + b = marshal.loads(d) + self.assertIsInstance(b, list) + self.assertIs(b[0], b) + + def test_reference_loop_dict(self): + a = {} + a[None] = a + for v in range(3): + self.assertRaises(ValueError, marshal.dumps, a, v) + for v in range(3, marshal.version + 1): + d = marshal.dumps(a, v) + b = marshal.loads(d) + self.assertIsInstance(b, dict) + self.assertIs(b[None], b) + + def test_reference_loop_tuple(self): + a = ([],) + a[0].append(a) + for v in range(3): + self.assertRaises(ValueError, marshal.dumps, a, v) + for v in range(3, marshal.version + 1): + d = marshal.dumps(a, v) + b = marshal.loads(d) + self.assertIsInstance(b, tuple) + self.assertIsInstance(b[0], list) + self.assertIs(b[0][0], b) + + def test_reference_loop_code(self): + def f(): + return 1234.5 + code = f.__code__ + a = [] + code = code.replace(co_consts=code.co_consts + (a,)) + # This test creates a reference loop which leads to reference leaks, + # so we need to break the loop manually. See gh-148722. + self.addCleanup(a.clear) + a.append(code) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, code, v) + + def test_reference_loop_slice(self): + a = slice([], None) + a.start.append(a) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, a, v) + + a = slice(None, []) + a.stop.append(a) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, a, v) + + a = slice(None, None, []) + a.step.append(a) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, a, v) + + def test_reference_loop_frozendict(self): + a = frozendict({None: []}) + a[None].append(a) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, a, v) + + def test_loads_reference_loop_list(self): + data = b'\xdb\x01\x00\x00\x00r\x00\x00\x00\x00' # [] + a = marshal.loads(data) + self.assertIsInstance(a, list) + self.assertIs(a[0], a) + + def test_loads_reference_loop_dict(self): + data = b'\xfbNr\x00\x00\x00\x000' # {None: } + a = marshal.loads(data) + self.assertIsInstance(a, dict) + self.assertIs(a[None], a) + + def test_loads_abnormal_reference_loops(self): + # Indirect self-references of tuples. + data = b'\xa8\x01\x00\x00\x00[\x01\x00\x00\x00r\x00\x00\x00\x00' # ([],) + a = marshal.loads(data) + self.assertIsInstance(a, tuple) + self.assertIsInstance(a[0], list) + self.assertIs(a[0][0], a) + + data = b'\xa8\x01\x00\x00\x00{Nr\x00\x00\x00\x000' # ({None: },) + a = marshal.loads(data) + self.assertIsInstance(a, tuple) + self.assertIsInstance(a[0], dict) + self.assertIs(a[0][None], a) + + # Direct self-reference which cannot be created in Python. + # This creates a reference loop which cannot be collected. + if False: + data = b'\xa8\x01\x00\x00\x00r\x00\x00\x00\x00' # (,) + a = marshal.loads(data) + self.assertIsInstance(a, tuple) + self.assertIs(a[0], a) + + # Direct self-references which cannot be created in Python + # because of unhashability. + data = b'\xfbr\x00\x00\x00\x00N0' # {: None} + self.assertRaises(TypeError, marshal.loads, data) + data = b'\xbc\x01\x00\x00\x00r\x00\x00\x00\x00' # {} + self.assertRaises(TypeError, marshal.loads, data) + + for data in [ + # Indirect self-references of immutable objects. + b'\xba[\x01\x00\x00\x00r\x00\x00\x00\x00NN', # slice([], None) + b'\xbaN[\x01\x00\x00\x00r\x00\x00\x00\x00N', # slice(None, []) + b'\xbaNN[\x01\x00\x00\x00r\x00\x00\x00\x00', # slice(None, None, []) + b'\xba{Nr\x00\x00\x00\x000NN', # slice({None: }, None) + b'\xbaN{Nr\x00\x00\x00\x000N', # slice(None, {None: }) + b'\xbaNN{Nr\x00\x00\x00\x000', # slice(None, None, {None: }) + b'\xfdN[\x01\x00\x00\x00r\x00\x00\x00\x000', # frozendict({None: []}) + b'\xfdN{Nr\x00\x00\x00\x0000', # frozendict({None: {None: }) + + # Direct self-references which cannot be created in Python. + b'\xbe\x01\x00\x00\x00r\x00\x00\x00\x00', # frozenset({}) + b'\xfdNr\x00\x00\x00\x000', # frozendict({None: }) + b'\xfdr\x00\x00\x00\x00N0', # frozendict({: None}) + b'\xbar\x00\x00\x00\x00NN', # slice(, None) + b'\xbaNr\x00\x00\x00\x00N', # slice(None, ) + b'\xbaNNr\x00\x00\x00\x00', # slice(None, None, ) + ]: + with self.subTest(data=data): + self.assertRaises(ValueError, marshal.loads, data) + def test_exact_type_match(self): # Former bug: # >>> class Int(int): pass diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index bc7af6e1538..b8861d09e15 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -196,13 +196,10 @@ def test_c_return_count(self): (E.BRANCH, "branch"), ] -EXCEPT_EVENTS = [ +SIMPLE_EVENTS = INSTRUMENTED_EVENTS + [ (E.RAISE, "raise"), - (E.PY_UNWIND, "unwind"), (E.EXCEPTION_HANDLED, "exception_handled"), -] - -SIMPLE_EVENTS = INSTRUMENTED_EVENTS + EXCEPT_EVENTS + [ + (E.PY_UNWIND, "unwind"), (E.C_RAISE, "c_raise"), (E.C_RETURN, "c_return"), ] @@ -738,18 +735,6 @@ def test_disable_legal_events(self): sys.monitoring.register_callback(TEST_TOOL, event, None) - def test_disable_illegal_events(self): - for event, name in EXCEPT_EVENTS: - try: - counter = CounterWithDisable() - counter.disable = True - sys.monitoring.register_callback(TEST_TOOL, event, counter) - sys.monitoring.set_events(TEST_TOOL, event) - with self.assertRaises(ValueError): - self.raise_handle_reraise() - finally: - sys.monitoring.set_events(TEST_TOOL, 0) - sys.monitoring.register_callback(TEST_TOOL, event, None) class ExceptionRecorder: @@ -1481,8 +1466,334 @@ def func3(): ('line', 'func3', 6)]) def test_set_non_local_event(self): + # C_RETURN/C_RAISE are ancillary (derived) events — not settable as local with self.assertRaises(ValueError): - sys.monitoring.set_local_events(TEST_TOOL, just_call.__code__, E.RAISE) + sys.monitoring.set_local_events(TEST_TOOL, just_call.__code__, E.C_RETURN) + + def test_local_reraise(self): + """RERAISE fires as a local event only for the instrumented code object.""" + + def foo(): + try: + raise RuntimeError("test") + except RuntimeError: + raise + + def bar(): + try: + raise RuntimeError("test") + except RuntimeError: + raise + + events = set() + + def callback(code, offset, exc): + events.add(code.co_name) + + try: + sys.monitoring.register_callback(TEST_TOOL, E.RERAISE, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.RERAISE) + try: + foo() + except RuntimeError: + pass + try: + bar() # should NOT trigger the callback + except RuntimeError: + pass + self.assertEqual(events, {'foo'}) + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.RERAISE, None) + + def test_local_reraise_disable(self): + """Returning DISABLE from a RERAISE callback disables it for that code object.""" + + call_count = 0 + + def foo(): + try: + raise RuntimeError("test") + except RuntimeError: + raise + + def callback(code, offset, exc): + nonlocal call_count + call_count += 1 + return sys.monitoring.DISABLE + + try: + sys.monitoring.register_callback(TEST_TOOL, E.RERAISE, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.RERAISE) + try: + foo() + except RuntimeError: + pass + self.assertEqual(call_count, 1) + try: + foo() + except RuntimeError: + pass + self.assertEqual(call_count, 1) # not fired again — disabled + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.RERAISE, None) + + def test_local_py_throw(self): + """PY_THROW fires as a local event only for the instrumented code object.""" + + def gen_foo(): + yield 1 + yield 2 + + def gen_bar(): + yield 1 + yield 2 + + events = [] + + def callback(code, offset, exc): + events.append(code.co_name) + + try: + sys.monitoring.register_callback(TEST_TOOL, E.PY_THROW, callback) + sys.monitoring.set_local_events(TEST_TOOL, gen_foo.__code__, E.PY_THROW) + + g = gen_foo() + next(g) + try: + g.throw(RuntimeError("test")) + except RuntimeError: + pass + + h = gen_bar() + next(h) + try: + h.throw(RuntimeError("test")) # should NOT trigger the callback + except RuntimeError: + pass + + self.assertEqual(events, ['gen_foo']) + finally: + sys.monitoring.set_local_events(TEST_TOOL, gen_foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_THROW, None) + + def test_local_py_throw_disable(self): + """Returning DISABLE from a PY_THROW callback disables it for that code object.""" + + call_count = 0 + + def gen_foo(): + yield 1 + yield 2 + + def callback(code, offset, exc): + nonlocal call_count + call_count += 1 + return sys.monitoring.DISABLE + + try: + sys.monitoring.register_callback(TEST_TOOL, E.PY_THROW, callback) + sys.monitoring.set_local_events(TEST_TOOL, gen_foo.__code__, E.PY_THROW) + + g = gen_foo() + next(g) + try: + g.throw(RuntimeError("test")) + except RuntimeError: + pass + self.assertEqual(call_count, 1) + + g2 = gen_foo() + next(g2) + try: + g2.throw(RuntimeError("test")) + except RuntimeError: + pass + self.assertEqual(call_count, 1) # not fired again — disabled + finally: + sys.monitoring.set_local_events(TEST_TOOL, gen_foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_THROW, None) + + def test_local_raise(self): + """RAISE fires as a local event only for the instrumented code object.""" + + def foo(): + try: + raise RuntimeError("test") + except RuntimeError: + pass + + def bar(): + try: + raise RuntimeError("test") + except RuntimeError: + pass + + events = [] + + def callback(code, offset, exc): + events.append(code.co_name) + + try: + sys.monitoring.register_callback(TEST_TOOL, E.RAISE, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.RAISE) + foo() + bar() # should NOT trigger the callback + self.assertEqual(events, ['foo']) + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.RAISE, None) + + def test_local_raise_disable(self): + """Returning DISABLE from a RAISE callback disables it for that code object.""" + + call_count = 0 + + def foo(): + try: + raise RuntimeError("test") + except RuntimeError: + pass + + def callback(code, offset, exc): + nonlocal call_count + call_count += 1 + return sys.monitoring.DISABLE + + try: + sys.monitoring.register_callback(TEST_TOOL, E.RAISE, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.RAISE) + foo() + self.assertEqual(call_count, 1) + foo() + self.assertEqual(call_count, 1) # not fired again — disabled + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.RAISE, None) + + def test_local_exception_handled(self): + """EXCEPTION_HANDLED fires as a local event only for the instrumented code object.""" + + def foo(): + try: + raise RuntimeError("test") + except RuntimeError: + pass + + def bar(): + try: + raise RuntimeError("test") + except RuntimeError: + pass + + events = [] + + def callback(code, offset, exc): + events.append(code.co_name) + + try: + sys.monitoring.register_callback(TEST_TOOL, E.EXCEPTION_HANDLED, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.EXCEPTION_HANDLED) + foo() + bar() # should NOT trigger the callback + self.assertEqual(events, ['foo']) + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.EXCEPTION_HANDLED, None) + + def test_local_exception_handled_disable(self): + """Returning DISABLE from an EXCEPTION_HANDLED callback disables it for that code object.""" + + call_count = 0 + + def foo(): + try: + raise RuntimeError("test") + except RuntimeError: + pass + + def callback(code, offset, exc): + nonlocal call_count + call_count += 1 + return sys.monitoring.DISABLE + + try: + sys.monitoring.register_callback(TEST_TOOL, E.EXCEPTION_HANDLED, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.EXCEPTION_HANDLED) + foo() + self.assertEqual(call_count, 1) + foo() + self.assertEqual(call_count, 1) # not fired again — disabled + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.EXCEPTION_HANDLED, None) + + def test_local_py_unwind(self): + """PY_UNWIND fires as a local event only for the instrumented code object.""" + + def foo(): + raise RuntimeError("test") + + def bar(): + raise RuntimeError("test") + + events = [] + + def callback(code, offset, exc): + events.append(code.co_name) + + try: + sys.monitoring.register_callback(TEST_TOOL, E.PY_UNWIND, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.PY_UNWIND) + + try: + foo() + except RuntimeError: + pass + + try: + bar() # should NOT trigger the callback + except RuntimeError: + pass + + self.assertEqual(events, ['foo']) + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_UNWIND, None) + + def test_local_py_unwind_disable(self): + """Returning DISABLE from a PY_UNWIND callback disables it for that code object.""" + + call_count = 0 + + def foo(): + raise RuntimeError("test") + + def callback(code, offset, exc): + nonlocal call_count + call_count += 1 + return sys.monitoring.DISABLE + + try: + sys.monitoring.register_callback(TEST_TOOL, E.PY_UNWIND, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.PY_UNWIND) + + try: + foo() + except RuntimeError: + pass + self.assertEqual(call_count, 1) # fired once + + try: + foo() + except RuntimeError: + pass + self.assertEqual(call_count, 1) # not fired again — disabled by DISABLE return + + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_UNWIND, None) def line_from_offset(code, offset): for start, end, line in code.co_lines(): diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py index 5d0857b059e..29cce4ee6d2 100644 --- a/Lib/test/test_patma.py +++ b/Lib/test/test_patma.py @@ -2762,6 +2762,96 @@ def test_patma_255(self): self.assertEqual(y, 1) self.assertIs(z, x) + def test_patma_256(self): + x = 0 + match x: + case +0: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_257(self): + x = 0 + match x: + case +0.0: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_258(self): + x = 0 + match x: + case +0j: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_259(self): + x = 0 + match x: + case +0.0j: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_260(self): + x = 1 + match x: + case +1: + y = 0 + self.assertEqual(x, 1) + self.assertEqual(y, 0) + + def test_patma_261(self): + x = 1.5 + match x: + case +1.5: + y = 0 + self.assertEqual(x, 1.5) + self.assertEqual(y, 0) + + def test_patma_262(self): + x = 1j + match x: + case +1j: + y = 0 + self.assertEqual(x, 1j) + self.assertEqual(y, 0) + + def test_patma_263(self): + x = 1.5j + match x: + case +1.5j: + y = 0 + self.assertEqual(x, 1.5j) + self.assertEqual(y, 0) + + def test_patma_264(self): + x = 0.25 + 1.75j + match x: + case +0.25 + 1.75j: + y = 0 + self.assertEqual(x, 0.25 + 1.75j) + self.assertEqual(y, 0) + + def test_patma_265(self): + x = 0.25 - 1.75j + match x: + case 0.25 - +1.75j: + y = 0 + self.assertEqual(x, 0.25 - 1.75j) + self.assertEqual(y, 0) + + def test_patma_266(self): + x = 0 + match x: + case +1e1000: + y = 0 + case 0: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 1) + def test_patma_runtime_checkable_protocol(self): # Runtime-checkable protocol from typing import Protocol, runtime_checkable diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 0e23cd66043..db900199755 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -6,6 +6,7 @@ import io import os import pdb +import re import sys import types import codecs @@ -4752,6 +4753,16 @@ def foo(self): stdout, stderr = self.run_pdb_script(script, commands) self.assertIn("The specified object 'C.foo' is not a function", stdout) + def test_pyrepl_available(self): + with patch.dict(os.environ, {"PYTHON_BASIC_REPL": "1"}): + self.assertFalse(pdb._pyrepl_available()) + + with patch.dict(os.environ, {}, clear=True): + mod = types.ModuleType("_pyrepl.main") + mod.CAN_USE_PYREPL = True + with patch.dict("sys.modules", {"_pyrepl.main": mod}): + self.assertTrue(pdb._pyrepl_available()) + class ChecklineTests(unittest.TestCase): def setUp(self): @@ -5006,6 +5017,20 @@ def setUpClass(cls): if readline.backend == "editline": raise unittest.SkipTest("libedit readline is not supported for pdb") + def _run_pty(self, script, input, env=None): + if env is None: + # By default, we use basic repl for the test. + # Subclass can overwrite this method and set env to use advanced REPL + env = os.environ | {'PYTHON_BASIC_REPL': '1'} + output = run_pty(script, input, env=env) + # filter all control characters + # Strip ANSI CSI sequences (good enough for most REPL/prompt output) + output = re.sub(r"\x1b\[[0-?]*[ -/]*[@-~]", "", output.decode("utf-8")) + return output + + def _pyrepl_available(self): + return pdb._pyrepl_available() + def test_basic_completion(self): script = textwrap.dedent(""" import pdb; pdb.Pdb().set_trace() @@ -5017,12 +5042,12 @@ def test_basic_completion(self): # then add ntin and complete 'contin' to 'continue' input = b"co\t\tntin\t\n" - output = run_pty(script, input) + output = self._run_pty(script, input) - self.assertIn(b'commands', output) - self.assertIn(b'condition', output) - self.assertIn(b'continue', output) - self.assertIn(b'hello!', output) + self.assertIn('commands', output) + self.assertIn('condition', output) + self.assertIn('continue', output) + self.assertIn('hello!', output) def test_expression_completion(self): script = textwrap.dedent(""" @@ -5039,11 +5064,11 @@ def test_expression_completion(self): # Continue input += b"c\n" - output = run_pty(script, input) + output = self._run_pty(script, input) - self.assertIn(b'special', output) - self.assertIn(b'species', output) - self.assertIn(b'$_frame', output) + self.assertIn('special', output) + self.assertIn('species', output) + self.assertIn('$_frame', output) def test_builtin_completion(self): script = textwrap.dedent(""" @@ -5057,9 +5082,9 @@ def test_builtin_completion(self): # Continue input += b"c\n" - output = run_pty(script, input) + output = self._run_pty(script, input) - self.assertIn(b'special', output) + self.assertIn('special', output) def test_convvar_completion(self): script = textwrap.dedent(""" @@ -5075,10 +5100,10 @@ def test_convvar_completion(self): # Continue input += b"c\n" - output = run_pty(script, input) + output = self._run_pty(script, input) - self.assertIn(b'", 0) + consts = meta["consts"] + self.assertIsInstance(consts, list) + _testinternalcapi.optimize_cfg(insts, consts, 0) + + def test_compiler_codegen_consts_include_none_required_for_implicit_return(self): + # Module "pass" only needs the const table entry for None once + # _PyCodegen_AddReturnAtEnd runs. If metadata["consts"] were taken + # before that, the list would not match LOAD_CONST opargs (here: 0 + # for None), and optimize_cfg would read out of range. + tree = ast.parse("pass", mode="exec", optimize=1) + insts, meta = _testinternalcapi.compiler_codegen(tree, "", 0) + consts = meta["consts"] + self.assertEqual(consts, [None]) + + load_const = opcode.opmap["LOAD_CONST"] + self.assertEqual( + [t[1] for t in insts.get_instructions() if t[0] == load_const], + [0], + ) + + # As if consts were snapshotted before AddReturnAtEnd: still LOAD_CONST 0, no row. + with self.assertRaisesRegex(ValueError, "out of range"): + _testinternalcapi.optimize_cfg(insts, [], 0) + + _testinternalcapi.optimize_cfg(insts, list(consts), 0) + def cfg_optimization_test(self, insts, expected_insts, consts=None, expected_consts=None, nlocals=0): diff --git a/Lib/test/test_perfmaps.py b/Lib/test/test_perfmaps.py index 647c32656ab..ee4eb50033c 100644 --- a/Lib/test/test_perfmaps.py +++ b/Lib/test/test_perfmaps.py @@ -1,4 +1,5 @@ import os +import sys import sysconfig import unittest @@ -17,6 +18,9 @@ def supports_trampoline_profiling(): raise unittest.SkipTest("perf trampoline profiling not supported") class TestPerfMapWriting(unittest.TestCase): + def tearDown(self): + perf_map_state_teardown() + def test_write_perf_map_entry(self): self.assertEqual(write_perf_map_entry(0x1234, 5678, "entry1"), 0) self.assertEqual(write_perf_map_entry(0x2345, 6789, "entry2"), 0) @@ -24,4 +28,15 @@ def test_write_perf_map_entry(self): perf_file_contents = f.read() self.assertIn("1234 162e entry1", perf_file_contents) self.assertIn("2345 1a85 entry2", perf_file_contents) - perf_map_state_teardown() + + @unittest.skipIf(sys.maxsize <= 2**32, "requires size_t wider than unsigned int") + def test_write_perf_map_entry_large_size(self): + code_addr = 0x3456 + code_size = 1 << 33 + entry_name = "entry_big" + + self.assertEqual(write_perf_map_entry(code_addr, code_size, entry_name), 0) + with open(f"/tmp/perf-{os.getpid()}.map") as f: + perf_file_contents = f.read() + self.assertIn(f"{code_addr:x} {code_size:x} {entry_name}", + perf_file_contents) diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index 57285ddf6eb..caf2d7ba6bf 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -160,6 +160,7 @@ def test_unknown_opcode_without_pos(self): next(it) +@support.force_not_colorized_test_class class DisTests(unittest.TestCase): maxDiff = None @@ -518,6 +519,7 @@ def test__all__(self): support.check__all__(self, pickletools, not_exported=not_exported) +@support.force_not_colorized_test_class class CommandLineTest(unittest.TestCase): def setUp(self): self.filename = tempfile.mktemp() diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py index 503430ddf02..240ec8a195c 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py @@ -436,6 +436,8 @@ def test_flamegraph_collector_basic(self): name = resolve_name(data, strings) self.assertTrue(name.startswith("Program Root: ")) self.assertIn("func2 (file.py:20)", name) + label = strings[data["label"]] + self.assertTrue(label.startswith("Program Root: ")) self.assertEqual(data["self"], 0) # non-leaf: no self time children = data.get("children", []) self.assertEqual(len(children), 1) diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 1e57b9244b4..dbd3b855f53 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -1075,6 +1075,12 @@ def test_avg_std(self): msg='%s%r' % (variate.__name__, args)) self.assertAlmostEqual(s2/(N-1), sigmasqrd, places=2, msg='%s%r' % (variate.__name__, args)) + def test_binomialvariate_log_zero(self): + # gh-149222: Variety random() return 0.0 no input Error + with unittest.mock.patch.object(random.Random, 'random', side_effect= [0.0] + [0.5] * 20): + result = random.binomialvariate(10, 0.5) + self.assertIsInstance(result, int) + self.assertIn(result, range(11)) def test_constant(self): g = random.Random() diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 27cd125078e..850cb66a89b 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -5,6 +5,7 @@ import subprocess import sys import unittest +from contextlib import contextmanager from functools import partial from textwrap import dedent from test import support @@ -67,6 +68,19 @@ def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, custom=F spawn_asyncio_repl = partial(spawn_repl, "-m", "asyncio", custom=True) +@contextmanager +def temp_pythonstartup(*, source: str, histfile: str = ".pythonhist"): + """Create environment variables for a PYTHONSTARTUP script in a temporary directory.""" + with os_helper.temp_dir() as tmpdir: + filename = os.path.join(tmpdir, "pythonstartup.py") + with open(filename, "w") as f: + f.write(source) + yield { + "PYTHONSTARTUP": filename, + "PYTHON_HISTORY": os.path.join(tmpdir, histfile) + } + + def run_on_interactive_mode(source): """Spawn a new Python interpreter, pass the given input source code from the stdin and return the @@ -276,8 +290,6 @@ def make_repl(env): """) % script self.assertIn(expected, output) - - def test_runsource_show_syntax_error_location(self): user_input = dedent("""def f(x, x): ... """) @@ -449,6 +461,33 @@ def test_quiet_mode(self): self.assertEqual(p.returncode, 0) self.assertEqual(output[:3], ">>>") + @support.force_not_colorized + @support.subTests( + ("startup_code", "expected_error"), + [ + ("some invalid syntax\n", "SyntaxError: invalid syntax"), + ("1/0\n", "ZeroDivisionError: division by zero"), + ], + ) + def test_pythonstartup_failure(self, startup_code, expected_error): + startup_env = self.enterContext( + temp_pythonstartup(source=startup_code, histfile=".asyncio_history")) + + p = spawn_repl( + "-qm", "asyncio", + env=os.environ | startup_env, + isolated=False, + custom=True) + p.stdin.write("print('user code', 'executed')\n") + output = kill_python(p) + self.assertEqual(p.returncode, 0) + + tb_hint = f'File "{startup_env["PYTHONSTARTUP"]}", line 1' + self.assertIn(tb_hint, output) + self.assertIn(expected_error, output) + + self.assertIn("user code executed", output) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index 9f3bc8973eb..55b9673ef6c 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -217,6 +217,25 @@ def test_invalid_names(self): # Package without __main__.py self.expect_import_error("multiprocessing") + def test_invalid_names_set_name_attribute(self): + cases = [ + # (mod_name, expected_name) -- comment indicates raise site + ("nonexistent_runpy_test_module", + "nonexistent_runpy_test_module"), # spec is None + ("sys.imp.eric", "sys.imp.eric"), # find_spec error + (".relative_name", ".relative_name"), # relative name rejected + ("sys", "sys"), # builtin: no code object + ("multiprocessing", "multiprocessing"), # package without __main__ + ] + for mod_name, expected_name in cases: + with self.subTest(mod_name=mod_name): + try: + run_module(mod_name) + except ImportError as exc: + self.assertEqual(exc.name, expected_name) + else: + self.fail("Expected ImportError for %r" % mod_name) + def test_library_module(self): self.assertEqual(run_module("runpy")["__name__"], "runpy") @@ -714,6 +733,17 @@ def test_directory_error(self): msg = "can't find '__main__' module in %r" % script_dir self._check_import_error(script_dir, msg) + def test_directory_error_sets_name_attribute(self): + with temp_dir() as script_dir: + self._make_test_script(script_dir, 'not_main') + try: + run_path(script_dir) + except ImportError as exc: + self.assertEqual(exc.name, '__main__') + else: + self.fail("Expected ImportError for directory without " + "__main__.py") + def test_zipfile(self): with temp_dir() as script_dir: mod_name = '__main__' diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index a4bd113bc7f..13a3487382d 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -2136,8 +2136,6 @@ def test_make_zipfile_rootdir_nodir(self): def check_unpack_archive(self, format, **kwargs): self.check_unpack_archive_with_converter( format, lambda path: path, **kwargs) - self.check_unpack_archive_with_converter( - format, FakePath, **kwargs) self.check_unpack_archive_with_converter(format, FakePath, **kwargs) def check_unpack_archive_with_converter(self, format, converter, **kwargs): @@ -2194,6 +2192,71 @@ def test_unpack_archive_zip(self): with self.assertRaises(TypeError): self.check_unpack_archive('zip', filter='data') + def test_unpack_archive_zip_badpaths(self): + srcdir = self.mkdtemp() + zipname = os.path.join(srcdir, 'test.zip') + abspath = os.path.join(srcdir, 'abspath') + with zipfile.ZipFile(zipname, 'w') as zf: + zf.writestr(abspath, 'badfile') + zf.writestr(os.sep + abspath, 'badfile') + zf.writestr('/abspath', 'badfile') + zf.writestr('C:/abspath', 'badfile') + zf.writestr('D:\\abspath', 'badfile') + zf.writestr('E:abspath', 'badfile') + zf.writestr('F:/G:/abspath', 'badfile') + zf.writestr('//server/share/abspath', 'badfile') + zf.writestr('\\\\server2\\share\\abspath', 'badfile') + zf.writestr('../relpath', 'badfile') + zf.writestr(os.pardir + os.sep + 'relpath2', 'badfile') + zf.writestr('good/file', 'goodfile') + zf.writestr('good..file', 'goodfile') + + dstdir = os.path.join(self.mkdtemp(), 'dst') + unpack_archive(zipname, dstdir) + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'good', 'file'))) + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'good..file'))) + self.assertFalse(os.path.exists(abspath)) + self.assertFalse(os.path.exists(os.path.join(dstdir, 'abspath'))) + self.assertFalse(os.path.exists(os.path.join(dstdir, 'G_'))) + self.assertFalse(os.path.exists(os.path.join(dstdir, 'server'))) + if os.name != 'nt': + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'C:', 'abspath'))) + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'D:\\abspath'))) + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'E:abspath'))) + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'F:', 'G:', 'abspath'))) + self.assertTrue(os.path.isfile(os.path.join(dstdir, '\\\\server2\\share\\abspath'))) + if os.pardir == '..': + self.assertFalse(os.path.exists(os.path.join(dstdir, '..', 'relpath'))) + self.assertFalse(os.path.exists(os.path.join(dstdir, 'relpath'))) + else: + self.assertTrue(os.path.isfile(os.path.join(dstdir, '..', 'relpath'))) + self.assertFalse(os.path.exists(os.path.join(dstdir, os.pardir, 'relpath2'))) + self.assertFalse(os.path.exists(os.path.join(dstdir, 'relpath2'))) + + dstdir2 = os.path.join(self.mkdtemp(), 'dst') + os.mkdir(dstdir2) + with os_helper.change_cwd(dstdir2): + unpack_archive(zipname, '') + self.assertTrue(os.path.isfile(os.path.join('good', 'file'))) + self.assertTrue(os.path.isfile('good..file')) + self.assertFalse(os.path.exists(abspath)) + self.assertFalse(os.path.exists('abspath')) + self.assertFalse(os.path.exists('C_')) + self.assertFalse(os.path.exists('server')) + if os.name != 'nt': + self.assertTrue(os.path.isfile(os.path.join('C:', 'abspath'))) + self.assertTrue(os.path.isfile('D:\\abspath')) + self.assertTrue(os.path.isfile('E:abspath')) + self.assertTrue(os.path.isfile(os.path.join('F:', 'G:', 'abspath'))) + self.assertTrue(os.path.isfile('\\\\server2\\share\\abspath')) + if os.pardir == '..': + self.assertFalse(os.path.exists(os.path.join('..', 'relpath'))) + self.assertFalse(os.path.exists('relpath')) + else: + self.assertTrue(os.path.isfile(os.path.join('..', 'relpath'))) + self.assertFalse(os.path.exists(os.path.join(os.pardir, 'relpath2'))) + self.assertFalse(os.path.exists('relpath2')) + def test_unpack_registry(self): formats = get_unpack_formats() diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 0c567961184..1a3db527d3d 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -22,6 +22,7 @@ import sysconfig import select import shutil +import socket import threading import gc import textwrap @@ -1044,19 +1045,49 @@ def test_communicate_timeout_large_input(self): # On Windows, stdin writing must also honor the timeout rather than # blocking indefinitely when the pipe buffer fills. - # Input larger than typical pipe buffer (4-64KB on Windows) - input_data = b"x" * (128 * 1024) + input_data = b"x" * (128 * 1024) # > typical pipe buffer + + # Cross-platform wake mechanism: the slow reader connects to a + # loopback TCP socket and blocks in select() on it (capped at 9s + # as a safety net we don't expect to hit). After phase 1 raises + # TimeoutExpired, the parent sends a byte to release the child so + # it drains stdin. A socket (rather than a raw pipe) is required + # because Windows select() only supports sockets, not arbitrary + # file descriptors. + server = socket.create_server(('127.0.0.1', 0), backlog=1) + server.settimeout(10) # bound the accept() if the child fails to start + port = server.getsockname()[1] + # The child sends one byte (low byte of its PID) first so the parent + # can detect the rare case of an unrelated process on the same host + # connecting to our ephemeral port before our child does. A single + # byte gives 1/256 collision odds, which is plenty for flake-prevention. + slow_reader = ( + "import os, socket, sys, select; " + f"s = socket.create_connection(('127.0.0.1', {port}), timeout=9); " + "s.sendall(bytes([os.getpid() & 0xff])); " + "select.select([s], [], [], 9); " + "sys.stdout.buffer.write(sys.stdin.buffer.read())" + ) p = subprocess.Popen( - [sys.executable, "-c", - "import sys, time; " - "time.sleep(30); " # Don't read stdin for a long time - "sys.stdout.buffer.write(sys.stdin.buffer.read())"], + [sys.executable, "-c", slow_reader], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + conn = None try: + conn, _ = server.accept() + server.close() + server = None + + conn.settimeout(5) + peer_byte = conn.recv(1) + conn.settimeout(None) + self.assertEqual(peer_byte, bytes([p.pid & 0xff]), + f"loopback handshake byte {peer_byte!r} != " + f"low byte of child PID {p.pid} ({p.pid & 0xff:#x})") + timeout = 0.2 start = time.monotonic() try: @@ -1065,7 +1096,7 @@ def test_communicate_timeout_large_input(self): elapsed = time.monotonic() - start self.fail( f"TimeoutExpired not raised. communicate() completed in " - f"{elapsed:.2f}s, but subprocess sleeps for 30s. " + f"{elapsed:.2f}s, but slow reader stalls for up to 9s. " "Stdin writing blocked without enforcing timeout.") except subprocess.TimeoutExpired: elapsed = time.monotonic() - start @@ -1073,11 +1104,16 @@ def test_communicate_timeout_large_input(self): # Timeout should occur close to the specified timeout value, # not after waiting for the subprocess to finish sleeping. # Allow generous margin for slow CI, but must be well under - # the subprocess sleep time. + # the slow-reader's stall cap. self.assertLess(elapsed, 5.0, f"TimeoutExpired raised after {elapsed:.2f}s; expected ~{timeout}s. " "Stdin writing blocked without checking timeout.") + # Release the slow reader so it stops blocking and drains stdin. + conn.sendall(b'go') + conn.close() + conn = None + # After timeout, continue communication. The remaining input # should be sent and we should receive all data back. stdout, stderr = p.communicate() @@ -1086,6 +1122,43 @@ def test_communicate_timeout_large_input(self): self.assertEqual(len(stdout), len(input_data), f"Expected {len(input_data)} bytes output but got {len(stdout)}") self.assertEqual(stdout, input_data) + finally: + if conn is not None: + conn.close() + if server is not None: + server.close() + p.kill() + p.wait() + + def test_communicate_timeout_resume_partial_write(self): + """Resume writing input after a partial-write TimeoutExpired. + + Exercises the _input_offset bookkeeping across the + _communicate_io_posix factoring: a first communicate() must time out + mid-write, and a subsequent communicate() must finish delivering the + remaining bytes so the child receives the full input intact. + """ + # 1 MiB easily exceeds typical pipe buffers (~64 KiB) so writing + # blocks once the buffer fills before the child starts reading. + input_data = bytes(range(256)) * 4096 # 1 MiB, distinctive pattern + self.assertEqual(len(input_data), 1024 * 1024) + + p = subprocess.Popen( + [sys.executable, "-c", + "import sys, time; " + "time.sleep(0.5); " + "sys.stdout.buffer.write(sys.stdin.buffer.read())"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + try: + with self.assertRaises(subprocess.TimeoutExpired): + p.communicate(input_data, timeout=0.05) + + # Resume: no new input, generous timeout to avoid CI flakes. + stdout, stderr = p.communicate(timeout=support.LONG_TIMEOUT) + self.assertEqual(len(stdout), len(input_data)) + self.assertEqual(stdout, input_data) finally: p.kill() p.wait() diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index f2babaacc27..e270cbb22e2 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -10,6 +10,7 @@ import re import warnings import stat +import time import unittest import unittest.mock @@ -1828,6 +1829,19 @@ def test_source_directory_not_leaked(self): payload = pathlib.Path(tmpname).read_text(encoding='latin-1') assert os.path.dirname(tmpname) not in payload + def test_create_with_mtime(self): + tarfile.open(tmpname, self.mode, mtime=0).close() + with self.open(tmpname, 'r') as fobj: + fobj.read() + self.assertEqual(fobj.mtime, 0) + + def test_create_without_mtime(self): + before = int(time.time()) + tarfile.open(tmpname, self.mode).close() + after = int(time.time()) + with self.open(tmpname, 'r') as fobj: + fobj.read() + self.assertTrue(before <= fobj.mtime <= after) class Bz2StreamWriteTest(Bz2Test, StreamWriteTest): decompressor = bz2.BZ2Decompressor if bz2 else None @@ -2134,6 +2148,19 @@ def test_create_with_compresslevel(self): with tarfile.open(tmpname, 'r:gz', compresslevel=1) as tobj: pass + def test_create_with_mtime(self): + tarfile.open(tmpname, self.mode, mtime=0).close() + with self.open(tmpname, 'rb') as fobj: + fobj.read() + self.assertEqual(fobj.mtime, 0) + + def test_create_without_mtime(self): + before = int(time.time()) + tarfile.open(tmpname, self.mode).close() + after = int(time.time()) + with self.open(tmpname, 'r') as fobj: + fobj.read() + self.assertTrue(before <= fobj.mtime <= after) class Bz2CreateTest(Bz2Test, CreateTest): diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 0ca91ce0d78..3d01804513b 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -2368,6 +2368,231 @@ class BarrierTests(lock_tests.BarrierTests): barriertype = staticmethod(threading.Barrier) +## Test Synchronization tools for iterators ################ + +class ThreadingIteratorToolsTests(BaseTestCase): + def test_serialize_serializes_concurrent_iteration(self): + limit = 10_000 + workers_count = 10 + result = 0 + result_lock = threading.Lock() + start = threading.Event() + + def producer(limit): + for x in range(limit): + yield x + + def consumer(iterator): + nonlocal result + start.wait() + total = 0 + for x in iterator: + total += x + with result_lock: + result += total + + iterator = threading.serialize_iterator(producer(limit)) + workers = [ + threading.Thread(target=consumer, args=(iterator,)) + for _ in range(workers_count) + ] + with threading_helper.wait_threads_exit(): + for worker in workers: + worker.start() + for worker in workers: + # Wait for the worker thread to actually start. + while worker.ident is None: + time.sleep(0.1) + start.set() + for worker in workers: + worker.join() + + self.assertEqual(result, limit * (limit - 1) // 2) + + def test_serialize_generator_methods(self): + # A generator that yields and receives + def echo(): + try: + while True: + val = yield "ready" + yield f"received {val}" + except ValueError: + yield "caught" + + it = threading.serialize_iterator(echo()) + + # Test __next__ + self.assertEqual(next(it), "ready") + + # Test send() + self.assertEqual(it.send("hello"), "received hello") + self.assertEqual(next(it), "ready") + + # Test throw() + self.assertEqual(it.throw(ValueError), "caught") + + # Test close() + it.close() + with self.assertRaises(StopIteration): + next(it) + + def test_serialize_methods_attribute_error(self): + # A standard iterator that does not have send/throw/close + # should raise AttributeError when called. + standard_it = threading.serialize_iterator([1, 2, 3]) + + with self.assertRaises(AttributeError): + standard_it.send("foo") + + with self.assertRaises(AttributeError): + standard_it.throw(ValueError) + + with self.assertRaises(AttributeError): + standard_it.close() + + def test_serialize_generator_methods_locking(self): + # Verifies that generator methods also acquire the lock. + # We can test this by checking if the lock is held during the call. + + class LockCheckingGenerator: + def __init__(self, lock): + self.lock = lock + def __iter__(self): + return self + def send(self, value): + if not self.lock.locked(): + raise RuntimeError("Lock not held during send()") + return value + def throw(self, *args): + if not self.lock.locked(): + raise RuntimeError("Lock not held during throw()") + def close(self): + if not self.lock.locked(): + raise RuntimeError("Lock not held during close()") + + # Manually create the serialize object to inspect the lock + it = threading.serialize_iterator([]) + mock_gen = LockCheckingGenerator(it._lock) + it._iterator = mock_gen + + # These should not raise RuntimeError + it.send(1) + it.throw(ValueError) + it.close() + + def test_serialize_next_exception(self): + # Verify exception pass through for calls to next() + + def f(): + raise RuntimeError + yield None + + g = threading.serialize_iterator(f()) + with self.assertRaises(RuntimeError): + next(g) + + def test_synchronized_serializes_generator_instances(self): + unique = 10 + repetitions = 5 + limit = 100 + start = threading.Event() + + @threading.synchronized_iterator + def atomic_counter(): + # The sleep widens the race window that would exist without + # synchronization between yielding a value and advancing state. + i = 0 + while True: + yield i + time.sleep(0.0005) + i += 1 + + def consumer(counter): + start.wait() + for _ in range(limit): + next(counter) + + unique_counters = [atomic_counter() for _ in range(unique)] + counters = unique_counters * repetitions + workers = [ + threading.Thread(target=consumer, args=(counter,)) + for counter in counters + ] + with threading_helper.wait_threads_exit(): + for worker in workers: + worker.start() + start.set() + for worker in workers: + worker.join() + + self.assertEqual( + {next(counter) for counter in unique_counters}, + {limit * repetitions}, + ) + + def test_synchronized_preserves_wrapped_metadata(self): + def gen(): + yield 1 + + wrapped = threading.synchronized_iterator(gen) + + self.assertEqual(wrapped.__name__, gen.__name__) + self.assertIs(wrapped.__wrapped__, gen) + self.assertEqual(list(wrapped()), [1]) + + def test_concurrent_tee_supports_concurrent_consumers(self): + limit = 5_000 + num_threads = 25 + successes = 0 + failures = [] + result_lock = threading.Lock() + start = threading.Event() + expected = list(range(limit)) + + def producer(limit): + for x in range(limit): + yield x + + def consumer(iterator): + nonlocal successes + start.wait() + items = list(iterator) + with result_lock: + if items == expected: + successes += 1 + else: + failures.append(items[:20]) + + tees = threading.concurrent_tee(producer(limit), n=num_threads) + workers = [ + threading.Thread(target=consumer, args=(iterator,)) + for iterator in tees + ] + with threading_helper.wait_threads_exit(): + for worker in workers: + worker.start() + start.set() + for worker in workers: + worker.join() + + self.assertEqual(failures, []) + self.assertEqual(successes, len(tees)) + + # Verify that locks are shared + self.assertEqual(len({id(t_obj.lock) for t_obj in tees}), 1) + + def test_concurrent_tee_zero_iterators(self): + self.assertEqual(threading.concurrent_tee(range(10), n=0), ()) + + def test_concurrent_tee_negative_n(self): + with self.assertRaises(ValueError): + threading.concurrent_tee(range(10), n=-1) + + +################# + + + class MiscTestCase(unittest.TestCase): def test__all__(self): restore_default_excepthook(self) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index ca67e381958..ab53a20cff5 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -3326,6 +3326,7 @@ def test_newline_at_the_end_of_buffer(self): run_test_script(file_name) +@support.force_not_colorized_test_class class CommandLineTest(unittest.TestCase): def setUp(self): self.filename = tempfile.mktemp() diff --git a/Lib/test/test_tomllib/test_misc.py b/Lib/test/test_tomllib/test_misc.py index 118fde24d88..abd0842d10b 100644 --- a/Lib/test/test_tomllib/test_misc.py +++ b/Lib/test/test_tomllib/test_misc.py @@ -9,8 +9,10 @@ from pathlib import Path import sys import tempfile +import textwrap import unittest from test import support +from test.support.script_helper import assert_python_ok from . import tomllib @@ -124,3 +126,20 @@ def test_types_import(self): never imported by tests. """ importlib.import_module(f"{tomllib.__name__}._types") + + def test_lazy_import(self): + # Test the TOML file can be parsed without importing regular + # expressions (tomllib._re) + code = textwrap.dedent(""" + import sys, tomllib, textwrap + document = textwrap.dedent(''' + [metadata] + key = "text" + array = ["array", "of", "text"] + booleans = [true, false] + ''') + tomllib.loads(document) + print("lazy import?", 'tomllib._re' not in sys.modules) + """) + proc = assert_python_ok("-c", code) + self.assertIn(b"lazy import? True", proc.out) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 9c0172f6ba7..5d19e370680 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -14,7 +14,7 @@ import re import sys import warnings -from unittest import TestCase, main, skip +from unittest import TestCase, main from unittest.mock import patch from copy import copy, deepcopy @@ -29,7 +29,7 @@ from typing import assert_type, cast, runtime_checkable from typing import get_type_hints from typing import get_origin, get_args, get_protocol_members -from typing import override +from typing import override, disjoint_base from typing import is_typeddict, is_protocol from typing import reveal_type from typing import dataclass_transform @@ -780,7 +780,7 @@ def test_typevartuple_none(self): self.assertIs(U_None.__default__, None) self.assertIs(U_None.has_default(), True) - class X[**Ts]: ... + class X[*Ts]: ... Ts, = X.__type_params__ self.assertIs(Ts.__default__, NoDefault) self.assertIs(Ts.has_default(), False) @@ -1288,6 +1288,57 @@ def test_cannot_call_instance(self): with self.assertRaises(TypeError): Ts() + def test_default_variance(self): + Ts = TypeVarTuple('Ts') + self.assertIs(Ts.__covariant__, False) + self.assertIs(Ts.__contravariant__, False) + self.assertIs(Ts.__infer_variance__, False) + self.assertIsNone(Ts.__bound__) + + def test_covariant(self): + Ts_co = TypeVarTuple('Ts_co', covariant=True) + self.assertIs(Ts_co.__covariant__, True) + self.assertIs(Ts_co.__contravariant__, False) + self.assertIs(Ts_co.__infer_variance__, False) + + def test_contravariant(self): + Ts_contra = TypeVarTuple('Ts_contra', contravariant=True) + self.assertIs(Ts_contra.__covariant__, False) + self.assertIs(Ts_contra.__contravariant__, True) + self.assertIs(Ts_contra.__infer_variance__, False) + + def test_infer_variance(self): + Ts = TypeVarTuple('Ts', infer_variance=True) + self.assertIs(Ts.__covariant__, False) + self.assertIs(Ts.__contravariant__, False) + self.assertIs(Ts.__infer_variance__, True) + + def test_bound(self): + Ts_bound = TypeVarTuple('Ts_bound', bound=int) + self.assertIs(Ts_bound.__bound__, int) + Ts_no_bound = TypeVarTuple('Ts_no_bound') + self.assertIsNone(Ts_no_bound.__bound__) + + def test_no_bivariant(self): + with self.assertRaises(ValueError): + TypeVarTuple('Ts', covariant=True, contravariant=True) + + def test_cannot_combine_explicit_and_infer(self): + with self.assertRaises(ValueError): + TypeVarTuple('Ts', covariant=True, infer_variance=True) + with self.assertRaises(ValueError): + TypeVarTuple('Ts', contravariant=True, infer_variance=True) + + def test_repr_with_variance(self): + Ts = TypeVarTuple('Ts') + self.assertEqual(repr(Ts), '~Ts') + Ts_co = TypeVarTuple('Ts_co', covariant=True) + self.assertEqual(repr(Ts_co), '+Ts_co') + Ts_contra = TypeVarTuple('Ts_contra', contravariant=True) + self.assertEqual(repr(Ts_contra), '-Ts_contra') + Ts_infer = TypeVarTuple('Ts_infer', infer_variance=True) + self.assertEqual(repr(Ts_infer), 'Ts_infer') + def test_unpacked_typevartuple_is_equal_to_itself(self): Ts = TypeVarTuple('Ts') self.assertEqual((*Ts,)[0], (*Ts,)[0]) @@ -1427,16 +1478,16 @@ def test_repr_is_correct(self): class G1(Generic[*Ts]): pass class G2(Generic[Unpack[Ts]]): pass - self.assertEqual(repr(Ts), 'Ts') + self.assertEqual(repr(Ts), '~Ts') - self.assertEqual(repr((*Ts,)[0]), 'typing.Unpack[Ts]') - self.assertEqual(repr(Unpack[Ts]), 'typing.Unpack[Ts]') + self.assertEqual(repr((*Ts,)[0]), 'typing.Unpack[~Ts]') + self.assertEqual(repr(Unpack[Ts]), 'typing.Unpack[~Ts]') - self.assertEqual(repr(tuple[*Ts]), 'tuple[typing.Unpack[Ts]]') - self.assertEqual(repr(Tuple[Unpack[Ts]]), 'typing.Tuple[typing.Unpack[Ts]]') + self.assertEqual(repr(tuple[*Ts]), 'tuple[typing.Unpack[~Ts]]') + self.assertEqual(repr(Tuple[Unpack[Ts]]), 'typing.Tuple[typing.Unpack[~Ts]]') - self.assertEqual(repr(*tuple[*Ts]), '*tuple[typing.Unpack[Ts]]') - self.assertEqual(repr(Unpack[Tuple[Unpack[Ts]]]), 'typing.Unpack[typing.Tuple[typing.Unpack[Ts]]]') + self.assertEqual(repr(*tuple[*Ts]), '*tuple[typing.Unpack[~Ts]]') + self.assertEqual(repr(Unpack[Tuple[Unpack[Ts]]]), 'typing.Unpack[typing.Tuple[typing.Unpack[~Ts]]]') def test_variadic_class_repr_is_correct(self): Ts = TypeVarTuple('Ts') @@ -1475,61 +1526,61 @@ def test_variadic_class_alias_repr_is_correct(self): class A(Generic[Unpack[Ts]]): pass B = A[*Ts] - self.assertEndsWith(repr(B), 'A[typing.Unpack[Ts]]') + self.assertEndsWith(repr(B), 'A[typing.Unpack[~Ts]]') self.assertEndsWith(repr(B[()]), 'A[()]') self.assertEndsWith(repr(B[float]), 'A[float]') self.assertEndsWith(repr(B[float, str]), 'A[float, str]') C = A[Unpack[Ts]] - self.assertEndsWith(repr(C), 'A[typing.Unpack[Ts]]') + self.assertEndsWith(repr(C), 'A[typing.Unpack[~Ts]]') self.assertEndsWith(repr(C[()]), 'A[()]') self.assertEndsWith(repr(C[float]), 'A[float]') self.assertEndsWith(repr(C[float, str]), 'A[float, str]') D = A[*Ts, int] - self.assertEndsWith(repr(D), 'A[typing.Unpack[Ts], int]') + self.assertEndsWith(repr(D), 'A[typing.Unpack[~Ts], int]') self.assertEndsWith(repr(D[()]), 'A[int]') self.assertEndsWith(repr(D[float]), 'A[float, int]') self.assertEndsWith(repr(D[float, str]), 'A[float, str, int]') E = A[Unpack[Ts], int] - self.assertEndsWith(repr(E), 'A[typing.Unpack[Ts], int]') + self.assertEndsWith(repr(E), 'A[typing.Unpack[~Ts], int]') self.assertEndsWith(repr(E[()]), 'A[int]') self.assertEndsWith(repr(E[float]), 'A[float, int]') self.assertEndsWith(repr(E[float, str]), 'A[float, str, int]') F = A[int, *Ts] - self.assertEndsWith(repr(F), 'A[int, typing.Unpack[Ts]]') + self.assertEndsWith(repr(F), 'A[int, typing.Unpack[~Ts]]') self.assertEndsWith(repr(F[()]), 'A[int]') self.assertEndsWith(repr(F[float]), 'A[int, float]') self.assertEndsWith(repr(F[float, str]), 'A[int, float, str]') G = A[int, Unpack[Ts]] - self.assertEndsWith(repr(G), 'A[int, typing.Unpack[Ts]]') + self.assertEndsWith(repr(G), 'A[int, typing.Unpack[~Ts]]') self.assertEndsWith(repr(G[()]), 'A[int]') self.assertEndsWith(repr(G[float]), 'A[int, float]') self.assertEndsWith(repr(G[float, str]), 'A[int, float, str]') H = A[int, *Ts, str] - self.assertEndsWith(repr(H), 'A[int, typing.Unpack[Ts], str]') + self.assertEndsWith(repr(H), 'A[int, typing.Unpack[~Ts], str]') self.assertEndsWith(repr(H[()]), 'A[int, str]') self.assertEndsWith(repr(H[float]), 'A[int, float, str]') self.assertEndsWith(repr(H[float, str]), 'A[int, float, str, str]') I = A[int, Unpack[Ts], str] - self.assertEndsWith(repr(I), 'A[int, typing.Unpack[Ts], str]') + self.assertEndsWith(repr(I), 'A[int, typing.Unpack[~Ts], str]') self.assertEndsWith(repr(I[()]), 'A[int, str]') self.assertEndsWith(repr(I[float]), 'A[int, float, str]') self.assertEndsWith(repr(I[float, str]), 'A[int, float, str, str]') J = A[*Ts, *tuple[str, ...]] - self.assertEndsWith(repr(J), 'A[typing.Unpack[Ts], *tuple[str, ...]]') + self.assertEndsWith(repr(J), 'A[typing.Unpack[~Ts], *tuple[str, ...]]') self.assertEndsWith(repr(J[()]), 'A[*tuple[str, ...]]') self.assertEndsWith(repr(J[float]), 'A[float, *tuple[str, ...]]') self.assertEndsWith(repr(J[float, str]), 'A[float, str, *tuple[str, ...]]') K = A[Unpack[Ts], Unpack[Tuple[str, ...]]] - self.assertEndsWith(repr(K), 'A[typing.Unpack[Ts], typing.Unpack[typing.Tuple[str, ...]]]') + self.assertEndsWith(repr(K), 'A[typing.Unpack[~Ts], typing.Unpack[typing.Tuple[str, ...]]]') self.assertEndsWith(repr(K[()]), 'A[typing.Unpack[typing.Tuple[str, ...]]]') self.assertEndsWith(repr(K[float]), 'A[float, typing.Unpack[typing.Tuple[str, ...]]]') self.assertEndsWith(repr(K[float, str]), 'A[float, str, typing.Unpack[typing.Tuple[str, ...]]]') @@ -1550,9 +1601,9 @@ class G(type(Unpack[Ts])): pass with self.assertRaisesRegex(TypeError, r'Cannot subclass typing\.Unpack'): class H(Unpack): pass - with self.assertRaisesRegex(TypeError, r'Cannot subclass typing.Unpack\[Ts\]'): + with self.assertRaisesRegex(TypeError, r'Cannot subclass typing.Unpack\[~Ts\]'): class I(*Ts): pass - with self.assertRaisesRegex(TypeError, r'Cannot subclass typing.Unpack\[Ts\]'): + with self.assertRaisesRegex(TypeError, r'Cannot subclass typing.Unpack\[~Ts\]'): class J(Unpack[Ts]): pass def test_variadic_class_args_are_correct(self): @@ -5596,13 +5647,13 @@ class TsP(Generic[*Ts, P]): MyCallable[[int], bool]: "MyCallable[[int], bool]", MyCallable[[int, str], bool]: "MyCallable[[int, str], bool]", MyCallable[[int, list[int]], bool]: "MyCallable[[int, list[int]], bool]", - MyCallable[Concatenate[*Ts, P], T]: "MyCallable[typing.Concatenate[typing.Unpack[Ts], ~P], ~T]", + MyCallable[Concatenate[*Ts, P], T]: "MyCallable[typing.Concatenate[typing.Unpack[~Ts], ~P], ~T]", DoubleSpec[P2, P, T]: "DoubleSpec[~P2, ~P, ~T]", DoubleSpec[[int], [str], bool]: "DoubleSpec[[int], [str], bool]", DoubleSpec[[int, int], [str, str], bool]: "DoubleSpec[[int, int], [str, str], bool]", - TsP[*Ts, P]: "TsP[typing.Unpack[Ts], ~P]", + TsP[*Ts, P]: "TsP[typing.Unpack[~Ts], ~P]", TsP[int, str, list[int], []]: "TsP[int, str, list[int], []]", TsP[int, [str, list[int]]]: "TsP[int, [str, list[int]]]", @@ -6745,11 +6796,7 @@ def test_get_type_hints_modules(self): self.assertEqual(gth(ann_module2), {}) self.assertEqual(gth(ann_module3), {}) - @skip("known bug") def test_get_type_hints_modules_forwardref(self): - # FIXME: This currently exposes a bug in typing. Cached forward references - # don't account for the case where there are multiple types of the same - # name coming from different modules in the same program. mgc_hints = {'default_a': Optional[mod_generics_cache.A], 'default_b': Optional[mod_generics_cache.B]} self.assertEqual(gth(mod_generics_cache), mgc_hints) @@ -6837,6 +6884,24 @@ def test_get_type_hints_wrapped_decoratored_func(self): self.assertEqual(gth(ForRefExample.func), expects) self.assertEqual(gth(ForRefExample.nested), expects) + def test_get_type_hints_wrapped_cycle_self(self): + # gh-146553: __wrapped__ self-reference must raise ValueError, + # not loop forever. + def f(x: int) -> str: ... + f.__wrapped__ = f + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + get_type_hints(f) + + def test_get_type_hints_wrapped_cycle_mutual(self): + # gh-146553: mutual __wrapped__ cycle (a -> b -> a) must raise + # ValueError, not loop forever. + def a(): ... + def b(): ... + a.__wrapped__ = b + b.__wrapped__ = a + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + get_type_hints(a) + def test_get_type_hints_annotated(self): def foobar(x: List['X']): ... X = Annotated[int, (1, 10)] @@ -10920,6 +10985,18 @@ def bar(self): self.assertNotIn('__magic__', dir_items) +class DisjointBaseTests(BaseTestCase): + def test_disjoint_base_unmodified(self): + class C: ... + self.assertIs(C, disjoint_base(C)) + + def test_dunder_disjoint_base(self): + @disjoint_base + class C: ... + + self.assertIs(C.__disjoint_base__, True) + + class RevealTypeTests(BaseTestCase): def test_reveal_type(self): obj = object() diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 5f9ab048cde..055be2994bf 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -1182,6 +1182,47 @@ def test_cli_name_required_for_uuid3(self, mock_err): self.assertEqual(cm.exception.code, 2) self.assertIn("error: Incorrect number of arguments", mock_err.getvalue()) + @mock.patch.object(sys, "argv", + ["", "-u", "uuid3", "-n", "@dns", "-N", "python.org"]) + def test_cli_uuid3_outputted_with_valid_namespace_and_name(self): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self.uuid.main() + + output = stdout.getvalue().strip() + uuid_output = self.uuid.UUID(output) + + # Output should be in the form of uuid3 + self.assertEqual(output, str(uuid_output)) + self.assertEqual(uuid_output.version, 3) + + @mock.patch.object(sys, "argv", + ["", "-u", "uuid3", "-n", + "0d6a16cc-34a7-47d8-b660-214d0ae184d2", + "-N", "some.user"]) + def test_cli_uuid3_outputted_with_custom_namespace_and_name(self): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self.uuid.main() + + output = stdout.getvalue().strip() + uuid_output = self.uuid.UUID(output) + + # Output should be in the form of uuid3 + self.assertEqual(output, str(uuid_output)) + self.assertEqual(uuid_output.version, 3) + + @mock.patch.object(sys, "argv", + ["", "-u", "uuid3", "-n", "any UUID", "-N", "python.org"]) + @mock.patch('sys.stderr', new_callable=io.StringIO) + def test_cli_uuid3_with_invalid_namespace(self, mock_err): + with self.assertRaises(SystemExit) as cm: + self.uuid.main() + # Check that exception code is the same as argparse.ArgumentParser.error + self.assertEqual(cm.exception.code, 2) + self.assertIn("error: badly formed hexadecimal UUID string", + mock_err.getvalue()) + @mock.patch.object(sys, "argv", [""]) def test_cli_uuid4_outputted_with_no_args(self): stdout = io.StringIO() @@ -1209,23 +1250,9 @@ def test_cli_uuid4_outputted_with_count(self): uuid_output = self.uuid.UUID(o) self.assertEqual(uuid_output.version, 4) - @mock.patch.object(sys, "argv", - ["", "-u", "uuid3", "-n", "@dns", "-N", "python.org"]) - def test_cli_uuid3_ouputted_with_valid_namespace_and_name(self): - stdout = io.StringIO() - with contextlib.redirect_stdout(stdout): - self.uuid.main() - - output = stdout.getvalue().strip() - uuid_output = self.uuid.UUID(output) - - # Output should be in the form of uuid5 - self.assertEqual(output, str(uuid_output)) - self.assertEqual(uuid_output.version, 3) - @mock.patch.object(sys, "argv", ["", "-u", "uuid5", "-n", "@dns", "-N", "python.org"]) - def test_cli_uuid5_ouputted_with_valid_namespace_and_name(self): + def test_cli_uuid5_outputted_with_valid_namespace_and_name(self): stdout = io.StringIO() with contextlib.redirect_stdout(stdout): self.uuid.main() @@ -1237,6 +1264,33 @@ def test_cli_uuid5_ouputted_with_valid_namespace_and_name(self): self.assertEqual(output, str(uuid_output)) self.assertEqual(uuid_output.version, 5) + @mock.patch.object(sys, "argv", + ["", "-u", "uuid5", "-n", + "0d6a16cc-34a7-47d8-b660-214d0ae184d2", + "-N", "some.user"]) + def test_cli_uuid5_ouputted_with_custom_namespace_and_name(self): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self.uuid.main() + + output = stdout.getvalue().strip() + uuid_output = self.uuid.UUID(output) + + # Output should be in the form of uuid5 + self.assertEqual(output, str(uuid_output)) + self.assertEqual(uuid_output.version, 5) + + @mock.patch.object(sys, "argv", + ["", "-u", "uuid5", "-n", "any UUID", "-N", "python.org"]) + @mock.patch('sys.stderr', new_callable=io.StringIO) + def test_cli_uuid5_with_invalid_namespace(self, mock_err): + with self.assertRaises(SystemExit) as cm: + self.uuid.main() + # Check that exception code is the same as argparse.ArgumentParser.error + self.assertEqual(cm.exception.code, 2) + self.assertIn("error: badly formed hexadecimal UUID string", + mock_err.getvalue()) + @mock.patch.object(sys, "argv", ["", "-u", "uuid6"]) def test_cli_uuid6(self): self.do_test_standalone_uuid(6) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index b380d0276b0..43864820688 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -3190,6 +3190,19 @@ def __deepcopy__(self, memo): self.assertEqual([c.tag for c in children[3:]], [a.tag, b.tag, a.tag, b.tag]) + @support.skip_if_unlimited_stack_size + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() + def test_deeply_nested_deepcopy(self): + # This should raise a RecursionError and not crash. + # See https://github.com/python/cpython/issues/148801. + root = cur = ET.Element('s') + for _ in range(150_000): + cur = ET.SubElement(cur, 'u') + with support.infinite_recursion(): + with self.assertRaises(RecursionError): + copy.deepcopy(root) + class MutationDeleteElementPath(str): def __new__(cls, elem, *args): @@ -3258,6 +3271,16 @@ def test_findtext_with_mutating(self): e.extend([ET.Element('bar')]) e.findtext(cls(e, 'x')) + def test_findtext_with_mutating_non_none_text(self): + for cls in [MutationDeleteElementPath, MutationClearElementPath]: + with self.subTest(cls): + e = ET.Element('foo') + child = ET.Element('bar') + child.text = str(object()) + e.append(child) + del child + repr(e.findtext(cls(e, 'x'))) + def test_findtext_with_error(self): e = ET.Element('foo') e.extend([ET.Element('bar')]) diff --git a/Lib/threading.py b/Lib/threading.py index 4ebceae7029..abac31e2588 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -29,6 +29,7 @@ 'Barrier', 'BrokenBarrierError', 'Timer', 'ThreadError', 'setprofile', 'settrace', 'local', 'stack_size', 'excepthook', 'ExceptHookArgs', 'gettrace', 'getprofile', + 'serialize_iterator', 'synchronized_iterator', 'concurrent_tee', 'setprofile_all_threads','settrace_all_threads'] # Rename some stuff so "from threading import *" is safe @@ -842,6 +843,148 @@ class BrokenBarrierError(RuntimeError): pass +## Synchronization tools for iterators ##################### + +class serialize_iterator: + """Wrap a non-concurrent iterator with a lock to enforce sequential access. + + Applies a non-reentrant lock around calls to __next__. If the + wrapped iterator also defines send(), throw(), or close(), those + calls are serialized as well. + + Allows iterator and generator instances to be shared by multiple consumer + threads. + + For example, itertools.count does not make thread-safe instances, + but that is easily fixed with: + + atomic_counter = serialize_iterator(itertools.count()) + + """ + + __slots__ = ('_iterator', '_lock') + + def __init__(self, iterable): + self._iterator = iter(iterable) + self._lock = Lock() + + def __iter__(self): + return self + + def __next__(self): + with self._lock: + return next(self._iterator) + + def send(self, value, /): + """Send a value to a generator. + + Raises AttributeError if not a generator. + """ + with self._lock: + return self._iterator.send(value) + + def throw(self, *args): + """Call throw() on a generator. + + Raises AttributeError if not a generator. + """ + with self._lock: + return self._iterator.throw(*args) + + def close(self): + """Call close() on a generator. + + Raises AttributeError if not a generator. + """ + with self._lock: + return self._iterator.close() + + +def synchronized_iterator(func): + """Wrap an iterator-returning callable to make its iterators thread-safe. + + Existing itertools and more-itertools can be wrapped so that their + iterator instances are serialized. + + For example, itertools.count does not make thread-safe instances, + but that is easily fixed with: + + atomic_counter = synchronized_iterator(itertools.count) + + Can also be used as a decorator for generator function definitions + so that the generator instances are serialized:: + + import time + + @synchronized_iterator + def enumerate_and_timestamp(iterable): + for count, value in enumerate(iterable): + yield count, time.time_ns(), value + + """ + + from functools import wraps + + @wraps(func) + def inner(*args, **kwargs): + iterator = func(*args, **kwargs) + return serialize_iterator(iterator) + + return inner + + +def concurrent_tee(iterable, n=2): + """Variant of itertools.tee() but with guaranteed threading semantics. + + Takes a non-threadsafe iterator as an input and creates concurrent + tee objects for other threads to have reliable independent copies of + the data stream. + + The new iterators are only thread-safe if consumed within a single thread. + To share just one of the new iterators across multiple threads, wrap it + with threading.serialize_iterator(). + """ + + if n < 0: + raise ValueError("n must be a non-negative integer") + if n == 0: + return () + iterator = _concurrent_tee(iterable) + result = [iterator] + for _ in range(n - 1): + result.append(_concurrent_tee(iterator)) + return tuple(result) + + +class _concurrent_tee: + __slots__ = ('iterator', 'link', 'lock') + + def __init__(self, iterable): + if isinstance(iterable, _concurrent_tee): + self.iterator = iterable.iterator + self.link = iterable.link + self.lock = iterable.lock + else: + self.iterator = iter(iterable) + self.link = [None, None] + self.lock = Lock() + + def __iter__(self): + return self + + def __next__(self): + link = self.link + if link[1] is None: + with self.lock: + if link[1] is None: + link[0] = next(self.iterator) + link[1] = [None, None] + value, self.link = link + return value + +############################################################ + + # Helper to generate new thread names _counter = _count(1).__next__ def _newname(name_template): diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 11c134482db..52cf3f0b7cc 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -35,6 +35,7 @@ from token import * from token import EXACT_TOKEN_TYPES import _tokenize +lazy import _colorize cookie_re = re.compile(br'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) @@ -505,6 +506,56 @@ def generate_tokens(readline): """ return _generate_tokens_from_c_tokenizer(readline, extra_tokens=True) + +def _get_token_colors(syntax, tokenize): + """Map token type numbers to theme colors.""" + return frozendict({ + COMMENT: syntax.comment, + DEDENT: tokenize.whitespace, + ENCODING: tokenize.whitespace, + ENDMARKER: tokenize.whitespace, + ERRORTOKEN: tokenize.error, + FSTRING_START: syntax.string, + FSTRING_MIDDLE: syntax.string, + FSTRING_END: syntax.string, + INDENT: tokenize.whitespace, + NAME: syntax.reset, + NEWLINE: tokenize.whitespace, + NL: tokenize.whitespace, + NUMBER: syntax.number, + OP: syntax.op, + SOFT_KEYWORD: syntax.soft_keyword, + STRING: syntax.string, + TSTRING_START: syntax.string, + TSTRING_MIDDLE: syntax.string, + TSTRING_END: syntax.string, + }) + + +def _format_tokens(tokens, *, color=False, exact=False): + theme = _colorize.get_theme(force_no_color=not color) + s = theme.syntax + t = theme.tokenize + token_colors = _get_token_colors(s, t) + for token in tokens: + token_range = ( + f"{t.position}{token.start[0]}" + f"{t.delimiter},{t.position}{token.start[1]}" + f"{t.delimiter}-" + f"{t.position}{token.end[0]}" + f"{t.delimiter},{t.position}{token.end[1]}" + f"{t.delimiter}:" + ) + token_color = token_colors.get(token.type, s.reset) + token_name = tok_name[token.exact_type if exact else token.type] + visible_range = f"{token.start[0]},{token.start[1]}-{token.end[0]},{token.end[1]}:" + yield ( + f"{token_range}{' ' * (20 - len(visible_range))}" + f"{token_color}{token_name:<15}" + f"{s.reset}{token.string!r:<15}" + ) + + def _main(args=None): import argparse @@ -524,7 +575,7 @@ def error(message, filename=None, location=None): sys.exit(1) # Parse the arguments and options - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument(dest='filename', nargs='?', metavar='filename.py', help='the file to tokenize; defaults to stdin') @@ -545,13 +596,8 @@ def error(message, filename=None, location=None): # Output the tokenization - for token in tokens: - token_type = token.type - if args.exact: - token_type = token.exact_type - token_range = "%d,%d-%d,%d:" % (token.start + token.end) - print("%-20s%-15s%-15r" % - (token_range, tok_name[token_type], token.string)) + for line in _format_tokens(tokens, color=True, exact=args.exact): + print(line) except IndentationError as err: line, column = err.args[1][1:3] error(err.args[0], filename, (line, column)) diff --git a/Lib/tomllib/_parser.py b/Lib/tomllib/_parser.py index b59d0f7d54b..8aa01301dce 100644 --- a/Lib/tomllib/_parser.py +++ b/Lib/tomllib/_parser.py @@ -4,7 +4,11 @@ from __future__ import annotations -from types import MappingProxyType +# Defer loading regular expressions until we actually need them in +# parse_value(). +__lazy_modules__ = ["tomllib._re"] + +import sys from ._re import ( RE_DATETIME, @@ -15,6 +19,9 @@ match_to_number, ) +if sys.version_info < (3, 15): + from types import MappingProxyType as frozendict + TYPE_CHECKING = False if TYPE_CHECKING: from collections.abc import Iterable @@ -42,7 +49,7 @@ KEY_INITIAL_CHARS: Final = BARE_KEY_CHARS | frozenset("\"'") HEXDIGIT_CHARS: Final = frozenset("abcdef" "ABCDEF" "0123456789") -BASIC_STR_ESCAPE_REPLACEMENTS: Final = MappingProxyType( +BASIC_STR_ESCAPE_REPLACEMENTS: Final = frozendict( { "\\b": "\u0008", # backspace "\\t": "\u0009", # tab diff --git a/Lib/typing.py b/Lib/typing.py index e78fb8b71a9..e7563a53878 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -126,6 +126,7 @@ 'cast', 'clear_overloads', 'dataclass_transform', + 'disjoint_base', 'evaluate_forward_ref', 'final', 'get_args', @@ -2485,8 +2486,12 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, else: nsobj = obj # Find globalns for the unwrapped object. + seen = {id(nsobj)} while hasattr(nsobj, '__wrapped__'): nsobj = nsobj.__wrapped__ + if id(nsobj) in seen: + raise ValueError(f'wrapper loop when unwrapping {obj!r}') + seen.add(id(nsobj)) globalns = getattr(nsobj, '__globals__', {}) if localns is None: localns = globalns @@ -2794,6 +2799,29 @@ class Other(Leaf): # Error reported by type checker return f +def disjoint_base(cls): + """This decorator marks a class as a disjoint base. + + Child classes of a disjoint base cannot inherit from other disjoint bases that are + not parent or child classes of the disjoint base. + + For example: + + @disjoint_base + class Disjoint1: pass + + @disjoint_base + class Disjoint2: pass + + class Disjoint3(Disjoint1, Disjoint2): pass # Type checker error + + Type checkers can use knowledge of disjoint bases to detect unreachable code + and determine when two types can overlap. + """ + cls.__disjoint_base__ = True + return cls + + # Some unconstrained type variables. These were initially used by the container types. # They were never meant for export and are now unused, but we keep them around to # avoid breaking compatibility with users who import them. @@ -3122,31 +3150,7 @@ def _namedtuple_mro_entries(bases): NamedTuple.__mro_entries__ = _namedtuple_mro_entries -class _SingletonMeta(type): - def __setattr__(cls, attr, value): - # TypeError is consistent with the behavior of NoneType - raise TypeError( - f"cannot set {attr!r} attribute of immutable type {cls.__name__!r}" - ) - - -class _NoExtraItemsType(metaclass=_SingletonMeta): - """The type of the NoExtraItems singleton.""" - - __slots__ = () - - def __new__(cls): - return globals().get("NoExtraItems") or object.__new__(cls) - - def __repr__(self): - return 'typing.NoExtraItems' - - def __reduce__(self): - return 'NoExtraItems' - -NoExtraItems = _NoExtraItemsType() -del _NoExtraItemsType -del _SingletonMeta +NoExtraItems = sentinel("NoExtraItems") def _get_typeddict_qualifiers(annotation_type): @@ -3588,7 +3592,7 @@ def isatty(self) -> bool: pass @abstractmethod - def read(self, n: int = -1) -> AnyStr: + def read(self, n: int = -1, /) -> AnyStr: pass @abstractmethod @@ -3596,15 +3600,15 @@ def readable(self) -> bool: pass @abstractmethod - def readline(self, limit: int = -1) -> AnyStr: + def readline(self, limit: int = -1, /) -> AnyStr: pass @abstractmethod - def readlines(self, hint: int = -1) -> list[AnyStr]: + def readlines(self, hint: int = -1, /) -> list[AnyStr]: pass @abstractmethod - def seek(self, offset: int, whence: int = 0) -> int: + def seek(self, offset: int, whence: int = 0, /) -> int: pass @abstractmethod @@ -3616,7 +3620,7 @@ def tell(self) -> int: pass @abstractmethod - def truncate(self, size: int | None = None) -> int: + def truncate(self, size: int | None = None, /) -> int: pass @abstractmethod @@ -3624,11 +3628,11 @@ def writable(self) -> bool: pass @abstractmethod - def write(self, s: AnyStr) -> int: + def write(self, s: AnyStr, /) -> int: pass @abstractmethod - def writelines(self, lines: list[AnyStr]) -> None: + def writelines(self, lines: list[AnyStr], /) -> None: pass @abstractmethod @@ -3636,7 +3640,7 @@ def __enter__(self) -> IO[AnyStr]: pass @abstractmethod - def __exit__(self, type, value, traceback) -> None: + def __exit__(self, type, value, traceback, /) -> None: pass @@ -3646,7 +3650,7 @@ class BinaryIO(IO[bytes]): __slots__ = () @abstractmethod - def write(self, s: bytes | bytearray) -> int: + def write(self, s: bytes | bytearray, /) -> int: pass @abstractmethod diff --git a/Lib/uuid.py b/Lib/uuid.py index c0150a59d7c..8c59581464b 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -962,7 +962,7 @@ def main(): default="uuid4", help="function to generate the UUID") parser.add_argument("-n", "--namespace", - choices=["any UUID", *namespaces.keys()], + metavar=f"{{any UUID,{','.join(namespaces)}}}", help="uuid3/uuid5 only: " "a UUID, or a well-known predefined UUID addressed " "by namespace name") @@ -984,7 +984,13 @@ def main(): f"{args.uuid} requires a namespace and a name. " "Run 'python -m uuid -h' for more information." ) - namespace = namespaces[namespace] if namespace in namespaces else UUID(namespace) + if namespace in namespaces: + namespace = namespaces[namespace] + else: + try: + namespace = UUID(namespace) + except ValueError as exc: + parser.error(f"{exc}: {args.namespace!r}") for _ in range(args.count): print(uuid_func(namespace, name)) else: diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 51e0ce9fa36..1e0cc5f6234 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -1410,6 +1410,7 @@ class ZipFile: fp = None # Set here since __del__ checks it _windows_illegal_name_trans_table = None + _ignore_invalid_names = False def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, compresslevel=None, *, strict_timestamps=True, metadata_encoding=None): @@ -1890,21 +1891,31 @@ def _extract_member(self, member, targetpath, pwd): # build the destination pathname, replacing # forward slashes to platform specific separators. - arcname = member.filename.replace('/', os.path.sep) - - if os.path.altsep: + arcname = member.filename + if os.path.sep != '/': + arcname = arcname.replace('/', os.path.sep) + if os.path.altsep and os.path.altsep != '/': arcname = arcname.replace(os.path.altsep, os.path.sep) # interpret absolute pathname as relative, remove drive letter or # UNC path, redundant separators, "." and ".." components. - arcname = os.path.splitdrive(arcname)[1] + drive, root, arcname = os.path.splitroot(arcname) + if self._ignore_invalid_names and (drive or root): + return None + if self._ignore_invalid_names and os.path.pardir in arcname.split(os.path.sep): + return None invalid_path_parts = ('', os.path.curdir, os.path.pardir) arcname = os.path.sep.join(x for x in arcname.split(os.path.sep) if x not in invalid_path_parts) if os.path.sep == '\\': # filter illegal characters on Windows - arcname = self._sanitize_windows_name(arcname, os.path.sep) + arcname2 = self._sanitize_windows_name(arcname, os.path.sep) + if self._ignore_invalid_names and arcname2 != arcname: + return None + arcname = arcname2 if not arcname and not member.is_dir(): + if self._ignore_invalid_names: + return None raise ValueError("Empty filename.") targetpath = os.path.join(targetpath, arcname) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index cd5f4c71b00..c5f92a99a1e 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 3.5.5", - url="https://github.com/openssl/openssl/releases/download/openssl-3.5.5/openssl-3.5.5.tar.gz", - checksum="b28c91532a8b65a1f983b4c28b7488174e4a01008e29ce8e69bd789f28bc2a89", + name="OpenSSL 3.5.6", + url="https://github.com/openssl/openssl/releases/download/openssl-3.5.6/openssl-3.5.6.tar.gz", + checksum="deae7c80cba99c4b4f940ecadb3c3338b13cb77418409238e57d7f31f2a3b736", buildrecipe=build_universal_openssl, configure=None, install=None, diff --git a/Makefile.pre.in b/Makefile.pre.in index f869c1f7c93..fc44399434f 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -290,6 +290,7 @@ LDLIBRARYDIR= @LDLIBRARYDIR@ INSTSONAME= @INSTSONAME@ LIBRARY_DEPS= @LIBRARY_DEPS@ LINK_PYTHON_DEPS=@LINK_PYTHON_DEPS@ +JIT_OBJS= @JIT_SHIM_O@ PY_ENABLE_SHARED= @PY_ENABLE_SHARED@ STATIC_LIBPYTHON= @STATIC_LIBPYTHON@ @@ -469,6 +470,7 @@ PYTHON_OBJS= \ Python/instruction_sequence.o \ Python/intrinsics.o \ Python/jit.o \ + $(JIT_OBJS) \ Python/legacy_tracing.o \ Python/lock.o \ Python/marshal.o \ @@ -512,6 +514,7 @@ PYTHON_OBJS= \ Python/suggestions.o \ Python/perf_trampoline.o \ Python/perf_jit_trampoline.o \ + Python/jit_unwind.o \ Python/remote_debugging.o \ Python/$(DYNLOADFILE) \ $(LIBOBJS) \ @@ -558,6 +561,7 @@ OBJECT_OBJS= \ Objects/obmalloc.o \ Objects/picklebufobject.o \ Objects/rangeobject.o \ + Objects/sentinelobject.o \ Objects/setobject.o \ Objects/sliceobject.o \ Objects/structseq.o \ @@ -1214,6 +1218,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/osdefs.h \ $(srcdir)/Include/osmodule.h \ $(srcdir)/Include/patchlevel.h \ + $(srcdir)/Include/pyabi.h \ $(srcdir)/Include/pyatomic.h \ $(srcdir)/Include/pybuffer.h \ $(srcdir)/Include/pycapsule.h \ @@ -1304,6 +1309,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/pystats.h \ $(srcdir)/Include/cpython/pythonrun.h \ $(srcdir)/Include/cpython/pythread.h \ + $(srcdir)/Include/cpython/sentinelobject.h \ $(srcdir)/Include/cpython/setobject.h \ $(srcdir)/Include/cpython/sliceobject.h \ $(srcdir)/Include/cpython/structseq.h \ @@ -3203,21 +3209,41 @@ Python/emscripten_trampoline_inner.wasm: $(srcdir)/Python/emscripten_trampoline_ Python/emscripten_trampoline_wasm.c: Python/emscripten_trampoline_inner.wasm $(PYTHON_FOR_REGEN) $(srcdir)/Platforms/emscripten/prepare_external_wasm.py $< $@ getWasmTrampolineModule +JIT_SHIM_BUILD_OBJS= @JIT_SHIM_BUILD_O@ +JIT_UNWIND_INFO_H= $(if $(JIT_OBJS),jit_unwind_info.h $(patsubst jit_stencils-%.h,jit_unwind_info-%.h,@JIT_STENCILS_H@)) +JIT_BUILD_TARGETS= jit_stencils.h @JIT_STENCILS_H@ $(JIT_UNWIND_INFO_H) $(JIT_SHIM_BUILD_OBJS) +JIT_TARGETS= $(JIT_BUILD_TARGETS) $(filter-out $(JIT_SHIM_BUILD_OBJS),$(JIT_OBJS)) +JIT_GENERATED_STAMP= .jit-stamp + JIT_DEPS = \ $(srcdir)/Tools/jit/*.c \ + $(srcdir)/Tools/jit/*.h \ $(srcdir)/Tools/jit/*.py \ $(srcdir)/Python/executor_cases.c.h \ pyconfig.h -jit_stencils.h @JIT_STENCILS_H@: $(JIT_DEPS) +$(JIT_GENERATED_STAMP): $(JIT_DEPS) @REGEN_JIT_COMMAND@ + @touch $@ + +$(JIT_BUILD_TARGETS): $(JIT_GENERATED_STAMP) + @if test ! -f "$@"; then \ + rm -f $(JIT_GENERATED_STAMP); \ + $(MAKE) $(JIT_GENERATED_STAMP); \ + test -f "$@"; \ + fi + +jit_shim-universal2-apple-darwin.o: jit_shim-aarch64-apple-darwin.o jit_shim-x86_64-apple-darwin.o + lipo -create -output $@ jit_shim-aarch64-apple-darwin.o jit_shim-x86_64-apple-darwin.o Python/jit.o: $(srcdir)/Python/jit.c @JIT_STENCILS_H@ $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< +Python/jit_unwind.o: $(srcdir)/Python/jit_unwind.c $(JIT_UNWIND_INFO_H) + $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< + .PHONY: regen-jit -regen-jit: - @REGEN_JIT_COMMAND@ +regen-jit: $(JIT_TARGETS) # Some make's put the object file in the current directory .c.o: @@ -3341,7 +3367,7 @@ clean-profile: clean-retain-profile clean-bolt # gh-141808: The JIT stencils are deliberately kept in clean-profile .PHONY: clean-jit-stencils clean-jit-stencils: - -rm -f jit_stencils*.h + -rm -f $(JIT_TARGETS) $(JIT_GENERATED_STAMP) jit_stencils*.h jit_unwind_info*.h jit_shim*.o .PHONY: clean clean: clean-profile clean-jit-stencils diff --git a/Misc/NEWS.d/3.15.0a8.rst b/Misc/NEWS.d/3.15.0a8.rst index ed37988f6ab..ff7930aeb29 100644 --- a/Misc/NEWS.d/3.15.0a8.rst +++ b/Misc/NEWS.d/3.15.0a8.rst @@ -185,8 +185,8 @@ dealing with contradictions in ``make_bottom``. .. nonce: 6wDI6S .. section: Core and Builtins -Ensure ``-X lazy_imports=none``` and ``PYTHON_LAZY_IMPORTS=none``` override -``__lazy_modules__``. Patch by Hugo van Kemenade. +Ensure ``-X lazy_imports=none`` and ``PYTHON_LAZY_IMPORTS=none`` override +:attr:`~module.__lazy_modules__`. Patch by Hugo van Kemenade. .. diff --git a/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst b/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst new file mode 100644 index 00000000000..d83aee08025 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst @@ -0,0 +1 @@ +Allow for custom LLVM path using ``LLVM_TOOLS_INSTALL_DIR`` during JIT build. diff --git a/Misc/NEWS.d/next/Build/2026-04-17-21-45-32.gh-issue-148644.vwkknh.rst b/Misc/NEWS.d/next/Build/2026-04-17-21-45-32.gh-issue-148644.vwkknh.rst new file mode 100644 index 00000000000..a0cc9c9358c --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-04-17-21-45-32.gh-issue-148644.vwkknh.rst @@ -0,0 +1 @@ +Errors during the PGO training job on Windows are no longer ignored, and a non-zero return code will cause the build to fail. diff --git a/Misc/NEWS.d/next/Build/2026-04-30-08-43-47.gh-issue-146475.1cL4hX.rst b/Misc/NEWS.d/next/Build/2026-04-30-08-43-47.gh-issue-146475.1cL4hX.rst new file mode 100644 index 00000000000..225c659393f --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-04-30-08-43-47.gh-issue-146475.1cL4hX.rst @@ -0,0 +1,2 @@ +Block Apple Clang from being used to build the JIT as it ships without +required LLVM tools. diff --git a/Misc/NEWS.d/next/Build/2026-05-01-20-01-32.gh-issue-149252.4W_0-w.rst b/Misc/NEWS.d/next/Build/2026-05-01-20-01-32.gh-issue-149252.4W_0-w.rst new file mode 100644 index 00000000000..646a8e33732 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-05-01-20-01-32.gh-issue-149252.4W_0-w.rst @@ -0,0 +1 @@ +Update to WASI SDK 33. diff --git a/Misc/NEWS.d/next/C_API/2026-04-09-14-45-44.gh-issue-148267.p84kG_.rst b/Misc/NEWS.d/next/C_API/2026-04-09-14-45-44.gh-issue-148267.p84kG_.rst new file mode 100644 index 00000000000..1ec1afd2cbf --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-04-09-14-45-44.gh-issue-148267.p84kG_.rst @@ -0,0 +1,2 @@ +Using :c:macro:`Py_LIMITED_API` on a non-Windows free-threaded build no +longer needs an extra :c:macro:`Py_GIL_DISABLED`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-26-10-03-18.gh-issue-134690.mUMT16.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-26-10-03-18.gh-issue-134690.mUMT16.rst new file mode 100644 index 00000000000..d26fa590b35 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-26-10-03-18.gh-issue-134690.mUMT16.rst @@ -0,0 +1,2 @@ +Removed deprecated in :pep:`626` since Python 3.12 +:attr:`!codeobject.co_lnotab` from :class:`types.CodeType`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-10-17-30-55.gh-issue-135357.sUXU1W.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-10-17-30-55.gh-issue-135357.sUXU1W.rst new file mode 100644 index 00000000000..378bb59de79 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-10-17-30-55.gh-issue-135357.sUXU1W.rst @@ -0,0 +1 @@ +Add support for :data:`!socket.SO_PASSRIGHTS` on Linux. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-09-19-00-36.gh-issue-137600.p_p6OU.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-09-19-00-36.gh-issue-137600.p_p6OU.rst new file mode 100644 index 00000000000..d1d1e36215d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-09-19-00-36.gh-issue-137600.p_p6OU.rst @@ -0,0 +1,4 @@ +:mod:`ast`: The constructors of AST nodes now raise a :exc:`TypeError` when +a required argument is omitted or when a keyword argument that does not map to +a field on the AST node is passed. These cases had previously raised a +:exc:`DeprecationWarning` since Python 3.13. Patch by Brian Schubert. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-17-20-30-17.gh-issue-126910.NaUwmD.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-17-20-30-17.gh-issue-126910.NaUwmD.rst new file mode 100644 index 00000000000..4d2634d0dd1 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-17-20-30-17.gh-issue-126910.NaUwmD.rst @@ -0,0 +1 @@ +Add support for unwinding JIT frames using GDB. Patch by Diego Russo and Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-23-11-34-37.gh-issue-142186.v8Yp3W.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-23-11-34-37.gh-issue-142186.v8Yp3W.rst new file mode 100644 index 00000000000..4a04658551c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-23-11-34-37.gh-issue-142186.v8Yp3W.rst @@ -0,0 +1,3 @@ +Global :mod:`sys.monitoring` events can now be turned on and disabled on a +per code object basis. Returning ``DISABLE`` from a callback disables the +event for the entire code object (for the current tool). diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-26-08-49-35.gh-issue-146455.f54083a9.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-26-08-49-35.gh-issue-146455.f54083a9.rst new file mode 100644 index 00000000000..4d7537f2529 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-26-08-49-35.gh-issue-146455.f54083a9.rst @@ -0,0 +1 @@ +Fix O(N²) compile-time regression in constant folding after it was moved from AST to CFG optimizer. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-05-16-10-00.gh-issue-149202.W8sQeR.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-05-16-10-00.gh-issue-149202.W8sQeR.rst new file mode 100644 index 00000000000..f82ca91f5ba --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-05-16-10-00.gh-issue-149202.W8sQeR.rst @@ -0,0 +1,4 @@ +Enable frame pointers by default for GCC-compatible CPython builds, including +``-mno-omit-leaf-frame-pointer`` when the compiler supports it, so profilers +and debuggers can unwind native interpreter frames more reliably. Users can pass +``--without-frame-pointers`` to opt out. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-07-20-37-23.gh-issue-148222.uF4D4E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-07-20-37-23.gh-issue-148222.uF4D4E.rst new file mode 100644 index 00000000000..2c273fc4dab --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-07-20-37-23.gh-issue-148222.uF4D4E.rst @@ -0,0 +1 @@ +Fix vectorcall support in :class:`types.GenericAlias` when the underlying type does not support the vectorcall protocol. Fix possible leaks in :class:`types.GenericAlias` and :class:`types.UnionType` in case of memory error. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-13-23-21-45.gh-issue-145239.pL8qRt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-13-23-21-45.gh-issue-145239.pL8qRt.rst new file mode 100644 index 00000000000..282b9917664 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-13-23-21-45.gh-issue-145239.pL8qRt.rst @@ -0,0 +1,3 @@ +Unary plus is now accepted in :keyword:`match` literal patterns, mirroring +the existing support for unary minus. +Patch by Bartosz Sławecki. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-17-11-30-00.gh-issue-142516.GcGen315.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-17-11-30-00.gh-issue-142516.GcGen315.rst new file mode 100644 index 00000000000..5abb2485c20 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-17-11-30-00.gh-issue-142516.GcGen315.rst @@ -0,0 +1 @@ +Forward-port the generational cycle garbage collector to the default 3.15 build, replacing the incremental collector while leaving the free-threaded collector unchanged. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-17-20-37-02.gh-issue-148653.nbbHMh.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-17-20-37-02.gh-issue-148653.nbbHMh.rst new file mode 100644 index 00000000000..d3242235c60 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-17-20-37-02.gh-issue-148653.nbbHMh.rst @@ -0,0 +1,2 @@ +Forbid :mod:`marshalling ` recursive code objects, :class:`slice` +and :class:`frozendict` objects which cannot be correctly unmarshalled. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-18-16-41-04.gh-issue-148571.Q6WB3A.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-18-16-41-04.gh-issue-148571.Q6WB3A.rst new file mode 100644 index 00000000000..70eeada3432 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-18-16-41-04.gh-issue-148571.Q6WB3A.rst @@ -0,0 +1 @@ +Fix a crash in the JIT optimizer when specialized opcode families inherited incompatible recorded operand layouts. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-06-43-32.gh-issue-148829.GtIrYO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-06-43-32.gh-issue-148829.GtIrYO.rst new file mode 100644 index 00000000000..3d9b4faa6ca --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-06-43-32.gh-issue-148829.GtIrYO.rst @@ -0,0 +1,2 @@ +Add :class:`sentinel`, implementing :pep:`661`. PEP by Tal Einat; patch by +Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-14-36-44.gh-issue-148820.XhOGhA.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-14-36-44.gh-issue-148820.XhOGhA.rst new file mode 100644 index 00000000000..392becaffb7 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-14-36-44.gh-issue-148820.XhOGhA.rst @@ -0,0 +1,5 @@ +Fix a race in :c:type:`!_PyRawMutex` on the free-threaded build where a +``Py_PARK_INTR`` return from ``_PySemaphore_Wait`` could let the waiter +destroy its semaphore before the unlocking thread's +``_PySemaphore_Wakeup`` completed, causing a fatal ``ReleaseSemaphore`` +error. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-19-29-29.gh-issue-148850.MSH0J_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-19-29-29.gh-issue-148850.MSH0J_.rst new file mode 100644 index 00000000000..324d1610310 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-19-29-29.gh-issue-148850.MSH0J_.rst @@ -0,0 +1 @@ +Fix the memory sanitizer false positive in :func:`os.getrandom`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-22-14-55-18.gh-issue-113956.0VEXd6.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-22-14-55-18.gh-issue-113956.0VEXd6.rst new file mode 100644 index 00000000000..54c04bbc28d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-22-14-55-18.gh-issue-113956.0VEXd6.rst @@ -0,0 +1,4 @@ +Fix a data race in :func:`sys.intern` in the free-threaded build when +interning a string owned by another thread. An interned copy owned by the +current thread is used instead when it is not safe to immortalize the +original. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-28-05-59-17.gh-issue-83065.f0UPNE.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-28-05-59-17.gh-issue-83065.f0UPNE.rst new file mode 100644 index 00000000000..81bfa45c069 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-28-05-59-17.gh-issue-83065.f0UPNE.rst @@ -0,0 +1,7 @@ +Fix a deadlock that could occur when one thread is importing a submodule +(for example ``import pkg.sub.mod``) while another thread is importing one +of its parent packages (for example ``import pkg.sub``) and that parent's +``__init__.py`` itself imports the submodule. The import system now +acquires module locks in hierarchical (parent-before-child) order so the +two threads serialise instead of raising +``_DeadlockError``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-28-21-19-21.gh-issue-149049.98u2Ib.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-28-21-19-21.gh-issue-149049.98u2Ib.rst new file mode 100644 index 00000000000..4c8f7e08a44 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-28-21-19-21.gh-issue-149049.98u2Ib.rst @@ -0,0 +1 @@ +Fix stack underflow for ``BINARY_OP`` in tier 2. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-29-14-06-00.gh-issue-149122.P8k2Lm.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-29-14-06-00.gh-issue-149122.P8k2Lm.rst new file mode 100644 index 00000000000..f34b6ea857a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-29-14-06-00.gh-issue-149122.P8k2Lm.rst @@ -0,0 +1,4 @@ +Fix a crash in optimized calls to :func:`all`, :func:`any`, :func:`tuple`, +:func:`list`, and :func:`set` with an async generator expression argument +(for example, ``tuple(await x for x in y)``). These calls now correctly raise +``TypeError`` instead of crashing. diff --git a/Misc/NEWS.d/next/Documentation/2026-04-17-02-28-55.gh-issue-148663.MHIbRB.rst b/Misc/NEWS.d/next/Documentation/2026-04-17-02-28-55.gh-issue-148663.MHIbRB.rst new file mode 100644 index 00000000000..0fbe5a699ef --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2026-04-17-02-28-55.gh-issue-148663.MHIbRB.rst @@ -0,0 +1,2 @@ +Document that :class:`calendar.IllegalMonthError` is a subclass of both +:exc:`ValueError` and :exc:`IndexError` since Python 3.12. diff --git a/Misc/NEWS.d/next/Library/2023-09-08-13-10-32.gh-issue-83281.2Plpcj.rst b/Misc/NEWS.d/next/Library/2023-09-08-13-10-32.gh-issue-83281.2Plpcj.rst new file mode 100644 index 00000000000..cf2ae770bd1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-08-13-10-32.gh-issue-83281.2Plpcj.rst @@ -0,0 +1,2 @@ +:mod:`email`: improve handling trailing garbage in address lists to avoid throwing +AttributeError in certain edge cases diff --git a/Misc/NEWS.d/next/Library/2023-12-25-19-14-07.gh-issue-113471.ZQMpbI.rst b/Misc/NEWS.d/next/Library/2023-12-25-19-14-07.gh-issue-113471.ZQMpbI.rst new file mode 100644 index 00000000000..99ba9bd1820 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-25-19-14-07.gh-issue-113471.ZQMpbI.rst @@ -0,0 +1,2 @@ +Allow :mod:`http.server` to set a default content-type when serving +files with an unknown or missing extension. diff --git a/Misc/NEWS.d/next/Library/2024-07-30-19-19-33.gh-issue-81074.YAeWNf.rst b/Misc/NEWS.d/next/Library/2024-07-30-19-19-33.gh-issue-81074.YAeWNf.rst new file mode 100644 index 00000000000..87de4fade14 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-30-19-19-33.gh-issue-81074.YAeWNf.rst @@ -0,0 +1,8 @@ +The :mod:`email` module no longer treats email addresses with non-ASCII +characters as defects when parsing a Unicode string or in the ``addr_spec`` +parameter to :class:`email.headerregistry.Address`. :rfc:`5322` permits such +addresses, and they were already supported when parsing bytes and in the Address +``username`` parameter. + +The (undocumented) :exc:`!email.errors.NonASCIILocalPartDefect` is no longer +used and should be considered deprecated. diff --git a/Misc/NEWS.d/next/Library/2024-07-31-17-22-10.gh-issue-83938.TtUa-c.rst b/Misc/NEWS.d/next/Library/2024-07-31-17-22-10.gh-issue-83938.TtUa-c.rst new file mode 100644 index 00000000000..7082c72f685 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-31-17-22-10.gh-issue-83938.TtUa-c.rst @@ -0,0 +1,8 @@ +The :mod:`email` module no longer incorrectly uses :rfc:`2047` encoding for +a mailbox with non-ASCII characters in its domain. Under a policy with +:attr:`~email.policy.EmailPolicy.utf8` set ``False``, attempting to serialize +such a message will now raise an :exc:`~email.errors.HeaderWriteError`. +Either apply an appropriate IDNA encoding to convert the domain to ASCII before +serialization, or use :data:`email.policy.SMTPUTF8` (or another policy with +``utf8=True``) to correctly pass through the internationalized domain name +as Unicode characters. diff --git a/Misc/NEWS.d/next/Library/2024-07-31-17-23-06.gh-issue-122476.TtUa-c.rst b/Misc/NEWS.d/next/Library/2024-07-31-17-23-06.gh-issue-122476.TtUa-c.rst new file mode 100644 index 00000000000..29c076d3a74 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-31-17-23-06.gh-issue-122476.TtUa-c.rst @@ -0,0 +1,7 @@ +The :mod:`email` module no longer incorrectly uses :rfc:`2047` encoding for +a mailbox with non-ASCII characters in its local-part. Under a policy with +:attr:`~email.policy.EmailPolicy.utf8` set ``False``, attempting to serialize +such a message will now raise an :exc:`~email.errors.HeaderWriteError`. +There is no valid 7-bit encoding for an internationalized local-part. Use +:data:`email.policy.SMTPUTF8` (or another policy with ``utf8=True``) to +correctly pass through the local-part as Unicode characters. diff --git a/Misc/NEWS.d/next/Library/2024-09-09-12-48-37.gh-issue-123853.e-zFxb.rst b/Misc/NEWS.d/next/Library/2024-09-09-12-48-37.gh-issue-123853.e-zFxb.rst new file mode 100644 index 00000000000..d7204c28936 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-09-12-48-37.gh-issue-123853.e-zFxb.rst @@ -0,0 +1,3 @@ +Update the table of Windows language code identifiers (LCIDs) used by +:func:`locale.getdefaultlocale` on Windows to protocol version 16.0 +(2024-04-23). diff --git a/Misc/NEWS.d/next/Library/2024-11-24-07-18-40.gh-issue-108951.jyKygP.rst b/Misc/NEWS.d/next/Library/2024-11-24-07-18-40.gh-issue-108951.jyKygP.rst new file mode 100644 index 00000000000..0e0280c9b6b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-24-07-18-40.gh-issue-108951.jyKygP.rst @@ -0,0 +1,2 @@ +:mod:`asyncio`: Add :meth:`TaskGroup.cancel ` which cancels +unfinished tasks and exits the group without raising :exc:`asyncio.CancelledError`. diff --git a/Misc/NEWS.d/next/Library/2025-04-17-15-26-35.gh-issue-132631.IDFZfb.rst b/Misc/NEWS.d/next/Library/2025-04-17-15-26-35.gh-issue-132631.IDFZfb.rst new file mode 100644 index 00000000000..9cc1d5a389c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-17-15-26-35.gh-issue-132631.IDFZfb.rst @@ -0,0 +1,2 @@ +Fix "I/O operation on closed file" when parsing JSON Lines file with +:mod:`JSON CLI `. diff --git a/Misc/NEWS.d/next/Library/2025-06-22-16-29-10.gh-issue-135528.Rt_QhR.rst b/Misc/NEWS.d/next/Library/2025-06-22-16-29-10.gh-issue-135528.Rt_QhR.rst new file mode 100644 index 00000000000..ab3855582c7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-22-16-29-10.gh-issue-135528.Rt_QhR.rst @@ -0,0 +1 @@ +:mod:`http.cookiejar`: add "tv", "or", "nom", "sch", and "web" to the default list of supported country code second-level domains. diff --git a/Misc/NEWS.d/next/Library/2025-07-02-17-01-17.gh-issue-125862.WgFYj3.rst b/Misc/NEWS.d/next/Library/2025-07-02-17-01-17.gh-issue-125862.WgFYj3.rst new file mode 100644 index 00000000000..1ccc91d55ec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-02-17-01-17.gh-issue-125862.WgFYj3.rst @@ -0,0 +1,4 @@ +The :func:`contextlib.contextmanager` and +:func:`contextlib.asynccontextmanager` decorators now work correctly with +generators, coroutine functions, and async generators when the wrapped +callables are used as decorators. diff --git a/Misc/NEWS.d/next/Library/2025-08-24-15-09-30.gh-issue-75707.GOWZrC.rst b/Misc/NEWS.d/next/Library/2025-08-24-15-09-30.gh-issue-75707.GOWZrC.rst new file mode 100644 index 00000000000..b2ff8a0cdf6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-24-15-09-30.gh-issue-75707.GOWZrC.rst @@ -0,0 +1 @@ +Add optional ``mtime`` argument to :func:`tarfile.open`, for setting the ``mtime`` header field in ``.tar.gz`` archives. diff --git a/Misc/NEWS.d/next/Library/2025-10-18-12-13-39.gh-issue-140287.49iU-4.rst b/Misc/NEWS.d/next/Library/2025-10-18-12-13-39.gh-issue-140287.49iU-4.rst new file mode 100644 index 00000000000..09643956d98 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-18-12-13-39.gh-issue-140287.49iU-4.rst @@ -0,0 +1,2 @@ +The :mod:`asyncio` REPL now handles exceptions when executing :envvar:`PYTHONSTARTUP` scripts. +Patch by Bartosz Sławecki. diff --git a/Misc/NEWS.d/next/Library/2025-12-17-02-55-03.gh-issue-108411.up7MAc.rst b/Misc/NEWS.d/next/Library/2025-12-17-02-55-03.gh-issue-108411.up7MAc.rst new file mode 100644 index 00000000000..95aa41e9226 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-17-02-55-03.gh-issue-108411.up7MAc.rst @@ -0,0 +1,2 @@ +``typing.IO`` and ``typing.BinaryIO`` method arguments are now +positional-only. diff --git a/Misc/NEWS.d/next/Library/2026-02-12-18-05-16.gh-issue-137855.2_PTbg.rst b/Misc/NEWS.d/next/Library/2026-02-12-18-05-16.gh-issue-137855.2_PTbg.rst new file mode 100644 index 00000000000..586c7d3495a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-02-12-18-05-16.gh-issue-137855.2_PTbg.rst @@ -0,0 +1 @@ +Reduce the import time of :mod:`dataclasses` module by ~20%. diff --git a/Misc/NEWS.d/next/Library/2026-02-22-00-00-00.gh-issue-145105.csv-reader-reentrant.rst b/Misc/NEWS.d/next/Library/2026-02-22-00-00-00.gh-issue-145105.csv-reader-reentrant.rst index bc61cc43a5a..1c2e06c86f6 100644 --- a/Misc/NEWS.d/next/Library/2026-02-22-00-00-00.gh-issue-145105.csv-reader-reentrant.rst +++ b/Misc/NEWS.d/next/Library/2026-02-22-00-00-00.gh-issue-145105.csv-reader-reentrant.rst @@ -1,2 +1,2 @@ -Fix crash in :mod:`csv` reader when iterating with a re-entrant iterator -that calls :func:`next` on the same reader from within ``__next__``. +Fix crash in :mod:`csv` reader when iterating with a re-entrant iterator +that calls :func:`next` on the same reader from within ``__next__``. diff --git a/Misc/NEWS.d/next/Library/2026-02-22-19-36-00.gh-issue-145056.TH8nX4.rst b/Misc/NEWS.d/next/Library/2026-02-22-19-36-00.gh-issue-145056.TH8nX4.rst new file mode 100644 index 00000000000..45be0109677 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-02-22-19-36-00.gh-issue-145056.TH8nX4.rst @@ -0,0 +1 @@ +Add support for :class:`frozendict` in :meth:`dataclasses.asdict` and :meth:`dataclasses.astuple`. diff --git a/Misc/NEWS.d/next/Library/2026-03-01-01-58-10.gh-issue-145378.oy6rb9.rst b/Misc/NEWS.d/next/Library/2026-03-01-01-58-10.gh-issue-145378.oy6rb9.rst new file mode 100644 index 00000000000..b6a6273d882 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-01-01-58-10.gh-issue-145378.oy6rb9.rst @@ -0,0 +1 @@ +Use ``PyREPL`` as the default input console for :mod:`pdb` diff --git a/Misc/NEWS.d/next/Library/2026-04-02-05-06-34.gh-issue-147991.2ANtR5.rst b/Misc/NEWS.d/next/Library/2026-04-02-05-06-34.gh-issue-147991.2ANtR5.rst new file mode 100644 index 00000000000..581c52926c3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-02-05-06-34.gh-issue-147991.2ANtR5.rst @@ -0,0 +1,2 @@ +Improve :mod:`tomllib` import time (up to 10x faster). Patch by Victor +Stinner. diff --git a/Misc/NEWS.d/next/Library/2026-04-07-12-37-53.gh-issue-148207.YhGem4.rst b/Misc/NEWS.d/next/Library/2026-04-07-12-37-53.gh-issue-148207.YhGem4.rst new file mode 100644 index 00000000000..dd88be0ad25 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-07-12-37-53.gh-issue-148207.YhGem4.rst @@ -0,0 +1,3 @@ +:class:`typing.TypeVarTuple` now accepts ``bound``, ``covariant``, +``contravariant``, and ``infer_variance`` parameters, matching the interface +of :class:`typing.TypeVar` and :class:`typing.ParamSpec`. diff --git a/Misc/NEWS.d/next/Library/2026-04-13-06-22-27.gh-issue-148464.Bj_NZy.rst b/Misc/NEWS.d/next/Library/2026-04-13-06-22-27.gh-issue-148464.Bj_NZy.rst new file mode 100644 index 00000000000..85b99531d03 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-13-06-22-27.gh-issue-148464.Bj_NZy.rst @@ -0,0 +1,3 @@ +Add missing ``__ctype_le/be__`` attributes for +:class:`~ctypes.c_float_complex` and :class:`~ctypes.c_double_complex`. Patch +by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Library/2026-04-13-15-59-44.gh-issue-148518.RQdvsu.rst b/Misc/NEWS.d/next/Library/2026-04-13-15-59-44.gh-issue-148518.RQdvsu.rst new file mode 100644 index 00000000000..994e4ad7446 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-13-15-59-44.gh-issue-148518.RQdvsu.rst @@ -0,0 +1,4 @@ +If an email containing an address header that ended in an open double quote +was parsed with a non-``compat32`` policy, accessing the ``username`` attribute +of the mailbox accessed through that header object would result in an +``IndexError``. It now correctly returns an empty string as the result. diff --git a/Misc/NEWS.d/next/Library/2026-04-15-11-00-39.gh-issue-146553.VGOsoP.rst b/Misc/NEWS.d/next/Library/2026-04-15-11-00-39.gh-issue-146553.VGOsoP.rst new file mode 100644 index 00000000000..44216318d47 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-15-11-00-39.gh-issue-146553.VGOsoP.rst @@ -0,0 +1,2 @@ +Fix infinite loop in :func:`typing.get_type_hints` when ``__wrapped__`` +forms a cycle. Patch by Shamil Abdulaev. diff --git a/Misc/NEWS.d/next/Library/2026-04-15-20-32-55.gh-issue-148639.-dwsjB.rst b/Misc/NEWS.d/next/Library/2026-04-15-20-32-55.gh-issue-148639.-dwsjB.rst new file mode 100644 index 00000000000..d7acdb09838 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-15-20-32-55.gh-issue-148639.-dwsjB.rst @@ -0,0 +1,2 @@ +Implement :pep:`800`, adding the :deco:`typing.disjoint_base` decorator. +Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Library/2026-04-16-13-30-00.gh-issue-148651.ZsTdLk.rst b/Misc/NEWS.d/next/Library/2026-04-16-13-30-00.gh-issue-148651.ZsTdLk.rst new file mode 100644 index 00000000000..b69f94a1766 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-16-13-30-00.gh-issue-148651.ZsTdLk.rst @@ -0,0 +1,2 @@ +Fix reference leak in :class:`compression.zstd.ZstdDecompressor` when an +invalid option key is passed. diff --git a/Misc/NEWS.d/next/Library/2026-04-17-16-31-58.gh-issue-148688.vVugFn.rst b/Misc/NEWS.d/next/Library/2026-04-17-16-31-58.gh-issue-148688.vVugFn.rst new file mode 100644 index 00000000000..1e367716e5a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-17-16-31-58.gh-issue-148688.vVugFn.rst @@ -0,0 +1,2 @@ +:mod:`bz2`, :mod:`compression.zstd`, :mod:`lzma`, :mod:`zlib`: Fix a double +free on memory allocation failure. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2026-04-18-17-37-13.gh-issue-148740.sYnFi0.rst b/Misc/NEWS.d/next/Library/2026-04-18-17-37-13.gh-issue-148740.sYnFi0.rst new file mode 100644 index 00000000000..7e49cedda7b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-18-17-37-13.gh-issue-148740.sYnFi0.rst @@ -0,0 +1,2 @@ +Fix usage for :mod:`uuid` command-line interface to support a custom namespace be +provided for uuid3 and uuid5. diff --git a/Misc/NEWS.d/next/Library/2026-04-18-21-39-15.gh-issue-148735.siw6DG.rst b/Misc/NEWS.d/next/Library/2026-04-18-21-39-15.gh-issue-148735.siw6DG.rst new file mode 100644 index 00000000000..db5e94c0cca --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-18-21-39-15.gh-issue-148735.siw6DG.rst @@ -0,0 +1,3 @@ +:mod:`xml.etree.ElementTree`: Fix a use-after-free in +:meth:`Element.findtext ` when the +element tree is mutated concurrently during the search. diff --git a/Misc/NEWS.d/next/Library/2026-04-20-18-29-21.gh-issue-148801.ROeNqs.rst b/Misc/NEWS.d/next/Library/2026-04-20-18-29-21.gh-issue-148801.ROeNqs.rst new file mode 100644 index 00000000000..6fcd30e8f05 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-20-18-29-21.gh-issue-148801.ROeNqs.rst @@ -0,0 +1,2 @@ +:mod:`xml.etree.ElementTree`: Fix a crash in :meth:`Element.__deepcopy__ +` on deeply nested trees. diff --git a/Misc/NEWS.d/next/Library/2026-04-22-20-49-49.gh-issue-124397.plMglV.rst b/Misc/NEWS.d/next/Library/2026-04-22-20-49-49.gh-issue-124397.plMglV.rst new file mode 100644 index 00000000000..431448a484b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-22-20-49-49.gh-issue-124397.plMglV.rst @@ -0,0 +1,3 @@ +The threading module added tooling to support concurrent iterator access: +:class:`threading.serialize_iterator`, :func:`threading.synchronized_iterator`, +and :func:`threading.concurrent_tee`. diff --git a/Misc/NEWS.d/next/Library/2026-04-23-07-38-04.gh-issue-148680.___ePl.rst b/Misc/NEWS.d/next/Library/2026-04-23-07-38-04.gh-issue-148680.___ePl.rst new file mode 100644 index 00000000000..d3790079545 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-23-07-38-04.gh-issue-148680.___ePl.rst @@ -0,0 +1 @@ +``ForwardRef`` objects that contain internal names to represent known objects now show the ``type_repr`` of the known object rather than the internal ``__annotationlib_name_x__`` name when evaluated as strings. diff --git a/Misc/NEWS.d/next/Library/2026-04-23-21-47-49.gh-issue-148947.W4V2lG.rst b/Misc/NEWS.d/next/Library/2026-04-23-21-47-49.gh-issue-148947.W4V2lG.rst new file mode 100644 index 00000000000..f9783266f5c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-23-21-47-49.gh-issue-148947.W4V2lG.rst @@ -0,0 +1,2 @@ +Fix crash in :deco:`dataclasses.dataclass` with ``slots=True`` that occurred +when a function found within the class had an empty ``__class__`` cell. diff --git a/Misc/NEWS.d/next/Library/2026-04-25-11-56-05.gh-issue-146311.iHWO0v.rst b/Misc/NEWS.d/next/Library/2026-04-25-11-56-05.gh-issue-146311.iHWO0v.rst new file mode 100644 index 00000000000..4f4a8365b6c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-25-11-56-05.gh-issue-146311.iHWO0v.rst @@ -0,0 +1,7 @@ +Add a *canonical* keyword-only parameter to the base16, base32, base64, +base85, ascii85, and Z85 decoders in :mod:`base64` and :mod:`binascii`. +When true, encodings with non-zero padding bits (base16/32/64) or +non-canonical encodings (base85/ascii85) are rejected. Single-character +final groups in :func:`binascii.a2b_ascii85` and :func:`binascii.a2b_base85` +are now always rejected as encoding violations, regardless of *canonical*; +previously they were silently ignored and produced no output bytes. diff --git a/Misc/NEWS.d/next/Library/2026-04-25-12-50-46.gh-issue-148981.YMM4Y9.rst b/Misc/NEWS.d/next/Library/2026-04-25-12-50-46.gh-issue-148981.YMM4Y9.rst new file mode 100644 index 00000000000..e36c7745f40 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-25-12-50-46.gh-issue-148981.YMM4Y9.rst @@ -0,0 +1 @@ +Add *color* parameter to :func:`ast.dump`. diff --git a/Misc/NEWS.d/next/Library/2026-04-25-18-09-16.gh-issue-148991.AZ64Et.rst b/Misc/NEWS.d/next/Library/2026-04-25-18-09-16.gh-issue-148991.AZ64Et.rst new file mode 100644 index 00000000000..336ed42e51f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-25-18-09-16.gh-issue-148991.AZ64Et.rst @@ -0,0 +1 @@ +Add colour to :mod:`tokenize` CLI output. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2026-04-26-23-01-50.gh-issue-149026.Akk4Bc.rst b/Misc/NEWS.d/next/Library/2026-04-26-23-01-50.gh-issue-149026.Akk4Bc.rst new file mode 100644 index 00000000000..d12a92e9f53 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-26-23-01-50.gh-issue-149026.Akk4Bc.rst @@ -0,0 +1 @@ +Add colour to :mod:`pickletools` CLI output. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2026-04-27-17-12-11.gh-issue-148914.i5C3kW.rst b/Misc/NEWS.d/next/Library/2026-04-27-17-12-11.gh-issue-148914.i5C3kW.rst new file mode 100644 index 00000000000..8348aad0d89 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-27-17-12-11.gh-issue-148914.i5C3kW.rst @@ -0,0 +1,6 @@ +Fix memoization of in-band :class:`~pickle.PickleBuffer` in the Python +implementation of :mod:`pickle`. Previously, identical +:class:`!PickleBuffer`\ s did not preserve identity, and empty writable +:class:`!PickleBuffer` memoized an empty bytearray object in place of +``b''``, so the following references to ``b''`` were unpickled as an empty +bytearray object. diff --git a/Misc/NEWS.d/next/Library/2026-04-27-20-15-54.gh-issue-149083.BdrpU8.rst b/Misc/NEWS.d/next/Library/2026-04-27-20-15-54.gh-issue-149083.BdrpU8.rst new file mode 100644 index 00000000000..7ad81616802 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-27-20-15-54.gh-issue-149083.BdrpU8.rst @@ -0,0 +1,2 @@ +:data:`dataclasses.MISSING` and :data:`dataclasses.KW_ONLY` are now +instances of :class:`sentinel`. diff --git a/Misc/NEWS.d/next/Library/2026-04-28-16-30-48.gh-issue-149085.5aNgBD.rst b/Misc/NEWS.d/next/Library/2026-04-28-16-30-48.gh-issue-149085.5aNgBD.rst new file mode 100644 index 00000000000..a5b92287bd0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-28-16-30-48.gh-issue-149085.5aNgBD.rst @@ -0,0 +1,3 @@ +Add a *max_threads* keyword argument to :func:`faulthandler.dump_traceback`, +:func:`faulthandler.dump_traceback_later`, :func:`faulthandler.enable`, and +:func:`faulthandler.register`. diff --git a/Misc/NEWS.d/next/Library/2026-04-29-14-33-42.gh-issue-149148.EaiYvk.rst b/Misc/NEWS.d/next/Library/2026-04-29-14-33-42.gh-issue-149148.EaiYvk.rst new file mode 100644 index 00000000000..06186773474 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-29-14-33-42.gh-issue-149148.EaiYvk.rst @@ -0,0 +1,2 @@ +:mod:`ensurepip`: Upgrade bundled pip to 26.1. This version fixes +the :cve:`2026-3219` vulnerability. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2026-04-29-16-11-27.gh-issue-149117.yEeTYd.rst b/Misc/NEWS.d/next/Library/2026-04-29-16-11-27.gh-issue-149117.yEeTYd.rst new file mode 100644 index 00000000000..41223e90ed0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-29-16-11-27.gh-issue-149117.yEeTYd.rst @@ -0,0 +1,3 @@ +Fix :func:`runpy.run_module` and :func:`runpy.run_path` to set the +:attr:`~ImportError.name` attribute on the :exc:`ImportError` they +raise. diff --git a/Misc/NEWS.d/next/Library/2026-04-30-14-21-26.gh-issue-149173.KJqZm0.rst b/Misc/NEWS.d/next/Library/2026-04-30-14-21-26.gh-issue-149173.KJqZm0.rst new file mode 100644 index 00000000000..019ab76b863 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-30-14-21-26.gh-issue-149173.KJqZm0.rst @@ -0,0 +1,2 @@ +Fix inverted :envvar:`PYTHON_BASIC_REPL` environment check in +``pdb._pyrepl_available``. diff --git a/Misc/NEWS.d/next/Library/2026-05-02-01-09-29.gh-issue-149221.__KOks.rst b/Misc/NEWS.d/next/Library/2026-05-02-01-09-29.gh-issue-149221.__KOks.rst new file mode 100644 index 00000000000..fab2b0f6a23 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-02-01-09-29.gh-issue-149221.__KOks.rst @@ -0,0 +1 @@ +Catch rare math domain error for :func:`random.binomialvariate`. diff --git a/Misc/NEWS.d/next/Security/2026-03-29-12-51-33.gh-issue-146581.4vZfB0.rst b/Misc/NEWS.d/next/Security/2026-03-29-12-51-33.gh-issue-146581.4vZfB0.rst new file mode 100644 index 00000000000..98e65549d79 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-03-29-12-51-33.gh-issue-146581.4vZfB0.rst @@ -0,0 +1,5 @@ +Fix vulnerability in :func:`shutil.unpack_archive` for ZIP files on Windows +which allowed to write files outside of the destination tree if the patch in +the archive contains a Windows drive prefix. Now such invalid paths will be +skipped. Files containing ".." in the name (like "foo..bar") are no longer +skipped. diff --git a/Misc/NEWS.d/next/Security/2026-04-20-15-31-37.gh-issue-148808._Z8JL0.rst b/Misc/NEWS.d/next/Security/2026-04-20-15-31-37.gh-issue-148808._Z8JL0.rst new file mode 100644 index 00000000000..0b5cf85fedf --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-04-20-15-31-37.gh-issue-148808._Z8JL0.rst @@ -0,0 +1,3 @@ +Added buffer boundary check when using ``nbytes`` parameter with +:meth:`!asyncio.AbstractEventLoop.sock_recvfrom_into`. Only +relevant for Windows and the :class:`asyncio.ProactorEventLoop`. diff --git a/Misc/NEWS.d/next/Security/2026-04-21-13-46-30.gh-issue-90309.srvj9q.rst b/Misc/NEWS.d/next/Security/2026-04-21-13-46-30.gh-issue-90309.srvj9q.rst new file mode 100644 index 00000000000..d7d376737e4 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-04-21-13-46-30.gh-issue-90309.srvj9q.rst @@ -0,0 +1,3 @@ +Base64-encode values when embedding cookies to JavaScript using the +:meth:`http.cookies.BaseCookie.js_output` method to avoid injection +and escaping. diff --git a/Misc/NEWS.d/next/Security/2026-04-24-23-15-42.gh-issue-148252.8BLmzd.rst b/Misc/NEWS.d/next/Security/2026-04-24-23-15-42.gh-issue-148252.8BLmzd.rst new file mode 100644 index 00000000000..531ea2348ff --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-04-24-23-15-42.gh-issue-148252.8BLmzd.rst @@ -0,0 +1,3 @@ +Fixed string table and sample record bounds checks in :mod:`!_remote_debugging` +when decoding certain ``.pyb`` inputs on 32-bit builds. Patch by Maurycy +Pawłowski-Wieroński. diff --git a/Misc/NEWS.d/next/Security/2026-04-26-17-49-58.gh-issue-149017.EiVFPo.rst b/Misc/NEWS.d/next/Security/2026-04-26-17-49-58.gh-issue-149017.EiVFPo.rst new file mode 100644 index 00000000000..6aa7efb68a1 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-04-26-17-49-58.gh-issue-149017.EiVFPo.rst @@ -0,0 +1 @@ +Update bundled `libexpat `_ to version 2.8.0. diff --git a/Misc/NEWS.d/next/Security/2026-05-02-15-38-03.gh-issue-149254.0HOL0j.rst b/Misc/NEWS.d/next/Security/2026-05-02-15-38-03.gh-issue-149254.0HOL0j.rst new file mode 100644 index 00000000000..f3cf924db25 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-05-02-15-38-03.gh-issue-149254.0HOL0j.rst @@ -0,0 +1 @@ +Update Android and iOS installer to use OpenSSL 3.5.6. diff --git a/Misc/NEWS.d/next/macOS/2026-05-01-20-12-33.gh-issue-149254.kXdWpS.rst b/Misc/NEWS.d/next/macOS/2026-05-01-20-12-33.gh-issue-149254.kXdWpS.rst new file mode 100644 index 00000000000..278327c91f1 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2026-05-01-20-12-33.gh-issue-149254.kXdWpS.rst @@ -0,0 +1 @@ +Update macOS installer to use OpenSSL 3.5.6. diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index ed9c0801680..aaeffd58e79 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -48,11 +48,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "9dfd09a3be37618cbcea380c2374b2b8f0288f57" + "checksumValue": "5343adc95840915b022b1d4524d0acb66b369ba2" }, { "algorithm": "SHA256", - "checksumValue": "26805a0d1a7a6a5cd8ead9cf7f4da29f63f0547a9ad41e80dba4ed9fe1943140" + "checksumValue": "1ec3bad08b6864c2c479e1fd941038c2dcd24c6d9a16400f4da54912d95aa321" } ], "fileName": "Modules/expat/expat.h" @@ -62,11 +62,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "da0328279276800cc747ea7da23886a3f402ccb3" + "checksumValue": "d8f9211d52ff0384e229e4d4d56adae5db2d7f91" }, { "algorithm": "SHA256", - "checksumValue": "15a80e414e9e7c43edba64b1608a77c724387070138693f9e9bcca49c78a2df7" + "checksumValue": "b77f8192baf90aaa41f7023bc68fd1f22ab2552f98758271a1e090544537def5" } ], "fileName": "Modules/expat/expat_external.h" @@ -90,11 +90,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "6a4a232233ba1034c3f2b459159d502e9b2d413b" + "checksumValue": "2555e70b29c1efc0af40879daafd12f8b36aca2c" }, { "algorithm": "SHA256", - "checksumValue": "c803935722f0dbdeeede7f040028fb119135e96dfad949479f8a5304b885bdd6" + "checksumValue": "4feb1df53898a48ae0ae04b5d0352c90395c8e693e5c2675f8ced41903d6fa94" } ], "fileName": "Modules/expat/internal.h" @@ -174,11 +174,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "0c74fbd48dd515c58eeb65b7e71b29da94be4694" + "checksumValue": "cb0af01558ec7b6474d2bd0c9386380c82618e8f" }, { "algorithm": "SHA256", - "checksumValue": "861e7a50ce81f9f16b42d32a9caa4f817d962b274b2929b579511c6f76d348d4" + "checksumValue": "6745a6b8cdd7344d4bd8f27f605363ed746e57ff02d4ebce3eb1806579cd030f" } ], "fileName": "Modules/expat/xmlparse.c" @@ -188,11 +188,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "7cff4d7210f046144f5fa635113f9c26f30fe3d3" + "checksumValue": "c8769fcb93f00272a6e6ca560be633649c817ff7" }, { "algorithm": "SHA256", - "checksumValue": "eaa6c327f9db4a5cec768d0c01927fea212d3ef4d4f970ebc0a98b9f3602784c" + "checksumValue": "5b81f0eb0e144b611dbd1bc9e6037075a16bff94f823d57a81eb2a3e4999e91a" } ], "fileName": "Modules/expat/xmlrole.c" @@ -216,11 +216,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "48b7aa6503302d4157c61a8763629f3236c23502" + "checksumValue": "63e4766a09e63760c6518670509198f8d638f4ad" }, { "algorithm": "SHA256", - "checksumValue": "75da65603e99837fd3116f1453372efd556f9f97d8de73364594dd78b3c8ec54" + "checksumValue": "0ad3f915f2748dc91bf4e4b4a50cf40bf2c95769d0eca7e3b293a230d82bb779" } ], "fileName": "Modules/expat/xmltok.c" @@ -272,11 +272,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "705842f8a09b09cc021d82a71ab03344bfd07b0a" + "checksumValue": "41b8c8fc275882c76d4210b7d40a18e506b07147" }, { "algorithm": "SHA256", - "checksumValue": "f95a2b4b7efda40f5faf366537cb20a57dddbad9655859d2e304f5e75f6907cc" + "checksumValue": "b2188c7e5fa5b33e355cf6cf342dfb8f6e23859f2a6b1ddf79841d7f84f7b196" } ], "fileName": "Modules/expat/xmltok_ns.c" @@ -1730,14 +1730,14 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "9931f9860d18e6cf72d183eb8f309bfb96196c00e1d40caa978e95bc9aa978b6" + "checksumValue": "c7cec5f60ea3a42e7780781c6745255c19aa3dbfeeae58646b7132f88dc24780" } ], - "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_7_5/expat-2.7.5.tar.gz", + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_0/expat-2.8.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.7.5:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.8.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1745,7 +1745,7 @@ "name": "expat", "originator": "Organization: Expat development team", "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.7.5" + "versionInfo": "2.8.0" }, { "SPDXID": "SPDXRef-PACKAGE-hacl-star", diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index ebb1de6eacc..c16e84772b4 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2222,6 +2222,36 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) return NULL; } +static int +set_stginfo_ffi_type_pointer(StgInfo *stginfo, struct fielddesc *fmt) +{ +#if defined(_Py_FFI_SUPPORT_C_COMPLEX) + if (!fmt->pffi_type->elements) { + stginfo->ffi_type_pointer = *fmt->pffi_type; + } + else { + /* From primitive types - only complex types have the elements + struct field as non-NULL (two element array). */ + assert(fmt->pffi_type->type == FFI_TYPE_COMPLEX); + const size_t els_size = 2 * sizeof(ffi_type *); + stginfo->ffi_type_pointer.size = fmt->pffi_type->size; + stginfo->ffi_type_pointer.alignment = fmt->pffi_type->alignment; + stginfo->ffi_type_pointer.type = fmt->pffi_type->type; + stginfo->ffi_type_pointer.elements = PyMem_Malloc(els_size); + if (!stginfo->ffi_type_pointer.elements) { + PyErr_NoMemory(); + return -1; + } + memcpy(stginfo->ffi_type_pointer.elements, + fmt->pffi_type->elements, els_size); + } +#else + assert(!fmt->pffi_type->elements); + stginfo->ffi_type_pointer = *fmt->pffi_type; +#endif + return 0; +} + static PyMethodDef c_void_p_methods[] = {C_VOID_P_FROM_PARAM_METHODDEF {0}}; static PyMethodDef c_char_p_methods[] = {C_CHAR_P_FROM_PARAM_METHODDEF {0}}; static PyMethodDef c_wchar_p_methods[] = {C_WCHAR_P_FROM_PARAM_METHODDEF {0}}; @@ -2266,8 +2296,10 @@ static PyObject *CreateSwappedType(ctypes_state *st, PyTypeObject *type, Py_DECREF(result); return NULL; } - - stginfo->ffi_type_pointer = *fmt->pffi_type; + if (set_stginfo_ffi_type_pointer(stginfo, fmt)) { + Py_DECREF(result); + return NULL; + } stginfo->align = fmt->pffi_type->alignment; stginfo->length = 0; stginfo->size = fmt->pffi_type->size; @@ -2367,18 +2399,8 @@ PyCSimpleType_init(PyObject *self, PyObject *args, PyObject *kwds) if (!stginfo) { goto error; } - - if (!fmt->pffi_type->elements) { - stginfo->ffi_type_pointer = *fmt->pffi_type; - } - else { - const size_t els_size = sizeof(fmt->pffi_type->elements); - stginfo->ffi_type_pointer.size = fmt->pffi_type->size; - stginfo->ffi_type_pointer.alignment = fmt->pffi_type->alignment; - stginfo->ffi_type_pointer.type = fmt->pffi_type->type; - stginfo->ffi_type_pointer.elements = PyMem_Malloc(els_size); - memcpy(stginfo->ffi_type_pointer.elements, - fmt->pffi_type->elements, els_size); + if (set_stginfo_ffi_type_pointer(stginfo, fmt)) { + goto error; } stginfo->align = fmt->pffi_type->alignment; stginfo->length = 0; diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 00101131198..55853c35f39 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -792,7 +792,45 @@ Zd_get(void *ptr, Py_ssize_t size) return PyComplex_FromDoubles(x[0], x[1]); } -/* F: float complex */ +static PyObject * +Zd_set_sw(void *ptr, PyObject *value, Py_ssize_t size) +{ + assert(NUM_BITS(size) || (size == 2*sizeof(double))); + Py_complex c = PyComplex_AsCComplex(value); + + if (c.real == -1 && PyErr_Occurred()) { + return NULL; + } +#ifdef WORDS_BIGENDIAN + if (PyFloat_Pack8(c.real, ptr, 1) + || PyFloat_Pack8(c.imag, ptr + sizeof(double), 1)) + { + return NULL; + } +#else + if (PyFloat_Pack8(c.real, ptr, 0) + || PyFloat_Pack8(c.imag, ptr + sizeof(double), 0)) + { + return NULL; + } +#endif + _RET(value); +} + +static PyObject * +Zd_get_sw(void *ptr, Py_ssize_t size) +{ + assert(NUM_BITS(size) || (size == 2*sizeof(double))); +#ifdef WORDS_BIGENDIAN + return PyComplex_FromDoubles(PyFloat_Unpack8(ptr, 1), + PyFloat_Unpack8(ptr + sizeof(double), 1)); +#else + return PyComplex_FromDoubles(PyFloat_Unpack8(ptr, 0), + PyFloat_Unpack8(ptr + sizeof(double), 0)); +#endif +} + +/* Zf: float complex */ static PyObject * Zf_set(void *ptr, PyObject *value, Py_ssize_t size) { @@ -817,7 +855,45 @@ Zf_get(void *ptr, Py_ssize_t size) return PyComplex_FromDoubles(x[0], x[1]); } -/* G: long double complex */ +static PyObject * +Zf_set_sw(void *ptr, PyObject *value, Py_ssize_t size) +{ + assert(NUM_BITS(size) || (size == 2*sizeof(float))); + Py_complex c = PyComplex_AsCComplex(value); + + if (c.real == -1 && PyErr_Occurred()) { + return NULL; + } +#ifdef WORDS_BIGENDIAN + if (PyFloat_Pack4(c.real, ptr, 1) + || PyFloat_Pack4(c.imag, ptr + sizeof(float), 1)) + { + return NULL; + } +#else + if (PyFloat_Pack4(c.real, ptr, 0) + || PyFloat_Pack4(c.imag, ptr + sizeof(float), 0)) + { + return NULL; + } +#endif + _RET(value); +} + +static PyObject * +Zf_get_sw(void *ptr, Py_ssize_t size) +{ + assert(NUM_BITS(size) || (size == 2*sizeof(float))); +#ifdef WORDS_BIGENDIAN + return PyComplex_FromDoubles(PyFloat_Unpack4(ptr, 1), + PyFloat_Unpack4(ptr + sizeof(float), 1)); +#else + return PyComplex_FromDoubles(PyFloat_Unpack4(ptr, 0), + PyFloat_Unpack4(ptr + sizeof(float), 0)); +#endif +} + +/* Zg: long double complex */ static PyObject * Zg_set(void *ptr, PyObject *value, Py_ssize_t size) { @@ -841,7 +917,8 @@ Zg_get(void *ptr, Py_ssize_t size) memcpy(&x, ptr, sizeof(x)); return PyComplex_FromDoubles((double)x[0], (double)x[1]); } -#endif +#endif // _Py_FFI_SUPPORT_C_COMPLEX + /* d: double */ static PyObject * @@ -1602,7 +1679,9 @@ for base_code, base_c_type in [ #if defined(_Py_FFI_SUPPORT_C_COMPLEX) if (Py_FFI_COMPLEX_AVAILABLE) { TABLE_ENTRY(Zd, &ffi_type_complex_double); + TABLE_ENTRY_SW(Zd, &ffi_type_complex_double); TABLE_ENTRY(Zf, &ffi_type_complex_float); + TABLE_ENTRY_SW(Zf, &ffi_type_complex_float); TABLE_ENTRY(Zg, &ffi_type_complex_longdouble); } #endif diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index e2185c4bd03..cbd1e026df2 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -16,6 +16,7 @@ #endif #include "Python.h" +#include "pycore_ceval.h" // _Py_EnterRecursiveCall() #include "pycore_dict.h" // _PyDict_CopyAsDict() #include "pycore_pyhash.h" // _Py_HashSecret #include "pycore_tuple.h" // _PyTuple_FromPair @@ -572,7 +573,7 @@ element_get_attrib(ElementObject* self) LOCAL(PyObject*) element_get_text(ElementObject* self) { - /* return borrowed reference to text attribute */ + /* return new reference to text attribute */ PyObject *res = self->text; @@ -587,13 +588,13 @@ element_get_text(ElementObject* self) } } - return res; + return Py_NewRef(res); } LOCAL(PyObject*) element_get_tail(ElementObject* self) { - /* return borrowed reference to text attribute */ + /* return new reference to tail attribute */ PyObject *res = self->tail; @@ -608,7 +609,7 @@ element_get_tail(ElementObject* self) } } - return res; + return Py_NewRef(res); } static PyObject* @@ -811,26 +812,31 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) /*[clinic end generated code: output=eefc3df50465b642 input=a2d40348c0aade10]*/ { Py_ssize_t i; - ElementObject* element; + ElementObject* element = NULL; PyObject* tag; PyObject* attrib; PyObject* text; PyObject* tail; PyObject* id; + if (_Py_EnterRecursiveCall(" in Element.__deepcopy__")) { + return NULL; + } + PyTypeObject *tp = Py_TYPE(self); elementtreestate *st = get_elementtree_state_by_type(tp); // The deepcopy() helper takes care of incrementing the refcount // of the object to copy so to avoid use-after-frees. tag = deepcopy(st, self->tag, memo); - if (!tag) - return NULL; + if (!tag) { + goto error; + } if (self->extra && self->extra->attrib) { attrib = deepcopy(st, self->extra->attrib, memo); if (!attrib) { Py_DECREF(tag); - return NULL; + goto error; } } else { attrib = NULL; @@ -841,8 +847,9 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) Py_DECREF(tag); Py_XDECREF(attrib); - if (!element) - return NULL; + if (!element) { + goto error; + } text = deepcopy(st, JOIN_OBJ(self->text), memo); if (!text) @@ -904,10 +911,12 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) if (i < 0) goto error; + _Py_LeaveRecursiveCall(); return (PyObject*) element; error: - Py_DECREF(element); + _Py_LeaveRecursiveCall(); + Py_XDECREF(element); return NULL; } @@ -1350,9 +1359,9 @@ _elementtree_Element_findtext_impl(ElementObject *self, PyTypeObject *cls, PyObject *text = element_get_text((ElementObject *)item); Py_DECREF(item); if (text == Py_None) { + Py_DECREF(text); return Py_GetConstant(Py_CONSTANT_EMPTY_STR); } - Py_XINCREF(text); return text; } Py_DECREF(item); @@ -2055,16 +2064,14 @@ static PyObject* element_text_getter(PyObject *op, void *closure) { ElementObject *self = _Element_CAST(op); - PyObject *res = element_get_text(self); - return Py_XNewRef(res); + return element_get_text(self); } static PyObject* element_tail_getter(PyObject *op, void *closure) { ElementObject *self = _Element_CAST(op); - PyObject *res = element_get_tail(self); - return Py_XNewRef(res); + return element_get_tail(self); } static PyObject* @@ -2307,16 +2314,14 @@ elementiter_next(PyObject *op) continue; gettext: + Py_DECREF(elem); if (!text) { - Py_DECREF(elem); return NULL; } if (text == Py_None) { - Py_DECREF(elem); + Py_DECREF(text); } else { - Py_INCREF(text); - Py_DECREF(elem); rc = PyObject_IsTrue(text); if (rc > 0) return text; diff --git a/Modules/_remote_debugging/binary_io.h b/Modules/_remote_debugging/binary_io.h index d90546078bf..18f989f672e 100644 --- a/Modules/_remote_debugging/binary_io.h +++ b/Modules/_remote_debugging/binary_io.h @@ -61,6 +61,7 @@ extern "C" { #define HDR_SIZE_COMPRESSION 4 #define FILE_HEADER_SIZE (HDR_OFF_COMPRESSION + HDR_SIZE_COMPRESSION) #define FILE_HEADER_PLACEHOLDER_SIZE 64 +#define SAMPLE_HEADER_FIXED_SIZE (sizeof(uint64_t) + sizeof(uint32_t) + 1) static_assert(FILE_HEADER_SIZE <= FILE_HEADER_PLACEHOLDER_SIZE, "FILE_HEADER_SIZE exceeds FILE_HEADER_PLACEHOLDER_SIZE"); diff --git a/Modules/_remote_debugging/binary_io_reader.c b/Modules/_remote_debugging/binary_io_reader.c index aca93e9cb1a..6c32ef70ac3 100644 --- a/Modules/_remote_debugging/binary_io_reader.c +++ b/Modules/_remote_debugging/binary_io_reader.c @@ -258,7 +258,7 @@ reader_parse_string_table(BinaryReader *reader, const uint8_t *data, size_t file PyErr_SetString(PyExc_ValueError, "Malformed varint in string table"); return -1; } - if (offset + str_len > file_size) { + if (offset > file_size || str_len > file_size - offset) { PyErr_SetString(PyExc_ValueError, "String table overflow"); return -1; } @@ -976,8 +976,8 @@ binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progre } while (offset < reader->sample_data_size) { - /* Read thread_id (8 bytes) + interpreter_id (4 bytes) */ - if (offset + 13 > reader->sample_data_size) { + /* Read thread_id (8 bytes) + interpreter_id (4 bytes) + encoding byte */ + if (reader->sample_data_size - offset < SAMPLE_HEADER_FIXED_SIZE) { break; /* End of data */ } diff --git a/Modules/_remote_debugging/binary_io_writer.c b/Modules/_remote_debugging/binary_io_writer.c index c129c93efe2..0ac6c88d037 100644 --- a/Modules/_remote_debugging/binary_io_writer.c +++ b/Modules/_remote_debugging/binary_io_writer.c @@ -23,7 +23,6 @@ * ============================================================================ */ /* Sample header sizes */ -#define SAMPLE_HEADER_FIXED_SIZE 13 /* thread_id(8) + interpreter_id(4) + encoding(1) */ #define SAMPLE_HEADER_MAX_SIZE 26 /* fixed + max_varint(10) + status(1) + margin */ #define MAX_VARINT_SIZE 10 /* Maximum bytes for a varint64 */ #define MAX_VARINT_SIZE_U32 5 /* Maximum bytes for a varint32 */ @@ -653,10 +652,13 @@ write_sample_with_encoding(BinaryWriter *writer, ThreadEntry *entry, memcpy(header_buf, &entry->thread_id, 8); memcpy(header_buf + 8, &entry->interpreter_id, 4); header_buf[12] = (uint8_t)encoding_type; - size_t varint_len = encode_varint_u64(header_buf + 13, timestamp_delta); - header_buf[13 + varint_len] = status; + size_t varint_len = encode_varint_u64( + header_buf + SAMPLE_HEADER_FIXED_SIZE, + timestamp_delta); + header_buf[SAMPLE_HEADER_FIXED_SIZE + varint_len] = status; - if (writer_write_bytes(writer, header_buf, 14 + varint_len) < 0) { + if (writer_write_bytes(writer, header_buf, + SAMPLE_HEADER_FIXED_SIZE + varint_len + 1) < 0) { return -1; } diff --git a/Modules/_remote_debugging/threads.c b/Modules/_remote_debugging/threads.c index 07f8148d7c9..e303c667ea0 100644 --- a/Modules/_remote_debugging/threads.c +++ b/Modules/_remote_debugging/threads.c @@ -314,19 +314,6 @@ unwind_stack_for_thread( long tid = GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.native_thread_id); - // Read GC collecting state from the interpreter (before any skip checks) - uintptr_t interp_addr = GET_MEMBER(uintptr_t, ts, unwinder->debug_offsets.thread_state.interp); - - // Read the GC runtime state from the interpreter state - uintptr_t gc_addr = interp_addr + unwinder->debug_offsets.interpreter_state.gc; - char gc_state[SIZEOF_GC_RUNTIME_STATE]; - if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, gc_addr, unwinder->debug_offsets.gc.size, gc_state) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read GC state"); - goto error; - } - STATS_INC(unwinder, memory_reads); - STATS_ADD(unwinder, memory_bytes_read, unwinder->debug_offsets.gc.size); - // Calculate thread status using flags (always) int status_flags = 0; diff --git a/Modules/_struct.c b/Modules/_struct.c index f1a6ecada97..3a970d99bb3 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -1,7 +1,7 @@ /* struct module -- pack values into and (out of) bytes objects */ /* New version supporting byte order, alignment and size options, - character strings, and unsigned numbers */ + byte strings, and unsigned numbers */ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 @@ -2323,7 +2323,7 @@ Struct_iter_unpack_impl(PyStructObject *self, PyObject *buffer) * * Takes a struct object, a tuple of arguments, and offset in that tuple of * argument for where to start processing the arguments for packing, and a - * character buffer for writing the packed string. The caller must insure + * character buffer for writing the packed data. The caller must ensure * that the buffer may contain the required length for packing the arguments. * 0 is returned on success, 1 is returned if there is an error. * @@ -2800,8 +2800,8 @@ static struct PyMethodDef module_functions[] = { PyDoc_STRVAR(module_doc, "Functions to convert between Python values and C structs.\n\ -Python bytes objects are used to hold the data representing the C struct\n\ -and also as format strings (explained below) to describe the layout of data\n\ +Python bytes objects are used to hold the data representing the C struct.\n\ +The format string (explained below) describes the layout of data\n\ in the C struct.\n\ \n\ The optional first format char indicates byte order, size and alignment:\n\ @@ -2811,19 +2811,18 @@ The optional first format char indicates byte order, size and alignment:\n\ >: big-endian, std. size & alignment\n\ !: same as >\n\ \n\ -The remaining chars indicate types of args and must match exactly;\n\ +The remaining characters indicate types of args and must match exactly;\n\ these can be preceded by a decimal repeat count:\n\ - x: pad byte (no data); c:char; b:signed byte; B:unsigned byte;\n\ - ?:_Bool; h:short; H:unsigned short; i:int; I:unsigned int;\n\ - l:long; L:unsigned long; f:float; d:double; e:half-float.\n\ - F:float complex; D:double complex.\n\ + x: pad byte (no data); c: char; b: signed byte; B: unsigned byte;\n\ + ?: _Bool; h: short; H: unsigned short; i: int; I: unsigned int;\n\ + l: long; L: unsigned long; q: long long; Q: unsigned long long;\n\ + f: float; d: double; e: half-float;\n\ + F: float complex; D: double complex.\n\ Special cases (preceding decimal count indicates length):\n\ - s:string (array of char); p: pascal string (with count byte).\n\ + s: byte string (array of char); p: Pascal string (with count byte).\n\ Special cases (only available in native format):\n\ - n:ssize_t; N:size_t;\n\ - P:an integer type that is wide enough to hold a pointer.\n\ -Special case (not in native mode unless 'long long' in platform C):\n\ - q:long long; Q:unsigned long long\n\ + n: ssize_t; N: size_t;\n\ + P: an integer type that is wide enough to hold a pointer.\n\ Whitespace between formats is ignored.\n\ \n\ The variable struct.error is an exception raised on errors.\n"); diff --git a/Modules/_testcapi/object.c b/Modules/_testcapi/object.c index 9160005e006..6e5c8dcbb72 100644 --- a/Modules/_testcapi/object.c +++ b/Modules/_testcapi/object.c @@ -555,6 +555,23 @@ pyobject_dump(PyObject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject * +pysentinel_new(PyObject *self, PyObject *args) +{ + const char *name; + const char *module_name = NULL; + if (!PyArg_ParseTuple(args, "s|s", &name, &module_name)) { + return NULL; + } + return PySentinel_New(name, module_name); +} + +static PyObject * +pysentinel_check(PyObject *self, PyObject *obj) +{ + return PyBool_FromLong(PySentinel_Check(obj)); +} + static PyMethodDef test_methods[] = { {"call_pyobject_print", call_pyobject_print, METH_VARARGS}, @@ -585,6 +602,8 @@ static PyMethodDef test_methods[] = { {"clear_managed_dict", clear_managed_dict, METH_O, NULL}, {"is_uniquely_referenced", is_uniquely_referenced, METH_O}, {"pyobject_dump", pyobject_dump, METH_VARARGS}, + {"pysentinel_new", pysentinel_new, METH_VARARGS}, + {"pysentinel_check", pysentinel_check, METH_O}, {NULL}, }; diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index deac8570fe3..a07675bb66d 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1081,13 +1081,17 @@ _testinternalcapi.compiler_codegen -> object compile_mode: int = 0 Apply compiler code generation to an AST. + +Return (instruction_sequence, metadata). metadata maps "argcount", +"posonlyargcount", "kwonlyargcount" to ints and "consts" to the list of +constants in LOAD_CONST index order (for use with optimize_cfg). [clinic start generated code]*/ static PyObject * _testinternalcapi_compiler_codegen_impl(PyObject *module, PyObject *ast, PyObject *filename, int optimize, int compile_mode) -/*[clinic end generated code: output=40a68f6e13951cc8 input=a0e00784f1517cd7]*/ +/*[clinic end generated code: output=40a68f6e13951cc8 input=e0c65e5c80efe30e]*/ { PyCompilerFlags *flags = NULL; return _PyCompile_CodeGen(ast, filename, flags, optimize, compile_mode); @@ -1103,12 +1107,15 @@ _testinternalcapi.optimize_cfg -> object nlocals: int Apply compiler optimizations to an instruction list. + +consts must be a list aligned with LOAD_CONST opargs (the "consts" entry +from the metadata dict returned by compiler_codegen for the same unit). [clinic start generated code]*/ static PyObject * _testinternalcapi_optimize_cfg_impl(PyObject *module, PyObject *instructions, PyObject *consts, int nlocals) -/*[clinic end generated code: output=57c53c3a3dfd1df0 input=6a96d1926d58d7e5]*/ +/*[clinic end generated code: output=57c53c3a3dfd1df0 input=905c3d935e063b27]*/ { return _PyCompile_OptimizeCfg(instructions, consts, nlocals); } @@ -1249,16 +1256,22 @@ write_perf_map_entry(PyObject *self, PyObject *args) { PyObject *code_addr_v; const void *code_addr; - unsigned int code_size; + PyObject *code_size_s; + size_t code_size; const char *entry_name; - if (!PyArg_ParseTuple(args, "OIs", &code_addr_v, &code_size, &entry_name)) + if (!PyArg_ParseTuple(args, "OOs", &code_addr_v, &code_size_s, &entry_name)) return NULL; code_addr = PyLong_AsVoidPtr(code_addr_v); if (code_addr == NULL) { return NULL; } + code_size = PyLong_AsSize_t(code_size_s); + if (code_size == (size_t)-1 && PyErr_Occurred()) { + return NULL; + } + int ret = PyUnstable_WritePerfMapEntry(code_addr, code_size, entry_name); if (ret < 0) { PyErr_SetFromErrno(PyExc_OSError); @@ -2718,7 +2731,8 @@ has_deferred_refcount(PyObject *self, PyObject *op) static PyObject * get_tracked_heap_size(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return PyLong_FromInt64(PyInterpreterState_Get()->gc.heap_size); + // Generational GC doesn't track heap_size, return -1. + return PyLong_FromInt64(-1); } static PyObject * diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index cd579491e4c..8897854078b 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -5913,7 +5913,7 @@ int og_oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; if (_PyJit_EnterExecutorShouldStopTracing(og_opcode)) { - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + if (_PyOpcode_Caches[_PyOpcode_Deopt[og_opcode]]) { PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } opcode = og_opcode; @@ -12500,7 +12500,10 @@ tracer->prev_state.instr_frame = frame; tracer->prev_state.instr_oparg = oparg; tracer->prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + // Branch opcodes use the cache for branch history, not + // specialization counters. Don't reset it. + && !IS_CONDITIONAL_JUMP_OPCODE(opcode)) { (&next_instr[1])->counter = trigger_backoff_counter(); } const _PyOpcodeRecordEntry *record_entry = &_PyOpcode_RecordEntries[opcode]; diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index f90bc9c5ab5..8a3cd182ab1 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -74,7 +74,7 @@ zstd_contentsize_converter(PyObject *size, unsigned long long *p) if (PyErr_ExceptionMatches(PyExc_OverflowError)) { PyErr_Format(PyExc_ValueError, "size argument should be a positive int less " - "than %ull", ZSTD_CONTENTSIZE_ERROR); + "than %llu", ZSTD_CONTENTSIZE_ERROR); return 0; } return 0; @@ -83,7 +83,7 @@ zstd_contentsize_converter(PyObject *size, unsigned long long *p) *p = ZSTD_CONTENTSIZE_ERROR; PyErr_Format(PyExc_ValueError, "size argument should be a positive int less " - "than %ull", ZSTD_CONTENTSIZE_ERROR); + "than %llu", ZSTD_CONTENTSIZE_ERROR); return 0; } *p = pledged_size; diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index 0186ee92f5b..46682b483ad 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -111,6 +111,7 @@ _zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options) int key_v = PyLong_AsInt(key); Py_DECREF(key); if (key_v == -1 && PyErr_Occurred()) { + Py_DECREF(value); return -1; } diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 064e4c1ab3a..b614053f61b 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -3075,8 +3075,10 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) len = 0; a = newarrayobject(type, len, descr); - if (a == NULL) + if (a == NULL) { + Py_XDECREF(it); return NULL; + } if (len > 0 && !array_Check(initial, state)) { Py_ssize_t i; @@ -3085,11 +3087,13 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PySequence_GetItem(initial, i); if (v == NULL) { Py_DECREF(a); + Py_XDECREF(it); return NULL; } if (setarrayitem(a, i, v) != 0) { Py_DECREF(v); Py_DECREF(a); + Py_XDECREF(it); return NULL; } Py_DECREF(v); @@ -3101,6 +3105,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) v = array_array_frombytes((PyObject *)a, initial); if (v == NULL) { Py_DECREF(a); + Py_XDECREF(it); return NULL; } Py_DECREF(v); @@ -3111,6 +3116,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) wchar_t *ustr = PyUnicode_AsWideCharString(initial, &n); if (ustr == NULL) { Py_DECREF(a); + Py_XDECREF(it); return NULL; } @@ -3131,6 +3137,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_UCS4 *ustr = PyUnicode_AsUCS4Copy(initial); if (ustr == NULL) { Py_DECREF(a); + Py_XDECREF(it); return NULL; } @@ -3158,6 +3165,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return a; } } + Py_XDECREF(it); PyErr_SetString(PyExc_ValueError, "bad typecode (must be b, B, u, w, h, H, i, I, l, L, q, Q, f, d, F, D, Zd or Zf)"); return NULL; diff --git a/Modules/binascii.c b/Modules/binascii.c index b80bfbfffe4..7e6e9655f8d 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -244,6 +244,9 @@ static const _Py_ALIGNED_DEF(64, unsigned char) table_b2a_base85_a85[] = #define BASE85_A85_Z 0x00000000 #define BASE85_A85_Y 0x20202020 +/* 85**0 through 85**4, used for canonical encoding checks. */ +static const uint32_t pow85[] = {1, 85, 7225, 614125, 52200625}; + static const _Py_ALIGNED_DEF(64, unsigned char) table_a2b_base32[] = { -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, @@ -729,6 +732,8 @@ binascii.a2b_base64 ignorechars: Py_buffer = NULL A byte string containing characters to ignore from the input when strict_mode is true. + canonical: bool = False + When set to true, reject non-zero padding bits per RFC 4648 section 3.5. Decode a line of base64 data. [clinic start generated code]*/ @@ -736,8 +741,8 @@ Decode a line of base64 data. static PyObject * binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode, int padded, PyBytesObject *alphabet, - Py_buffer *ignorechars) -/*[clinic end generated code: output=525d840a299ff132 input=74a53dd3b23474b3]*/ + Py_buffer *ignorechars, int canonical) +/*[clinic end generated code: output=77c46dcbf4239527 input=c99096d071deeec8]*/ { assert(data->len >= 0); @@ -909,6 +914,16 @@ binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode, goto error_end; } + /* https://datatracker.ietf.org/doc/html/rfc4648.html#section-3.5 + * Decoders MAY reject non-zero padding bits. */ + if (canonical && leftchar != 0) { + state = get_binascii_state(module); + if (state) { + PyErr_SetString(state->Error, "Non-zero padding bits"); + } + goto error_end; + } + Py_XDECREF(table_obj); return PyBytesWriter_FinishWithPointer(writer, bin_data); @@ -1037,14 +1052,16 @@ binascii.a2b_ascii85 Expect data to be wrapped in '<~' and '~>' as in Adobe Ascii85. ignorechars: Py_buffer = b'' A byte string containing characters to ignore from the input. + canonical: bool = False + When set to true, reject non-canonical encodings. Decode Ascii85 data. [clinic start generated code]*/ static PyObject * binascii_a2b_ascii85_impl(PyObject *module, Py_buffer *data, int foldspaces, - int adobe, Py_buffer *ignorechars) -/*[clinic end generated code: output=599aa3e41095a651 input=f39abd11eab4bac0]*/ + int adobe, Py_buffer *ignorechars, int canonical) +/*[clinic end generated code: output=09b35f1eac531357 input=dd050604ed30199e]*/ { const unsigned char *ascii_data = data->buf; Py_ssize_t ascii_len = data->len; @@ -1107,6 +1124,7 @@ binascii_a2b_ascii85_impl(PyObject *module, Py_buffer *data, int foldspaces, uint32_t leftchar = 0; int group_pos = 0; + int from_z = 0; /* true when current group came from 'z' shorthand */ for (; ascii_len > 0 || group_pos != 0; ascii_len--, ascii_data++) { /* Shift (in radix-85) data or padding into our buffer. */ unsigned char this_digit; @@ -1142,6 +1160,7 @@ binascii_a2b_ascii85_impl(PyObject *module, Py_buffer *data, int foldspaces, goto error; } leftchar = this_ch == 'y' ? BASE85_A85_Y : BASE85_A85_Z; + from_z = (this_ch == 'z'); group_pos = 5; } else if (!ignorechar(this_ch, ignorechars, ignorecache)) { @@ -1159,11 +1178,62 @@ binascii_a2b_ascii85_impl(PyObject *module, Py_buffer *data, int foldspaces, } /* Write current chunk. */ - Py_ssize_t chunk_len = ascii_len < 1 ? 3 + ascii_len : 4; - for (Py_ssize_t i = 0; i < chunk_len; i++) { + int chunk_len = ascii_len < 1 ? 3 + (int)ascii_len : 4; + + /* A final partial 5-tuple containing only one character is an + * encoding violation per the PLRM spec; reject unconditionally. */ + if (chunk_len == 0) { + state = get_binascii_state(module); + if (state != NULL) { + PyErr_SetString(state->Error, + "Incomplete Ascii85 group"); + } + goto error; + } + + for (int i = 0; i < chunk_len; i++) { *bin_data++ = (leftchar >> (24 - 8 * i)) & 0xff; } + if (canonical) { + /* The PLRM spec requires all-zero groups to use the 'z' + * abbreviation. Reject '!!!!!' (five zero digits). */ + if (chunk_len == 4 && leftchar == 0 && !from_z) { + state = get_binascii_state(module); + if (state != NULL) { + PyErr_SetString(state->Error, + "Non-canonical encoding, " + "use 'z' for all-zero groups"); + } + goto error; + } + /* Reject non-canonical partial groups. + * + * A partial group of N chars (2-4) encodes N-1 bytes. + * The decoder pads missing chars with digit 84 (the max). + * The encoder produces the unique N chars for those bytes + * by zero-padding the bytes to a uint32 and taking the + * leading N base-85 digits. Two encodings are equivalent + * iff they yield the same quotient when divided by + * 85**(5-N). */ + if (chunk_len < 4) { + int n_pad = 4 - chunk_len; + uint32_t canonical_top = + (leftchar >> (n_pad * 8)) << (n_pad * 8); + if (canonical_top / pow85[n_pad] + != leftchar / pow85[n_pad]) + { + state = get_binascii_state(module); + if (state != NULL) { + PyErr_SetString(state->Error, + "Non-zero padding bits"); + } + goto error; + } + } + } + + from_z = 0; group_pos = 0; leftchar = 0; } @@ -1315,14 +1385,17 @@ binascii.a2b_base85 alphabet: PyBytesObject(c_default="NULL") = BASE85_ALPHABET ignorechars: Py_buffer = b'' A byte string containing characters to ignore from the input. + canonical: bool = False + When set to true, reject non-canonical encodings. Decode a line of Base85 data. [clinic start generated code]*/ static PyObject * binascii_a2b_base85_impl(PyObject *module, Py_buffer *data, - PyBytesObject *alphabet, Py_buffer *ignorechars) -/*[clinic end generated code: output=6a8d6eae798818d7 input=04d72a319712bdf3]*/ + PyBytesObject *alphabet, Py_buffer *ignorechars, + int canonical) +/*[clinic end generated code: output=90dfef0c6b51e5f3 input=2819dc8aeffee5a2]*/ { const unsigned char *ascii_data = data->buf; Py_ssize_t ascii_len = data->len; @@ -1403,11 +1476,41 @@ binascii_a2b_base85_impl(PyObject *module, Py_buffer *data, } /* Write current chunk. */ - Py_ssize_t chunk_len = ascii_len < 1 ? 3 + ascii_len : 4; - for (Py_ssize_t i = 0; i < chunk_len; i++) { + int chunk_len = ascii_len < 1 ? 3 + (int)ascii_len : 4; + + /* A 1-char final group is an encoding violation (no conforming + * encoder produces it); reject unconditionally. */ + if (chunk_len == 0) { + state = get_binascii_state(module); + if (state != NULL) { + PyErr_SetString(state->Error, + "Incomplete Base85 group"); + } + goto error; + } + + for (int i = 0; i < chunk_len; i++) { *bin_data++ = (leftchar >> (24 - 8 * i)) & 0xff; } + /* Reject non-canonical encodings in the final group. + * See the comment in a2b_ascii85 for the full explanation. */ + if (canonical && chunk_len < 4) { + int n_pad = 4 - chunk_len; + uint32_t canonical_top = + (leftchar >> (n_pad * 8)) << (n_pad * 8); + if (canonical_top / pow85[n_pad] + != leftchar / pow85[n_pad]) + { + state = get_binascii_state(module); + if (state != NULL) { + PyErr_SetString(state->Error, + "Non-zero padding bits"); + } + goto error; + } + } + group_pos = 0; leftchar = 0; } @@ -1535,14 +1638,17 @@ binascii.a2b_base32 alphabet: PyBytesObject(c_default="NULL") = BASE32_ALPHABET ignorechars: Py_buffer = b'' A byte string containing characters to ignore from the input. + canonical: bool = False + When set to true, reject non-zero padding bits per RFC 4648 section 3.5. Decode a line of base32 data. [clinic start generated code]*/ static PyObject * binascii_a2b_base32_impl(PyObject *module, Py_buffer *data, int padded, - PyBytesObject *alphabet, Py_buffer *ignorechars) -/*[clinic end generated code: output=7dbbaa816d956b1c input=07a3721acdf9b688]*/ + PyBytesObject *alphabet, Py_buffer *ignorechars, + int canonical) +/*[clinic end generated code: output=bc70f2bb6001fb55 input=5bfe6d1ea2f30e3b]*/ { const unsigned char *ascii_data = data->buf; Py_ssize_t ascii_len = data->len; @@ -1723,6 +1829,16 @@ binascii_a2b_base32_impl(PyObject *module, Py_buffer *data, int padded, goto error; } + /* https://datatracker.ietf.org/doc/html/rfc4648.html#section-3.5 + * Decoders MAY reject non-zero padding bits. */ + if (canonical && leftchar != 0) { + state = get_binascii_state(module); + if (state) { + PyErr_SetString(state->Error, "Non-zero padding bits"); + } + goto error; + } + Py_XDECREF(table_obj); return PyBytesWriter_FinishWithPointer(writer, bin_data); diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index 21f4ee3201e..85edc6fbb58 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -92,7 +92,11 @@ PyDoc_STRVAR(_testinternalcapi_compiler_codegen__doc__, "compiler_codegen($module, /, ast, filename, optimize, compile_mode=0)\n" "--\n" "\n" -"Apply compiler code generation to an AST."); +"Apply compiler code generation to an AST.\n" +"\n" +"Return (instruction_sequence, metadata). metadata maps \"argcount\",\n" +"\"posonlyargcount\", \"kwonlyargcount\" to ints and \"consts\" to the list of\n" +"constants in LOAD_CONST index order (for use with optimize_cfg)."); #define _TESTINTERNALCAPI_COMPILER_CODEGEN_METHODDEF \ {"compiler_codegen", _PyCFunction_CAST(_testinternalcapi_compiler_codegen), METH_FASTCALL|METH_KEYWORDS, _testinternalcapi_compiler_codegen__doc__}, @@ -169,7 +173,10 @@ PyDoc_STRVAR(_testinternalcapi_optimize_cfg__doc__, "optimize_cfg($module, /, instructions, consts, nlocals)\n" "--\n" "\n" -"Apply compiler optimizations to an instruction list."); +"Apply compiler optimizations to an instruction list.\n" +"\n" +"consts must be a list aligned with LOAD_CONST opargs (the \"consts\" entry\n" +"from the metadata dict returned by compiler_codegen for the same unit)."); #define _TESTINTERNALCAPI_OPTIMIZE_CFG_METHODDEF \ {"optimize_cfg", _PyCFunction_CAST(_testinternalcapi_optimize_cfg), METH_FASTCALL|METH_KEYWORDS, _testinternalcapi_optimize_cfg__doc__}, @@ -392,4 +399,4 @@ get_next_dict_keys_version(PyObject *module, PyObject *Py_UNUSED(ignored)) { return get_next_dict_keys_version_impl(module); } -/*[clinic end generated code: output=fbd8b7e0cae8bac7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ecb5d7ac85b153fa input=a9049054013a1b77]*/ diff --git a/Modules/clinic/binascii.c.h b/Modules/clinic/binascii.c.h index 0a2d33c428d..ed695758ef9 100644 --- a/Modules/clinic/binascii.c.h +++ b/Modules/clinic/binascii.c.h @@ -119,7 +119,7 @@ exit: PyDoc_STRVAR(binascii_a2b_base64__doc__, "a2b_base64($module, data, /, *, strict_mode=,\n" " padded=True, alphabet=BASE64_ALPHABET,\n" -" ignorechars=)\n" +" ignorechars=, canonical=False)\n" "--\n" "\n" "Decode a line of base64 data.\n" @@ -132,7 +132,9 @@ PyDoc_STRVAR(binascii_a2b_base64__doc__, " When set to false, padding in input is not required.\n" " ignorechars\n" " A byte string containing characters to ignore from the input when\n" -" strict_mode is true."); +" strict_mode is true.\n" +" canonical\n" +" When set to true, reject non-zero padding bits per RFC 4648 section 3.5."); #define BINASCII_A2B_BASE64_METHODDEF \ {"a2b_base64", _PyCFunction_CAST(binascii_a2b_base64), METH_FASTCALL|METH_KEYWORDS, binascii_a2b_base64__doc__}, @@ -140,7 +142,7 @@ PyDoc_STRVAR(binascii_a2b_base64__doc__, static PyObject * binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode, int padded, PyBytesObject *alphabet, - Py_buffer *ignorechars); + Py_buffer *ignorechars, int canonical); static PyObject * binascii_a2b_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -148,7 +150,7 @@ binascii_a2b_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 4 + #define NUM_KEYWORDS 5 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -157,7 +159,7 @@ binascii_a2b_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(strict_mode), &_Py_ID(padded), &_Py_ID(alphabet), &_Py_ID(ignorechars), }, + .ob_item = { &_Py_ID(strict_mode), &_Py_ID(padded), &_Py_ID(alphabet), &_Py_ID(ignorechars), &_Py_ID(canonical), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -166,20 +168,21 @@ binascii_a2b_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"", "strict_mode", "padded", "alphabet", "ignorechars", NULL}; + static const char * const _keywords[] = {"", "strict_mode", "padded", "alphabet", "ignorechars", "canonical", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "a2b_base64", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[5]; + PyObject *argsbuf[6]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; Py_buffer data = {NULL, NULL}; int strict_mode = -1; int padded = 1; PyBytesObject *alphabet = NULL; Py_buffer ignorechars = {NULL, NULL}; + int canonical = 0; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -220,11 +223,20 @@ binascii_a2b_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P goto skip_optional_kwonly; } } - if (PyObject_GetBuffer(args[4], &ignorechars, PyBUF_SIMPLE) != 0) { + if (args[4]) { + if (PyObject_GetBuffer(args[4], &ignorechars, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + canonical = PyObject_IsTrue(args[5]); + if (canonical < 0) { goto exit; } skip_optional_kwonly: - return_value = binascii_a2b_base64_impl(module, &data, strict_mode, padded, alphabet, &ignorechars); + return_value = binascii_a2b_base64_impl(module, &data, strict_mode, padded, alphabet, &ignorechars, canonical); exit: /* Cleanup for data */ @@ -352,7 +364,7 @@ exit: PyDoc_STRVAR(binascii_a2b_ascii85__doc__, "a2b_ascii85($module, data, /, *, foldspaces=False, adobe=False,\n" -" ignorechars=b\'\')\n" +" ignorechars=b\'\', canonical=False)\n" "--\n" "\n" "Decode Ascii85 data.\n" @@ -362,14 +374,16 @@ PyDoc_STRVAR(binascii_a2b_ascii85__doc__, " adobe\n" " Expect data to be wrapped in \'<~\' and \'~>\' as in Adobe Ascii85.\n" " ignorechars\n" -" A byte string containing characters to ignore from the input."); +" A byte string containing characters to ignore from the input.\n" +" canonical\n" +" When set to true, reject non-canonical encodings."); #define BINASCII_A2B_ASCII85_METHODDEF \ {"a2b_ascii85", _PyCFunction_CAST(binascii_a2b_ascii85), METH_FASTCALL|METH_KEYWORDS, binascii_a2b_ascii85__doc__}, static PyObject * binascii_a2b_ascii85_impl(PyObject *module, Py_buffer *data, int foldspaces, - int adobe, Py_buffer *ignorechars); + int adobe, Py_buffer *ignorechars, int canonical); static PyObject * binascii_a2b_ascii85(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -377,7 +391,7 @@ binascii_a2b_ascii85(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 3 + #define NUM_KEYWORDS 4 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -386,7 +400,7 @@ binascii_a2b_ascii85(PyObject *module, PyObject *const *args, Py_ssize_t nargs, } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(foldspaces), &_Py_ID(adobe), &_Py_ID(ignorechars), }, + .ob_item = { &_Py_ID(foldspaces), &_Py_ID(adobe), &_Py_ID(ignorechars), &_Py_ID(canonical), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -395,19 +409,20 @@ binascii_a2b_ascii85(PyObject *module, PyObject *const *args, Py_ssize_t nargs, # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"", "foldspaces", "adobe", "ignorechars", NULL}; + static const char * const _keywords[] = {"", "foldspaces", "adobe", "ignorechars", "canonical", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "a2b_ascii85", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[4]; + PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; Py_buffer data = {NULL, NULL}; int foldspaces = 0; int adobe = 0; Py_buffer ignorechars = {.buf = "", .obj = NULL, .len = 0}; + int canonical = 0; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -438,11 +453,20 @@ binascii_a2b_ascii85(PyObject *module, PyObject *const *args, Py_ssize_t nargs, goto skip_optional_kwonly; } } - if (PyObject_GetBuffer(args[3], &ignorechars, PyBUF_SIMPLE) != 0) { + if (args[3]) { + if (PyObject_GetBuffer(args[3], &ignorechars, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + canonical = PyObject_IsTrue(args[4]); + if (canonical < 0) { goto exit; } skip_optional_kwonly: - return_value = binascii_a2b_ascii85_impl(module, &data, foldspaces, adobe, &ignorechars); + return_value = binascii_a2b_ascii85_impl(module, &data, foldspaces, adobe, &ignorechars, canonical); exit: /* Cleanup for data */ @@ -573,20 +597,23 @@ exit: PyDoc_STRVAR(binascii_a2b_base85__doc__, "a2b_base85($module, data, /, *, alphabet=BASE85_ALPHABET,\n" -" ignorechars=b\'\')\n" +" ignorechars=b\'\', canonical=False)\n" "--\n" "\n" "Decode a line of Base85 data.\n" "\n" " ignorechars\n" -" A byte string containing characters to ignore from the input."); +" A byte string containing characters to ignore from the input.\n" +" canonical\n" +" When set to true, reject non-canonical encodings."); #define BINASCII_A2B_BASE85_METHODDEF \ {"a2b_base85", _PyCFunction_CAST(binascii_a2b_base85), METH_FASTCALL|METH_KEYWORDS, binascii_a2b_base85__doc__}, static PyObject * binascii_a2b_base85_impl(PyObject *module, Py_buffer *data, - PyBytesObject *alphabet, Py_buffer *ignorechars); + PyBytesObject *alphabet, Py_buffer *ignorechars, + int canonical); static PyObject * binascii_a2b_base85(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -594,7 +621,7 @@ binascii_a2b_base85(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -603,7 +630,7 @@ binascii_a2b_base85(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(alphabet), &_Py_ID(ignorechars), }, + .ob_item = { &_Py_ID(alphabet), &_Py_ID(ignorechars), &_Py_ID(canonical), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -612,18 +639,19 @@ binascii_a2b_base85(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"", "alphabet", "ignorechars", NULL}; + static const char * const _keywords[] = {"", "alphabet", "ignorechars", "canonical", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "a2b_base85", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; Py_buffer data = {NULL, NULL}; PyBytesObject *alphabet = NULL; Py_buffer ignorechars = {.buf = "", .obj = NULL, .len = 0}; + int canonical = 0; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -646,11 +674,20 @@ binascii_a2b_base85(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P goto skip_optional_kwonly; } } - if (PyObject_GetBuffer(args[2], &ignorechars, PyBUF_SIMPLE) != 0) { + if (args[2]) { + if (PyObject_GetBuffer(args[2], &ignorechars, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + canonical = PyObject_IsTrue(args[3]); + if (canonical < 0) { goto exit; } skip_optional_kwonly: - return_value = binascii_a2b_base85_impl(module, &data, alphabet, &ignorechars); + return_value = binascii_a2b_base85_impl(module, &data, alphabet, &ignorechars, canonical); exit: /* Cleanup for data */ @@ -768,7 +805,7 @@ exit: PyDoc_STRVAR(binascii_a2b_base32__doc__, "a2b_base32($module, data, /, *, padded=True, alphabet=BASE32_ALPHABET,\n" -" ignorechars=b\'\')\n" +" ignorechars=b\'\', canonical=False)\n" "--\n" "\n" "Decode a line of base32 data.\n" @@ -776,14 +813,17 @@ PyDoc_STRVAR(binascii_a2b_base32__doc__, " padded\n" " When set to false, padding in input is not required.\n" " ignorechars\n" -" A byte string containing characters to ignore from the input."); +" A byte string containing characters to ignore from the input.\n" +" canonical\n" +" When set to true, reject non-zero padding bits per RFC 4648 section 3.5."); #define BINASCII_A2B_BASE32_METHODDEF \ {"a2b_base32", _PyCFunction_CAST(binascii_a2b_base32), METH_FASTCALL|METH_KEYWORDS, binascii_a2b_base32__doc__}, static PyObject * binascii_a2b_base32_impl(PyObject *module, Py_buffer *data, int padded, - PyBytesObject *alphabet, Py_buffer *ignorechars); + PyBytesObject *alphabet, Py_buffer *ignorechars, + int canonical); static PyObject * binascii_a2b_base32(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -791,7 +831,7 @@ binascii_a2b_base32(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 3 + #define NUM_KEYWORDS 4 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -800,7 +840,7 @@ binascii_a2b_base32(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(padded), &_Py_ID(alphabet), &_Py_ID(ignorechars), }, + .ob_item = { &_Py_ID(padded), &_Py_ID(alphabet), &_Py_ID(ignorechars), &_Py_ID(canonical), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -809,19 +849,20 @@ binascii_a2b_base32(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"", "padded", "alphabet", "ignorechars", NULL}; + static const char * const _keywords[] = {"", "padded", "alphabet", "ignorechars", "canonical", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "a2b_base32", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[4]; + PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; Py_buffer data = {NULL, NULL}; int padded = 1; PyBytesObject *alphabet = NULL; Py_buffer ignorechars = {.buf = "", .obj = NULL, .len = 0}; + int canonical = 0; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -853,11 +894,20 @@ binascii_a2b_base32(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P goto skip_optional_kwonly; } } - if (PyObject_GetBuffer(args[3], &ignorechars, PyBUF_SIMPLE) != 0) { + if (args[3]) { + if (PyObject_GetBuffer(args[3], &ignorechars, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + canonical = PyObject_IsTrue(args[4]); + if (canonical < 0) { goto exit; } skip_optional_kwonly: - return_value = binascii_a2b_base32_impl(module, &data, padded, alphabet, &ignorechars); + return_value = binascii_a2b_base32_impl(module, &data, padded, alphabet, &ignorechars, canonical); exit: /* Cleanup for data */ @@ -1634,4 +1684,4 @@ exit: return return_value; } -/*[clinic end generated code: output=2acab1ceb0058b1a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b41544f39b0ef681 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/faulthandler.c.h b/Modules/clinic/faulthandler.c.h index de8280ce26b..e06cfdcfba2 100644 --- a/Modules/clinic/faulthandler.c.h +++ b/Modules/clinic/faulthandler.c.h @@ -6,23 +6,26 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_long.h" // _PyLong_UnsignedInt_Converter() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(faulthandler_dump_traceback_py__doc__, -"dump_traceback($module, /, file=sys.stderr, all_threads=True)\n" +"dump_traceback($module, /, file=sys.stderr, all_threads=True, *,\n" +" max_threads=100)\n" "--\n" "\n" "Dump the traceback of the current thread into file.\n" "\n" -"Dump the traceback of all threads if all_threads is true."); +"Dump the traceback of all threads if all_threads is true. max_threads\n" +"caps the number of threads dumped."); #define FAULTHANDLER_DUMP_TRACEBACK_PY_METHODDEF \ {"dump_traceback", _PyCFunction_CAST(faulthandler_dump_traceback_py), METH_FASTCALL|METH_KEYWORDS, faulthandler_dump_traceback_py__doc__}, static PyObject * faulthandler_dump_traceback_py_impl(PyObject *module, PyObject *file, - int all_threads); + int all_threads, Py_ssize_t max_threads); static PyObject * faulthandler_dump_traceback_py(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -30,7 +33,7 @@ faulthandler_dump_traceback_py(PyObject *module, PyObject *const *args, Py_ssize PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -39,7 +42,7 @@ faulthandler_dump_traceback_py(PyObject *module, PyObject *const *args, Py_ssize } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(file), &_Py_ID(all_threads), }, + .ob_item = { &_Py_ID(file), &_Py_ID(all_threads), &_Py_ID(max_threads), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -48,17 +51,18 @@ faulthandler_dump_traceback_py(PyObject *module, PyObject *const *args, Py_ssize # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"file", "all_threads", NULL}; + static const char * const _keywords[] = {"file", "all_threads", "max_threads", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "dump_traceback", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *file = NULL; int all_threads = 1; + Py_ssize_t max_threads = 100; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -74,12 +78,33 @@ faulthandler_dump_traceback_py(PyObject *module, PyObject *const *args, Py_ssize goto skip_optional_pos; } } - all_threads = PyObject_IsTrue(args[1]); - if (all_threads < 0) { - goto exit; + if (args[1]) { + all_threads = PyObject_IsTrue(args[1]); + if (all_threads < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } } skip_optional_pos: - return_value = faulthandler_dump_traceback_py_impl(module, file, all_threads); + if (!noptargs) { + goto skip_optional_kwonly; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[2]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + max_threads = ival; + } +skip_optional_kwonly: + return_value = faulthandler_dump_traceback_py_impl(module, file, all_threads, max_threads); exit: return return_value; @@ -149,7 +174,8 @@ exit: } PyDoc_STRVAR(faulthandler_py_enable__doc__, -"enable($module, /, file=sys.stderr, all_threads=True, c_stack=True)\n" +"enable($module, /, file=sys.stderr, all_threads=True, c_stack=True, *,\n" +" max_threads=100)\n" "--\n" "\n" "Enable the fault handler."); @@ -159,7 +185,8 @@ PyDoc_STRVAR(faulthandler_py_enable__doc__, static PyObject * faulthandler_py_enable_impl(PyObject *module, PyObject *file, - int all_threads, int c_stack); + int all_threads, int c_stack, + Py_ssize_t max_threads); static PyObject * faulthandler_py_enable(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -167,7 +194,7 @@ faulthandler_py_enable(PyObject *module, PyObject *const *args, Py_ssize_t nargs PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 3 + #define NUM_KEYWORDS 4 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -176,7 +203,7 @@ faulthandler_py_enable(PyObject *module, PyObject *const *args, Py_ssize_t nargs } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(file), &_Py_ID(all_threads), &_Py_ID(c_stack), }, + .ob_item = { &_Py_ID(file), &_Py_ID(all_threads), &_Py_ID(c_stack), &_Py_ID(max_threads), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -185,18 +212,19 @@ faulthandler_py_enable(PyObject *module, PyObject *const *args, Py_ssize_t nargs # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"file", "all_threads", "c_stack", NULL}; + static const char * const _keywords[] = {"file", "all_threads", "c_stack", "max_threads", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "enable", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *file = NULL; int all_threads = 1; int c_stack = 1; + Py_ssize_t max_threads = 100; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -221,12 +249,33 @@ faulthandler_py_enable(PyObject *module, PyObject *const *args, Py_ssize_t nargs goto skip_optional_pos; } } - c_stack = PyObject_IsTrue(args[2]); - if (c_stack < 0) { - goto exit; + if (args[2]) { + c_stack = PyObject_IsTrue(args[2]); + if (c_stack < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } } skip_optional_pos: - return_value = faulthandler_py_enable_impl(module, file, all_threads, c_stack); + if (!noptargs) { + goto skip_optional_kwonly; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[3]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + max_threads = ival; + } +skip_optional_kwonly: + return_value = faulthandler_py_enable_impl(module, file, all_threads, c_stack, max_threads); exit: return return_value; @@ -280,13 +329,14 @@ exit: PyDoc_STRVAR(faulthandler_dump_traceback_later__doc__, "dump_traceback_later($module, /, timeout, repeat=False,\n" -" file=sys.stderr, exit=False)\n" +" file=sys.stderr, exit=False, *, max_threads=100)\n" "--\n" "\n" "Dump the traceback of all threads in timeout seconds.\n" "\n" "If repeat is true, the tracebacks of all threads are dumped every timeout\n" -"seconds. If exit is true, call _exit(1) which is not safe."); +"seconds. If exit is true, call _exit(1) which is not safe. max_threads\n" +"caps the number of threads dumped."); #define FAULTHANDLER_DUMP_TRACEBACK_LATER_METHODDEF \ {"dump_traceback_later", _PyCFunction_CAST(faulthandler_dump_traceback_later), METH_FASTCALL|METH_KEYWORDS, faulthandler_dump_traceback_later__doc__}, @@ -294,7 +344,8 @@ PyDoc_STRVAR(faulthandler_dump_traceback_later__doc__, static PyObject * faulthandler_dump_traceback_later_impl(PyObject *module, PyObject *timeout_obj, int repeat, - PyObject *file, int exit); + PyObject *file, int exit, + Py_ssize_t max_threads); static PyObject * faulthandler_dump_traceback_later(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -302,7 +353,7 @@ faulthandler_dump_traceback_later(PyObject *module, PyObject *const *args, Py_ss PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 4 + #define NUM_KEYWORDS 5 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -311,7 +362,7 @@ faulthandler_dump_traceback_later(PyObject *module, PyObject *const *args, Py_ss } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(timeout), &_Py_ID(repeat), &_Py_ID(file), &_Py_ID(exit), }, + .ob_item = { &_Py_ID(timeout), &_Py_ID(repeat), &_Py_ID(file), &_Py_ID(exit), &_Py_ID(max_threads), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -320,19 +371,20 @@ faulthandler_dump_traceback_later(PyObject *module, PyObject *const *args, Py_ss # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"timeout", "repeat", "file", "exit", NULL}; + static const char * const _keywords[] = {"timeout", "repeat", "file", "exit", "max_threads", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "dump_traceback_later", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[4]; + PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *timeout_obj; int repeat = 0; PyObject *file = NULL; int exit = 0; + Py_ssize_t max_threads = 100; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -358,12 +410,33 @@ faulthandler_dump_traceback_later(PyObject *module, PyObject *const *args, Py_ss goto skip_optional_pos; } } - exit = PyObject_IsTrue(args[3]); - if (exit < 0) { - goto exit; + if (args[3]) { + exit = PyObject_IsTrue(args[3]); + if (exit < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } } skip_optional_pos: - return_value = faulthandler_dump_traceback_later_impl(module, timeout_obj, repeat, file, exit); + if (!noptargs) { + goto skip_optional_kwonly; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[4]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + max_threads = ival; + } +skip_optional_kwonly: + return_value = faulthandler_dump_traceback_later_impl(module, timeout_obj, repeat, file, exit, max_threads); exit: return return_value; @@ -391,20 +464,22 @@ faulthandler_cancel_dump_traceback_later_py(PyObject *module, PyObject *Py_UNUSE PyDoc_STRVAR(faulthandler_register_py__doc__, "register($module, /, signum, file=sys.stderr, all_threads=True,\n" -" chain=False)\n" +" chain=False, *, max_threads=100)\n" "--\n" "\n" "Register a handler for the signal \'signum\'.\n" "\n" "Dump the traceback of the current thread, or of all threads if\n" -"all_threads is True, into file."); +"all_threads is True, into file. max_threads caps the number of threads\n" +"dumped."); #define FAULTHANDLER_REGISTER_PY_METHODDEF \ {"register", _PyCFunction_CAST(faulthandler_register_py), METH_FASTCALL|METH_KEYWORDS, faulthandler_register_py__doc__}, static PyObject * faulthandler_register_py_impl(PyObject *module, int signum, PyObject *file, - int all_threads, int chain); + int all_threads, int chain, + Py_ssize_t max_threads); static PyObject * faulthandler_register_py(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -412,7 +487,7 @@ faulthandler_register_py(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 4 + #define NUM_KEYWORDS 5 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -421,7 +496,7 @@ faulthandler_register_py(PyObject *module, PyObject *const *args, Py_ssize_t nar } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(signum), &_Py_ID(file), &_Py_ID(all_threads), &_Py_ID(chain), }, + .ob_item = { &_Py_ID(signum), &_Py_ID(file), &_Py_ID(all_threads), &_Py_ID(chain), &_Py_ID(max_threads), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -430,19 +505,20 @@ faulthandler_register_py(PyObject *module, PyObject *const *args, Py_ssize_t nar # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"signum", "file", "all_threads", "chain", NULL}; + static const char * const _keywords[] = {"signum", "file", "all_threads", "chain", "max_threads", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "register", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[4]; + PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; int signum; PyObject *file = NULL; int all_threads = 1; int chain = 0; + Py_ssize_t max_threads = 100; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -471,12 +547,33 @@ faulthandler_register_py(PyObject *module, PyObject *const *args, Py_ssize_t nar goto skip_optional_pos; } } - chain = PyObject_IsTrue(args[3]); - if (chain < 0) { - goto exit; + if (args[3]) { + chain = PyObject_IsTrue(args[3]); + if (chain < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } } skip_optional_pos: - return_value = faulthandler_register_py_impl(module, signum, file, all_threads, chain); + if (!noptargs) { + goto skip_optional_kwonly; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[4]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + max_threads = ival; + } +skip_optional_kwonly: + return_value = faulthandler_register_py_impl(module, signum, file, all_threads, chain, max_threads); exit: return return_value; @@ -685,4 +782,4 @@ exit: #ifndef FAULTHANDLER__RAISE_EXCEPTION_METHODDEF #define FAULTHANDLER__RAISE_EXCEPTION_METHODDEF #endif /* !defined(FAULTHANDLER__RAISE_EXCEPTION_METHODDEF) */ -/*[clinic end generated code: output=31bf0149d0d02ccf input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2452d767c85130a6 input=a9049054013a1b77]*/ diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 18dbaebde29..79c609f19aa 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -45,6 +45,7 @@ #ifndef Expat_INCLUDED # define Expat_INCLUDED 1 +# include // for uint8_t # include # include "expat_external.h" @@ -917,10 +918,21 @@ XML_SetParamEntityParsing(XML_Parser parser, function behavior. This must be called before parsing is started. Returns 1 if successful, 0 when called after parsing has started. Note: If parser == NULL, the function will do nothing and return 0. + DEPRECATED since Expat 2.8.0. */ XMLPARSEAPI(int) XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt); +/* Sets the hash salt to use for internal hash calculations. + Helps in preventing DoS attacks based on predicting hash function behavior. + This must be called before parsing is started. + Returns XML_TRUE if successful, XML_FALSE when called after parsing has + started or when parser is NULL. + Added in Expat 2.8.0. +*/ +XMLPARSEAPI(XML_Bool) +XML_SetHashSalt16Bytes(XML_Parser parser, const uint8_t entropy[16]); + /* If XML_Parse or XML_ParseBuffer have returned XML_STATUS_ERROR, then XML_GetErrorCode returns information about the error. */ @@ -1081,8 +1093,8 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); See https://semver.org */ # define XML_MAJOR_VERSION 2 -# define XML_MINOR_VERSION 7 -# define XML_MICRO_VERSION 5 +# define XML_MINOR_VERSION 8 +# define XML_MICRO_VERSION 0 # ifdef __cplusplus } diff --git a/Modules/expat/expat_config.h b/Modules/expat/expat_config.h index 09d3161dbc0..70df73c8e00 100644 --- a/Modules/expat/expat_config.h +++ b/Modules/expat/expat_config.h @@ -22,5 +22,10 @@ // bpo-30947: Python uses best available entropy sources to // call XML_SetHashSalt(), expat entropy sources are not needed #define XML_POOR_ENTROPY 1 +#undef HAVE_ARC4RANDOM +#undef HAVE_ARC4RANDOM_BUF +#undef HAVE_GETENTROPY +#undef HAVE_GETRANDOM +#undef HAVE_SYSCALL_GETRANDOM #endif /* EXPAT_CONFIG_H */ diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h index cf4d445e68b..cc945c424e4 100644 --- a/Modules/expat/expat_external.h +++ b/Modules/expat/expat_external.h @@ -12,9 +12,10 @@ Copyright (c) 2001-2002 Greg Stein Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2016 Cristian Rodríguez - Copyright (c) 2016-2026 Sebastian Pipping + Copyright (c) 2016-2025 Sebastian Pipping Copyright (c) 2017 Rhodri James Copyright (c) 2018 Yury Gribov + Copyright (c) 2026 Matthew Fernandez Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -48,7 +49,7 @@ /* Expat tries very hard to make the API boundary very specifically defined. There are two macros defined to control this boundary; each of these can be defined before including this header to - achieve some different behavior, but doing so it not recommended or + achieve some different behavior, but doing so is not recommended or tested frequently. XMLCALL - The calling convention to use for all calls across the diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index 61266ebb772..420d4217a56 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -28,7 +28,7 @@ Copyright (c) 2002-2003 Fred L. Drake, Jr. Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2003 Greg Stein - Copyright (c) 2016-2025 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2018 Yury Gribov Copyright (c) 2019 David Loffredo Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow @@ -113,6 +113,7 @@ #if defined(_WIN32) \ && (! defined(__USE_MINGW_ANSI_STDIO) \ || (1 - __USE_MINGW_ANSI_STDIO - 1 == 0)) +# define EXPAT_FMT_LLX(midpart) "%" midpart "I64x" # define EXPAT_FMT_ULL(midpart) "%" midpart "I64u" # if defined(_WIN64) // Note: modifiers "td" and "zu" do not work for MinGW # define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "I64d" @@ -122,6 +123,7 @@ # define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" # endif #else +# define EXPAT_FMT_LLX(midpart) "%" midpart "llx" # define EXPAT_FMT_ULL(midpart) "%" midpart "llu" # if ! defined(ULONG_MAX) # error Compiler did not define ULONG_MAX for us diff --git a/Modules/expat/refresh.sh b/Modules/expat/refresh.sh index 779929fc6ed..774e0b89d94 100755 --- a/Modules/expat/refresh.sh +++ b/Modules/expat/refresh.sh @@ -12,9 +12,9 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. These values are used for verifying the SBOM, too. -expected_libexpat_tag="R_2_7_5" -expected_libexpat_version="2.7.5" -expected_libexpat_sha256="9931f9860d18e6cf72d183eb8f309bfb96196c00e1d40caa978e95bc9aa978b6" +expected_libexpat_tag="R_2_8_0" +expected_libexpat_version="2.8.0" +expected_libexpat_sha256="c7cec5f60ea3a42e7780781c6745255c19aa3dbfeeae58646b7132f88dc24780" expat_dir="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")" cd ${expat_dir} @@ -64,6 +64,18 @@ This may be due to source changes and will require updating this script" >&2 exit 1 fi +# Step 4: Skip the Windows rand_s entropy path in xmlparse.c when +# XML_POOR_ENTROPY is set. +sed -z -i 's|#if defined(_WIN32)\n# include "random_rand_s\.h"\n#endif /\* defined(_WIN32) \*/|#if defined(_WIN32) \&\& ! defined(XML_POOR_ENTROPY)\n# include "random_rand_s.h"\n#endif /* defined(_WIN32) \&\& ! defined(XML_POOR_ENTROPY) */|' xmlparse.c +sed -z -i 's|# ifdef _WIN32\n if (writeRandomBytes_rand_s|# if defined(_WIN32) \&\& ! defined(XML_POOR_ENTROPY)\n if (writeRandomBytes_rand_s|' xmlparse.c + +if ! grep -q '#if defined(_WIN32) && ! defined(XML_POOR_ENTROPY)' xmlparse.c; then + echo " +Error: rand_s gate not patched in xmlparse.c; +This may be due to source changes and will require updating this script" >&2 + exit 1 +fi + echo ' Updated! next steps: - Verify all is okay: diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index 0248b6651ff..e6842f3f0bf 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* 93c1caa66e2b0310459482516af05505b57c5cb7b96df777105308fc585c85d1 (2.7.5+) +/* a5d18f6a50f536615ac1c70304f87d94f99cc85a86b502188952440610ccf0f8 (2.8.0+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -41,10 +41,12 @@ Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow Copyright (c) 2024-2025 Berkay Eren Ürün Copyright (c) 2024 Hanno Böck - Copyright (c) 2025 Matthew Fernandez + Copyright (c) 2025-2026 Matthew Fernandez Copyright (c) 2025 Atrem Borovik Copyright (c) 2025 Alfonso Gregory Copyright (c) 2026 Rosen Penev + Copyright (c) 2026 Francesco Bertolaccini + Copyright (c) 2026 Christian Ng Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -84,28 +86,16 @@ # error XML_CONTEXT_BYTES must be defined, non-empty and >=0 (0 to disable, >=1 to enable; 1024 is a common default) #endif -#if defined(HAVE_SYSCALL_GETRANDOM) -# if ! defined(_GNU_SOURCE) -# define _GNU_SOURCE 1 /* syscall prototype */ -# endif -#endif - -#ifdef _WIN32 -/* force stdlib to define rand_s() */ -# if ! defined(_CRT_RAND_S) -# define _CRT_RAND_S -# endif -#endif - #include #include #include /* memset(), memcpy() */ #include #include /* INT_MAX, UINT_MAX */ #include /* fprintf */ -#include /* getenv, rand_s */ +#include /* getenv */ #include /* SIZE_MAX, uintptr_t */ #include /* isnan */ +#include #ifdef _WIN32 # define getpid GetCurrentProcessId @@ -125,26 +115,34 @@ #include "expat.h" #include "siphash.h" -#if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) -# if defined(HAVE_GETRANDOM) -# include /* getrandom */ -# else -# include /* syscall */ -# include /* SYS_getrandom */ -# endif -# if ! defined(GRND_NONBLOCK) -# define GRND_NONBLOCK 0x0001 -# endif /* defined(GRND_NONBLOCK) */ -#endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ +#if defined(HAVE_ARC4RANDOM) +# include "random_arc4random.h" +#endif /* defined(HAVE_ARC4RANDOM) */ -#if defined(_WIN32) && ! defined(LOAD_LIBRARY_SEARCH_SYSTEM32) -# define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800 -#endif +#if defined(HAVE_ARC4RANDOM_BUF) +# include "random_arc4random_buf.h" +#endif // defined(HAVE_ARC4RANDOM_BUF) + +#if defined(XML_DEV_URANDOM) +# include "random_dev_urandom.h" +#endif /* defined(XML_DEV_URANDOM) */ + +#if defined(HAVE_GETENTROPY) +# include "random_getentropy.h" +#endif // defined(HAVE_GETENTROPY) + +#if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) +# include "random_getrandom.h" +#endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ + +#if defined(_WIN32) && ! defined(XML_POOR_ENTROPY) +# include "random_rand_s.h" +#endif /* defined(_WIN32) && ! defined(XML_POOR_ENTROPY) */ #if ! defined(HAVE_GETRANDOM) && ! defined(HAVE_SYSCALL_GETRANDOM) \ && ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) \ - && ! defined(XML_DEV_URANDOM) && ! defined(_WIN32) \ - && ! defined(XML_POOR_ENTROPY) + && ! defined(HAVE_GETENTROPY) && ! defined(XML_DEV_URANDOM) \ + && ! defined(_WIN32) && ! defined(XML_POOR_ENTROPY) # error You do not have support for any sources of high quality entropy \ enabled. For end user security, that is probably not what you want. \ \ @@ -153,10 +151,11 @@ * Linux >=3.17 + glibc (including <2.25) (syscall SYS_getrandom): HAVE_SYSCALL_GETRANDOM, \ * BSD / macOS >=10.7 / glibc >=2.36 (arc4random_buf): HAVE_ARC4RANDOM_BUF, \ * BSD / macOS (including <10.7) / glibc >=2.36 (arc4random): HAVE_ARC4RANDOM, \ + * BSD / macOS >=10.12 / glibc >=2.25 (getentropy): HAVE_GETENTROPY, \ * Linux (including <3.17) / BSD / macOS (including <10.7) / Solaris >=8 (/dev/urandom): XML_DEV_URANDOM, \ * Windows >=Vista (rand_s): _WIN32. \ \ - If insist on not using any of these, bypass this error by defining \ + If you insist on not using any of these, bypass this error by defining \ XML_POOR_ENTROPY; you have been warned. \ \ If you have reasons to patch this detection code away or need changes \ @@ -604,7 +603,7 @@ static ELEMENT_TYPE *getElementType(XML_Parser parser, const ENCODING *enc, static XML_Char *copyString(const XML_Char *s, XML_Parser parser); -static unsigned long generate_hash_secret_salt(XML_Parser parser); +static struct sipkey generate_hash_secret_salt(void); static XML_Bool startParsing(XML_Parser parser); static XML_Parser parserCreate(const XML_Char *encodingName, @@ -777,7 +776,8 @@ struct XML_ParserStruct { XML_Bool m_useForeignDTD; enum XML_ParamEntityParsing m_paramEntityParsing; #endif - unsigned long m_hash_secret_salt; + struct sipkey m_hash_secret_salt_128; + XML_Bool m_hash_secret_salt_set; #if XML_GE == 1 ACCOUNTING m_accounting; MALLOC_TRACKER m_alloc_tracker; @@ -1036,135 +1036,6 @@ static const XML_Char implicitContext[] ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, '\0'}; -/* To avoid warnings about unused functions: */ -#if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) - -# if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) - -/* Obtain entropy on Linux 3.17+ */ -static int -writeRandomBytes_getrandom_nonblock(void *target, size_t count) { - int success = 0; /* full count bytes written? */ - size_t bytesWrittenTotal = 0; - const unsigned int getrandomFlags = GRND_NONBLOCK; - - do { - void *const currentTarget = (void *)((char *)target + bytesWrittenTotal); - const size_t bytesToWrite = count - bytesWrittenTotal; - - assert(bytesToWrite <= INT_MAX); - - const int bytesWrittenMore = -# if defined(HAVE_GETRANDOM) - (int)getrandom(currentTarget, bytesToWrite, getrandomFlags); -# else - (int)syscall(SYS_getrandom, currentTarget, bytesToWrite, - getrandomFlags); -# endif - - if (bytesWrittenMore > 0) { - bytesWrittenTotal += bytesWrittenMore; - if (bytesWrittenTotal >= count) - success = 1; - } - } while (! success && (errno == EINTR)); - - return success; -} - -# endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ - -# if ! defined(_WIN32) && defined(XML_DEV_URANDOM) - -/* Extract entropy from /dev/urandom */ -static int -writeRandomBytes_dev_urandom(void *target, size_t count) { - int success = 0; /* full count bytes written? */ - size_t bytesWrittenTotal = 0; - - const int fd = open("/dev/urandom", O_RDONLY); - if (fd < 0) { - return 0; - } - - do { - void *const currentTarget = (void *)((char *)target + bytesWrittenTotal); - const size_t bytesToWrite = count - bytesWrittenTotal; - - const ssize_t bytesWrittenMore = read(fd, currentTarget, bytesToWrite); - - if (bytesWrittenMore > 0) { - bytesWrittenTotal += bytesWrittenMore; - if (bytesWrittenTotal >= count) - success = 1; - } - } while (! success && (errno == EINTR)); - - close(fd); - return success; -} - -# endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */ - -#endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */ - -#if defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF) - -static void -writeRandomBytes_arc4random(void *target, size_t count) { - size_t bytesWrittenTotal = 0; - - while (bytesWrittenTotal < count) { - const uint32_t random32 = arc4random(); - size_t i = 0; - - for (; (i < sizeof(random32)) && (bytesWrittenTotal < count); - i++, bytesWrittenTotal++) { - const uint8_t random8 = (uint8_t)(random32 >> (i * 8)); - ((uint8_t *)target)[bytesWrittenTotal] = random8; - } - } -} - -#endif /* defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF) */ - -#ifdef _WIN32 - -/* Provide declaration of rand_s() for MinGW-32 (not 64, which has it), - as it didn't declare it in its header prior to version 5.3.0 of its - runtime package (mingwrt, containing stdlib.h). The upstream fix - was introduced at https://osdn.net/projects/mingw/ticket/39658 . */ -# if defined(__MINGW32__) && defined(__MINGW32_VERSION) \ - && __MINGW32_VERSION < 5003000L && ! defined(__MINGW64_VERSION_MAJOR) -__declspec(dllimport) int rand_s(unsigned int *); -# endif - -/* Obtain entropy on Windows using the rand_s() function which - * generates cryptographically secure random numbers. Internally it - * uses RtlGenRandom API which is present in Windows XP and later. - */ -static int -writeRandomBytes_rand_s(void *target, size_t count) { - size_t bytesWrittenTotal = 0; - - while (bytesWrittenTotal < count) { - unsigned int random32 = 0; - size_t i = 0; - - if (rand_s(&random32)) - return 0; /* failure */ - - for (; (i < sizeof(random32)) && (bytesWrittenTotal < count); - i++, bytesWrittenTotal++) { - const uint8_t random8 = (uint8_t)(random32 >> (i * 8)); - ((uint8_t *)target)[bytesWrittenTotal] = random8; - } - } - return 1; /* success */ -} - -#endif /* _WIN32 */ - #if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) static unsigned long @@ -1192,69 +1063,70 @@ gather_time_entropy(void) { #endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */ -static unsigned long -ENTROPY_DEBUG(const char *label, unsigned long entropy) { +static struct sipkey +ENTROPY_DEBUG(const char *label, struct sipkey entropy_128) { if (getDebugLevel("EXPAT_ENTROPY_DEBUG", 0) >= 1u) { - fprintf(stderr, "expat: Entropy: %s --> 0x%0*lx (%lu bytes)\n", label, - (int)sizeof(entropy) * 2, entropy, (unsigned long)sizeof(entropy)); + fprintf(stderr, + "expat: Entropy: %s --> [0x" EXPAT_FMT_LLX( + "016") ", 0x" EXPAT_FMT_LLX("016") "] (16 bytes)\n", + label, (unsigned long long)entropy_128.k[0], + (unsigned long long)entropy_128.k[1]); } - return entropy; + return entropy_128; } -static unsigned long -generate_hash_secret_salt(XML_Parser parser) { - unsigned long entropy; - (void)parser; +static struct sipkey +generate_hash_secret_salt(void) { + struct sipkey entropy; /* "Failproof" high quality providers: */ #if defined(HAVE_ARC4RANDOM_BUF) - arc4random_buf(&entropy, sizeof(entropy)); + writeRandomBytes_arc4random_buf(&entropy, sizeof(entropy)); return ENTROPY_DEBUG("arc4random_buf", entropy); #elif defined(HAVE_ARC4RANDOM) - writeRandomBytes_arc4random((void *)&entropy, sizeof(entropy)); + writeRandomBytes_arc4random(&entropy, sizeof(entropy)); return ENTROPY_DEBUG("arc4random", entropy); #else /* Try high quality providers first .. */ -# ifdef _WIN32 - if (writeRandomBytes_rand_s((void *)&entropy, sizeof(entropy))) { +# if defined(_WIN32) && ! defined(XML_POOR_ENTROPY) + if (writeRandomBytes_rand_s(&entropy, sizeof(entropy))) { return ENTROPY_DEBUG("rand_s", entropy); } +# elif defined(HAVE_GETENTROPY) + if (writeRandomBytes_getentropy(&entropy, sizeof(entropy))) { + return ENTROPY_DEBUG("getentropy", entropy); + } + errno = 0; # elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) - if (writeRandomBytes_getrandom_nonblock((void *)&entropy, sizeof(entropy))) { + if (writeRandomBytes_getrandom_nonblock(&entropy, sizeof(entropy))) { return ENTROPY_DEBUG("getrandom", entropy); } # endif # if ! defined(_WIN32) && defined(XML_DEV_URANDOM) - if (writeRandomBytes_dev_urandom((void *)&entropy, sizeof(entropy))) { + if (writeRandomBytes_dev_urandom(&entropy, sizeof(entropy))) { return ENTROPY_DEBUG("/dev/urandom", entropy); } # endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */ /* .. and self-made low quality for backup: */ - entropy = gather_time_entropy(); + entropy.k[0] = 0; + entropy.k[1] = gather_time_entropy(); # if ! defined(__wasi__) /* Process ID is 0 bits entropy if attacker has local access */ - entropy ^= getpid(); + entropy.k[1] ^= getpid(); # endif /* Factors are 2^31-1 and 2^61-1 (Mersenne primes M31 and M61) */ if (sizeof(unsigned long) == 4) { - return ENTROPY_DEBUG("fallback(4)", entropy * 2147483647); + entropy.k[1] *= 2147483647; + return ENTROPY_DEBUG("fallback(4)", entropy); } else { - return ENTROPY_DEBUG("fallback(8)", - entropy * (unsigned long)2305843009213693951ULL); + entropy.k[1] *= 2305843009213693951ULL; + return ENTROPY_DEBUG("fallback(8)", entropy); } #endif } -static unsigned long -get_hash_secret_salt(XML_Parser parser) { - const XML_Parser rootParser = getRootParserOf(parser, NULL); - assert(! rootParser->m_parentParser); - - return rootParser->m_hash_secret_salt; -} - static enum XML_Error callProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { @@ -1323,8 +1195,10 @@ callProcessor(XML_Parser parser, const char *start, const char *end, static XML_Bool /* only valid for root parser */ startParsing(XML_Parser parser) { /* hash functions must be initialized before setContext() is called */ - if (parser->m_hash_secret_salt == 0) - parser->m_hash_secret_salt = generate_hash_secret_salt(parser); + if (parser->m_hash_secret_salt_set != XML_TRUE) { + parser->m_hash_secret_salt_128 = generate_hash_secret_salt(); + parser->m_hash_secret_salt_set = XML_TRUE; + } if (parser->m_ns) { /* implicit context only set for root parser, since child parsers (i.e. external entity parsers) will inherit it @@ -1612,7 +1486,9 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_useForeignDTD = XML_FALSE; parser->m_paramEntityParsing = XML_PARAM_ENTITY_PARSING_NEVER; #endif - parser->m_hash_secret_salt = 0; + parser->m_hash_secret_salt_128.k[0] = 0; + parser->m_hash_secret_salt_128.k[1] = 0; + parser->m_hash_secret_salt_set = XML_FALSE; #if XML_GE == 1 memset(&parser->m_accounting, 0, sizeof(ACCOUNTING)); @@ -1779,7 +1655,8 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, from hash tables associated with either parser without us having to worry which hash secrets each table has. */ - unsigned long oldhash_secret_salt; + struct sipkey oldhash_secret_salt_128; + XML_Bool oldhash_secret_salt_set; XML_Bool oldReparseDeferralEnabled; /* Validate the oldParser parameter before we pull everything out of it */ @@ -1825,7 +1702,8 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, from hash tables associated with either parser without us having to worry which hash secrets each table has. */ - oldhash_secret_salt = parser->m_hash_secret_salt; + oldhash_secret_salt_128 = parser->m_hash_secret_salt_128; + oldhash_secret_salt_set = parser->m_hash_secret_salt_set; oldReparseDeferralEnabled = parser->m_reparseDeferralEnabled; #ifdef XML_DTD @@ -1880,7 +1758,8 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, parser->m_externalEntityRefHandlerArg = oldExternalEntityRefHandlerArg; parser->m_defaultExpandInternalEntities = oldDefaultExpandInternalEntities; parser->m_ns_triplets = oldns_triplets; - parser->m_hash_secret_salt = oldhash_secret_salt; + parser->m_hash_secret_salt_128 = oldhash_secret_salt_128; + parser->m_hash_secret_salt_set = oldhash_secret_salt_set; parser->m_reparseDeferralEnabled = oldReparseDeferralEnabled; parser->m_parentParser = oldParser; #ifdef XML_DTD @@ -2324,6 +2203,7 @@ XML_SetParamEntityParsing(XML_Parser parser, #endif } +// DEPRECATED since Expat 2.8.0. int XMLCALL XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) { if (parser == NULL) @@ -2335,10 +2215,46 @@ XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) { /* block after XML_Parse()/XML_ParseBuffer() has been called */ if (parserBusy(rootParser)) return 0; - rootParser->m_hash_secret_salt = hash_salt; + + rootParser->m_hash_secret_salt_128.k[0] = 0; + rootParser->m_hash_secret_salt_128.k[1] = hash_salt; + + if (hash_salt != 0) { // to remain backwards compatible + rootParser->m_hash_secret_salt_set = XML_TRUE; + + if (sizeof(unsigned long) == 4) + ENTROPY_DEBUG("explicit(4)", rootParser->m_hash_secret_salt_128); + else + ENTROPY_DEBUG("explicit(8)", rootParser->m_hash_secret_salt_128); + } + return 1; } +XML_Bool XMLCALL +XML_SetHashSalt16Bytes(XML_Parser parser, const uint8_t entropy[16]) { + if (parser == NULL) + return XML_FALSE; + + if (entropy == NULL) + return XML_FALSE; + + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(! rootParser->m_parentParser); + + /* block after XML_Parse()/XML_ParseBuffer() has been called */ + if (parserBusy(rootParser)) + return XML_FALSE; + + sip_tokey(&(rootParser->m_hash_secret_salt_128), entropy); + + rootParser->m_hash_secret_salt_set = XML_TRUE; + + ENTROPY_DEBUG("explicit(16)", rootParser->m_hash_secret_salt_128); + + return XML_TRUE; +} + enum XML_Status XMLCALL XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { if ((parser == NULL) || (len < 0) || ((s == NULL) && (len != 0))) { @@ -7842,8 +7758,10 @@ keylen(KEY s) { static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key) { - key->k[0] = 0; - key->k[1] = get_hash_secret_salt(parser); + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(! rootParser->m_parentParser); + + *key = rootParser->m_hash_secret_salt_128; } static unsigned long FASTCALL diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c index b1dfb456e5d..d56bee82dd2 100644 --- a/Modules/expat/xmlrole.c +++ b/Modules/expat/xmlrole.c @@ -12,7 +12,7 @@ Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2002-2003 Fred L. Drake, Jr. Copyright (c) 2005-2009 Steven Solie - Copyright (c) 2016-2026 Sebastian Pipping + Copyright (c) 2016-2023 Sebastian Pipping Copyright (c) 2017 Rhodri James Copyright (c) 2019 David Loffredo Copyright (c) 2021 Donghee Na diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index f6e5f742c92..32cd5f147e9 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -12,7 +12,7 @@ Copyright (c) 2002 Greg Stein Copyright (c) 2002-2016 Karl Waclawek Copyright (c) 2005-2009 Steven Solie - Copyright (c) 2016-2026 Sebastian Pipping + Copyright (c) 2016-2024 Sebastian Pipping Copyright (c) 2016 Pascal Cuoq Copyright (c) 2016 Don Lewis Copyright (c) 2017 Rhodri James diff --git a/Modules/expat/xmltok_ns.c b/Modules/expat/xmltok_ns.c index 1cd60de1e4f..810ca2c6d04 100644 --- a/Modules/expat/xmltok_ns.c +++ b/Modules/expat/xmltok_ns.c @@ -11,7 +11,7 @@ Copyright (c) 2002 Greg Stein Copyright (c) 2002 Fred L. Drake, Jr. Copyright (c) 2002-2006 Karl Waclawek - Copyright (c) 2017-2026 Sebastian Pipping + Copyright (c) 2017-2021 Sebastian Pipping Copyright (c) 2025 Alfonso Gregory Licensed under the MIT license: diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index bc7731c2588..923f6f5b56d 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -185,7 +185,8 @@ get_thread_state(void) static void faulthandler_dump_traceback(int fd, int all_threads, - PyInterpreterState *interp) + PyInterpreterState *interp, + Py_ssize_t max_threads) { static volatile int reentrant = 0; @@ -205,7 +206,7 @@ faulthandler_dump_traceback(int fd, int all_threads, PyThreadState *tstate = PyGILState_GetThisThreadState(); if (all_threads == 1) { - (void)_Py_DumpTracebackThreads(fd, NULL, tstate); + (void)_Py_DumpTracebackThreads(fd, NULL, tstate, max_threads); } else { if (all_threads == FT_IGNORE_ALL_THREADS) { @@ -243,16 +244,19 @@ faulthandler.dump_traceback as faulthandler_dump_traceback_py file: object(py_default="sys.stderr") = NULL all_threads: bool = True + * + max_threads: Py_ssize_t = 100 Dump the traceback of the current thread into file. -Dump the traceback of all threads if all_threads is true. +Dump the traceback of all threads if all_threads is true. max_threads +caps the number of threads dumped. [clinic start generated code]*/ static PyObject * faulthandler_dump_traceback_py_impl(PyObject *module, PyObject *file, - int all_threads) -/*[clinic end generated code: output=34efece0ca18314f input=b832ec55e27a7898]*/ + int all_threads, Py_ssize_t max_threads) +/*[clinic end generated code: output=ee1bbc2668e56e77 input=38630eb40e641de6]*/ { PyThreadState *tstate; const char *errmsg; @@ -273,7 +277,7 @@ faulthandler_dump_traceback_py_impl(PyObject *module, PyObject *file, /* gh-128400: Accessing other thread states while they're running * isn't safe if those threads are running. */ _PyEval_StopTheWorld(interp); - errmsg = _Py_DumpTracebackThreads(fd, NULL, tstate); + errmsg = _Py_DumpTracebackThreads(fd, NULL, tstate, max_threads); _PyEval_StartTheWorld(interp); if (errmsg != NULL) { PyErr_SetString(PyExc_RuntimeError, errmsg); @@ -409,7 +413,8 @@ faulthandler_fatal_error(int signum) } faulthandler_dump_traceback(fd, deduce_all_threads(), - fatal_error.interp); + fatal_error.interp, + fatal_error.max_threads); faulthandler_dump_c_stack(fd); _Py_DumpExtensionModules(fd, fatal_error.interp); @@ -485,7 +490,8 @@ faulthandler_exc_handler(struct _EXCEPTION_POINTERS *exc_info) } faulthandler_dump_traceback(fd, deduce_all_threads(), - fatal_error.interp); + fatal_error.interp, + fatal_error.max_threads); faulthandler_dump_c_stack(fd); /* call the next exception handler */ @@ -590,14 +596,17 @@ faulthandler.enable as faulthandler_py_enable file: object(py_default="sys.stderr") = NULL all_threads: bool = True c_stack: bool = True + * + max_threads: Py_ssize_t = 100 Enable the fault handler. [clinic start generated code]*/ static PyObject * faulthandler_py_enable_impl(PyObject *module, PyObject *file, - int all_threads, int c_stack) -/*[clinic end generated code: output=580d89b5eb62f1cb input=77277746a88b25ca]*/ + int all_threads, int c_stack, + Py_ssize_t max_threads) +/*[clinic end generated code: output=7ee655332317c47a input=e64759714f27b466]*/ { int fd; PyThreadState *tstate; @@ -617,6 +626,7 @@ faulthandler_py_enable_impl(PyObject *module, PyObject *file, fatal_error.all_threads = all_threads; fatal_error.interp = PyThreadState_GetInterpreter(tstate); fatal_error.c_stack = c_stack; + fatal_error.max_threads = max_threads; if (faulthandler_enable() < 0) { return NULL; @@ -703,7 +713,8 @@ faulthandler_thread(void *unused) (void)_Py_write_noraise(thread.fd, thread.header, (int)thread.header_len); - errmsg = _Py_DumpTracebackThreads(thread.fd, thread.interp, NULL); + errmsg = _Py_DumpTracebackThreads(thread.fd, thread.interp, NULL, + thread.max_threads); ok = (errmsg == NULL); if (thread.exit) @@ -777,18 +788,22 @@ faulthandler.dump_traceback_later repeat: bool = False file: object(py_default="sys.stderr") = NULL exit: bool = False + * + max_threads: Py_ssize_t = 100 Dump the traceback of all threads in timeout seconds. If repeat is true, the tracebacks of all threads are dumped every timeout -seconds. If exit is true, call _exit(1) which is not safe. +seconds. If exit is true, call _exit(1) which is not safe. max_threads +caps the number of threads dumped. [clinic start generated code]*/ static PyObject * faulthandler_dump_traceback_later_impl(PyObject *module, PyObject *timeout_obj, int repeat, - PyObject *file, int exit) -/*[clinic end generated code: output=a24d80d694d25ba2 input=fd005625ecc2ba9a]*/ + PyObject *file, int exit, + Py_ssize_t max_threads) +/*[clinic end generated code: output=543a0f3807113394 input=6836555ee157ddb4]*/ { PyTime_t timeout, timeout_us; int fd; @@ -861,6 +876,7 @@ faulthandler_dump_traceback_later_impl(PyObject *module, thread.exit = exit; thread.header = header; thread.header_len = header_len; + thread.max_threads = max_threads; /* Arm these locks to serve as events when released */ PyThread_acquire_lock(thread.running, 1); @@ -945,7 +961,8 @@ faulthandler_user(int signum) if (!user->enabled) return; - faulthandler_dump_traceback(user->fd, user->all_threads, user->interp); + faulthandler_dump_traceback(user->fd, user->all_threads, user->interp, + user->max_threads); #ifdef HAVE_SIGACTION if (user->chain) { @@ -995,17 +1012,21 @@ faulthandler.register as faulthandler_register_py file: object(py_default="sys.stderr") = NULL all_threads: bool = True chain: bool = False + * + max_threads: Py_ssize_t = 100 Register a handler for the signal 'signum'. Dump the traceback of the current thread, or of all threads if -all_threads is True, into file. +all_threads is True, into file. max_threads caps the number of threads +dumped. [clinic start generated code]*/ static PyObject * faulthandler_register_py_impl(PyObject *module, int signum, PyObject *file, - int all_threads, int chain) -/*[clinic end generated code: output=1f770cee150a56cd input=ae9de829e850907b]*/ + int all_threads, int chain, + Py_ssize_t max_threads) +/*[clinic end generated code: output=d63a5b4f388dee5f input=c75096a20de502fe]*/ { int fd; user_signal_t *user; @@ -1056,6 +1077,7 @@ faulthandler_register_py_impl(PyObject *module, int signum, PyObject *file, user->all_threads = all_threads; user->chain = chain; user->interp = PyThreadState_GetInterpreter(tstate); + user->max_threads = max_threads; user->enabled = 1; Py_RETURN_NONE; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 8da28130e9d..18bddf46a74 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -158,6 +158,15 @@ gc_set_threshold_impl(PyObject *module, int threshold0, int group_right_1, { GCState *gcstate = get_gc_state(); +#ifndef Py_GIL_DISABLED + gcstate->generations[0].threshold = threshold0; + if (group_right_1) { + gcstate->generations[1].threshold = threshold1; + } + if (group_right_2) { + gcstate->generations[2].threshold = threshold2; + } +#else gcstate->young.threshold = threshold0; if (group_right_1) { gcstate->old[0].threshold = threshold1; @@ -165,6 +174,7 @@ gc_set_threshold_impl(PyObject *module, int threshold0, int group_right_1, if (group_right_2) { gcstate->old[1].threshold = threshold2; } +#endif Py_RETURN_NONE; } @@ -179,10 +189,17 @@ gc_get_threshold_impl(PyObject *module) /*[clinic end generated code: output=7902bc9f41ecbbd8 input=286d79918034d6e6]*/ { GCState *gcstate = get_gc_state(); +#ifndef Py_GIL_DISABLED + return Py_BuildValue("(iii)", + gcstate->generations[0].threshold, + gcstate->generations[1].threshold, + gcstate->generations[2].threshold); +#else return Py_BuildValue("(iii)", gcstate->young.threshold, gcstate->old[0].threshold, - 0); + gcstate->old[1].threshold); +#endif } /*[clinic input] @@ -206,10 +223,17 @@ gc_get_count_impl(PyObject *module) gc->alloc_count = 0; #endif +#ifndef Py_GIL_DISABLED + return Py_BuildValue("(iii)", + gcstate->generations[0].count, + gcstate->generations[1].count, + gcstate->generations[2].count); +#else return Py_BuildValue("(iii)", gcstate->young.count, - gcstate->old[gcstate->visited_space].count, - gcstate->old[gcstate->visited_space^1].count); + gcstate->old[0].count, + gcstate->old[1].count); +#endif } /*[clinic input] diff --git a/Modules/overlapped.c b/Modules/overlapped.c index 822e1ce4bdc..51aee5afd35 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -1910,6 +1910,11 @@ _overlapped_Overlapped_WSARecvFromInto_impl(OverlappedObject *self, } #endif + if (bufobj->len < (Py_ssize_t)size) { + PyErr_SetString(PyExc_ValueError, "nbytes is greater than the length of the buffer"); + return NULL; + } + wsabuf.buf = bufobj->buf; wsabuf.len = size; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index e5ce487723b..5bd53c2146a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -20,6 +20,7 @@ #include "pycore_fileutils.h" // _Py_closerange() #include "pycore_import.h" // _PyImport_AcquireLock() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() +#include "pycore_jit_unwind.h" // _Py_jit_debug_mutex #include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_object.h" // _PyObject_LookupSpecial() @@ -758,6 +759,13 @@ PyOS_AfterFork_Child(void) goto fatal_error; } +#if defined(PY_HAVE_JIT_GDB_UNWIND) + // The child can inherit this mutex locked if another thread held it at + // fork(), but the child itself cannot be inside gdb_jit_register_code(). + // Reinitialize it before any executor cleanup can unregister JIT code. + _Py_jit_debug_mutex = (PyMutex){0}; +#endif + reset_remotedebug_data(tstate); reset_asyncio_state((_PyThreadStateImpl *)tstate); @@ -17195,6 +17203,8 @@ os_getrandom_impl(PyObject *module, Py_ssize_t size, int flags) goto error; } + _Py_MSAN_UNPOISON(data, size); + return PyBytesWriter_FinishWithSize(writer, n); error: diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index f1a55db229e..f5993fc8fda 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -8276,6 +8276,9 @@ socket_exec(PyObject *m) #ifdef SO_BINDTODEVICE ADD_INT_MACRO(m, SO_BINDTODEVICE); #endif +#ifdef SO_PASSRIGHTS + ADD_INT_MACRO(m, SO_PASSRIGHTS); +#endif #ifdef SO_BINDTOIFINDEX ADD_INT_MACRO(m, SO_BINDTOIFINDEX); #endif diff --git a/Objects/clinic/sentinelobject.c.h b/Objects/clinic/sentinelobject.c.h new file mode 100644 index 00000000000..51fd35a5979 --- /dev/null +++ b/Objects/clinic/sentinelobject.c.h @@ -0,0 +1,34 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + +static PyObject * +sentinel_new_impl(PyTypeObject *type, PyObject *name); + +static PyObject * +sentinel_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = &PySentinel_Type; + PyObject *name; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("sentinel", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("sentinel", PyTuple_GET_SIZE(args), 1, 1)) { + goto exit; + } + if (!PyUnicode_Check(PyTuple_GET_ITEM(args, 0))) { + _PyArg_BadArgument("sentinel", "argument 1", "str", PyTuple_GET_ITEM(args, 0)); + goto exit; + } + name = PyTuple_GET_ITEM(args, 0); + return_value = sentinel_new_impl(type, name); + +exit: + return return_value; +} +/*[clinic end generated code: output=7f28fc0bf0259cba input=a9049054013a1b77]*/ diff --git a/Objects/clinic/typevarobject.c.h b/Objects/clinic/typevarobject.c.h index bd4c7a0e64f..d2f350a3487 100644 --- a/Objects/clinic/typevarobject.c.h +++ b/Objects/clinic/typevarobject.c.h @@ -517,13 +517,15 @@ paramspec_has_default(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(typevartuple__doc__, -"typevartuple(name, *, default=typing.NoDefault)\n" +"typevartuple(name, *, bound=None, covariant=False, contravariant=False,\n" +" infer_variance=False, default=typing.NoDefault)\n" "--\n" "\n" "Create a new TypeVarTuple with the given name."); static PyObject * -typevartuple_impl(PyTypeObject *type, PyObject *name, +typevartuple_impl(PyTypeObject *type, PyObject *name, PyObject *bound, + int covariant, int contravariant, int infer_variance, PyObject *default_value); static PyObject * @@ -532,7 +534,7 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 6 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -541,7 +543,7 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(name), &_Py_ID(default), }, + .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), &_Py_ID(default), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -550,18 +552,22 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "default", NULL}; + static const char * const _keywords[] = {"name", "bound", "covariant", "contravariant", "infer_variance", "default", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "typevartuple", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[6]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *name; + PyObject *bound = Py_None; + int covariant = 0; + int contravariant = 0; + int infer_variance = 0; PyObject *default_value = &_Py_NoDefaultStruct; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, @@ -577,9 +583,42 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (!noptargs) { goto skip_optional_kwonly; } - default_value = fastargs[1]; + if (fastargs[1]) { + bound = fastargs[1]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[2]) { + covariant = PyObject_IsTrue(fastargs[2]); + if (covariant < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[3]) { + contravariant = PyObject_IsTrue(fastargs[3]); + if (contravariant < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[4]) { + infer_variance = PyObject_IsTrue(fastargs[4]); + if (infer_variance < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + default_value = fastargs[5]; skip_optional_kwonly: - return_value = typevartuple_impl(type, name, default_value); + return_value = typevartuple_impl(type, name, bound, covariant, contravariant, infer_variance, default_value); exit: return return_value; @@ -764,4 +803,4 @@ skip_optional_kwonly: exit: return return_value; } -/*[clinic end generated code: output=67ab9a5d1869f2c9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2e7dd170924d92e5 input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 2c3d6dc4b0f..50ebe657a0e 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1296,77 +1296,6 @@ _PyLineTable_NextAddressRange(PyCodeAddressRange *range) return 1; } -static int -emit_pair(PyObject **bytes, int *offset, int a, int b) -{ - Py_ssize_t len = PyBytes_GET_SIZE(*bytes); - if (*offset + 2 >= len) { - if (_PyBytes_Resize(bytes, len * 2) < 0) - return 0; - } - unsigned char *lnotab = (unsigned char *) PyBytes_AS_STRING(*bytes); - lnotab += *offset; - *lnotab++ = a; - *lnotab++ = b; - *offset += 2; - return 1; -} - -static int -emit_delta(PyObject **bytes, int bdelta, int ldelta, int *offset) -{ - while (bdelta > 255) { - if (!emit_pair(bytes, offset, 255, 0)) { - return 0; - } - bdelta -= 255; - } - while (ldelta > 127) { - if (!emit_pair(bytes, offset, bdelta, 127)) { - return 0; - } - bdelta = 0; - ldelta -= 127; - } - while (ldelta < -128) { - if (!emit_pair(bytes, offset, bdelta, -128)) { - return 0; - } - bdelta = 0; - ldelta += 128; - } - return emit_pair(bytes, offset, bdelta, ldelta); -} - -static PyObject * -decode_linetable(PyCodeObject *code) -{ - PyCodeAddressRange bounds; - PyObject *bytes; - int table_offset = 0; - int code_offset = 0; - int line = code->co_firstlineno; - bytes = PyBytes_FromStringAndSize(NULL, 64); - if (bytes == NULL) { - return NULL; - } - _PyCode_InitAddressRange(code, &bounds); - while (_PyLineTable_NextAddressRange(&bounds)) { - if (bounds.opaque.computed_line != line) { - int bdelta = bounds.ar_start - code_offset; - int ldelta = bounds.opaque.computed_line - line; - if (!emit_delta(&bytes, bdelta, ldelta, &table_offset)) { - Py_DECREF(bytes); - return NULL; - } - code_offset = bounds.ar_start; - line = bounds.opaque.computed_line; - } - } - _PyBytes_Resize(&bytes, table_offset); - return bytes; -} - typedef struct { PyObject_HEAD @@ -2739,18 +2668,6 @@ static PyMemberDef code_memberlist[] = { }; -static PyObject * -code_getlnotab(PyObject *self, void *closure) -{ - PyCodeObject *code = _PyCodeObject_CAST(self); - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "co_lnotab is deprecated, use co_lines instead.", - 1) < 0) { - return NULL; - } - return decode_linetable(code); -} - static PyObject * code_getvarnames(PyObject *self, void *closure) { @@ -2788,7 +2705,6 @@ code_getcode(PyObject *self, void *closure) } static PyGetSetDef code_getsetlist[] = { - {"co_lnotab", code_getlnotab, NULL, NULL}, {"_co_code_adaptive", code_getcodeadaptive, NULL, NULL}, // The following old names are kept for backward compatibility. {"co_varnames", code_getvarnames, NULL, NULL}, diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 7aef56cf4e9..e3bc8eb2739 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -242,7 +242,6 @@ _Py_make_parameters(PyObject *args) len += needed; if (_PyTuple_Resize(¶meters, len) < 0) { Py_DECREF(subparams); - Py_DECREF(parameters); Py_XDECREF(tuple_args); return NULL; } @@ -650,7 +649,7 @@ ga_vectorcall(PyObject *self, PyObject *const *args, size_t nargsf, PyObject *kwnames) { gaobject *alias = (gaobject *) self; - PyObject *obj = PyVectorcall_Function(alias->origin)(alias->origin, args, nargsf, kwnames); + PyObject *obj = PyObject_Vectorcall(alias->origin, args, nargsf, kwnames); return set_orig_class(obj, self); } diff --git a/Objects/genobject.c b/Objects/genobject.c index 2895833b4ff..d628889afc6 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -496,7 +496,7 @@ gen_close(PyObject *self, PyObject *args) } if (is_resume(frame->instr_ptr)) { - bool no_unwind_tools = _PyEval_NoToolsForUnwind(_PyThreadState_GET()); + bool no_unwind_tools = _PyEval_NoToolsForUnwind(_PyThreadState_GET(), frame); /* We can safely ignore the outermost try block * as it is automatically generated to handle * StopIteration. */ @@ -1110,9 +1110,6 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) return (PyObject *)gen; } -static PyObject * -compute_cr_origin(int origin_depth, _PyInterpreterFrame *current_frame); - PyObject * _Py_MakeCoro(PyFunctionObject *func) { @@ -1150,7 +1147,7 @@ _Py_MakeCoro(PyFunctionObject *func) assert(frame); assert(_PyFrame_IsIncomplete(frame)); frame = _PyFrame_GetFirstComplete(frame->previous); - PyObject *cr_origin = compute_cr_origin(origin_depth, frame); + PyObject *cr_origin = _PyCoro_ComputeOrigin(origin_depth, frame); ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin; if (!cr_origin) { Py_DECREF(coro); @@ -1535,8 +1532,8 @@ PyTypeObject _PyCoroWrapper_Type = { 0, /* tp_free */ }; -static PyObject * -compute_cr_origin(int origin_depth, _PyInterpreterFrame *current_frame) +PyObject * +_PyCoro_ComputeOrigin(int origin_depth, _PyInterpreterFrame *current_frame) { _PyInterpreterFrame *frame = current_frame; /* First count how many frames we have */ @@ -1581,7 +1578,7 @@ PyCoro_New(PyFrameObject *f, PyObject *name, PyObject *qualname) if (origin_depth == 0) { ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL; } else { - PyObject *cr_origin = compute_cr_origin(origin_depth, _PyEval_GetFrame()); + PyObject *cr_origin = _PyCoro_ComputeOrigin(origin_depth, _PyEval_GetFrame()); ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin; if (!cr_origin) { Py_DECREF(coro); diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt deleted file mode 100644 index 335e441cfde..00000000000 --- a/Objects/lnotab_notes.txt +++ /dev/null @@ -1,228 +0,0 @@ -Description of the internal format of the line number table in Python 3.10 -and earlier. - -(For 3.11 onwards, see InternalDocs/code_objects.md) - -Conceptually, the line number table consists of a sequence of triples: - start-offset (inclusive), end-offset (exclusive), line-number. - -Note that not all byte codes have a line number so we need handle `None` for the line-number. - -However, storing the above sequence directly would be very inefficient as we would need 12 bytes per entry. - -First, note that the end of one entry is the same as the start of the next, so we can overlap entries. -Second, we don't really need arbitrary access to the sequence, so we can store deltas. - -We just need to store (end - start, line delta) pairs. The start offset of the first entry is always zero. - -Third, most deltas are small, so we can use a single byte for each value, as long we allow several entries for the same line. - -Consider the following table - Start End Line - 0 6 1 - 6 50 2 - 50 350 7 - 350 360 No line number - 360 376 8 - 376 380 208 - -Stripping the redundant ends gives: - - End-Start Line-delta - 6 +1 - 44 +1 - 300 +5 - 10 No line number - 16 +1 - 4 +200 - - -Note that the end - start value is always positive. - -Finally, in order to fit into a single byte we need to convert start deltas to the range 0 <= delta <= 254, -and line deltas to the range -127 <= delta <= 127. -A line delta of -128 is used to indicate no line number. -Also note that a delta of zero indicates that there are no bytecodes in the given range, -which means we can use an invalid line number for that range. - -Final form: - - Start delta Line delta - 6 +1 - 44 +1 - 254 +5 - 46 0 - 10 -128 (No line number, treated as a delta of zero) - 16 +1 - 0 +127 (line 135, but the range is empty as no bytecodes are at line 135) - 4 +73 - -Iterating over the table. -------------------------- - -For the `co_lines` method we want to emit the full form, omitting the (350, 360, No line number) and empty entries. - -The code is as follows: - -def co_lines(code): - line = code.co_firstlineno - end = 0 - table_iter = iter(code.internal_line_table): - for sdelta, ldelta in table_iter: - if ldelta == 0: # No change to line number, just accumulate changes to end - end += sdelta - continue - start = end - end = start + sdelta - if ldelta == -128: # No valid line number -- skip entry - continue - line += ldelta - if end == start: # Empty range, omit. - continue - yield start, end, line - - - - -The historical co_lnotab format -------------------------------- - -prior to 3.10 code objects stored a field named co_lnotab. -This was an array of unsigned bytes disguised as a Python bytes object. - -The old co_lnotab did not account for the presence of bytecodes without a line number, -nor was it well suited to tracing as a number of workarounds were required. - -The old format can still be accessed via `code.co_lnotab`, which is lazily computed from the new format. - -Below is the description of the old co_lnotab format: - - -The array is conceptually a compressed list of - (bytecode offset increment, line number increment) -pairs. The details are important and delicate, best illustrated by example: - - byte code offset source code line number - 0 1 - 6 2 - 50 7 - 350 207 - 361 208 - -Instead of storing these numbers literally, we compress the list by storing only -the difference from one row to the next. Conceptually, the stored list might -look like: - - 0, 1, 6, 1, 44, 5, 300, 200, 11, 1 - -The above doesn't really work, but it's a start. An unsigned byte (byte code -offset) can't hold negative values, or values larger than 255, a signed byte -(line number) can't hold values larger than 127 or less than -128, and the -above example contains two such values. (Note that before 3.6, line number -was also encoded by an unsigned byte.) So we make two tweaks: - - (a) there's a deep assumption that byte code offsets increase monotonically, - and - (b) if byte code offset jumps by more than 255 from one row to the next, or if - source code line number jumps by more than 127 or less than -128 from one row - to the next, more than one pair is written to the table. In case #b, - there's no way to know from looking at the table later how many were written. - That's the delicate part. A user of co_lnotab desiring to find the source - line number corresponding to a bytecode address A should do something like - this: - - lineno = addr = 0 - for addr_incr, line_incr in co_lnotab: - addr += addr_incr - if addr > A: - return lineno - if line_incr >= 0x80: - line_incr -= 0x100 - lineno += line_incr - -(In C, this is implemented by PyCode_Addr2Line().) In order for this to work, -when the addr field increments by more than 255, the line # increment in each -pair generated must be 0 until the remaining addr increment is < 256. So, in -the example above, assemble_lnotab in compile.c should not (as was actually done -until 2.2) expand 300, 200 to - 255, 255, 45, 45, -but to - 255, 0, 45, 127, 0, 73. - -The above is sufficient to reconstruct line numbers for tracebacks, but not for -line tracing. Tracing is handled by PyCode_CheckLineNumber() in codeobject.c -and maybe_call_line_trace() in ceval.c. - -*** Tracing *** - -To a first approximation, we want to call the tracing function when the line -number of the current instruction changes. Re-computing the current line for -every instruction is a little slow, though, so each time we compute the line -number we save the bytecode indices where it's valid: - - *instr_lb <= frame->f_lasti < *instr_ub - -is true so long as execution does not change lines. That is, *instr_lb holds -the first bytecode index of the current line, and *instr_ub holds the first -bytecode index of the next line. As long as the above expression is true, -maybe_call_line_trace() does not need to call PyCode_CheckLineNumber(). Note -that the same line may appear multiple times in the lnotab, either because the -bytecode jumped more than 255 indices between line number changes or because -the compiler inserted the same line twice. Even in that case, *instr_ub holds -the first index of the next line. - -However, we don't *always* want to call the line trace function when the above -test fails. - -Consider this code: - -1: def f(a): -2: while a: -3: print(1) -4: break -5: else: -6: print(2) - -which compiles to this: - - 2 0 SETUP_LOOP 26 (to 28) - >> 2 LOAD_FAST 0 (a) - 4 POP_JUMP_IF_FALSE 18 - - 3 6 LOAD_GLOBAL 0 (print) - 8 LOAD_CONST 1 (1) - 10 CALL_NO_KW 1 - 12 POP_TOP - - 4 14 BREAK_LOOP - 16 JUMP_ABSOLUTE 2 - >> 18 POP_BLOCK - - 6 20 LOAD_GLOBAL 0 (print) - 22 LOAD_CONST 2 (2) - 24 CALL_NO_KW 1 - 26 POP_TOP - >> 28 LOAD_CONST 0 (None) - 30 RETURN_VALUE - -If 'a' is false, execution will jump to the POP_BLOCK instruction at offset 18 -and the co_lnotab will claim that execution has moved to line 4, which is wrong. -In this case, we could instead associate the POP_BLOCK with line 5, but that -would break jumps around loops without else clauses. - -We fix this by only calling the line trace function for a forward jump if the -co_lnotab indicates we have jumped to the *start* of a line, i.e. if the current -instruction offset matches the offset given for the start of a line by the -co_lnotab. For backward jumps, however, we always call the line trace function, -which lets a debugger stop on every evaluation of a loop guard (which usually -won't be the first opcode in a line). - -Why do we set f_lineno when tracing, and only just before calling the trace -function? Well, consider the code above when 'a' is true. If stepping through -this with 'n' in pdb, you would stop at line 1 with a "call" type event, then -line events on lines 2, 3, and 4, then a "return" type event -- but because the -code for the return actually falls in the range of the "line 6" opcodes, you -would be shown line 6 during this event. This is a change from the behaviour in -2.2 and before, and I've found it confusing in practice. By setting and using -f_lineno when tracing, one can report a line number different from that -suggested by f_lasti on this one occasion where it's desirable. diff --git a/Objects/object.c b/Objects/object.c index 3166254f6f6..e0e26bb50d3 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2526,7 +2526,7 @@ extern PyTypeObject _PyMemoryIter_Type; extern PyTypeObject _PyPositionsIterator; extern PyTypeObject _Py_GenericAliasIterType; -static PyTypeObject* static_types[] = { +static PyTypeObject* static_types[_Py_NUM_MANAGED_PREINITIALIZED_TYPES] = { // The two most important base types: must be initialized first and // deallocated last. &PyBaseObject_Type, @@ -2597,6 +2597,7 @@ static PyTypeObject* static_types[] = { &PyRange_Type, &PyReversed_Type, &PySTEntry_Type, + &PySentinel_Type, &PySeqIter_Type, &PySetIter_Type, &PySet_Type, @@ -2643,6 +2644,9 @@ static PyTypeObject* static_types[] = { &_PyUnion_Type, #ifdef _Py_TIER2 &_PyUOpExecutor_Type, +#else + // The array should have the same size on all builds; see gh-149139 + NULL, #endif &_PyWeakref_CallableProxyType, &_PyWeakref_ProxyType, @@ -2667,6 +2671,9 @@ _PyTypes_InitTypes(PyInterpreterState *interp) // All other static types (unless initialized elsewhere) for (size_t i=0; i < Py_ARRAY_LENGTH(static_types); i++) { PyTypeObject *type = static_types[i]; + if (type == NULL) { + continue; + } if (_PyStaticType_InitBuiltin(interp, type) < 0) { return _PyStatus_ERR("Can't initialize builtin type"); } @@ -2707,6 +2714,9 @@ _PyTypes_FiniTypes(PyInterpreterState *interp) // their base classes. for (Py_ssize_t i=Py_ARRAY_LENGTH(static_types)-1; i>=0; i--) { PyTypeObject *type = static_types[i]; + if (type == NULL) { + continue; + } _PyStaticType_FiniBuiltin(interp, type); } } @@ -2768,9 +2778,9 @@ _Py_SetImmortalUntracked(PyObject *op) return; } #ifdef Py_GIL_DISABLED - op->ob_tid = _Py_UNOWNED_TID; - op->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; - op->ob_ref_shared = 0; + _Py_atomic_store_uintptr_relaxed(&op->ob_tid, _Py_UNOWNED_TID); + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, _Py_IMMORTAL_REFCNT_LOCAL); + _Py_atomic_store_ssize_relaxed(&op->ob_ref_shared, 0); _Py_atomic_or_uint8(&op->ob_gc_bits, _PyGC_BITS_DEFERRED); #elif SIZEOF_VOID_P > 4 op->ob_flags = _Py_IMMORTAL_FLAGS; diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index e2d5b012955..1809bd30451 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -665,6 +665,9 @@ _PyMem_ArenaAlloc(void *Py_UNUSED(ctx), size_t size) if (ptr == MAP_FAILED) return NULL; assert(ptr != NULL); +#ifdef MADV_HUGEPAGE + (void)madvise(ptr, size, MADV_HUGEPAGE); +#endif (void)_PyAnnotateMemoryMap(ptr, size, "cpython:pymalloc"); return ptr; #else diff --git a/Objects/sentinelobject.c b/Objects/sentinelobject.c new file mode 100644 index 00000000000..e7e9f60e3ed --- /dev/null +++ b/Objects/sentinelobject.c @@ -0,0 +1,196 @@ +/* Sentinel object implementation */ + +#include "Python.h" +#include "descrobject.h" // PyMemberDef +#include "pycore_ceval.h" // _PyThreadState_GET() +#include "pycore_interpframe.h" // _PyFrame_IsIncomplete() +#include "pycore_object.h" // _PyObject_GC_TRACK/UNTRACK() +#include "pycore_stackref.h" // PyStackRef_AsPyObjectBorrow() +#include "pycore_tuple.h" // _PyTuple_FromPair +#include "pycore_typeobject.h" // _Py_BaseObject_RichCompare() +#include "pycore_unionobject.h" // _Py_union_type_or() + +typedef struct { + PyObject_HEAD + PyObject *name; + PyObject *module; +} sentinelobject; + +#define sentinelobject_CAST(op) \ + (assert(PySentinel_Check(op)), _Py_CAST(sentinelobject *, (op))) + +/*[clinic input] +class sentinel "sentinelobject *" "&PySentinel_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=8b88f8268d3b5775]*/ + +#include "clinic/sentinelobject.c.h" + + +static PyObject * +caller(void) +{ + _PyInterpreterFrame *f = _PyThreadState_GET()->current_frame; + if (f == NULL || PyStackRef_IsNull(f->f_funcobj)) { + assert(!PyErr_Occurred()); + Py_RETURN_NONE; + } + PyFunctionObject *func = _PyFrame_GetFunction(f); + assert(PyFunction_Check(func)); + PyObject *r = PyFunction_GetModule((PyObject *)func); + if (!r) { + assert(!PyErr_Occurred()); + Py_RETURN_NONE; + } + return Py_NewRef(r); +} + +static PyObject * +sentinel_new_with_module(PyTypeObject *type, PyObject *name, PyObject *module) +{ + assert(PyUnicode_Check(name)); + + sentinelobject *self = PyObject_GC_New(sentinelobject, type); + if (self == NULL) { + return NULL; + } + self->name = Py_NewRef(name); + self->module = Py_NewRef(module); + _PyObject_GC_TRACK(self); + return (PyObject *)self; +} + +/*[clinic input] +@classmethod +sentinel.__new__ as sentinel_new + + name: object(subclass_of='&PyUnicode_Type') + / +[clinic start generated code]*/ + +static PyObject * +sentinel_new_impl(PyTypeObject *type, PyObject *name) +/*[clinic end generated code: output=4af55c6048bed30d input=3ab75704f39c119c]*/ +{ + PyObject *module = caller(); + PyObject *self = sentinel_new_with_module(type, name, module); + Py_DECREF(module); + return self; +} + +PyObject * +PySentinel_New(const char *name, const char *module_name) +{ + PyObject *name_obj = PyUnicode_FromString(name); + if (name_obj == NULL) { + return NULL; + } + PyObject *module_obj = module_name == NULL + ? Py_None + : PyUnicode_FromString(module_name); + if (module_obj == NULL) { + Py_DECREF(name_obj); + return NULL; + } + + PyObject *sentinel = sentinel_new_with_module( + &PySentinel_Type, name_obj, module_obj); + Py_DECREF(module_obj); + Py_DECREF(name_obj); + return sentinel; +} + +static int +sentinel_clear(PyObject *op) +{ + sentinelobject *self = sentinelobject_CAST(op); + Py_CLEAR(self->name); + Py_CLEAR(self->module); + return 0; +} + +static void +sentinel_dealloc(PyObject *op) +{ + _PyObject_GC_UNTRACK(op); + (void)sentinel_clear(op); + Py_TYPE(op)->tp_free(op); +} + +static int +sentinel_traverse(PyObject *op, visitproc visit, void *arg) +{ + sentinelobject *self = sentinelobject_CAST(op); + Py_VISIT(self->name); + Py_VISIT(self->module); + return 0; +} + +static PyObject * +sentinel_repr(PyObject *op) +{ + sentinelobject *self = sentinelobject_CAST(op); + return Py_NewRef(self->name); +} + +static PyObject * +sentinel_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return Py_NewRef(self); +} + +static PyObject * +sentinel_deepcopy(PyObject *self, PyObject *Py_UNUSED(memo)) +{ + return Py_NewRef(self); +} + +static PyObject * +sentinel_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) +{ + sentinelobject *self = sentinelobject_CAST(op); + return Py_NewRef(self->name); +} + +static PyMethodDef sentinel_methods[] = { + {"__copy__", sentinel_copy, METH_NOARGS, NULL}, + {"__deepcopy__", sentinel_deepcopy, METH_O, NULL}, + {"__reduce__", sentinel_reduce, METH_NOARGS, NULL}, + {NULL, NULL} +}; + +static PyMemberDef sentinel_members[] = { + {"__name__", Py_T_OBJECT_EX, offsetof(sentinelobject, name), Py_READONLY}, + {"__module__", Py_T_OBJECT_EX, offsetof(sentinelobject, module), Py_READONLY}, + {NULL} +}; + +static PyNumberMethods sentinel_as_number = { + .nb_or = _Py_union_type_or, +}; + +PyDoc_STRVAR(sentinel_doc, +"sentinel(name, /)\n" +"--\n\n" +"Create a unique sentinel object with the given name."); + +PyTypeObject PySentinel_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "sentinel", + .tp_basicsize = sizeof(sentinelobject), + .tp_dealloc = sentinel_dealloc, + .tp_repr = sentinel_repr, + .tp_as_number = &sentinel_as_number, + .tp_hash = PyObject_GenericHash, + .tp_getattro = PyObject_GenericGetAttr, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HAVE_GC, + .tp_doc = sentinel_doc, + .tp_traverse = sentinel_traverse, + .tp_clear = sentinel_clear, + .tp_richcompare = _Py_BaseObject_RichCompare, + .tp_methods = sentinel_methods, + .tp_members = sentinel_members, + .tp_new = sentinel_new, + .tp_free = PyObject_GC_Del, +}; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 08b95cfbc6c..fb3c7101410 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5878,7 +5878,13 @@ PyType_GetModuleByToken_DuringGC(PyTypeObject *type, const void *token) PyObject * PyType_GetModuleByToken(PyTypeObject *type, const void *token) { - PyObject *mod = PyType_GetModuleByToken_DuringGC(type, token); + return Py_XNewRef(PyType_GetModuleByDef(type, (PyModuleDef *)token)); +} + +PyObject * +PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) +{ + PyObject *mod = PyType_GetModuleByToken_DuringGC(type, def); if (!mod) { PyErr_Format( PyExc_TypeError, @@ -5886,14 +5892,6 @@ PyType_GetModuleByToken(PyTypeObject *type, const void *token) type->tp_name); return NULL; } - return Py_NewRef(mod); -} - -PyObject * -PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) -{ - PyObject *mod = PyType_GetModuleByToken(type, def); - Py_XDECREF(mod); // return borrowed ref return mod; } diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index c2b8ee43119..cdc0ea42eac 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -36,8 +36,12 @@ typedef struct { typedef struct { PyObject_HEAD PyObject *name; + PyObject *bound; PyObject *default_value; PyObject *evaluate_default; + bool covariant; + bool contravariant; + bool infer_variance; } typevartupleobject; typedef struct { @@ -1524,6 +1528,7 @@ typevartuple_dealloc(PyObject *self) typevartupleobject *tvt = typevartupleobject_CAST(self); Py_XDECREF(tvt->name); + Py_XDECREF(tvt->bound); Py_XDECREF(tvt->default_value); Py_XDECREF(tvt->evaluate_default); PyObject_ClearManagedDict(self); @@ -1555,16 +1560,28 @@ static PyObject * typevartuple_repr(PyObject *self) { typevartupleobject *tvt = typevartupleobject_CAST(self); - return Py_NewRef(tvt->name); + + if (tvt->infer_variance) { + return Py_NewRef(tvt->name); + } + + char variance = tvt->covariant ? '+' : tvt->contravariant ? '-' : '~'; + return PyUnicode_FromFormat("%c%U", variance, tvt->name); } static PyMemberDef typevartuple_members[] = { {"__name__", _Py_T_OBJECT, offsetof(typevartupleobject, name), Py_READONLY}, + {"__bound__", _Py_T_OBJECT, offsetof(typevartupleobject, bound), Py_READONLY}, + {"__covariant__", Py_T_BOOL, offsetof(typevartupleobject, covariant), Py_READONLY}, + {"__contravariant__", Py_T_BOOL, offsetof(typevartupleobject, contravariant), Py_READONLY}, + {"__infer_variance__", Py_T_BOOL, offsetof(typevartupleobject, infer_variance), Py_READONLY}, {0} }; static typevartupleobject * -typevartuple_alloc(PyObject *name, PyObject *module, PyObject *default_value) +typevartuple_alloc(PyObject *name, PyObject *bound, PyObject *default_value, + bool covariant, bool contravariant, bool infer_variance, + PyObject *module) { PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.typevartuple_type; typevartupleobject *tvt = PyObject_GC_New(typevartupleobject, tp); @@ -1572,6 +1589,10 @@ typevartuple_alloc(PyObject *name, PyObject *module, PyObject *default_value) return NULL; } tvt->name = Py_NewRef(name); + tvt->bound = Py_XNewRef(bound); + tvt->covariant = covariant; + tvt->contravariant = contravariant; + tvt->infer_variance = infer_variance; tvt->default_value = Py_XNewRef(default_value); tvt->evaluate_default = NULL; _PyObject_GC_TRACK(tvt); @@ -1590,21 +1611,46 @@ typevartuple.__new__ name: object(subclass_of="&PyUnicode_Type") * + bound: object = None + covariant: bool = False + contravariant: bool = False + infer_variance: bool = False default as default_value: object(c_default="&_Py_NoDefaultStruct") = typing.NoDefault Create a new TypeVarTuple with the given name. [clinic start generated code]*/ static PyObject * -typevartuple_impl(PyTypeObject *type, PyObject *name, +typevartuple_impl(PyTypeObject *type, PyObject *name, PyObject *bound, + int covariant, int contravariant, int infer_variance, PyObject *default_value) -/*[clinic end generated code: output=9d6b76dfe95aae51 input=e149739929a866d0]*/ +/*[clinic end generated code: output=40bc9ca10f64e392 input=56e28c725a8da40b]*/ { - PyObject *module = caller(); - if (module == NULL) { + if (covariant && contravariant) { + PyErr_SetString(PyExc_ValueError, "Bivariant types are not supported."); return NULL; } - PyObject *result = (PyObject *)typevartuple_alloc(name, module, default_value); + if (infer_variance && (covariant || contravariant)) { + PyErr_SetString(PyExc_ValueError, "Variance cannot be specified with infer_variance."); + return NULL; + } + if (Py_IsNone(bound)) { + bound = NULL; + } + if (bound != NULL) { + bound = type_check(bound, "Bound must be a type."); + if (bound == NULL) { + return NULL; + } + } + PyObject *module = caller(); + if (module == NULL) { + Py_XDECREF(bound); + return NULL; + } + PyObject *result = (PyObject *)typevartuple_alloc( + name, bound, default_value, covariant, contravariant, infer_variance, module); + Py_XDECREF(bound); Py_DECREF(module); return result; } @@ -1688,6 +1734,7 @@ typevartuple_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(Py_TYPE(self)); typevartupleobject *tvt = typevartupleobject_CAST(self); Py_VISIT(tvt->name); + Py_VISIT(tvt->bound); Py_VISIT(tvt->default_value); Py_VISIT(tvt->evaluate_default); return PyObject_VisitManagedDict(self, visit, arg); @@ -1698,6 +1745,7 @@ typevartuple_clear(PyObject *self) { typevartupleobject *tvt = typevartupleobject_CAST(self); Py_CLEAR(tvt->name); + Py_CLEAR(tvt->bound); Py_CLEAR(tvt->default_value); Py_CLEAR(tvt->evaluate_default); PyObject_ClearManagedDict(self); @@ -1829,7 +1877,7 @@ PyObject * _Py_make_typevartuple(PyThreadState *Py_UNUSED(ignored), PyObject *v) { assert(PyUnicode_Check(v)); - return (PyObject *)typevartuple_alloc(v, NULL, NULL); + return (PyObject *)typevartuple_alloc(v, NULL, NULL, false, false, true, NULL); } static PyObject * diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index d2569132998..9aee7120c81 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -589,6 +589,14 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) { #define CHECK(expr) \ do { if (!(expr)) { _PyObject_ASSERT_FAILED_MSG(op, Py_STRINGIFY(expr)); } } while (0) +#ifdef Py_GIL_DISABLED +# define CHECK_IF_GIL(expr) (void)(expr) +# define CHECK_IF_FT(expr) CHECK(expr) +#else +# define CHECK_IF_GIL(expr) CHECK(expr) +# define CHECK_IF_FT(expr) (void)(expr) +#endif + assert(op != NULL); CHECK(PyUnicode_Check(op)); @@ -669,11 +677,9 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) /* Check interning state */ #ifdef Py_DEBUG - // Note that we do not check `_Py_IsImmortal(op)`, since stable ABI - // extensions can make immortal strings mortal (but with a high enough - // refcount). - // The other way is extremely unlikely (worth a potential failed assertion - // in a debug build), so we do check `!_Py_IsImmortal(op)`. + // Note that we do not check `_Py_IsImmortal(op)` in the GIL-enabled build + // since stable ABI extensions can make immortal strings mortal (but with a + // high enough refcount). switch (PyUnicode_CHECK_INTERNED(op)) { case SSTATE_NOT_INTERNED: if (ascii->state.statically_allocated) { @@ -683,18 +689,20 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) // are static but use SSTATE_NOT_INTERNED } else { - CHECK(!_Py_IsImmortal(op)); + CHECK_IF_GIL(!_Py_IsImmortal(op)); } break; case SSTATE_INTERNED_MORTAL: CHECK(!ascii->state.statically_allocated); - CHECK(!_Py_IsImmortal(op)); + CHECK_IF_GIL(!_Py_IsImmortal(op)); break; case SSTATE_INTERNED_IMMORTAL: CHECK(!ascii->state.statically_allocated); + CHECK_IF_FT(_Py_IsImmortal(op)); break; case SSTATE_INTERNED_IMMORTAL_STATIC: CHECK(ascii->state.statically_allocated); + CHECK_IF_FT(_Py_IsImmortal(op)); break; default: Py_UNREACHABLE(); @@ -14208,6 +14216,18 @@ immortalize_interned(PyObject *s) FT_ATOMIC_STORE_UINT8(_PyUnicode_STATE(s).interned, SSTATE_INTERNED_IMMORTAL); } +#ifdef Py_GIL_DISABLED +static bool +can_immortalize_safely(PyObject *s) +{ + if (_Py_IsOwnedByCurrentThread(s) || _Py_IsImmortal(s)) { + return true; + } + Py_ssize_t shared = _Py_atomic_load_ssize(&s->ob_ref_shared); + return _Py_REF_IS_MERGED(shared); +} +#endif + static /* non-null */ PyObject* intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, bool immortalize) @@ -14236,11 +14256,16 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, // no, go on break; case SSTATE_INTERNED_MORTAL: +#ifndef Py_GIL_DISABLED // yes but we might need to make it immortal if (immortalize) { immortalize_interned(s); } return s; +#else + // not fully interned yet; fall through to the locking path + break; +#endif default: // all done return s; @@ -14305,6 +14330,23 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, Py_DECREF(r); } #endif + +#ifdef Py_GIL_DISABLED + // Immortalization writes to the refcount fields non-atomically. That + // races with Py_INCREF / Py_DECREF on the thread that owns `s`. If we + // don't own it (and its refcount hasn't been merged), intern a copy + // we own instead. + if (!can_immortalize_safely(s)) { + PyObject *copy = _PyUnicode_Copy(s); + if (copy == NULL) { + PyErr_Clear(); + return s; + } + Py_DECREF(s); + s = copy; + } +#endif + FT_MUTEX_LOCK(INTERN_MUTEX); PyObject *t; { diff --git a/Objects/unionobject.c b/Objects/unionobject.c index d33d581f049..0f6b1e44bc2 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -245,6 +245,7 @@ is_unionable(PyObject *obj) { if (obj == Py_None || PyType_Check(obj) || + PySentinel_Check(obj) || _PyGenericAlias_Check(obj) || _PyUnion_Check(obj) || Py_IS_TYPE(obj, &_PyTypeAlias_Type)) { diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 38236922a52..953973a2ad3 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -158,6 +158,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 73861dbb0c9..13db4d93f54 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -400,6 +400,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 8fb2f096c93..9d2f032f5a9 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -170,16 +170,20 @@ if "%do_pgo%"=="true" ( del /s "%dir%\*.pgc" del /s "%dir%\..\Lib\*.pyc" set conf=PGUpdate - if "%clean%"=="false" ( - echo on - call "%dir%\..\python.bat" %pgo_job% - @echo off - call :Kill - set target=Build - ) + if "%clean%"=="false" goto :RunPgoJob ) goto :Build +:RunPgoJob +echo on +call "%dir%\..\python.bat" %pgo_job% +@echo off +set pgo_errorlevel=%ERRORLEVEL% +call :Kill +if %pgo_errorlevel% NEQ 0 exit /B %pgo_errorlevel% +set target=Build +goto :Build + :Kill echo on %MSBUILD% "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^ diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 94ae718d58c..f79608e1d58 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -12,8 +12,9 @@ $(IntDir.Replace(`\\`, `\`)) $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_frozen\ $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\zlib-ng\ - $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_$(Configuration) - $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_PGInstrument + $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_$(Configuration)\ + $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_PGInstrument\ + $(GeneratedJitStencilsDir.Replace(`\\`, `\`)) $(ProjectName) $(TargetName)$(PyDebugExt) false diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 61bee29c0af..fae4a90b453 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -115,6 +115,9 @@ version.lib;ws2_32.lib;pathcch.lib;bcrypt.lib;%(AdditionalDependencies) zlib-ng$(PyDebugExt).lib;%(AdditionalDependencies) + $(GeneratedJitStencilsDir)jit_shim-aarch64-pc-windows-msvc.o;%(AdditionalDependencies) + $(GeneratedJitStencilsDir)jit_shim-i686-pc-windows-msvc.o;%(AdditionalDependencies) + $(GeneratedJitStencilsDir)jit_shim-x86_64-pc-windows-msvc.o;%(AdditionalDependencies) @@ -192,6 +195,7 @@ + @@ -359,6 +363,7 @@ + @@ -557,6 +562,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 664788e69af..04b6641ae30 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -156,6 +156,9 @@ Include + + Include + Include @@ -516,6 +519,9 @@ Include\cpython + + Include + Include\cpython @@ -1271,6 +1277,9 @@ Objects + + Objects + Objects diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 41af9cacfb9..9552e73ef6a 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -35,6 +35,9 @@ <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-aarch64-pc-windows-msvc.h" Condition="$(Platform) == 'ARM64'"/> <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-i686-pc-windows-msvc.h" Condition="$(Platform) == 'Win32'"/> <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-x86_64-pc-windows-msvc.h" Condition="$(Platform) == 'x64'"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_shim-aarch64-pc-windows-msvc.o" Condition="$(Platform) == 'ARM64'"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_shim-i686-pc-windows-msvc.o" Condition="$(Platform) == 'Win32'"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_shim-x86_64-pc-windows-msvc.o" Condition="$(Platform) == 'x64'"/> <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/> <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/> <_SbomSources Include="$(PySourcePath)PCbuild\get_externals.bat" /> @@ -129,7 +132,7 @@ x86_64-pc-windows-msvc $(JITArgs) --debug - + diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 71a164fbec5..df4454cf948 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -873,6 +873,70 @@ def visitModule(self, mod): return 0; } +/* + * Format the names in the set 'missing' into a natural language list, + * sorted in the order in which they appear in 'fields'. + * + * Similar to format_missing() from 'Python/ceval.c'. + * + * Parameters + * + * missing Set of missing field names to render. + * fields Sequence of AST node field names (self._fields). + */ +static PyObject * +format_missing(PyObject *missing, PyObject *fields) +{ + Py_ssize_t num_fields, num_total, num_left; + num_fields = PySequence_Size(fields); + if (num_fields == -1) { + return NULL; + } + num_total = num_left = PySet_GET_SIZE(missing); + PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); + if (writer == NULL) { + goto error; + } + // Iterate all AST node fields in order so that the missing positional + // arguments are rendered in the order in which __init__ expects them. + for (Py_ssize_t i = 0; i < num_fields; i++) { + PyObject *name = PySequence_GetItem(fields, i); + if (name == NULL) { + goto error; + } + int contains = PySet_Contains(missing, name); + if (contains == -1) { + Py_DECREF(name); + goto error; + } + else if (contains == 1) { + const char* fmt = NULL; + if (num_left == 1) { + fmt = "'%U'"; + } + else if (num_total == 2) { + fmt = "'%U' and "; + } + else if (num_left == 2) { + fmt = "'%U', and "; + } + else { + fmt = "'%U', "; + } + num_left--; + if (PyUnicodeWriter_Format(writer, fmt, name) < 0) { + Py_DECREF(name); + goto error; + } + } + Py_DECREF(name); + } + return PyUnicodeWriter_Finish(writer); +error: + PyUnicodeWriter_Discard(writer); + return NULL; +} + static int ast_type_init(PyObject *self, PyObject *args, PyObject *kw) { @@ -942,8 +1006,8 @@ def visitModule(self, mod): } if (p == 0) { PyErr_Format(PyExc_TypeError, - "%.400s got multiple values for argument %R", - Py_TYPE(self)->tp_name, key); + "%T got multiple values for argument %R", + self, key); res = -1; goto cleanup; } @@ -963,16 +1027,11 @@ def visitModule(self, mod): goto cleanup; } else if (contains == 0) { - if (PyErr_WarnFormat( - PyExc_DeprecationWarning, 1, - "%.400s.__init__ got an unexpected keyword argument %R. " - "Support for arbitrary keyword arguments is deprecated " - "and will be removed in Python 3.15.", - Py_TYPE(self)->tp_name, key - ) < 0) { - res = -1; - goto cleanup; - } + PyErr_Format(PyExc_TypeError, + "%T.__init__ got an unexpected keyword argument %R", + self, key); + res = -1; + goto cleanup; } } res = PyObject_SetAttr(self, key, value); @@ -982,7 +1041,7 @@ def visitModule(self, mod): } } Py_ssize_t size = PySet_Size(remaining_fields); - PyObject *field_types = NULL, *remaining_list = NULL; + PyObject *field_types = NULL, *remaining_list = NULL, *missing_names = NULL; if (size > 0) { if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), &_Py_ID(_field_types), &field_types) < 0) { @@ -999,6 +1058,10 @@ def visitModule(self, mod): if (!remaining_list) { goto set_remaining_cleanup; } + missing_names = PySet_New(NULL); + if (!missing_names) { + goto set_remaining_cleanup; + } for (Py_ssize_t i = 0; i < size; i++) { PyObject *name = PyList_GET_ITEM(remaining_list, i); PyObject *type = PyDict_GetItemWithError(field_types, name); @@ -1007,14 +1070,10 @@ def visitModule(self, mod): goto set_remaining_cleanup; } else { - if (PyErr_WarnFormat( - PyExc_DeprecationWarning, 1, - "Field %R is missing from %.400s._field_types. " - "This will become an error in Python 3.15.", - name, Py_TYPE(self)->tp_name - ) < 0) { - goto set_remaining_cleanup; - } + PyErr_Format(PyExc_TypeError, + "Field %R is missing from %T._field_types", + name, self); + goto set_remaining_cleanup; } } else if (_PyUnion_Check(type)) { @@ -1042,16 +1101,25 @@ def visitModule(self, mod): } else { // simple field (e.g., identifier) - if (PyErr_WarnFormat( - PyExc_DeprecationWarning, 1, - "%.400s.__init__ missing 1 required positional argument: %R. " - "This will become an error in Python 3.15.", - Py_TYPE(self)->tp_name, name - ) < 0) { + res = PySet_Add(missing_names, name); + if (res < 0) { goto set_remaining_cleanup; } } } + Py_ssize_t num_missing = PySet_GET_SIZE(missing_names); + if (num_missing > 0) { + PyObject *name_str = format_missing(missing_names, fields); + if (!name_str) { + goto set_remaining_cleanup; + } + PyErr_Format(PyExc_TypeError, + "%T.__init__ missing %d required positional argument%s: %U", + self, num_missing, num_missing == 1 ? "" : "s", name_str); + Py_DECREF(name_str); + goto set_remaining_cleanup; + } + Py_DECREF(missing_names); Py_DECREF(remaining_list); Py_DECREF(field_types); } @@ -1061,6 +1129,7 @@ def visitModule(self, mod): Py_XDECREF(remaining_fields); return res; set_remaining_cleanup: + Py_XDECREF(missing_names); Py_XDECREF(remaining_list); Py_XDECREF(field_types); res = -1; @@ -1144,182 +1213,6 @@ def visitModule(self, mod): return result; } -/* - * Perform the following validations: - * - * - All keyword arguments are known 'fields' or 'attributes'. - * - No field or attribute would be left unfilled after copy.replace(). - * - * On success, this returns 1. Otherwise, set a TypeError - * exception and returns -1 (no exception is set if some - * other internal errors occur). - * - * Parameters - * - * self The AST node instance. - * dict The AST node instance dictionary (self.__dict__). - * fields The list of fields (self._fields). - * attributes The list of attributes (self._attributes). - * kwargs Keyword arguments passed to ast_type_replace(). - * - * The 'dict', 'fields', 'attributes' and 'kwargs' arguments can be NULL. - * - * Note: this function can be removed in 3.15 since the verification - * will be done inside the constructor. - */ -static inline int -ast_type_replace_check(PyObject *self, - PyObject *dict, - PyObject *fields, - PyObject *attributes, - PyObject *kwargs) -{ - // While it is possible to make some fast paths that would avoid - // allocating objects on the stack, this would cost us readability. - // For instance, if 'fields' and 'attributes' are both empty, and - // 'kwargs' is not empty, we could raise a TypeError immediately. - PyObject *expecting = PySet_New(fields); - if (expecting == NULL) { - return -1; - } - if (attributes) { - if (_PySet_Update(expecting, attributes) < 0) { - Py_DECREF(expecting); - return -1; - } - } - // Any keyword argument that is neither a field nor attribute is rejected. - // We first need to check whether a keyword argument is accepted or not. - // If all keyword arguments are accepted, we compute the required fields - // and attributes. A field or attribute is not needed if: - // - // 1) it is given in 'kwargs', or - // 2) it already exists on 'self'. - if (kwargs) { - Py_ssize_t pos = 0; - PyObject *key, *value; - while (PyDict_Next(kwargs, &pos, &key, &value)) { - int rc = PySet_Discard(expecting, key); - if (rc < 0) { - Py_DECREF(expecting); - return -1; - } - if (rc == 0) { - PyErr_Format(PyExc_TypeError, - "%.400s.__replace__ got an unexpected keyword " - "argument %R.", Py_TYPE(self)->tp_name, key); - Py_DECREF(expecting); - return -1; - } - } - } - // check that the remaining fields or attributes would be filled - if (dict) { - Py_ssize_t pos = 0; - PyObject *key, *value; - while (PyDict_Next(dict, &pos, &key, &value)) { - // Mark fields or attributes that are found on the instance - // as non-mandatory. If they are not given in 'kwargs', they - // will be shallow-coied; otherwise, they would be replaced - // (not in this function). - if (PySet_Discard(expecting, key) < 0) { - Py_DECREF(expecting); - return -1; - } - } - if (attributes) { - // Some attributes may or may not be present at runtime. - // In particular, now that we checked whether 'kwargs' - // is correct or not, we allow any attribute to be missing. - // - // Note that fields must still be entirely determined when - // calling the constructor later. - PyObject *unused = PyObject_CallMethodOneArg(expecting, - &_Py_ID(difference_update), - attributes); - if (unused == NULL) { - Py_DECREF(expecting); - return -1; - } - Py_DECREF(unused); - } - } - - // Discard fields from 'expecting' that default to None - PyObject *field_types = NULL; - if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), - &_Py_ID(_field_types), - &field_types) < 0) - { - Py_DECREF(expecting); - return -1; - } - if (field_types != NULL) { - Py_ssize_t pos = 0; - PyObject *field_name, *field_type; - while (PyDict_Next(field_types, &pos, &field_name, &field_type)) { - if (_PyUnion_Check(field_type)) { - // optional field - if (PySet_Discard(expecting, field_name) < 0) { - Py_DECREF(expecting); - Py_DECREF(field_types); - return -1; - } - } - } - Py_DECREF(field_types); - } - - // Now 'expecting' contains the fields or attributes - // that would not be filled inside ast_type_replace(). - Py_ssize_t m = PySet_GET_SIZE(expecting); - if (m > 0) { - PyObject *names = PyList_New(m); - if (names == NULL) { - Py_DECREF(expecting); - return -1; - } - Py_ssize_t i = 0, pos = 0; - PyObject *item; - Py_hash_t hash; - while (_PySet_NextEntry(expecting, &pos, &item, &hash)) { - PyObject *name = PyObject_Repr(item); - if (name == NULL) { - Py_DECREF(expecting); - Py_DECREF(names); - return -1; - } - // steal the reference 'name' - PyList_SET_ITEM(names, i++, name); - } - Py_DECREF(expecting); - if (PyList_Sort(names) < 0) { - Py_DECREF(names); - return -1; - } - PyObject *sep = PyUnicode_FromString(", "); - if (sep == NULL) { - Py_DECREF(names); - return -1; - } - PyObject *str_names = PyUnicode_Join(sep, names); - Py_DECREF(sep); - Py_DECREF(names); - if (str_names == NULL) { - return -1; - } - PyErr_Format(PyExc_TypeError, - "%.400s.__replace__ missing %ld keyword argument%s: %U.", - Py_TYPE(self)->tp_name, m, m == 1 ? "" : "s", str_names); - Py_DECREF(str_names); - return -1; - } - else { - Py_DECREF(expecting); - return 1; - } -} - /* * Python equivalent: * @@ -1409,9 +1302,6 @@ def visitModule(self, mod): if (PyObject_GetOptionalAttr(self, state->__dict__, &dict) < 0) { goto cleanup; } - if (ast_type_replace_check(self, dict, fields, attributes, kwargs) < 0) { - goto cleanup; - } empty_tuple = PyTuple_New(0); if (empty_tuple == NULL) { goto cleanup; diff --git a/Parser/parser.c b/Parser/parser.c index f853d309de9..c55c081dfc3 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -9066,7 +9066,7 @@ complex_number_rule(Parser *p) return _res; } -// signed_number: NUMBER | '-' NUMBER +// signed_number: NUMBER | '+' NUMBER | '-' NUMBER static expr_ty signed_number_rule(Parser *p) { @@ -9107,6 +9107,33 @@ signed_number_rule(Parser *p) D(fprintf(stderr, "%*c%s signed_number[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NUMBER")); } + { // '+' NUMBER + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> signed_number[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+' NUMBER")); + Token * _literal; + expr_ty number; + if ( + (_literal = _PyPegen_expect_token(p, 14)) // token='+' + && + (number = _PyPegen_number_token(p)) // NUMBER + ) + { + D(fprintf(stderr, "%*c+ signed_number[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+' NUMBER")); + _res = number; + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s signed_number[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+' NUMBER")); + } { // '-' NUMBER if (p->error_indicator) { p->level--; @@ -9149,7 +9176,7 @@ signed_number_rule(Parser *p) return _res; } -// signed_real_number: real_number | '-' real_number +// signed_real_number: real_number | '+' real_number | '-' real_number static expr_ty signed_real_number_rule(Parser *p) { @@ -9190,6 +9217,33 @@ signed_real_number_rule(Parser *p) D(fprintf(stderr, "%*c%s signed_real_number[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "real_number")); } + { // '+' real_number + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> signed_real_number[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+' real_number")); + Token * _literal; + expr_ty real; + if ( + (_literal = _PyPegen_expect_token(p, 14)) // token='+' + && + (real = real_number_rule(p)) // real_number + ) + { + D(fprintf(stderr, "%*c+ signed_real_number[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+' real_number")); + _res = real; + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s signed_real_number[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+' real_number")); + } { // '-' real_number if (p->error_indicator) { p->level--; @@ -9275,7 +9329,7 @@ real_number_rule(Parser *p) return _res; } -// imaginary_number: NUMBER +// imaginary_number: NUMBER | '+' NUMBER static expr_ty imaginary_number_rule(Parser *p) { @@ -9312,6 +9366,33 @@ imaginary_number_rule(Parser *p) D(fprintf(stderr, "%*c%s imaginary_number[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NUMBER")); } + { // '+' NUMBER + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> imaginary_number[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+' NUMBER")); + Token * _literal; + expr_ty imag; + if ( + (_literal = _PyPegen_expect_token(p, 14)) // token='+' + && + (imag = _PyPegen_number_token(p)) // NUMBER + ) + { + D(fprintf(stderr, "%*c+ imaginary_number[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+' NUMBER")); + _res = _PyPegen_ensure_imaginary ( p , imag ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s imaginary_number[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+' NUMBER")); + } _res = NULL; done: p->level--; diff --git a/Platforms/Android/__main__.py b/Platforms/Android/__main__.py index 315632ea12c..d2546cf76c2 100755 --- a/Platforms/Android/__main__.py +++ b/Platforms/Android/__main__.py @@ -216,8 +216,14 @@ def make_build_python(context): def unpack_deps(host, prefix_dir, cache_dir): os.chdir(prefix_dir) deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" - for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.5.5-0", - "sqlite-3.50.4-0", "xz-5.4.6-1", "zstd-1.5.7-2"]: + for name_ver in [ + "bzip2-1.0.8-3", + "libffi-3.4.4-3", + "openssl-3.5.6-0", + "sqlite-3.50.4-0", + "xz-5.4.6-1", + "zstd-1.5.7-2" + ]: filename = f"{name_ver}-{host}.tar.gz" out_path = download(f"{deps_url}/{name_ver}/{filename}", cache_dir) shutil.unpack_archive(out_path) diff --git a/Platforms/Apple/__main__.py b/Platforms/Apple/__main__.py index 44a991c6c20..d94198a309f 100644 --- a/Platforms/Apple/__main__.py +++ b/Platforms/Apple/__main__.py @@ -319,7 +319,7 @@ def unpack_deps( for name_ver in [ "BZip2-1.0.8-2", "libFFI-3.4.7-2", - "OpenSSL-3.5.5-1", + "OpenSSL-3.5.6-1", "XZ-5.6.4-2", "mpdecimal-4.0.0-2", "zstd-1.5.7-1", diff --git a/Platforms/WASI/config.toml b/Platforms/WASI/config.toml index 31ec2b8023d..6a6d5713ee9 100644 --- a/Platforms/WASI/config.toml +++ b/Platforms/WASI/config.toml @@ -2,5 +2,5 @@ # This allows for blanket copying of the WASI build code between supported # Python versions. [targets] -wasi-sdk = 32 +wasi-sdk = 33 host-triple = "wasm32-wasip1" diff --git a/Platforms/emscripten/README.md b/Platforms/emscripten/README.md index 017bb3c8977..ce230c4b74a 100644 --- a/Platforms/emscripten/README.md +++ b/Platforms/emscripten/README.md @@ -186,8 +186,8 @@ #### In the browser are not shipped. All other modules are bundled as pre-compiled ``pyc`` files. - In-memory file system (MEMFS) is not persistent and limited. -- Test modules are disabled by default. Use ``--enable-test-modules`` build - test modules like ``_testcapi``. +- Test modules are built by default. Use ``--disable-test-modules`` to disable + building test modules like ``_testcapi``. ## Detecting Emscripten builds diff --git a/Platforms/emscripten/web_example/index.html b/Platforms/emscripten/web_example/index.html index 9c89c9c0ed3..3a207b92015 100644 --- a/Platforms/emscripten/web_example/index.html +++ b/Platforms/emscripten/web_example/index.html @@ -663,9 +663,9 @@

Simple REPL for Python WASM

The simple REPL provides a limited Python experience in the browser. - Tools/wasm/README.md + Platforms/emscripten/README.md contains a list of known limitations and issues. Networking, subprocesses, and threading are not available. @@ -679,9 +679,9 @@

Simple REPL for Python WASM

your browser instead of using server.py as described in - Tools/wasm/README.md + Platforms/emscripten/README.md .

diff --git a/Python/Python-ast.c b/Python/Python-ast.c index dad1530e343..6bcf57bdd6b 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -5197,6 +5197,70 @@ ast_clear(PyObject *op) return 0; } +/* + * Format the names in the set 'missing' into a natural language list, + * sorted in the order in which they appear in 'fields'. + * + * Similar to format_missing() from 'Python/ceval.c'. + * + * Parameters + * + * missing Set of missing field names to render. + * fields Sequence of AST node field names (self._fields). + */ +static PyObject * +format_missing(PyObject *missing, PyObject *fields) +{ + Py_ssize_t num_fields, num_total, num_left; + num_fields = PySequence_Size(fields); + if (num_fields == -1) { + return NULL; + } + num_total = num_left = PySet_GET_SIZE(missing); + PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); + if (writer == NULL) { + goto error; + } + // Iterate all AST node fields in order so that the missing positional + // arguments are rendered in the order in which __init__ expects them. + for (Py_ssize_t i = 0; i < num_fields; i++) { + PyObject *name = PySequence_GetItem(fields, i); + if (name == NULL) { + goto error; + } + int contains = PySet_Contains(missing, name); + if (contains == -1) { + Py_DECREF(name); + goto error; + } + else if (contains == 1) { + const char* fmt = NULL; + if (num_left == 1) { + fmt = "'%U'"; + } + else if (num_total == 2) { + fmt = "'%U' and "; + } + else if (num_left == 2) { + fmt = "'%U', and "; + } + else { + fmt = "'%U', "; + } + num_left--; + if (PyUnicodeWriter_Format(writer, fmt, name) < 0) { + Py_DECREF(name); + goto error; + } + } + Py_DECREF(name); + } + return PyUnicodeWriter_Finish(writer); +error: + PyUnicodeWriter_Discard(writer); + return NULL; +} + static int ast_type_init(PyObject *self, PyObject *args, PyObject *kw) { @@ -5266,8 +5330,8 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) } if (p == 0) { PyErr_Format(PyExc_TypeError, - "%.400s got multiple values for argument %R", - Py_TYPE(self)->tp_name, key); + "%T got multiple values for argument %R", + self, key); res = -1; goto cleanup; } @@ -5287,16 +5351,11 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) goto cleanup; } else if (contains == 0) { - if (PyErr_WarnFormat( - PyExc_DeprecationWarning, 1, - "%.400s.__init__ got an unexpected keyword argument %R. " - "Support for arbitrary keyword arguments is deprecated " - "and will be removed in Python 3.15.", - Py_TYPE(self)->tp_name, key - ) < 0) { - res = -1; - goto cleanup; - } + PyErr_Format(PyExc_TypeError, + "%T.__init__ got an unexpected keyword argument %R", + self, key); + res = -1; + goto cleanup; } } res = PyObject_SetAttr(self, key, value); @@ -5306,7 +5365,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) } } Py_ssize_t size = PySet_Size(remaining_fields); - PyObject *field_types = NULL, *remaining_list = NULL; + PyObject *field_types = NULL, *remaining_list = NULL, *missing_names = NULL; if (size > 0) { if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), &_Py_ID(_field_types), &field_types) < 0) { @@ -5323,6 +5382,10 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) if (!remaining_list) { goto set_remaining_cleanup; } + missing_names = PySet_New(NULL); + if (!missing_names) { + goto set_remaining_cleanup; + } for (Py_ssize_t i = 0; i < size; i++) { PyObject *name = PyList_GET_ITEM(remaining_list, i); PyObject *type = PyDict_GetItemWithError(field_types, name); @@ -5331,14 +5394,10 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) goto set_remaining_cleanup; } else { - if (PyErr_WarnFormat( - PyExc_DeprecationWarning, 1, - "Field %R is missing from %.400s._field_types. " - "This will become an error in Python 3.15.", - name, Py_TYPE(self)->tp_name - ) < 0) { - goto set_remaining_cleanup; - } + PyErr_Format(PyExc_TypeError, + "Field %R is missing from %T._field_types", + name, self); + goto set_remaining_cleanup; } } else if (_PyUnion_Check(type)) { @@ -5366,16 +5425,25 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) } else { // simple field (e.g., identifier) - if (PyErr_WarnFormat( - PyExc_DeprecationWarning, 1, - "%.400s.__init__ missing 1 required positional argument: %R. " - "This will become an error in Python 3.15.", - Py_TYPE(self)->tp_name, name - ) < 0) { + res = PySet_Add(missing_names, name); + if (res < 0) { goto set_remaining_cleanup; } } } + Py_ssize_t num_missing = PySet_GET_SIZE(missing_names); + if (num_missing > 0) { + PyObject *name_str = format_missing(missing_names, fields); + if (!name_str) { + goto set_remaining_cleanup; + } + PyErr_Format(PyExc_TypeError, + "%T.__init__ missing %d required positional argument%s: %U", + self, num_missing, num_missing == 1 ? "" : "s", name_str); + Py_DECREF(name_str); + goto set_remaining_cleanup; + } + Py_DECREF(missing_names); Py_DECREF(remaining_list); Py_DECREF(field_types); } @@ -5385,6 +5453,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) Py_XDECREF(remaining_fields); return res; set_remaining_cleanup: + Py_XDECREF(missing_names); Py_XDECREF(remaining_list); Py_XDECREF(field_types); res = -1; @@ -5468,182 +5537,6 @@ cleanup: return result; } -/* - * Perform the following validations: - * - * - All keyword arguments are known 'fields' or 'attributes'. - * - No field or attribute would be left unfilled after copy.replace(). - * - * On success, this returns 1. Otherwise, set a TypeError - * exception and returns -1 (no exception is set if some - * other internal errors occur). - * - * Parameters - * - * self The AST node instance. - * dict The AST node instance dictionary (self.__dict__). - * fields The list of fields (self._fields). - * attributes The list of attributes (self._attributes). - * kwargs Keyword arguments passed to ast_type_replace(). - * - * The 'dict', 'fields', 'attributes' and 'kwargs' arguments can be NULL. - * - * Note: this function can be removed in 3.15 since the verification - * will be done inside the constructor. - */ -static inline int -ast_type_replace_check(PyObject *self, - PyObject *dict, - PyObject *fields, - PyObject *attributes, - PyObject *kwargs) -{ - // While it is possible to make some fast paths that would avoid - // allocating objects on the stack, this would cost us readability. - // For instance, if 'fields' and 'attributes' are both empty, and - // 'kwargs' is not empty, we could raise a TypeError immediately. - PyObject *expecting = PySet_New(fields); - if (expecting == NULL) { - return -1; - } - if (attributes) { - if (_PySet_Update(expecting, attributes) < 0) { - Py_DECREF(expecting); - return -1; - } - } - // Any keyword argument that is neither a field nor attribute is rejected. - // We first need to check whether a keyword argument is accepted or not. - // If all keyword arguments are accepted, we compute the required fields - // and attributes. A field or attribute is not needed if: - // - // 1) it is given in 'kwargs', or - // 2) it already exists on 'self'. - if (kwargs) { - Py_ssize_t pos = 0; - PyObject *key, *value; - while (PyDict_Next(kwargs, &pos, &key, &value)) { - int rc = PySet_Discard(expecting, key); - if (rc < 0) { - Py_DECREF(expecting); - return -1; - } - if (rc == 0) { - PyErr_Format(PyExc_TypeError, - "%.400s.__replace__ got an unexpected keyword " - "argument %R.", Py_TYPE(self)->tp_name, key); - Py_DECREF(expecting); - return -1; - } - } - } - // check that the remaining fields or attributes would be filled - if (dict) { - Py_ssize_t pos = 0; - PyObject *key, *value; - while (PyDict_Next(dict, &pos, &key, &value)) { - // Mark fields or attributes that are found on the instance - // as non-mandatory. If they are not given in 'kwargs', they - // will be shallow-coied; otherwise, they would be replaced - // (not in this function). - if (PySet_Discard(expecting, key) < 0) { - Py_DECREF(expecting); - return -1; - } - } - if (attributes) { - // Some attributes may or may not be present at runtime. - // In particular, now that we checked whether 'kwargs' - // is correct or not, we allow any attribute to be missing. - // - // Note that fields must still be entirely determined when - // calling the constructor later. - PyObject *unused = PyObject_CallMethodOneArg(expecting, - &_Py_ID(difference_update), - attributes); - if (unused == NULL) { - Py_DECREF(expecting); - return -1; - } - Py_DECREF(unused); - } - } - - // Discard fields from 'expecting' that default to None - PyObject *field_types = NULL; - if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), - &_Py_ID(_field_types), - &field_types) < 0) - { - Py_DECREF(expecting); - return -1; - } - if (field_types != NULL) { - Py_ssize_t pos = 0; - PyObject *field_name, *field_type; - while (PyDict_Next(field_types, &pos, &field_name, &field_type)) { - if (_PyUnion_Check(field_type)) { - // optional field - if (PySet_Discard(expecting, field_name) < 0) { - Py_DECREF(expecting); - Py_DECREF(field_types); - return -1; - } - } - } - Py_DECREF(field_types); - } - - // Now 'expecting' contains the fields or attributes - // that would not be filled inside ast_type_replace(). - Py_ssize_t m = PySet_GET_SIZE(expecting); - if (m > 0) { - PyObject *names = PyList_New(m); - if (names == NULL) { - Py_DECREF(expecting); - return -1; - } - Py_ssize_t i = 0, pos = 0; - PyObject *item; - Py_hash_t hash; - while (_PySet_NextEntry(expecting, &pos, &item, &hash)) { - PyObject *name = PyObject_Repr(item); - if (name == NULL) { - Py_DECREF(expecting); - Py_DECREF(names); - return -1; - } - // steal the reference 'name' - PyList_SET_ITEM(names, i++, name); - } - Py_DECREF(expecting); - if (PyList_Sort(names) < 0) { - Py_DECREF(names); - return -1; - } - PyObject *sep = PyUnicode_FromString(", "); - if (sep == NULL) { - Py_DECREF(names); - return -1; - } - PyObject *str_names = PyUnicode_Join(sep, names); - Py_DECREF(sep); - Py_DECREF(names); - if (str_names == NULL) { - return -1; - } - PyErr_Format(PyExc_TypeError, - "%.400s.__replace__ missing %ld keyword argument%s: %U.", - Py_TYPE(self)->tp_name, m, m == 1 ? "" : "s", str_names); - Py_DECREF(str_names); - return -1; - } - else { - Py_DECREF(expecting); - return 1; - } -} - /* * Python equivalent: * @@ -5733,9 +5626,6 @@ ast_type_replace(PyObject *self, PyObject *args, PyObject *kwargs) if (PyObject_GetOptionalAttr(self, state->__dict__, &dict) < 0) { goto cleanup; } - if (ast_type_replace_check(self, dict, fields, attributes, kwargs) < 0) { - goto cleanup; - } empty_tuple = PyTuple_New(0); if (empty_tuple == NULL) { goto cleanup; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index fec64e1ff9d..35b30a24331 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1840,15 +1840,17 @@ hash as builtin_hash obj: object / -Return the hash value for the given object. +Return the integer hash value for the given object. -Two objects that compare equal must also have the same hash value, but the -reverse is not necessarily true. +Two objects that compare equal must also have the same hash value, but +the reverse is not necessarily true. Hash values may differ between +Python processes. Not all objects are hashable; calling hash() on an +unhashable object raises TypeError. [clinic start generated code]*/ static PyObject * builtin_hash(PyObject *module, PyObject *obj) -/*[clinic end generated code: output=237668e9d7688db7 input=58c48be822bf9c54]*/ +/*[clinic end generated code: output=237668e9d7688db7 input=70a242ff65f6717c]*/ { Py_hash_t x; @@ -3553,6 +3555,7 @@ _PyBuiltin_Init(PyInterpreterState *interp) SETBUILTIN("object", &PyBaseObject_Type); SETBUILTIN("range", &PyRange_Type); SETBUILTIN("reversed", &PyReversed_Type); + SETBUILTIN("sentinel", &PySentinel_Type); SETBUILTIN("set", &PySet_Type); SETBUILTIN("slice", &PySlice_Type); SETBUILTIN("staticmethod", &PyStaticMethod_Type); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7de889b93b7..d485172c82f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3529,7 +3529,7 @@ dummy_func( int og_oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; if (_PyJit_EnterExecutorShouldStopTracing(og_opcode)) { - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + if (_PyOpcode_Caches[_PyOpcode_Deopt[og_opcode]]) { PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } opcode = og_opcode; @@ -6107,11 +6107,11 @@ dummy_func( value = PyStackRef_FromPyObjectBorrow(ptr); } - tier2 op(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW, (ptr/4, callable, null, arg -- res, a, c)) { - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - c = callable; - INPUTS_DEAD(); + tier2 pure op(_RROT_3, (bottom, middle, top -- bottom, middle, top)) { + _PyStackRef temp = top; + top = middle; + middle = bottom; + bottom = temp; } tier2 op(_START_EXECUTOR, (executor/4 --)) { @@ -6541,7 +6541,10 @@ dummy_func( tracer->prev_state.instr_frame = frame; tracer->prev_state.instr_oparg = oparg; tracer->prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + // Branch opcodes use the cache for branch history, not + // specialization counters. Don't reset it. + && !IS_CONDITIONAL_JUMP_OPCODE(opcode)) { (&next_instr[1])->counter = trigger_backoff_counter(); } diff --git a/Python/ceval.c b/Python/ceval.c index 03bc5229565..28087ba58d4 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1305,7 +1305,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } #ifdef _Py_TIER2 #ifdef _Py_JIT -_PyJitEntryFuncPtr _Py_jit_entry = _Py_LazyJitShim; +_PyJitEntryFuncPtr _Py_jit_entry = _PyJIT_Entry; #else _PyJitEntryFuncPtr _Py_jit_entry = _PyTier2Interpreter; #endif @@ -2406,15 +2406,16 @@ void _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RAISE)) { + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_RAISE)) { return; } do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RAISE); } bool -_PyEval_NoToolsForUnwind(PyThreadState *tstate) { - return no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_UNWIND); +_PyEval_NoToolsForUnwind(PyThreadState *tstate, _PyInterpreterFrame *frame) +{ + return no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_PY_UNWIND); } @@ -3408,7 +3409,7 @@ _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwarg _PyErr_Format( tstate, PyExc_TypeError, "%V got multiple values for keyword argument '%S'", - funcstr, "finction", dupkey); + funcstr, "function", dupkey); Py_XDECREF(funcstr); return; } diff --git a/Python/ceval.h b/Python/ceval.h index bb5f7ddb857..0437ab85c5a 100644 --- a/Python/ceval.h +++ b/Python/ceval.h @@ -367,7 +367,7 @@ no_tools_for_global_event(PyThreadState *tstate, int event) static inline bool no_tools_for_local_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event) { - assert(event < _PY_MONITORING_LOCAL_EVENTS); + assert(event < _PY_MONITORING_UNGROUPED_EVENTS); _PyCoMonitoringData *data = _PyFrame_GetCode(frame)->_co_monitoring; if (data) { return data->active_monitors.tools[event] == 0; @@ -382,7 +382,7 @@ monitor_handled(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *exc) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { return 0; } return _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); @@ -393,7 +393,7 @@ monitor_throw(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_THROW)) { + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_PY_THROW)) { return; } do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW); @@ -403,7 +403,7 @@ static void monitor_reraise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RERAISE)) { + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_RERAISE)) { return; } do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RERAISE); @@ -431,7 +431,7 @@ monitor_unwind(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_UNWIND)) { + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_PY_UNWIND)) { return; } do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_UNWIND); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index a7d63fd3b82..a4e9980589e 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -168,7 +168,6 @@ #define STOP_TRACING() ((void)(0)); #endif - /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ #ifdef Py_DEBUG #define PRE_DISPATCH_GOTO() if (frame->lltrace >= 5) { \ diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index c8c141f863d..e6b845cd375 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -826,10 +826,12 @@ PyDoc_STRVAR(builtin_hash__doc__, "hash($module, obj, /)\n" "--\n" "\n" -"Return the hash value for the given object.\n" +"Return the integer hash value for the given object.\n" "\n" -"Two objects that compare equal must also have the same hash value, but the\n" -"reverse is not necessarily true."); +"Two objects that compare equal must also have the same hash value, but\n" +"the reverse is not necessarily true. Hash values may differ between\n" +"Python processes. Not all objects are hashable; calling hash() on an\n" +"unhashable object raises TypeError."); #define BUILTIN_HASH_METHODDEF \ {"hash", (PyCFunction)builtin_hash, METH_O, builtin_hash__doc__}, @@ -1380,4 +1382,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=1c3327da8885bb8e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f1fc836a63d89826 input=a9049054013a1b77]*/ diff --git a/Python/codegen.c b/Python/codegen.c index 389e7cf85d3..a77451152c6 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -3959,6 +3959,14 @@ maybe_optimize_function_call(compiler *c, expr_ty e, jump_target_label end) return 0; } + expr_ty generator_exp = asdl_seq_GET(args, 0); + PySTEntryObject *generator_entry = _PySymtable_Lookup(SYMTABLE(c), (void *)generator_exp); + if (generator_entry->ste_coroutine) { + Py_DECREF(generator_entry); + return 0; + } + Py_DECREF(generator_entry); + location loc = LOC(func); int optimized = 0; @@ -3998,7 +4006,6 @@ maybe_optimize_function_call(compiler *c, expr_ty e, jump_target_label end) } else if (const_oparg == CONSTANT_BUILTIN_SET) { ADDOP_I(c, loc, BUILD_SET, 0); } - expr_ty generator_exp = asdl_seq_GET(args, 0); VISIT(c, expr, generator_exp); NEW_JUMP_TARGET_LABEL(c, loop); diff --git a/Python/compile.c b/Python/compile.c index 5f82641a394..eb9fc827bea 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1658,6 +1658,7 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, { PyObject *res = NULL; PyObject *metadata = NULL; + PyObject *consts_list = NULL; if (!PyAST_Check(ast)) { PyErr_SetString(PyExc_TypeError, "expected an AST"); @@ -1712,12 +1713,23 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, } if (_PyInstructionSequence_ApplyLabelMap(_PyCompile_InstrSequence(c)) < 0) { - return NULL; + goto finally; } + + /* After AddReturnAtEnd: co_consts indices match the final instruction stream. */ + consts_list = consts_dict_keys_inorder(umd->u_consts); + if (consts_list == NULL) { + goto finally; + } + if (PyDict_SetItemString(metadata, "consts", consts_list) < 0) { + goto finally; + } + /* Allocate a copy of the instruction sequence on the heap */ res = _PyTuple_FromPair((PyObject *)_PyCompile_InstrSequence(c), metadata); finally: + Py_XDECREF(consts_list); Py_XDECREF(metadata); _PyCompile_ExitScope(c); compiler_free(c); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 93d39bee1b9..f8fc35de9d7 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -22337,23 +22337,22 @@ break; } - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03: { + case _RROT_3_r03: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef a; - _PyStackRef c; - arg = stack_pointer[-1]; - callable = stack_pointer[-3]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - c = callable; - _tos_cache2 = c; - _tos_cache1 = a; - _tos_cache0 = res; + _PyStackRef top; + _PyStackRef middle; + _PyStackRef bottom; + top = stack_pointer[-1]; + middle = stack_pointer[-2]; + bottom = stack_pointer[-3]; + _PyStackRef temp = top; + top = middle; + middle = bottom; + bottom = temp; + _tos_cache2 = top; + _tos_cache1 = middle; + _tos_cache0 = bottom; SET_CURRENT_CACHED_VALUES(3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -22361,24 +22360,23 @@ break; } - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13: { + case _RROT_3_r13: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef a; - _PyStackRef c; + _PyStackRef top; + _PyStackRef middle; + _PyStackRef bottom; _PyStackRef _stack_item_0 = _tos_cache0; - arg = _stack_item_0; - callable = stack_pointer[-2]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - c = callable; - _tos_cache2 = c; - _tos_cache1 = a; - _tos_cache0 = res; + top = _stack_item_0; + middle = stack_pointer[-1]; + bottom = stack_pointer[-2]; + _PyStackRef temp = top; + top = middle; + middle = bottom; + bottom = temp; + _tos_cache2 = top; + _tos_cache1 = middle; + _tos_cache0 = bottom; SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -22386,25 +22384,24 @@ break; } - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23: { + case _RROT_3_r23: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef a; - _PyStackRef c; + _PyStackRef top; + _PyStackRef middle; + _PyStackRef bottom; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - arg = _stack_item_1; - callable = stack_pointer[-1]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - c = callable; - _tos_cache2 = c; - _tos_cache1 = a; - _tos_cache0 = res; + top = _stack_item_1; + middle = _stack_item_0; + bottom = stack_pointer[-1]; + _PyStackRef temp = top; + top = middle; + middle = bottom; + bottom = temp; + _tos_cache2 = top; + _tos_cache1 = middle; + _tos_cache0 = bottom; SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -22412,26 +22409,25 @@ break; } - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33: { + case _RROT_3_r33: { CHECK_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef a; - _PyStackRef c; + _PyStackRef top; + _PyStackRef middle; + _PyStackRef bottom; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; - arg = _stack_item_2; - callable = _stack_item_0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - c = callable; - _tos_cache2 = c; - _tos_cache1 = a; - _tos_cache0 = res; + top = _stack_item_2; + middle = _stack_item_1; + bottom = _stack_item_0; + _PyStackRef temp = top; + top = middle; + middle = bottom; + bottom = temp; + _tos_cache2 = top; + _tos_cache1 = middle; + _tos_cache0 = bottom; SET_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; diff --git a/Python/flowgraph.c b/Python/flowgraph.c index c234fa3d8c3..2cb2d32a410 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -6,6 +6,7 @@ #include "pycore_intrinsics.h" #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_long.h" // _PY_IS_SMALL_INT() +#include "pycore_hashtable.h" // _Py_hashtable_t #include "pycore_opcode_utils.h" #include "pycore_opcode_metadata.h" // OPCODE_HAS_ARG, etc @@ -1309,6 +1310,14 @@ get_const_value(int opcode, int oparg, PyObject *co_consts) PyObject *constant = NULL; assert(loads_const(opcode)); if (opcode == LOAD_CONST) { + assert(PyList_Check(co_consts)); + Py_ssize_t n = PyList_GET_SIZE(co_consts); + if (oparg < 0 || oparg >= n) { + PyErr_Format(PyExc_ValueError, + "LOAD_CONST index %d is out of range for consts (len=%zd)", + oparg, n); + return NULL; + } constant = PyList_GET_ITEM(co_consts, oparg); } if (opcode == LOAD_SMALL_INT) { @@ -1325,30 +1334,38 @@ get_const_value(int opcode, int oparg, PyObject *co_consts) // Steals a reference to newconst. static int -add_const(PyObject *newconst, PyObject *consts, PyObject *const_cache) +add_const(PyObject *newconst, PyObject *consts, PyObject *const_cache, + _Py_hashtable_t *consts_index) { if (_PyCompile_ConstCacheMergeOne(const_cache, &newconst) < 0) { Py_DECREF(newconst); return -1; } - Py_ssize_t index; - for (index = 0; index < PyList_GET_SIZE(consts); index++) { - if (PyList_GET_ITEM(consts, index) == newconst) { - break; - } + _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(consts_index, (void *)newconst); + if (entry != NULL) { + Py_DECREF(newconst); + return (int)(uintptr_t)entry->value; } - if (index == PyList_GET_SIZE(consts)) { - if ((size_t)index >= (size_t)INT_MAX - 1) { - PyErr_SetString(PyExc_OverflowError, "too many constants"); - Py_DECREF(newconst); - return -1; - } - if (PyList_Append(consts, newconst)) { - Py_DECREF(newconst); - return -1; - } + + Py_ssize_t index = PyList_GET_SIZE(consts); + if ((size_t)index >= (size_t)INT_MAX - 1) { + PyErr_SetString(PyExc_OverflowError, "too many constants"); + Py_DECREF(newconst); + return -1; } + if (PyList_Append(consts, newconst)) { + Py_DECREF(newconst); + return -1; + } + + if (_Py_hashtable_set(consts_index, (void *)newconst, (void *)(uintptr_t)index) < 0) { + PyList_SetSlice(consts, index, index + 1, NULL); + Py_DECREF(newconst); + PyErr_NoMemory(); + return -1; + } + Py_DECREF(newconst); return (int)index; } @@ -1424,7 +1441,8 @@ maybe_instr_make_load_smallint(cfg_instr *instr, PyObject *newconst, /* Steals reference to "newconst" */ static int instr_make_load_const(cfg_instr *instr, PyObject *newconst, - PyObject *consts, PyObject *const_cache) + PyObject *consts, PyObject *const_cache, + _Py_hashtable_t *consts_index) { int res = maybe_instr_make_load_smallint(instr, newconst, consts, const_cache); if (res < 0) { @@ -1434,7 +1452,7 @@ instr_make_load_const(cfg_instr *instr, PyObject *newconst, if (res > 0) { return SUCCESS; } - int oparg = add_const(newconst, consts, const_cache); + int oparg = add_const(newconst, consts, const_cache, consts_index); RETURN_IF_ERROR(oparg); INSTR_SET_OP1(instr, LOAD_CONST, oparg); return SUCCESS; @@ -1447,7 +1465,8 @@ instr_make_load_const(cfg_instr *instr, PyObject *newconst, Called with codestr pointing to the first LOAD_CONST. */ static int -fold_tuple_of_constants(basicblock *bb, int i, PyObject *consts, PyObject *const_cache) +fold_tuple_of_constants(basicblock *bb, int i, PyObject *consts, + PyObject *const_cache, _Py_hashtable_t *consts_index) { /* Pre-conditions */ assert(PyDict_CheckExact(const_cache)); @@ -1484,7 +1503,7 @@ fold_tuple_of_constants(basicblock *bb, int i, PyObject *consts, PyObject *const } nop_out(const_instrs, seq_size); - return instr_make_load_const(instr, const_tuple, consts, const_cache); + return instr_make_load_const(instr, const_tuple, consts, const_cache, consts_index); } /* Replace: @@ -1502,7 +1521,8 @@ fold_tuple_of_constants(basicblock *bb, int i, PyObject *consts, PyObject *const */ static int fold_constant_intrinsic_list_to_tuple(basicblock *bb, int i, - PyObject *consts, PyObject *const_cache) + PyObject *consts, PyObject *const_cache, + _Py_hashtable_t *consts_index) { assert(PyDict_CheckExact(const_cache)); assert(PyList_CheckExact(consts)); @@ -1554,7 +1574,7 @@ fold_constant_intrinsic_list_to_tuple(basicblock *bb, int i, nop_out(&instr, 1); } assert(consts_found == 0); - return instr_make_load_const(intrinsic, newconst, consts, const_cache); + return instr_make_load_const(intrinsic, newconst, consts, const_cache, consts_index); } if (expect_append) { @@ -1590,7 +1610,8 @@ Optimize lists and sets for: */ static int optimize_lists_and_sets(basicblock *bb, int i, int nextop, - PyObject *consts, PyObject *const_cache) + PyObject *consts, PyObject *const_cache, + _Py_hashtable_t *consts_index) { assert(PyDict_CheckExact(const_cache)); assert(PyList_CheckExact(consts)); @@ -1640,7 +1661,7 @@ optimize_lists_and_sets(basicblock *bb, int i, int nextop, Py_SETREF(const_result, frozenset); } - int index = add_const(const_result, consts, const_cache); + int index = add_const(const_result, consts, const_cache, consts_index); RETURN_IF_ERROR(index); nop_out(const_instrs, seq_size); @@ -1837,7 +1858,8 @@ eval_const_binop(PyObject *left, int op, PyObject *right) } static int -fold_const_binop(basicblock *bb, int i, PyObject *consts, PyObject *const_cache) +fold_const_binop(basicblock *bb, int i, PyObject *consts, + PyObject *const_cache, _Py_hashtable_t *consts_index) { #define BINOP_OPERAND_COUNT 2 assert(PyDict_CheckExact(const_cache)); @@ -1879,7 +1901,7 @@ fold_const_binop(basicblock *bb, int i, PyObject *consts, PyObject *const_cache) } nop_out(operands_instrs, BINOP_OPERAND_COUNT); - return instr_make_load_const(binop, newconst, consts, const_cache); + return instr_make_load_const(binop, newconst, consts, const_cache, consts_index); } static PyObject * @@ -1925,7 +1947,8 @@ eval_const_unaryop(PyObject *operand, int opcode, int oparg) } static int -fold_const_unaryop(basicblock *bb, int i, PyObject *consts, PyObject *const_cache) +fold_const_unaryop(basicblock *bb, int i, PyObject *consts, + PyObject *const_cache, _Py_hashtable_t *consts_index) { #define UNARYOP_OPERAND_COUNT 1 assert(PyDict_CheckExact(const_cache)); @@ -1962,7 +1985,7 @@ fold_const_unaryop(basicblock *bb, int i, PyObject *consts, PyObject *const_cach assert(PyBool_Check(newconst)); } nop_out(&operand_instr, UNARYOP_OPERAND_COUNT); - return instr_make_load_const(unaryop, newconst, consts, const_cache); + return instr_make_load_const(unaryop, newconst, consts, const_cache, consts_index); } #define VISITED (-1) @@ -2157,7 +2180,8 @@ apply_static_swaps(basicblock *block, int i) } static int -basicblock_optimize_load_const(PyObject *const_cache, basicblock *bb, PyObject *consts) +basicblock_optimize_load_const(PyObject *const_cache, basicblock *bb, + PyObject *consts, _Py_hashtable_t *consts_index) { assert(PyDict_CheckExact(const_cache)); assert(PyList_CheckExact(consts)); @@ -2167,6 +2191,9 @@ basicblock_optimize_load_const(PyObject *const_cache, basicblock *bb, PyObject * cfg_instr *inst = &bb->b_instr[i]; if (inst->i_opcode == LOAD_CONST) { PyObject *constant = get_const_value(inst->i_opcode, inst->i_oparg, consts); + if (constant == NULL) { + return ERROR; + } int res = maybe_instr_make_load_smallint(inst, constant, consts, const_cache); Py_DECREF(constant); if (res < 0) { @@ -2272,7 +2299,7 @@ basicblock_optimize_load_const(PyObject *const_cache, basicblock *bb, PyObject * return ERROR; } cnt = PyBool_FromLong(is_true); - int index = add_const(cnt, consts, const_cache); + int index = add_const(cnt, consts, const_cache, consts_index); if (index < 0) { return ERROR; } @@ -2286,15 +2313,17 @@ basicblock_optimize_load_const(PyObject *const_cache, basicblock *bb, PyObject * } static int -optimize_load_const(PyObject *const_cache, cfg_builder *g, PyObject *consts) { +optimize_load_const(PyObject *const_cache, cfg_builder *g, PyObject *consts, + _Py_hashtable_t *consts_index) { for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - RETURN_IF_ERROR(basicblock_optimize_load_const(const_cache, b, consts)); + RETURN_IF_ERROR(basicblock_optimize_load_const(const_cache, b, consts, consts_index)); } return SUCCESS; } static int -optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) +optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts, + _Py_hashtable_t *consts_index) { assert(PyDict_CheckExact(const_cache)); assert(PyList_CheckExact(consts)); @@ -2334,11 +2363,11 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) continue; } } - RETURN_IF_ERROR(fold_tuple_of_constants(bb, i, consts, const_cache)); + RETURN_IF_ERROR(fold_tuple_of_constants(bb, i, consts, const_cache, consts_index)); break; case BUILD_LIST: case BUILD_SET: - RETURN_IF_ERROR(optimize_lists_and_sets(bb, i, nextop, consts, const_cache)); + RETURN_IF_ERROR(optimize_lists_and_sets(bb, i, nextop, consts, const_cache, consts_index)); break; case POP_JUMP_IF_NOT_NONE: case POP_JUMP_IF_NONE: @@ -2473,7 +2502,7 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) _Py_FALLTHROUGH; case UNARY_INVERT: case UNARY_NEGATIVE: - RETURN_IF_ERROR(fold_const_unaryop(bb, i, consts, const_cache)); + RETURN_IF_ERROR(fold_const_unaryop(bb, i, consts, const_cache, consts_index)); break; case CALL_INTRINSIC_1: if (oparg == INTRINSIC_LIST_TO_TUPLE) { @@ -2481,15 +2510,15 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) INSTR_SET_OP0(inst, NOP); } else { - RETURN_IF_ERROR(fold_constant_intrinsic_list_to_tuple(bb, i, consts, const_cache)); + RETURN_IF_ERROR(fold_constant_intrinsic_list_to_tuple(bb, i, consts, const_cache, consts_index)); } } else if (oparg == INTRINSIC_UNARY_POSITIVE) { - RETURN_IF_ERROR(fold_const_unaryop(bb, i, consts, const_cache)); + RETURN_IF_ERROR(fold_const_unaryop(bb, i, consts, const_cache, consts_index)); } break; case BINARY_OP: - RETURN_IF_ERROR(fold_const_binop(bb, i, consts, const_cache)); + RETURN_IF_ERROR(fold_const_binop(bb, i, consts, const_cache, consts_index)); break; } } @@ -2534,16 +2563,17 @@ remove_redundant_nops_and_jumps(cfg_builder *g) NOPs. Later those NOPs are removed. */ static int -optimize_cfg(cfg_builder *g, PyObject *consts, PyObject *const_cache, int firstlineno) +optimize_cfg(cfg_builder *g, PyObject *consts, PyObject *const_cache, + _Py_hashtable_t *consts_index, int firstlineno) { assert(PyDict_CheckExact(const_cache)); RETURN_IF_ERROR(check_cfg(g)); RETURN_IF_ERROR(inline_small_or_no_lineno_blocks(g->g_entryblock)); RETURN_IF_ERROR(remove_unreachable(g->g_entryblock)); RETURN_IF_ERROR(resolve_line_numbers(g, firstlineno)); - RETURN_IF_ERROR(optimize_load_const(const_cache, g, consts)); + RETURN_IF_ERROR(optimize_load_const(const_cache, g, consts, consts_index)); for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - RETURN_IF_ERROR(optimize_basic_block(const_cache, b, consts)); + RETURN_IF_ERROR(optimize_basic_block(const_cache, b, consts, consts_index)); } RETURN_IF_ERROR(remove_redundant_nops_and_pairs(g->g_entryblock)); RETURN_IF_ERROR(remove_unreachable(g->g_entryblock)); @@ -3663,7 +3693,33 @@ _PyCfg_OptimizeCodeUnit(cfg_builder *g, PyObject *consts, PyObject *const_cache, RETURN_IF_ERROR(label_exception_targets(g->g_entryblock)); /** Optimization **/ - RETURN_IF_ERROR(optimize_cfg(g, consts, const_cache, firstlineno)); + + _Py_hashtable_t *consts_index = _Py_hashtable_new( + _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct); + if (consts_index == NULL) { + PyErr_NoMemory(); + return ERROR; + } + + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(consts); i++) { + PyObject *item = PyList_GET_ITEM(consts, i); + if (_Py_hashtable_get_entry(consts_index, (void *)item) != NULL) { + continue; + } + if (_Py_hashtable_set(consts_index, (void *)item, + (void *)(uintptr_t)i) < 0) { + _Py_hashtable_destroy(consts_index); + PyErr_NoMemory(); + return ERROR; + } + } + + int ret = optimize_cfg(g, consts, const_cache, consts_index, firstlineno); + + _Py_hashtable_destroy(consts_index); + + RETURN_IF_ERROR(ret); + RETURN_IF_ERROR(remove_unused_consts(g->g_entryblock, consts)); RETURN_IF_ERROR( add_checks_for_loads_of_uninitialized_variables( @@ -4064,6 +4120,10 @@ _PyCompile_OptimizeCfg(PyObject *seq, PyObject *consts, int nlocals) PyErr_SetString(PyExc_ValueError, "expected an instruction sequence"); return NULL; } + if (!PyList_Check(consts)) { + PyErr_SetString(PyExc_TypeError, "consts must be a list"); + return NULL; + } PyObject *const_cache = PyDict_New(); if (const_cache == NULL) { return NULL; diff --git a/Python/gc.c b/Python/gc.c index 284ac725d37..59bed10c1fb 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -6,17 +6,18 @@ #include "pycore_ceval.h" // _Py_set_eval_breaker_bit() #include "pycore_dict.h" // _PyInlineValuesSize() #include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_context.h" #include "pycore_interp.h" // PyInterpreterState.gc #include "pycore_interpframe.h" // _PyFrame_GetLocalsArray() +#include "pycore_object.h" #include "pycore_object_alloc.h" // _PyObject_MallocWithType() +#include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_tuple.h" // _PyTuple_MaybeUntrack() #include "pycore_weakref.h" // _PyWeakref_ClearRef() - #include "pydtrace.h" - -#ifndef Py_GIL_DISABLED +#if !defined(Py_GIL_DISABLED) typedef struct _gc_runtime_state GCState; @@ -24,10 +25,6 @@ typedef struct _gc_runtime_state GCState; # define GC_DEBUG #endif -// Define this when debugging the GC -// #define GC_EXTRA_DEBUG - - #define GC_NEXT _PyGCHead_NEXT #define GC_PREV _PyGCHead_PREV @@ -50,7 +47,7 @@ typedef struct _gc_runtime_state GCState; // move_legacy_finalizers() removes this flag instead. // Between them, unreachable list is not normal list and we can not use // most gc_list_* functions for it. -#define NEXT_MASK_UNREACHABLE 2 +#define NEXT_MASK_UNREACHABLE (1) #define AS_GC(op) _Py_AS_GC(op) #define FROM_GC(gc) _Py_FROM_GC(gc) @@ -100,48 +97,9 @@ gc_decref(PyGC_Head *g) g->_gc_prev -= 1 << _PyGC_PREV_SHIFT; } -static inline int -gc_old_space(PyGC_Head *g) -{ - return g->_gc_next & _PyGC_NEXT_MASK_OLD_SPACE_1; -} -static inline int -other_space(int space) -{ - assert(space == 0 || space == 1); - return space ^ _PyGC_NEXT_MASK_OLD_SPACE_1; -} +#define GEN_HEAD(gcstate, n) (&(gcstate)->generations[n].head) -static inline void -gc_flip_old_space(PyGC_Head *g) -{ - g->_gc_next ^= _PyGC_NEXT_MASK_OLD_SPACE_1; -} - -static inline void -gc_set_old_space(PyGC_Head *g, int space) -{ - assert(space == 0 || space == _PyGC_NEXT_MASK_OLD_SPACE_1); - g->_gc_next &= ~_PyGC_NEXT_MASK_OLD_SPACE_1; - g->_gc_next |= space; -} - -static PyGC_Head * -GEN_HEAD(GCState *gcstate, int n) -{ - assert((gcstate->visited_space & (~1)) == 0); - switch(n) { - case 0: - return &gcstate->young.head; - case 1: - return &gcstate->old[gcstate->visited_space].head; - case 2: - return &gcstate->old[gcstate->visited_space^1].head; - default: - Py_UNREACHABLE(); - } -} static GCState * get_gc_state(void) @@ -160,12 +118,11 @@ _PyGC_InitState(GCState *gcstate) GEN.head._gc_prev = (uintptr_t)&GEN.head; \ } while (0) - assert(gcstate->young.count == 0); - assert(gcstate->old[0].count == 0); - assert(gcstate->old[1].count == 0); - INIT_HEAD(gcstate->young); - INIT_HEAD(gcstate->old[0]); - INIT_HEAD(gcstate->old[1]); + for (int i = 0; i < NUM_GENERATIONS; i++) { + assert(gcstate->generations[i].count == 0); + INIT_HEAD(gcstate->generations[i]); + }; + gcstate->generation0 = GEN_HEAD(gcstate, 0); INIT_HEAD(gcstate->permanent_generation); #undef INIT_HEAD @@ -191,7 +148,6 @@ _PyGC_Init(PyInterpreterState *interp) if (gcstate->callbacks == NULL) { return _PyStatus_NO_MEMORY(); } - gcstate->heap_size = 0; return _PyStatus_OK(); } @@ -269,7 +225,6 @@ gc_list_is_empty(PyGC_Head *list) static inline void gc_list_append(PyGC_Head *node, PyGC_Head *list) { - assert((list->_gc_prev & ~_PyGC_PREV_MASK) == 0); PyGC_Head *last = (PyGC_Head *)list->_gc_prev; // last <-> node @@ -327,8 +282,6 @@ gc_list_merge(PyGC_Head *from, PyGC_Head *to) PyGC_Head *from_tail = GC_PREV(from); assert(from_head != from); assert(from_tail != from); - assert(gc_list_is_empty(to) || - gc_old_space(to_tail) == gc_old_space(from_tail)); _PyGCHead_SET_NEXT(to_tail, from_head); _PyGCHead_SET_PREV(from_head, to_tail); @@ -397,8 +350,8 @@ enum flagstates {collecting_clear_unreachable_clear, static void validate_list(PyGC_Head *head, enum flagstates flags) { - assert((head->_gc_prev & ~_PyGC_PREV_MASK) == 0); - assert((head->_gc_next & ~_PyGC_PREV_MASK) == 0); + assert((head->_gc_prev & PREV_MASK_COLLECTING) == 0); + assert((head->_gc_next & NEXT_MASK_UNREACHABLE) == 0); uintptr_t prev_value = 0, next_value = 0; switch (flags) { case collecting_clear_unreachable_clear: @@ -420,7 +373,7 @@ validate_list(PyGC_Head *head, enum flagstates flags) PyGC_Head *gc = GC_NEXT(head); while (gc != head) { PyGC_Head *trueprev = GC_PREV(gc); - PyGC_Head *truenext = GC_NEXT(gc); + PyGC_Head *truenext = (PyGC_Head *)(gc->_gc_next & ~NEXT_MASK_UNREACHABLE); assert(truenext != NULL); assert(trueprev == prev); assert((gc->_gc_prev & PREV_MASK_COLLECTING) == prev_value); @@ -430,58 +383,10 @@ validate_list(PyGC_Head *head, enum flagstates flags) } assert(prev == GC_PREV(head)); } - #else #define validate_list(x, y) do{}while(0) #endif -#ifdef GC_EXTRA_DEBUG - - -static void -gc_list_validate_space(PyGC_Head *head, int space) { - PyGC_Head *gc = GC_NEXT(head); - while (gc != head) { - assert(gc_old_space(gc) == space); - gc = GC_NEXT(gc); - } -} - -static void -validate_spaces(GCState *gcstate) -{ - int visited = gcstate->visited_space; - int not_visited = other_space(visited); - gc_list_validate_space(&gcstate->young.head, not_visited); - for (int space = 0; space < 2; space++) { - gc_list_validate_space(&gcstate->old[space].head, space); - } - gc_list_validate_space(&gcstate->permanent_generation.head, visited); -} - -static void -validate_consistent_old_space(PyGC_Head *head) -{ - PyGC_Head *gc = GC_NEXT(head); - if (gc == head) { - return; - } - int old_space = gc_old_space(gc); - while (gc != head) { - PyGC_Head *truenext = GC_NEXT(gc); - assert(truenext != NULL); - assert(gc_old_space(gc) == old_space); - gc = truenext; - } -} - - -#else -#define validate_spaces(g) do{}while(0) -#define validate_consistent_old_space(l) do{}while(0) -#define gc_list_validate_space(l, s) do{}while(0) -#endif - /*** end of list stuff ***/ @@ -501,8 +406,8 @@ update_refs(PyGC_Head *containers) if (_Py_IsImmortal(op)) { assert(!_Py_IsStaticImmortal(op)); _PyObject_GC_UNTRACK(op); - gc = next; - continue; + gc = next; + continue; } gc_reset_refs(gc, Py_REFCNT(op)); /* Python's cyclic gc should never see an incoming refcount @@ -530,19 +435,12 @@ update_refs(PyGC_Head *containers) return candidates; } -struct visit_decref_context { - PyObject *parent; - struct gc_generation_stats *stats; -}; - /* A traversal callback for subtract_refs. */ static int -visit_decref(PyObject *op, void *arg) +visit_decref(PyObject *op, void *parent) { OBJECT_STAT_INC(object_visits); - struct visit_decref_context *ctx = (struct visit_decref_context *)arg; - ctx->stats->object_visits += 1; - _PyObject_ASSERT(ctx->parent, !_PyObject_IsFreed(op)); + _PyObject_ASSERT(_PyObject_CAST(parent), !_PyObject_IsFreed(op)); if (_PyObject_IS_GC(op)) { PyGC_Head *gc = AS_GC(op); @@ -589,35 +487,24 @@ _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg) * reachable from outside containers, and so can't be collected. */ static void -subtract_refs(PyGC_Head *containers, struct gc_generation_stats *stats) +subtract_refs(PyGC_Head *containers) { traverseproc traverse; PyGC_Head *gc = GC_NEXT(containers); for (; gc != containers; gc = GC_NEXT(gc)) { PyObject *op = FROM_GC(gc); traverse = Py_TYPE(op)->tp_traverse; - struct visit_decref_context ctx = { - .parent = op, - .stats = stats - }; (void) traverse(op, visit_decref, - &ctx); + op); } } -struct visit_reachable_context { - PyGC_Head *head; - struct gc_generation_stats *stats; -}; - /* A traversal callback for move_unreachable. */ static int visit_reachable(PyObject *op, void *arg) { - struct visit_reachable_context *ctx = (struct visit_reachable_context *)arg; - ctx->stats->object_visits += 1; - PyGC_Head *reachable = ctx->head; + PyGC_Head *reachable = arg; OBJECT_STAT_INC(object_visits); if (!_PyObject_IS_GC(op)) { return 0; @@ -647,13 +534,12 @@ visit_reachable(PyObject *op, void *arg) // Manually unlink gc from unreachable list because the list functions // don't work right in the presence of NEXT_MASK_UNREACHABLE flags. PyGC_Head *prev = GC_PREV(gc); - PyGC_Head *next = GC_NEXT(gc); + PyGC_Head *next = (PyGC_Head*)(gc->_gc_next & ~NEXT_MASK_UNREACHABLE); _PyObject_ASSERT(FROM_GC(prev), prev->_gc_next & NEXT_MASK_UNREACHABLE); _PyObject_ASSERT(FROM_GC(next), next->_gc_next & NEXT_MASK_UNREACHABLE); - prev->_gc_next = gc->_gc_next; // copy flag bits - gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; + prev->_gc_next = gc->_gc_next; // copy NEXT_MASK_UNREACHABLE _PyGCHead_SET_PREV(next, prev); gc_list_append(gc, reachable); @@ -690,7 +576,7 @@ visit_reachable(PyObject *op, void *arg) * So we can not gc_list_* functions for unreachable until we remove the flag. */ static void -move_unreachable(PyGC_Head *young, PyGC_Head *unreachable, struct gc_generation_stats *stats) +move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) { // previous elem in the young list, used for restore gc_prev. PyGC_Head *prev = young; @@ -705,14 +591,6 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable, struct gc_generation_ * or to the right have been scanned yet. */ - struct visit_reachable_context ctx = { - .head = young, - .stats = stats - }; - - validate_consistent_old_space(young); - /* Record which old space we are in, and set NEXT_MASK_UNREACHABLE bit for convenience */ - uintptr_t flags = NEXT_MASK_UNREACHABLE | (gc->_gc_next & _PyGC_NEXT_MASK_OLD_SPACE_1); while (gc != young) { if (gc_get_refs(gc)) { /* gc is definitely reachable from outside the @@ -731,7 +609,7 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable, struct gc_generation_ // young->_gc_prev == gc. Don't do gc = GC_NEXT(gc) before! (void) traverse(op, visit_reachable, - &ctx); + (void *)young); // relink gc_prev to prev element. _PyGCHead_SET_PREV(gc, prev); // gc is not COLLECTING state after here. @@ -758,18 +636,17 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable, struct gc_generation_ // But this may pollute the unreachable list head's 'next' pointer // too. That's semantically senseless but expedient here - the // damage is repaired when this function ends. - last->_gc_next = flags | (uintptr_t)gc; + last->_gc_next = (NEXT_MASK_UNREACHABLE | (uintptr_t)gc); _PyGCHead_SET_PREV(gc, last); - gc->_gc_next = flags | (uintptr_t)unreachable; + gc->_gc_next = (NEXT_MASK_UNREACHABLE | (uintptr_t)unreachable); unreachable->_gc_prev = (uintptr_t)gc; } - gc = _PyGCHead_NEXT(prev); + gc = (PyGC_Head*)prev->_gc_next; } // young->_gc_prev must be last element remained in the list. young->_gc_prev = (uintptr_t)prev; - young->_gc_next &= _PyGC_PREV_MASK; // don't let the pollution of the list head's next pointer leak - unreachable->_gc_next &= _PyGC_PREV_MASK; + unreachable->_gc_next &= ~NEXT_MASK_UNREACHABLE; } /* In theory, all tuples should be younger than the @@ -825,8 +702,8 @@ move_legacy_finalizers(PyGC_Head *unreachable, PyGC_Head *finalizers) PyObject *op = FROM_GC(gc); _PyObject_ASSERT(op, gc->_gc_next & NEXT_MASK_UNREACHABLE); - next = GC_NEXT(gc); gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; + next = (PyGC_Head*)gc->_gc_next; if (has_legacy_finalizer(op)) { gc_clear_collecting(gc); @@ -849,8 +726,8 @@ clear_unreachable_mask(PyGC_Head *unreachable) PyGC_Head *gc, *next; for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { _PyObject_ASSERT((PyObject*)FROM_GC(gc), gc->_gc_next & NEXT_MASK_UNREACHABLE); - next = GC_NEXT(gc); gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; + next = (PyGC_Head*)gc->_gc_next; } validate_list(unreachable, collecting_set_unreachable_clear); } @@ -859,9 +736,7 @@ clear_unreachable_mask(PyGC_Head *unreachable) static int visit_move(PyObject *op, void *arg) { - struct visit_reachable_context *ctx = (struct visit_reachable_context *)arg; - PyGC_Head *tolist = ctx->head; - ctx->stats->object_visits += 1; + PyGC_Head *tolist = arg; OBJECT_STAT_INC(object_visits); if (_PyObject_IS_GC(op)) { PyGC_Head *gc = AS_GC(op); @@ -877,12 +752,8 @@ visit_move(PyObject *op, void *arg) * into finalizers set. */ static void -move_legacy_finalizer_reachable(PyGC_Head *finalizers, struct gc_generation_stats *stats) +move_legacy_finalizer_reachable(PyGC_Head *finalizers) { - struct visit_reachable_context ctx = { - .head = finalizers, - .stats = stats - }; traverseproc traverse; PyGC_Head *gc = GC_NEXT(finalizers); for (; gc != finalizers; gc = GC_NEXT(gc)) { @@ -890,7 +761,7 @@ move_legacy_finalizer_reachable(PyGC_Head *finalizers, struct gc_generation_stat traverse = Py_TYPE(FROM_GC(gc))->tp_traverse; (void) traverse(FROM_GC(gc), visit_move, - &ctx); + (void *)finalizers); } } @@ -1035,7 +906,6 @@ handle_weakref_callbacks(PyGC_Head *unreachable, PyGC_Head *old) /* Invoke the callbacks we decided to honor. It's safe to invoke them * because they can't reference unreachable objects. */ - int visited_space = get_gc_state()->visited_space; while (! gc_list_is_empty(&wrcb_to_call)) { PyObject *temp; PyObject *callback; @@ -1071,7 +941,6 @@ handle_weakref_callbacks(PyGC_Head *unreachable, PyGC_Head *old) Py_DECREF(op); if (wrcb_to_call._gc_next == (uintptr_t)gc) { /* object is still alive -- move it */ - gc_set_old_space(gc, visited_space); gc_list_move(gc, old); } else { @@ -1250,6 +1119,25 @@ delete_garbage(PyThreadState *tstate, GCState *gcstate, } +// Show stats for objects in each generations +static void +show_stats_each_generations(GCState *gcstate) +{ + char buf[100]; + size_t pos = 0; + + for (int i = 0; i < NUM_GENERATIONS && pos < sizeof(buf); i++) { + pos += PyOS_snprintf(buf+pos, sizeof(buf)-pos, + " %zd", + gc_list_size(GEN_HEAD(gcstate, i))); + } + + PySys_FormatStderr( + "gc: objects in each generation:%s\n" + "gc: objects in permanent generation: %zd\n", + buf, gc_list_size(&gcstate->permanent_generation.head)); +} + /* Deduce which objects among "base" are unreachable from outside the list and move them to 'unreachable'. The process consist in the following steps: @@ -1278,7 +1166,7 @@ flag is cleared (for example, by using 'clear_unreachable_mask' function or by a call to 'move_legacy_finalizers'), the 'unreachable' list is not a normal list and we can not use most gc_list_* functions for it. */ static inline Py_ssize_t -deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable, struct gc_generation_stats *stats) { +deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable) { validate_list(base, collecting_clear_unreachable_clear); /* Using ob_refcnt and gc_refs, calculate which objects in the * container set are reachable from outside the set (i.e., have a @@ -1286,7 +1174,7 @@ deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable, struct gc_generation * set are taken into account). */ Py_ssize_t candidates = update_refs(base); // gc_prev is used for gc_refs - subtract_refs(base, stats); + subtract_refs(base); /* Leave everything reachable from outside base in base, and move * everything else (in base) to unreachable. @@ -1323,7 +1211,8 @@ deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable, struct gc_generation * the reachable objects instead. But this is a one-time cost, probably not * worth complicating the code to speed just a little. */ - move_unreachable(base, unreachable, stats); // gc_prev is pointer again + gc_list_init(unreachable); + move_unreachable(base, unreachable); // gc_prev is pointer again validate_list(base, collecting_clear_unreachable_clear); validate_list(unreachable, collecting_set_unreachable_set); return candidates; @@ -1344,8 +1233,7 @@ PREV_MARK_COLLECTING set, but the objects in this set are going to be removed so we can skip the expense of clearing the flag to avoid extra iteration. */ static inline void handle_resurrected_objects(PyGC_Head *unreachable, PyGC_Head* still_unreachable, - PyGC_Head *old_generation, - struct gc_generation_stats *stats) + PyGC_Head *old_generation) { // Remove the PREV_MASK_COLLECTING from unreachable // to prepare it for a new call to 'deduce_unreachable' @@ -1355,49 +1243,126 @@ handle_resurrected_objects(PyGC_Head *unreachable, PyGC_Head* still_unreachable, // have the PREV_MARK_COLLECTING set, but the objects are going to be // removed so we can skip the expense of clearing the flag. PyGC_Head* resurrected = unreachable; - deduce_unreachable(resurrected, still_unreachable, stats); + deduce_unreachable(resurrected, still_unreachable); clear_unreachable_mask(still_unreachable); // Move the resurrected objects to the old generation for future collection. gc_list_merge(resurrected, old_generation); } -static void -gc_collect_region(PyThreadState *tstate, - PyGC_Head *from, - PyGC_Head *to, - struct gc_generation_stats *stats); -static inline Py_ssize_t -gc_list_set_space(PyGC_Head *list, int space) +/* Invoke progress callbacks to notify clients that garbage collection + * is starting or stopping + */ +static void +invoke_gc_callback(PyThreadState *tstate, const char *phase, + int generation, struct gc_generation_stats *stats) { - Py_ssize_t size = 0; - PyGC_Head *gc; - for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(gc)) { - gc_set_old_space(gc, space); - size++; + assert(!_PyErr_Occurred(tstate)); + + /* we may get called very early */ + GCState *gcstate = &tstate->interp->gc; + if (gcstate->callbacks == NULL) { + return; } - return size; + + /* The local variable cannot be rebound, check it for sanity */ + assert(PyList_CheckExact(gcstate->callbacks)); + PyObject *info = NULL; + if (PyList_GET_SIZE(gcstate->callbacks) != 0) { + info = Py_BuildValue("{sisnsnsnsd}", + "generation", generation, + "collected", stats->collected, + "uncollectable", stats->uncollectable, + "candidates", stats->candidates, + "duration", stats->duration); + if (info == NULL) { + PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); + return; + } + } + + PyObject *phase_obj = PyUnicode_FromString(phase); + if (phase_obj == NULL) { + Py_XDECREF(info); + PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); + return; + } + + PyObject *stack[] = {phase_obj, info}; + for (Py_ssize_t i=0; icallbacks); i++) { + PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i); + Py_INCREF(cb); /* make sure cb doesn't go away */ + r = PyObject_Vectorcall(cb, stack, 2, NULL); + if (r == NULL) { + PyErr_FormatUnraisable("Exception ignored while " + "calling GC callback %R", cb); + } + else { + Py_DECREF(r); + } + Py_DECREF(cb); + } + Py_DECREF(phase_obj); + Py_XDECREF(info); + assert(!_PyErr_Occurred(tstate)); } -/* Making progress in the incremental collector - * In order to eventually collect all cycles - * the incremental collector must progress through the old - * space faster than objects are added to the old space. - * - * Each young or incremental collection adds a number of - * objects, S (for survivors) to the old space, and - * incremental collectors scan I objects from the old space. - * I > S must be true. We also want I > S * N to be where - * N > 1. Higher values of N mean that the old space is - * scanned more rapidly. - * The default incremental threshold of 10 translates to - * N == 1.4 (1 + 4/threshold) - */ -/* Divide by 10, so that the default incremental threshold of 10 - * scans objects at 1% of the heap size */ -#define SCAN_RATE_DIVISOR 10 +/* Find the oldest generation (highest numbered) where the count + * exceeds the threshold. Objects in the that generation and + * generations younger than it will be collected. */ +static int +gc_select_generation(GCState *gcstate) +{ + for (int i = NUM_GENERATIONS-1; i >= 0; i--) { + if (gcstate->generations[i].count > gcstate->generations[i].threshold) { + /* Avoid quadratic performance degradation in number + of tracked objects (see also issue #4074): + + To limit the cost of garbage collection, there are two strategies; + - make each collection faster, e.g. by scanning fewer objects + - do less collections + This heuristic is about the latter strategy. + + In addition to the various configurable thresholds, we only trigger a + full collection if the ratio + + long_lived_pending / long_lived_total + + is above a given value (hardwired to 25%). + + The reason is that, while "non-full" collections (i.e., collections of + the young and middle generations) will always examine roughly the same + number of objects -- determined by the aforementioned thresholds --, + the cost of a full collection is proportional to the total number of + long-lived objects, which is virtually unbounded. + + Indeed, it has been remarked that doing a full collection every + of object creations entails a dramatic performance + degradation in workloads which consist in creating and storing lots of + long-lived objects (e.g. building a large list of GC-tracked objects would + show quadratic performance, instead of linear as expected: see issue #4074). + + Using the above ratio, instead, yields amortized linear performance in + the total number of objects (the effect of which can be summarized + thusly: "each full garbage collection is more and more costly as the + number of objects grows, but we do fewer and fewer of them"). + + This heuristic was suggested by Martin von Löwis on python-dev in + June 2008. His original analysis and proposal can be found at: + http://mail.python.org/pipermail/python-dev/2008-June/080579.html + */ + if (i == NUM_GENERATIONS - 1 + && gcstate->long_lived_pending < gcstate->long_lived_total / 4) + { + continue; + } + return i; + } + } + return -1; +} static struct gc_generation_stats * gc_get_stats(GCState *gcstate, int gen) @@ -1441,421 +1406,120 @@ add_stats(GCState *gcstate, int gen, struct gc_generation_stats *stats) cur_stats->ts_start = stats->ts_start; cur_stats->ts_stop = stats->ts_stop; - cur_stats->heap_size = stats->heap_size; - cur_stats->work_to_do = stats->work_to_do; cur_stats->collections += 1; - cur_stats->object_visits += stats->object_visits; cur_stats->collected += stats->collected; cur_stats->uncollectable += stats->uncollectable; cur_stats->candidates += stats->candidates; - cur_stats->objects_transitively_reachable += stats->objects_transitively_reachable; - cur_stats->objects_not_transitively_reachable += stats->objects_not_transitively_reachable; - cur_stats->duration += stats->duration; } -static void -gc_collect_young(PyThreadState *tstate, - struct gc_generation_stats *stats) -{ - GCState *gcstate = &tstate->interp->gc; - validate_spaces(gcstate); - PyGC_Head *young = &gcstate->young.head; - PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; - untrack_tuples(young); - - PyGC_Head survivors; - gc_list_init(&survivors); - gc_list_set_space(young, gcstate->visited_space); - gc_collect_region(tstate, young, &survivors, stats); - gc_list_merge(&survivors, visited); - validate_spaces(gcstate); - gcstate->young.count = 0; - gcstate->old[gcstate->visited_space].count++; - validate_spaces(gcstate); -} - -#ifndef NDEBUG -static inline int -IS_IN_VISITED(PyGC_Head *gc, int visited_space) -{ - assert(visited_space == 0 || other_space(visited_space) == 0); - return gc_old_space(gc) == visited_space; -} -#endif - -struct container_and_flag { - PyGC_Head *container; - int visited_space; - intptr_t size; - struct gc_generation_stats *stats; -}; - -/* A traversal callback for adding to container) */ -static int -visit_add_to_container(PyObject *op, void *arg) -{ - OBJECT_STAT_INC(object_visits); - struct container_and_flag *cf = (struct container_and_flag *)arg; - cf->stats->object_visits += 1; - int visited = cf->visited_space; - assert(visited == get_gc_state()->visited_space); - if (!_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { - PyGC_Head *gc = AS_GC(op); - if (_PyObject_GC_IS_TRACKED(op) && - gc_old_space(gc) != visited) { - gc_flip_old_space(gc); - gc_list_move(gc, cf->container); - cf->size++; - } - } - return 0; -} - -static intptr_t -expand_region_transitively_reachable(PyGC_Head *container, - PyGC_Head *gc, - GCState *gcstate, - struct gc_generation_stats *stats) -{ - struct container_and_flag arg = { - .container = container, - .visited_space = gcstate->visited_space, - .size = 0, - .stats = stats - }; - assert(GC_NEXT(gc) == container); - while (gc != container) { - /* Survivors will be moved to visited space, so they should - * have been marked as visited */ - assert(IS_IN_VISITED(gc, gcstate->visited_space)); - PyObject *op = FROM_GC(gc); - assert(_PyObject_GC_IS_TRACKED(op)); - if (_Py_IsImmortal(op)) { - PyGC_Head *next = GC_NEXT(gc); - gc_list_move(gc, &gcstate->permanent_generation.head); - gc = next; - continue; - } - traverseproc traverse = Py_TYPE(op)->tp_traverse; - (void) traverse(op, - visit_add_to_container, - &arg); - gc = GC_NEXT(gc); - } - return arg.size; -} - -/* Do bookkeeping for a completed GC cycle */ -static void -completed_scavenge(GCState *gcstate) -{ - /* We must observe two invariants: - * 1. Members of the permanent generation must be marked visited. - * 2. We cannot touch members of the permanent generation. */ - int visited; - if (gc_list_is_empty(&gcstate->permanent_generation.head)) { - /* Permanent generation is empty so we can flip spaces bit */ - int not_visited = gcstate->visited_space; - visited = other_space(not_visited); - gcstate->visited_space = visited; - /* Make sure all objects have visited bit set correctly */ - gc_list_set_space(&gcstate->young.head, not_visited); - } - else { - /* We must move the objects from visited to pending space. */ - visited = gcstate->visited_space; - int not_visited = other_space(visited); - assert(gc_list_is_empty(&gcstate->old[not_visited].head)); - gc_list_merge(&gcstate->old[visited].head, &gcstate->old[not_visited].head); - gc_list_set_space(&gcstate->old[not_visited].head, not_visited); - } - assert(gc_list_is_empty(&gcstate->old[visited].head)); - gcstate->work_to_do = 0; - gcstate->phase = GC_PHASE_MARK; -} - -static intptr_t -move_to_reachable(PyObject *op, PyGC_Head *reachable, int visited_space) -{ - if (op != NULL && !_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { - PyGC_Head *gc = AS_GC(op); - if (_PyObject_GC_IS_TRACKED(op) && - gc_old_space(gc) != visited_space) { - gc_flip_old_space(gc); - gc_list_move(gc, reachable); - return 1; - } - } - return 0; -} - -static intptr_t -mark_all_reachable(PyGC_Head *reachable, PyGC_Head *visited, int visited_space, struct gc_generation_stats *stats) -{ - // Transitively traverse all objects from reachable, until empty - struct container_and_flag arg = { - .container = reachable, - .visited_space = visited_space, - .size = 0, - .stats = stats - }; - while (!gc_list_is_empty(reachable)) { - PyGC_Head *gc = _PyGCHead_NEXT(reachable); - assert(gc_old_space(gc) == visited_space); - gc_list_move(gc, visited); - PyObject *op = FROM_GC(gc); - traverseproc traverse = Py_TYPE(op)->tp_traverse; - (void) traverse(op, - visit_add_to_container, - &arg); - } - gc_list_validate_space(visited, visited_space); - return arg.size; -} - -static intptr_t -mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, bool start, struct gc_generation_stats *stats) -{ - PyGC_Head reachable; - gc_list_init(&reachable); - Py_ssize_t objects_marked = 0; - // Move all objects on stacks to reachable - _PyRuntimeState *runtime = &_PyRuntime; - HEAD_LOCK(runtime); - PyThreadState* ts = PyInterpreterState_ThreadHead(interp); - HEAD_UNLOCK(runtime); - while (ts) { - _PyInterpreterFrame *frame = ts->current_frame; - while (frame) { - if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { - frame = frame->previous; - continue; - } - _PyStackRef *locals = frame->localsplus; - _PyStackRef *sp = frame->stackpointer; - objects_marked += move_to_reachable(frame->f_locals, &reachable, visited_space); - PyObject *func = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - objects_marked += move_to_reachable(func, &reachable, visited_space); - while (sp > locals) { - sp--; - if (PyStackRef_IsNullOrInt(*sp)) { - continue; - } - PyObject *op = PyStackRef_AsPyObjectBorrow(*sp); - if (_Py_IsImmortal(op)) { - continue; - } - if (_PyObject_IS_GC(op)) { - PyGC_Head *gc = AS_GC(op); - if (_PyObject_GC_IS_TRACKED(op) && - gc_old_space(gc) != visited_space) { - gc_flip_old_space(gc); - objects_marked++; - gc_list_move(gc, &reachable); - } - } - } - if (!start && frame->visited) { - // If this frame has already been visited, then the lower frames - // will have already been visited and will not have changed - break; - } - frame->visited = 1; - frame = frame->previous; - } - HEAD_LOCK(runtime); - ts = PyThreadState_Next(ts); - HEAD_UNLOCK(runtime); - } - objects_marked += mark_all_reachable(&reachable, visited, visited_space, stats); - assert(gc_list_is_empty(&reachable)); - return objects_marked; -} - -static intptr_t -mark_global_roots(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, struct gc_generation_stats *stats) -{ - PyGC_Head reachable; - gc_list_init(&reachable); - Py_ssize_t objects_marked = 0; - objects_marked += move_to_reachable(interp->sysdict, &reachable, visited_space); - objects_marked += move_to_reachable(interp->builtins, &reachable, visited_space); - objects_marked += move_to_reachable(interp->dict, &reachable, visited_space); - struct types_state *types = &interp->types; - for (int i = 0; i < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; i++) { - objects_marked += move_to_reachable(types->builtins.initialized[i].tp_dict, &reachable, visited_space); - objects_marked += move_to_reachable(types->builtins.initialized[i].tp_subclasses, &reachable, visited_space); - } - for (int i = 0; i < _Py_MAX_MANAGED_STATIC_EXT_TYPES; i++) { - objects_marked += move_to_reachable(types->for_extensions.initialized[i].tp_dict, &reachable, visited_space); - objects_marked += move_to_reachable(types->for_extensions.initialized[i].tp_subclasses, &reachable, visited_space); - } - objects_marked += mark_all_reachable(&reachable, visited, visited_space, stats); - assert(gc_list_is_empty(&reachable)); - return objects_marked; -} - -static intptr_t -mark_at_start(PyThreadState *tstate, struct gc_generation_stats *stats) -{ - // TO DO -- Make this incremental - GCState *gcstate = &tstate->interp->gc; - PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; - Py_ssize_t objects_marked = mark_global_roots(tstate->interp, visited, gcstate->visited_space, stats); - objects_marked += mark_stacks(tstate->interp, visited, gcstate->visited_space, true, stats); - gcstate->work_to_do -= objects_marked; - gcstate->phase = GC_PHASE_COLLECT; - validate_spaces(gcstate); - return objects_marked; -} - -static intptr_t -assess_work_to_do(GCState *gcstate) -{ - /* The amount of work we want to do depends on three things. - * 1. The number of new objects created - * 2. The growth in heap size since the last collection - * 3. The heap size (up to the number of new objects, to avoid quadratic effects) - * - * For a steady state heap, the amount of work to do is three times the number - * of new objects added to the heap. This ensures that we stay ahead in the - * worst case of all new objects being garbage. - * - * This could be improved by tracking survival rates, but it is still a - * large improvement on the non-marking approach. - */ - intptr_t scale_factor = gcstate->old[0].threshold; - if (scale_factor < 2) { - scale_factor = 2; - } - intptr_t new_objects = gcstate->young.count; - intptr_t max_heap_fraction = new_objects*2; - intptr_t heap_fraction = gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; - if (heap_fraction > max_heap_fraction) { - heap_fraction = max_heap_fraction; - } - gcstate->young.count = 0; - return new_objects + heap_fraction; -} - -static void -gc_collect_increment(PyThreadState *tstate, struct gc_generation_stats *stats) -{ - GCState *gcstate = &tstate->interp->gc; - gcstate->work_to_do += assess_work_to_do(gcstate); - if (gcstate->work_to_do < 0) { - return; - } - untrack_tuples(&gcstate->young.head); - if (gcstate->phase == GC_PHASE_MARK) { - Py_ssize_t objects_marked = mark_at_start(tstate, stats); - stats->objects_transitively_reachable += objects_marked; - stats->candidates += objects_marked; - gcstate->work_to_do -= objects_marked; - validate_spaces(gcstate); - return; - } - PyGC_Head *not_visited = &gcstate->old[gcstate->visited_space^1].head; - PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; - PyGC_Head increment; - gc_list_init(&increment); - int scale_factor = gcstate->old[0].threshold; - if (scale_factor < 2) { - scale_factor = 2; - } - intptr_t objects_marked = mark_stacks(tstate->interp, visited, gcstate->visited_space, false, stats); - stats->objects_transitively_reachable += objects_marked; - gcstate->work_to_do -= objects_marked; - gc_list_set_space(&gcstate->young.head, gcstate->visited_space); - gc_list_merge(&gcstate->young.head, &increment); - gc_list_validate_space(&increment, gcstate->visited_space); - Py_ssize_t increment_size = gc_list_size(&increment); - while (increment_size < gcstate->work_to_do) { - if (gc_list_is_empty(not_visited)) { - break; - } - PyGC_Head *gc = _PyGCHead_NEXT(not_visited); - gc_list_move(gc, &increment); - increment_size++; - assert(!_Py_IsImmortal(FROM_GC(gc))); - gc_set_old_space(gc, gcstate->visited_space); - increment_size += expand_region_transitively_reachable(&increment, gc, gcstate, stats); - } - stats->objects_not_transitively_reachable += increment_size; - validate_list(&increment, collecting_clear_unreachable_clear); - gc_list_validate_space(&increment, gcstate->visited_space); - PyGC_Head survivors; - gc_list_init(&survivors); - gc_collect_region(tstate, &increment, &survivors, stats); - gc_list_merge(&survivors, visited); - assert(gc_list_is_empty(&increment)); - gcstate->work_to_do -= increment_size; - - if (gc_list_is_empty(not_visited)) { - completed_scavenge(gcstate); - } - validate_spaces(gcstate); -} - -static void -gc_collect_full(PyThreadState *tstate, - struct gc_generation_stats *stats) -{ - GCState *gcstate = &tstate->interp->gc; - validate_spaces(gcstate); - PyGC_Head *young = &gcstate->young.head; - PyGC_Head *pending = &gcstate->old[gcstate->visited_space^1].head; - PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; - untrack_tuples(young); - /* merge all generations into visited */ - gc_list_merge(young, pending); - gc_list_validate_space(pending, 1-gcstate->visited_space); - gc_list_set_space(pending, gcstate->visited_space); - gcstate->young.count = 0; - gc_list_merge(pending, visited); - validate_spaces(gcstate); - - gc_collect_region(tstate, visited, visited, - stats); - validate_spaces(gcstate); - gcstate->young.count = 0; - gcstate->old[0].count = 0; - gcstate->old[1].count = 0; - completed_scavenge(gcstate); - _PyGC_ClearAllFreeLists(tstate->interp); - validate_spaces(gcstate); -} - -/* This is the main function. Read this to understand how the +/* This is the main function. Read this to understand how the * collection process works. */ -static void -gc_collect_region(PyThreadState *tstate, - PyGC_Head *from, - PyGC_Head *to, - struct gc_generation_stats *stats) +static Py_ssize_t +gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) { + int i; + PyGC_Head *young; /* the generation we are examining */ + PyGC_Head *old; /* next older generation */ PyGC_Head unreachable; /* non-problematic unreachable trash */ PyGC_Head finalizers; /* objects with, & reachable from, __del__ */ - PyGC_Head *gc; /* initialize to prevent a compiler warning */ + PyGC_Head *gc; GCState *gcstate = &tstate->interp->gc; + // gc_collect_main() must not be called before _PyGC_Init + // or after _PyGC_Fini() assert(gcstate->garbage != NULL); assert(!_PyErr_Occurred(tstate)); - gc_list_init(&unreachable); - stats->candidates = deduce_unreachable(from, &unreachable, stats); - validate_consistent_old_space(from); - untrack_tuples(from); - - /* Move reachable objects to next generation. */ - validate_consistent_old_space(to); - if (from != to) { - gc_list_merge(from, to); + int expected = 0; + if (!_Py_atomic_compare_exchange_int(&gcstate->collecting, &expected, 1)) { + // Don't start a garbage collection if one is already in progress. + return 0; + } + gcstate->frame = tstate->current_frame; + + if (generation == GENERATION_AUTO) { + // Select the oldest generation that needs collecting. We will collect + // objects from that generation and all generations younger than it. + generation = gc_select_generation(gcstate); + if (generation < 0) { + // No generation needs to be collected. + _Py_atomic_store_int(&gcstate->collecting, 0); + return 0; + } + } + + assert(generation >= 0 && generation < NUM_GENERATIONS); + +#ifdef Py_STATS + if (_Py_stats) { + _Py_stats->object_stats.object_visits = 0; + } +#endif + GC_STAT_ADD(generation, collections, 1); + + struct gc_generation_stats stats = { 0 }; + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(tstate, "start", generation, &stats); + } + + // ignore error: don't interrupt the GC if reading the clock fails + (void)PyTime_PerfCounterRaw(&stats.ts_start); + if (gcstate->debug & _PyGC_DEBUG_STATS) { + PySys_WriteStderr("gc: collecting generation %d...\n", generation); + show_stats_each_generations(gcstate); + } + + if (PyDTrace_GC_START_ENABLED()) { + PyDTrace_GC_START(generation); + } + + /* update collection and allocation counters */ + if (generation+1 < NUM_GENERATIONS) { + gcstate->generations[generation+1].count += 1; + } + for (i = 0; i <= generation; i++) { + gcstate->generations[i].count = 0; + } + + /* merge younger generations with one we are currently collecting */ + for (i = 0; i < generation; i++) { + gc_list_merge(GEN_HEAD(gcstate, i), GEN_HEAD(gcstate, generation)); + } + + /* handy references */ + young = GEN_HEAD(gcstate, generation); + if (generation < NUM_GENERATIONS-1) { + old = GEN_HEAD(gcstate, generation+1); + } + else { + old = young; + } + validate_list(old, collecting_clear_unreachable_clear); + + stats.candidates = deduce_unreachable(young, &unreachable); + + untrack_tuples(young); + /* Move reachable objects to next generation. */ + if (young != old) { + if (generation == NUM_GENERATIONS - 2) { + gcstate->long_lived_pending += gc_list_size(young); + } + gc_list_merge(young, old); + } + else { + // In Python <= 3.13, we called untrack_dicts(young) here to untrack + // atomic-only dicts (see issue #14775). Python 3.14 removed the lazy + // dict tracking machinery entirely (GH-127010) -- dicts are always + // tracked from creation and never untracked by GC. That way, we don't + // have to restore MAINTAIN_TRACKING across every PyDict_SetItem call + // site; the cost is slightly more work for full collections on dicts + // with only atomic values. + gcstate->long_lived_pending = 0; + gcstate->long_lived_total = gc_list_size(young); } - validate_consistent_old_space(to); /* All objects in unreachable are trash, but objects reachable from * legacy finalizers (e.g. tp_del) can't safely be deleted. @@ -1868,9 +1532,11 @@ gc_collect_region(PyThreadState *tstate, * unreachable objects reachable *from* those are also uncollectable, * and we move those into the finalizers list too. */ - move_legacy_finalizer_reachable(&finalizers, stats); + move_legacy_finalizer_reachable(&finalizers); + validate_list(&finalizers, collecting_clear_unreachable_clear); validate_list(&unreachable, collecting_set_unreachable_clear); + /* Print debugging information. */ if (gcstate->debug & _PyGC_DEBUG_COLLECTABLE) { for (gc = GC_NEXT(&unreachable); gc != &unreachable; gc = GC_NEXT(gc)) { @@ -1878,23 +1544,25 @@ gc_collect_region(PyThreadState *tstate, } } - /* Invoke weakref callbacks as necessary. */ - stats->collected += handle_weakref_callbacks(&unreachable, to); - gc_list_validate_space(to, gcstate->visited_space); - validate_list(to, collecting_clear_unreachable_clear); + /* Clear weakrefs and invoke callbacks as necessary. */ + stats.collected += handle_weakref_callbacks(&unreachable, old); + validate_list(old, collecting_clear_unreachable_clear); validate_list(&unreachable, collecting_set_unreachable_clear); /* Call tp_finalize on objects which have one. */ finalize_garbage(tstate, &unreachable); + /* Handle any objects that may have resurrected after the call * to 'finalize_garbage' and continue the collection with the * objects that are still unreachable */ PyGC_Head final_unreachable; - gc_list_init(&final_unreachable); - handle_resurrected_objects(&unreachable, &final_unreachable, to, stats); + handle_resurrected_objects(&unreachable, &final_unreachable, old); - /* Clear weakrefs to objects in the unreachable set. See the comments - * above handle_weakref_callbacks() for details. + /* Clear weakrefs to objects in the unreachable set. No Python-level + * code must be allowed to access those unreachable objects. During + * delete_garbage(), finalizers outside the unreachable set might run + * and create new weakrefs. If those weakrefs were not cleared, they + * could reveal unreachable objects. Callbacks are not executed. */ clear_weakrefs(&final_unreachable); @@ -1902,8 +1570,8 @@ gc_collect_region(PyThreadState *tstate, * the reference cycles to be broken. It may also cause some objects * in finalizers to be freed. */ - stats->collected += gc_list_size(&final_unreachable); - delete_garbage(tstate, gcstate, &final_unreachable, to); + stats.collected += gc_list_size(&final_unreachable); + delete_garbage(tstate, gcstate, &final_unreachable, old); /* Collect statistics on uncollectable objects found and print * debugging information. */ @@ -1913,75 +1581,61 @@ gc_collect_region(PyThreadState *tstate, if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) debug_cycle("uncollectable", FROM_GC(gc)); } - stats->uncollectable = n; + stats.uncollectable = n; + (void)PyTime_PerfCounterRaw(&stats.ts_stop); + stats.duration = PyTime_AsSecondsDouble(stats.ts_stop - stats.ts_start); + if (gcstate->debug & _PyGC_DEBUG_STATS) { + PySys_WriteStderr( + "gc: done, %zd unreachable, %zd uncollectable, %.4fs elapsed\n", + stats.uncollectable+stats.collected, stats.uncollectable, + stats.duration); + } + /* Append instances in the uncollectable set to a Python * reachable list of garbage. The programmer has to deal with * this if they insist on creating this type of structure. */ - handle_legacy_finalizers(tstate, gcstate, &finalizers, to); - gc_list_validate_space(to, gcstate->visited_space); - validate_list(to, collecting_clear_unreachable_clear); -} + handle_legacy_finalizers(tstate, gcstate, &finalizers, old); + validate_list(old, collecting_clear_unreachable_clear); -/* Invoke progress callbacks to notify clients that garbage collection - * is starting or stopping - */ -static void -do_gc_callback(GCState *gcstate, const char *phase, - int generation, struct gc_generation_stats *stats) -{ - assert(!PyErr_Occurred()); - - /* The local variable cannot be rebound, check it for sanity */ - assert(PyList_CheckExact(gcstate->callbacks)); - PyObject *info = NULL; - if (PyList_GET_SIZE(gcstate->callbacks) != 0) { - info = Py_BuildValue("{sisnsnsnsd}", - "generation", generation, - "collected", stats->collected, - "uncollectable", stats->uncollectable, - "candidates", stats->candidates, - "duration", stats->duration); - if (info == NULL) { - PyErr_FormatUnraisable("Exception ignored while invoking gc callbacks"); - return; - } + /* Clear free list only during the collection of the highest + * generation */ + if (generation == NUM_GENERATIONS-1) { + _PyGC_ClearAllFreeLists(tstate->interp); } - PyObject *phase_obj = PyUnicode_FromString(phase); - if (phase_obj == NULL) { - Py_XDECREF(info); - PyErr_FormatUnraisable("Exception ignored while invoking gc callbacks"); - return; - } - - PyObject *stack[] = {phase_obj, info}; - for (Py_ssize_t i=0; icallbacks); i++) { - PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i); - Py_INCREF(cb); /* make sure cb doesn't go away */ - r = PyObject_Vectorcall(cb, stack, 2, NULL); - if (r == NULL) { - PyErr_FormatUnraisable("Exception ignored while " - "calling GC callback %R", cb); + if (_PyErr_Occurred(tstate)) { + if (reason == _Py_GC_REASON_SHUTDOWN) { + _PyErr_Clear(tstate); } else { - Py_DECREF(r); + PyErr_FormatUnraisable("Exception ignored in garbage collection"); } - Py_DECREF(cb); } - Py_DECREF(phase_obj); - Py_XDECREF(info); - assert(!PyErr_Occurred()); -} -static void -invoke_gc_callback(GCState *gcstate, const char *phase, - int generation, struct gc_generation_stats *stats) -{ - if (gcstate->callbacks == NULL) { - return; + /* Update stats */ + add_stats(gcstate, generation, &stats); + GC_STAT_ADD(generation, objects_collected, m); +#ifdef Py_STATS + if (_Py_stats) { + GC_STAT_ADD(generation, object_visits, + _Py_stats->object_stats.object_visits); + _Py_stats->object_stats.object_visits = 0; } - do_gc_callback(gcstate, phase, generation, stats); +#endif + + if (PyDTrace_GC_DONE_ENABLED()) { + PyDTrace_GC_DONE(stats.uncollectable + stats.collected); + } + + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(tstate, "stop", generation, &stats); + } + + assert(!_PyErr_Occurred(tstate)); + gcstate->frame = NULL; + _Py_atomic_store_int(&gcstate->collecting, 0); + return stats.uncollectable + stats.collected; } static int @@ -2043,25 +1697,20 @@ _PyGC_GetObjects(PyInterpreterState *interp, int generation) GCState *gcstate = &interp->gc; PyObject *result = PyList_New(0); - /* Generation: - * -1: Return all objects - * 0: All young objects - * 1: No objects - * 2: All old objects - */ - if (result == NULL || generation == 1) { - return result; + if (result == NULL) { + return NULL; } - if (generation <= 0) { - if (append_objects(result, &gcstate->young.head)) { - goto error; + + if (generation == -1) { + /* If generation is -1, get all objects from all generations */ + for (int i = 0; i < NUM_GENERATIONS; i++) { + if (append_objects(result, GEN_HEAD(gcstate, i))) { + goto error; + } } } - if (generation != 0) { - if (append_objects(result, &gcstate->old[0].head)) { - goto error; - } - if (append_objects(result, &gcstate->old[1].head)) { + else { + if (append_objects(result, GEN_HEAD(gcstate, generation))) { goto error; } } @@ -2076,23 +1725,10 @@ void _PyGC_Freeze(PyInterpreterState *interp) { GCState *gcstate = &interp->gc; - /* The permanent_generation must be visited */ - gc_list_set_space(&gcstate->young.head, gcstate->visited_space); - gc_list_merge(&gcstate->young.head, &gcstate->permanent_generation.head); - gcstate->young.count = 0; - PyGC_Head*old0 = &gcstate->old[0].head; - PyGC_Head*old1 = &gcstate->old[1].head; - if (gcstate->visited_space) { - gc_list_set_space(old0, 1); + for (int i = 0; i < NUM_GENERATIONS; ++i) { + gc_list_merge(GEN_HEAD(gcstate, i), &gcstate->permanent_generation.head); + gcstate->generations[i].count = 0; } - else { - gc_list_set_space(old1, 0); - } - gc_list_merge(old0, &gcstate->permanent_generation.head); - gcstate->old[0].count = 0; - gc_list_merge(old1, &gcstate->permanent_generation.head); - gcstate->old[1].count = 0; - validate_spaces(gcstate); } void @@ -2100,8 +1736,7 @@ _PyGC_Unfreeze(PyInterpreterState *interp) { GCState *gcstate = &interp->gc; gc_list_merge(&gcstate->permanent_generation.head, - &gcstate->old[gcstate->visited_space].head); - validate_spaces(gcstate); + GEN_HEAD(gcstate, NUM_GENERATIONS-1)); } Py_ssize_t @@ -2137,103 +1772,29 @@ PyGC_IsEnabled(void) return gcstate->enabled; } -// Show stats for objects in each generations -static void -show_stats_each_generations(GCState *gcstate) +/* Public API to invoke gc.collect() from C */ +Py_ssize_t +PyGC_Collect(void) { - char buf[100]; - size_t pos = 0; + PyThreadState *tstate = _PyThreadState_GET(); + GCState *gcstate = &tstate->interp->gc; - for (int i = 0; i < NUM_GENERATIONS && pos < sizeof(buf); i++) { - pos += PyOS_snprintf(buf+pos, sizeof(buf)-pos, - " %zd", - gc_list_size(GEN_HEAD(gcstate, i))); + if (!gcstate->enabled) { + return 0; } - PySys_FormatStderr( - "gc: objects in each generation:%s\n" - "gc: objects in permanent generation: %zd\n", - buf, gc_list_size(&gcstate->permanent_generation.head)); + + Py_ssize_t n; + PyObject *exc = _PyErr_GetRaisedException(tstate); + n = gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_MANUAL); + _PyErr_SetRaisedException(tstate, exc); + + return n; } Py_ssize_t _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) { - GCState *gcstate = &tstate->interp->gc; - assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL); - - int expected = 0; - if (!_Py_atomic_compare_exchange_int(&gcstate->collecting, &expected, 1)) { - // Don't start a garbage collection if one is already in progress. - return 0; - } - gcstate->frame = tstate->current_frame; - - struct gc_generation_stats stats = { 0 }; - if (reason != _Py_GC_REASON_SHUTDOWN) { - invoke_gc_callback(gcstate, "start", generation, &stats); - } - if (gcstate->debug & _PyGC_DEBUG_STATS) { - PySys_WriteStderr("gc: collecting generation %d...\n", generation); - show_stats_each_generations(gcstate); - } - if (PyDTrace_GC_START_ENABLED()) { - PyDTrace_GC_START(generation); - } - stats.heap_size = gcstate->heap_size; - stats.work_to_do = gcstate->work_to_do; - (void)PyTime_PerfCounterRaw(&stats.ts_start); - PyObject *exc = _PyErr_GetRaisedException(tstate); - switch(generation) { - case 0: - gc_collect_young(tstate, &stats); - break; - case 1: - gc_collect_increment(tstate, &stats); - break; - case 2: - gc_collect_full(tstate, &stats); - break; - default: - Py_UNREACHABLE(); - } - (void)PyTime_PerfCounterRaw(&stats.ts_stop); - stats.duration = PyTime_AsSecondsDouble(stats.ts_stop - stats.ts_start); - add_stats(gcstate, generation, &stats); - if (PyDTrace_GC_DONE_ENABLED()) { - PyDTrace_GC_DONE(stats.uncollectable + stats.collected); - } - if (reason != _Py_GC_REASON_SHUTDOWN) { - invoke_gc_callback(gcstate, "stop", generation, &stats); - } - _PyErr_SetRaisedException(tstate, exc); - GC_STAT_ADD(generation, objects_collected, stats.collected); -#ifdef Py_STATS - PyStats *s = _PyStats_GET(); - if (s) { - GC_STAT_ADD(generation, object_visits, - s->object_stats.object_visits); - s->object_stats.object_visits = 0; - } -#endif - validate_spaces(gcstate); - gcstate->frame = NULL; - _Py_atomic_store_int(&gcstate->collecting, 0); - - if (gcstate->debug & _PyGC_DEBUG_STATS) { - PySys_WriteStderr( - "gc: done, %zd unreachable, %zd uncollectable, %.4fs elapsed\n", - stats.collected + stats.uncollectable, stats.uncollectable, stats.duration - ); - } - - return stats.uncollectable + stats.collected; -} - -/* Public API to invoke gc.collect() from C */ -Py_ssize_t -PyGC_Collect(void) -{ - return _PyGC_Collect(_PyThreadState_GET(), 2, _Py_GC_REASON_MANUAL); + return gc_collect_main(tstate, generation, reason); } void @@ -2245,7 +1806,7 @@ _PyGC_CollectNoFail(PyThreadState *tstate) during interpreter shutdown (and then never finish it). See http://bugs.python.org/issue8713#msg195178 for an example. */ - _PyGC_Collect(_PyThreadState_GET(), 2, _Py_GC_REASON_SHUTDOWN); + gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_SHUTDOWN); } void @@ -2304,8 +1865,6 @@ _PyGC_Fini(PyInterpreterState *interp) GCState *gcstate = &interp->gc; Py_CLEAR(gcstate->garbage); Py_CLEAR(gcstate->callbacks); - PyMem_RawFree(gcstate->generation_stats); - gcstate->generation_stats = NULL; /* Prevent a subtle bug that affects sub-interpreters that use basic * single-phase init extensions (m_size == -1). Those extensions cause objects @@ -2322,9 +1881,9 @@ _PyGC_Fini(PyInterpreterState *interp) * This bug was originally fixed when reported as gh-90228. The bug was * re-introduced in gh-94673. */ - finalize_unlink_gc_head(&gcstate->young.head); - finalize_unlink_gc_head(&gcstate->old[0].head); - finalize_unlink_gc_head(&gcstate->old[1].head); + for (int i = 0; i < NUM_GENERATIONS; i++) { + finalize_unlink_gc_head(&gcstate->generations[i].head); + } finalize_unlink_gc_head(&gcstate->permanent_generation.head); } @@ -2399,11 +1958,20 @@ _Py_ScheduleGC(PyThreadState *tstate) } void -_Py_TriggerGC(struct _gc_runtime_state *gcstate) +_PyObject_GC_Link(PyObject *op) { + PyGC_Head *gc = AS_GC(op); + // gc must be correctly aligned + _PyObject_ASSERT(op, ((uintptr_t)gc & (sizeof(uintptr_t)-1)) == 0); + PyThreadState *tstate = _PyThreadState_GET(); - if (gcstate->enabled && - gcstate->young.threshold != 0 && + GCState *gcstate = &tstate->interp->gc; + gc->_gc_next = 0; + gc->_gc_prev = 0; + gcstate->generations[0].count++; /* number of allocated GC objects */ + if (gcstate->generations[0].count > gcstate->generations[0].threshold && + gcstate->enabled && + gcstate->generations[0].threshold && !_Py_atomic_load_int_relaxed(&gcstate->collecting) && !_PyErr_Occurred(tstate)) { @@ -2411,23 +1979,14 @@ _Py_TriggerGC(struct _gc_runtime_state *gcstate) } } -void -_PyObject_GC_Link(PyObject *op) -{ - PyGC_Head *gc = AS_GC(op); - // gc must be correctly aligned - _PyObject_ASSERT(op, ((uintptr_t)gc & (sizeof(uintptr_t)-1)) == 0); - gc->_gc_next = 0; - gc->_gc_prev = 0; - -} - void _Py_RunGC(PyThreadState *tstate) { - if (tstate->interp->gc.enabled) { - _PyGC_Collect(tstate, 1, _Py_GC_REASON_HEAP); + GCState *gcstate = get_gc_state(); + if (!gcstate->enabled) { + return; } + gc_collect_main(tstate, GENERATION_AUTO, _Py_GC_REASON_HEAP); } static PyObject * @@ -2528,11 +2087,6 @@ PyObject_GC_Del(void *op) PyGC_Head *g = AS_GC(op); if (_PyObject_GC_IS_TRACKED(op)) { gc_list_remove(g); - GCState *gcstate = get_gc_state(); - if (gcstate->young.count > 0) { - gcstate->young.count--; - } - gcstate->heap_size--; #ifdef Py_DEBUG PyObject *exc = PyErr_GetRaisedException(); if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, @@ -2546,6 +2100,10 @@ PyObject_GC_Del(void *op) PyErr_SetRaisedException(exc); #endif } + GCState *gcstate = get_gc_state(); + if (gcstate->generations[0].count > 0) { + gcstate->generations[0].count--; + } PyObject_Free(((char *)op)-presize); } @@ -2590,18 +2148,14 @@ PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) GCState *gcstate = get_gc_state(); int original_state = gcstate->enabled; gcstate->enabled = 0; - if (visit_generation(callback, arg, &gcstate->young) < 0) { - goto done; - } - if (visit_generation(callback, arg, &gcstate->old[0]) < 0) { - goto done; - } - if (visit_generation(callback, arg, &gcstate->old[1]) < 0) { - goto done; + for (size_t i = 0; i < NUM_GENERATIONS; i++) { + if (visit_generation(callback, arg, &gcstate->generations[i]) < 0) { + goto done; + } } visit_generation(callback, arg, &gcstate->permanent_generation); done: gcstate->enabled = original_state; } -#endif // Py_GIL_DISABLED +#endif // !Py_GIL_DISABLED diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e84886ed040..dccee0e4a3b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5913,7 +5913,7 @@ int og_oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; if (_PyJit_EnterExecutorShouldStopTracing(og_opcode)) { - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + if (_PyOpcode_Caches[_PyOpcode_Deopt[og_opcode]]) { PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } opcode = og_opcode; @@ -12497,7 +12497,10 @@ tracer->prev_state.instr_frame = frame; tracer->prev_state.instr_oparg = oparg; tracer->prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + // Branch opcodes use the cache for branch history, not + // specialization counters. Don't reset it. + && !IS_CONDITIONAL_JUMP_OPCODE(opcode)) { (&next_instr[1])->counter = trigger_backoff_counter(); } const _PyOpcodeRecordEntry *record_entry = &_PyOpcode_RecordEntries[opcode]; diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 256e2a3d3a2..51bcbfdb3b6 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -185,6 +185,12 @@ opcode_has_event(int opcode) ); } +uint8_t +_PyCode_Deinstrument(uint8_t opcode) +{ + return DE_INSTRUMENT[opcode]; +} + static inline bool is_instrumented(int opcode) { @@ -197,7 +203,7 @@ is_instrumented(int opcode) static inline bool monitors_equals(_Py_LocalMonitors a, _Py_LocalMonitors b) { - for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { if (a.tools[i] != b.tools[i]) { return false; } @@ -210,7 +216,7 @@ static inline _Py_LocalMonitors monitors_sub(_Py_LocalMonitors a, _Py_LocalMonitors b) { _Py_LocalMonitors res; - for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { res.tools[i] = a.tools[i] & ~b.tools[i]; } return res; @@ -221,7 +227,7 @@ static inline _Py_LocalMonitors monitors_and(_Py_LocalMonitors a, _Py_LocalMonitors b) { _Py_LocalMonitors res; - for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { res.tools[i] = a.tools[i] & b.tools[i]; } return res; @@ -237,7 +243,7 @@ static inline _Py_LocalMonitors local_union(_Py_GlobalMonitors a, _Py_LocalMonitors b) { _Py_LocalMonitors res; - for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { res.tools[i] = a.tools[i] | b.tools[i]; } return res; @@ -246,7 +252,7 @@ local_union(_Py_GlobalMonitors a, _Py_LocalMonitors b) static inline bool monitors_are_empty(_Py_LocalMonitors m) { - for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { if (m.tools[i]) { return false; } @@ -257,7 +263,7 @@ monitors_are_empty(_Py_LocalMonitors m) static inline bool multiple_tools(_Py_LocalMonitors *m) { - for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { if (_Py_popcount32(m->tools[i]) > 1) { return true; } @@ -269,7 +275,7 @@ static inline _PyMonitoringEventSet get_local_events(_Py_LocalMonitors *m, int tool_id) { _PyMonitoringEventSet result = 0; - for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) { + for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) { if ((m->tools[e] >> tool_id) & 1) { result |= (1 << e); } @@ -330,12 +336,6 @@ _PyInstruction_GetLength(PyCodeObject *code, int offset) return 1 + _PyOpcode_Caches[inst.op.code]; } -static inline uint8_t -get_original_opcode(_PyCoLineInstrumentationData *line_data, int index) -{ - return line_data->data[index*line_data->bytes_per_entry]; -} - static inline uint8_t * get_original_opcode_ptr(_PyCoLineInstrumentationData *line_data, int index) { @@ -401,7 +401,7 @@ dump_instrumentation_data_lines(PyCodeObject *code, _PyCoLineInstrumentationData fprintf(out, ", lines = NULL"); } else { - int opcode = get_original_opcode(lines, i); + int opcode = _PyCode_GetOriginalOpcode(lines, i); int line_delta = get_line_delta(lines, i); if (opcode == 0) { fprintf(out, ", lines = {original_opcode = No LINE (0), line_delta = %d)", line_delta); @@ -453,7 +453,7 @@ static void dump_local_monitors(const char *prefix, _Py_LocalMonitors monitors, FILE*out) { fprintf(out, "%s monitors:\n", prefix); - for (int event = 0; event < _PY_MONITORING_LOCAL_EVENTS; event++) { + for (int event = 0; event < _PY_MONITORING_UNGROUPED_EVENTS; event++) { fprintf(out, " Event %d: Tools %x\n", event, monitors.tools[event]); } } @@ -571,7 +571,7 @@ sanity_check_instrumentation(PyCodeObject *code) } if (opcode == INSTRUMENTED_LINE) { CHECK(data->lines); - opcode = get_original_opcode(data->lines, i); + opcode = _PyCode_GetOriginalOpcode(data->lines, i); CHECK(valid_opcode(opcode)); CHECK(opcode != END_FOR); CHECK(opcode != RESUME); @@ -588,7 +588,7 @@ sanity_check_instrumentation(PyCodeObject *code) * *and* we are executing a INSTRUMENTED_LINE instruction * that has de-instrumented itself, then we will execute * an invalid INSTRUMENTED_INSTRUCTION */ - CHECK(get_original_opcode(data->lines, i) != INSTRUMENTED_INSTRUCTION); + CHECK(_PyCode_GetOriginalOpcode(data->lines, i) != INSTRUMENTED_INSTRUCTION); } if (opcode == INSTRUMENTED_INSTRUCTION) { CHECK(data->per_instruction_opcodes[i] != 0); @@ -603,7 +603,7 @@ sanity_check_instrumentation(PyCodeObject *code) } CHECK(active_monitors.tools[event] != 0); } - if (data->lines && get_original_opcode(data->lines, i)) { + if (data->lines && _PyCode_GetOriginalOpcode(data->lines, i)) { int line1 = compute_line(code, get_line_delta(data->lines, i)); int line2 = _PyCode_CheckLineNumber(i*sizeof(_Py_CODEUNIT), &range); CHECK(line1 == line2); @@ -655,7 +655,7 @@ _Py_GetBaseCodeUnit(PyCodeObject *code, int i) return inst; } if (opcode == INSTRUMENTED_LINE) { - opcode = get_original_opcode(code->_co_monitoring->lines, i); + opcode = _PyCode_GetOriginalOpcode(code->_co_monitoring->lines, i); } if (opcode == INSTRUMENTED_INSTRUCTION) { opcode = code->_co_monitoring->per_instruction_opcodes[i]; @@ -714,7 +714,7 @@ de_instrument_line(PyCodeObject *code, _Py_CODEUNIT *bytecode, _PyCoMonitoringDa return; } _PyCoLineInstrumentationData *lines = monitoring->lines; - int original_opcode = get_original_opcode(lines, i); + int original_opcode = _PyCode_GetOriginalOpcode(lines, i); if (original_opcode == INSTRUMENTED_INSTRUCTION) { set_original_opcode(lines, i, monitoring->per_instruction_opcodes[i]); } @@ -1102,8 +1102,10 @@ get_tools_for_instruction(PyCodeObject *code, PyInterpreterState *interp, int i, event == PY_MONITORING_EVENT_C_RETURN); event = PY_MONITORING_EVENT_CALL; } + assert(_PY_MONITORING_IS_UNGROUPED_EVENT(event)); + CHECK(debug_check_sanity(interp, code)); if (PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) { - CHECK(debug_check_sanity(interp, code)); + /* Instrumented events use per-instruction tool bitmaps. */ if (code->_co_monitoring->tools) { tools = code->_co_monitoring->tools[i]; } @@ -1112,7 +1114,9 @@ get_tools_for_instruction(PyCodeObject *code, PyInterpreterState *interp, int i, } } else { - tools = interp->monitors.tools[event]; + /* Other (non-instrumented) events are not tied to specific instructions; + * use the code-object-level active_monitors bitmap instead. */ + tools = code->_co_monitoring->active_monitors.tools[event]; } return tools; } @@ -1139,6 +1143,25 @@ static const char *const event_names [] = { [PY_MONITORING_EVENT_STOP_ITERATION] = "STOP_ITERATION", }; +/* Disable an "other" (non-instrumented) event (e.g. PY_UNWIND) for a single + * tool on this code object. Must be called with the world stopped or the + * code lock held. */ +static void +remove_local_tool(PyCodeObject *code, PyInterpreterState *interp, + int event, int tool) +{ + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + assert(_PY_MONITORING_IS_UNGROUPED_EVENT(event)); + assert(!PY_MONITORING_IS_INSTRUMENTED_EVENT(event)); + assert(code->_co_monitoring); + code->_co_monitoring->local_monitors.tools[event] &= ~(1 << tool); + /* Recompute active_monitors for this event as the union of global and + * (now updated) local monitors. */ + code->_co_monitoring->active_monitors.tools[event] = + interp->monitors.tools[event] | + code->_co_monitoring->local_monitors.tools[event]; +} + static int call_instrumentation_vector( _Py_CODEUNIT *instr, PyThreadState *tstate, int event, @@ -1183,7 +1206,18 @@ call_instrumentation_vector( } else { /* DISABLE */ - if (!PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) { + if (PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) { + _PyEval_StopTheWorld(interp); + remove_tools(code, offset, event, 1 << tool); + _PyEval_StartTheWorld(interp); + } + else if (_PY_MONITORING_IS_UNGROUPED_EVENT(event)) { + /* Other (non-instrumented) event: disable for this code object. */ + _PyEval_StopTheWorld(interp); + remove_local_tool(code, interp, event, tool); + _PyEval_StartTheWorld(interp); + } + else { PyErr_Format(PyExc_ValueError, "Cannot disable %s events. Callback removed.", event_names[event]); @@ -1192,12 +1226,6 @@ call_instrumentation_vector( err = -1; break; } - else { - PyInterpreterState *interp = tstate->interp; - _PyEval_StopTheWorld(interp); - remove_tools(code, offset, event, 1 << tool); - _PyEval_StartTheWorld(interp); - } } } Py_DECREF(arg2_obj); @@ -1391,7 +1419,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, Py_DECREF(line_obj); uint8_t original_opcode; done: - original_opcode = get_original_opcode(line_data, i); + original_opcode = _PyCode_GetOriginalOpcode(line_data, i); assert(original_opcode != 0); assert(original_opcode != INSTRUMENTED_LINE); assert(_PyOpcode_Deopt[original_opcode] == original_opcode); @@ -1464,7 +1492,7 @@ initialize_tools(PyCodeObject *code) int opcode = instr->op.code; assert(opcode != ENTER_EXECUTOR); if (opcode == INSTRUMENTED_LINE) { - opcode = get_original_opcode(code->_co_monitoring->lines, i); + opcode = _PyCode_GetOriginalOpcode(code->_co_monitoring->lines, i); } if (opcode == INSTRUMENTED_INSTRUCTION) { opcode = code->_co_monitoring->per_instruction_opcodes[i]; @@ -1681,7 +1709,7 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) _Py_LocalMonitors *local_monitors = &code->_co_monitoring->local_monitors; for (int i = 0; i < PY_MONITORING_TOOL_IDS; i++) { if (code->_co_monitoring->tool_versions[i] != interp->monitoring_tool_versions[i]) { - for (int j = 0; j < _PY_MONITORING_LOCAL_EVENTS; j++) { + for (int j = 0; j < _PY_MONITORING_UNGROUPED_EVENTS; j++) { local_monitors->tools[j] &= ~(1 << i); } } @@ -1849,7 +1877,7 @@ force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) if (removed_line_tools) { _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; for (int i = code->_co_firsttraceable; i < code_len;) { - if (get_original_opcode(line_data, i)) { + if (_PyCode_GetOriginalOpcode(line_data, i)) { remove_line_tools(code, i, removed_line_tools); } i += _PyInstruction_GetLength(code, i); @@ -1876,7 +1904,7 @@ force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) if (new_line_tools) { _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; for (int i = code->_co_firsttraceable; i < code_len;) { - if (get_original_opcode(line_data, i)) { + if (_PyCode_GetOriginalOpcode(line_data, i)) { add_line_tools(code, i, new_line_tools); } i += _PyInstruction_GetLength(code, i); @@ -1977,7 +2005,7 @@ static void set_local_events(_Py_LocalMonitors *m, int tool_id, _PyMonitoringEventSet events) { assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); - for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) { + for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) { uint8_t *tools = &m->tools[e]; int val = (events >> e) & 1; *tools &= ~(1 << tool_id); @@ -2037,7 +2065,7 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); PyInterpreterState *interp = _PyInterpreterState_GET(); - assert(events < (1 << _PY_MONITORING_LOCAL_EVENTS)); + assert(events < (1 << _PY_MONITORING_UNGROUPED_EVENTS)); if (code->_co_firsttraceable >= Py_SIZE(code)) { PyErr_Format(PyExc_SystemError, "cannot instrument shim code object '%U'", code->co_name); return -1; @@ -2373,7 +2401,7 @@ monitoring_get_local_events_impl(PyObject *module, int tool_id, _PyMonitoringEventSet event_set = 0; _PyCoMonitoringData *data = ((PyCodeObject *)code)->_co_monitoring; if (data != NULL) { - for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) { + for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) { if ((data->local_monitors.tools[e] >> tool_id) & 1) { event_set |= (1 << e); } @@ -2416,7 +2444,7 @@ monitoring_set_local_events_impl(PyObject *module, int tool_id, event_set &= ~(1 << PY_MONITORING_EVENT_BRANCH); event_set |= (1 << PY_MONITORING_EVENT_BRANCH_RIGHT) | (1 << PY_MONITORING_EVENT_BRANCH_LEFT); } - if (event_set < 0 || event_set >= (1 << _PY_MONITORING_LOCAL_EVENTS)) { + if (event_set < 0 || event_set >= (1 << _PY_MONITORING_UNGROUPED_EVENTS)) { PyErr_Format(PyExc_ValueError, "invalid local event set 0x%x", event_set); return NULL; } diff --git a/Python/jit.c b/Python/jit.c index af75acf1ff2..8b555105129 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -15,6 +15,7 @@ #include "pycore_interpframe.h" #include "pycore_interpolation.h" #include "pycore_intrinsics.h" +#include "pycore_jit_unwind.h" #include "pycore_lazyimportobject.h" #include "pycore_list.h" #include "pycore_long.h" @@ -60,7 +61,39 @@ jit_error(const char *message) PyErr_Format(PyExc_RuntimeWarning, "JIT %s (%d)", message, hint); } -static size_t _Py_jit_shim_size = 0; +/* + * Publish JIT code to optional tooling backends. + * + * The return value is a backend-specific deregistration handle, not a + * success/failure indicator. NULL means there is nothing to unregister later: + * perf does not need a handle, and GDB registration failures are intentionally + * non-fatal because tooling support must not make JIT compilation fail. + */ +static void * +jit_record_code(const void *code_addr, size_t code_size, + const char *entry, const char *filename) +{ +#ifdef PY_HAVE_PERF_TRAMPOLINE + _PyPerf_Callbacks callbacks; + _PyPerfTrampoline_GetCallbacks(&callbacks); + if (callbacks.write_state == _Py_perfmap_jit_callbacks.write_state) { + _PyPerfJit_WriteNamedCode( + code_addr, code_size, entry, filename); + return NULL; + } +#endif + +#if defined(PY_HAVE_JIT_GDB_UNWIND) + return _PyJitUnwind_GdbRegisterCode( + code_addr, code_size, entry, filename); +#else + (void)code_addr; + (void)code_size; + (void)entry; + (void)filename; + return NULL; +#endif +} static int address_in_executor_array(_PyExecutorObject **ptrs, size_t count, uintptr_t addr) @@ -104,13 +137,6 @@ _PyJIT_AddressInJitCode(PyInterpreterState *interp, uintptr_t addr) if (interp == NULL) { return 0; } - if (_Py_jit_entry != _Py_LazyJitShim && _Py_jit_shim_size != 0) { - uintptr_t start = (uintptr_t)_Py_jit_entry; - uintptr_t end = start + _Py_jit_shim_size; - if (addr >= start && addr < end) { - return 1; - } - } if (address_in_executor_array(interp->executor_ptrs, interp->executor_count, addr)) { return 1; } @@ -724,78 +750,13 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz } executor->jit_code = memory; executor->jit_size = total_size; + executor->jit_gdb_handle = jit_record_code(memory, + code_size + state.trampolines.size, + "jit", + "executor"); return 0; } -/* One-off compilation of the jit entry shim - * We compile this once only as it effectively a normal - * function, but we need to use the JIT because it needs - * to understand the jit-specific calling convention. - * Don't forget to call _PyJIT_Fini later! - */ -static _PyJitEntryFuncPtr -compile_shim(void) -{ - _PyExecutorObject dummy; - const StencilGroup *group; - size_t code_size = 0; - size_t data_size = 0; - jit_state state = {0}; - group = &shim; - code_size += group->code_size; - data_size += group->data_size; - combine_symbol_mask(group->trampoline_mask, state.trampolines.mask); - combine_symbol_mask(group->got_mask, state.got_symbols.mask); - // Round up to the nearest page: - size_t page_size = get_page_size(); - assert((page_size & (page_size - 1)) == 0); - size_t code_padding = DATA_ALIGN - ((code_size + state.trampolines.size) & (DATA_ALIGN - 1)); - size_t padding = page_size - ((code_size + state.trampolines.size + code_padding + data_size + state.got_symbols.size) & (page_size - 1)); - size_t total_size = code_size + state.trampolines.size + code_padding + data_size + state.got_symbols.size + padding; - unsigned char *memory = jit_alloc(total_size); - if (memory == NULL) { - return NULL; - } - unsigned char *code = memory; - state.trampolines.mem = memory + code_size; - unsigned char *data = memory + code_size + state.trampolines.size + code_padding; - state.got_symbols.mem = data + data_size; - // Compile the shim, which handles converting between the native - // calling convention and the calling convention used by jitted code - // (which may be different for efficiency reasons). - group = &shim; - group->emit(code, data, &dummy, NULL, &state); - code += group->code_size; - data += group->data_size; - assert(code == memory + code_size); - assert(data == memory + code_size + state.trampolines.size + code_padding + data_size); - if (mark_executable(memory, total_size)) { - jit_free(memory, total_size); - return NULL; - } - _Py_jit_shim_size = total_size; - return (_PyJitEntryFuncPtr)memory; -} - -static PyMutex lazy_jit_mutex = { 0 }; - -_Py_CODEUNIT * -_Py_LazyJitShim( - _PyExecutorObject *executor, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate -) { - PyMutex_Lock(&lazy_jit_mutex); - if (_Py_jit_entry == _Py_LazyJitShim) { - _PyJitEntryFuncPtr shim = compile_shim(); - if (shim == NULL) { - PyMutex_Unlock(&lazy_jit_mutex); - Py_FatalError("Cannot allocate core JIT code"); - } - _Py_jit_entry = shim; - } - PyMutex_Unlock(&lazy_jit_mutex); - return _Py_jit_entry(executor, frame, stack_pointer, tstate); -} - // Free executor's memory allocated with _PyJIT_Compile void _PyJIT_Free(_PyExecutorObject *executor) @@ -805,6 +766,12 @@ _PyJIT_Free(_PyExecutorObject *executor) if (memory) { executor->jit_code = NULL; executor->jit_size = 0; +#if defined(PY_HAVE_JIT_GDB_UNWIND) + if (executor->jit_gdb_handle != NULL) { + _PyJitUnwind_GdbUnregisterCode(executor->jit_gdb_handle); + executor->jit_gdb_handle = NULL; + } +#endif if (jit_free(memory, size)) { PyErr_FormatUnraisable("Exception ignored while " "freeing JIT memory"); @@ -812,22 +779,4 @@ _PyJIT_Free(_PyExecutorObject *executor) } } -// Free shim memory allocated with compile_shim -void -_PyJIT_Fini(void) -{ - PyMutex_Lock(&lazy_jit_mutex); - unsigned char *memory = (unsigned char *)_Py_jit_entry; - size_t size = _Py_jit_shim_size; - if (size) { - _Py_jit_entry = _Py_LazyJitShim; - _Py_jit_shim_size = 0; - if (jit_free(memory, size)) { - PyErr_FormatUnraisable("Exception ignored while " - "freeing JIT entry code"); - } - } - PyMutex_Unlock(&lazy_jit_mutex); -} - #endif // _Py_JIT diff --git a/Python/jit_unwind.c b/Python/jit_unwind.c new file mode 100644 index 00000000000..09002feafec --- /dev/null +++ b/Python/jit_unwind.c @@ -0,0 +1,986 @@ +/* + * Python JIT - DWARF .eh_frame builder + * + * This file contains the DWARF CFI generator used to build .eh_frame + * data for JIT code (perf jitdump and other unwinders). + */ + +#include "Python.h" +#include "pycore_jit_unwind.h" +#include "pycore_lock.h" + +#if defined(PY_HAVE_JIT_GDB_UNWIND) +# include "jit_unwind_info.h" +# if !JIT_UNWIND_INFO_SUPPORTED +# error "JIT unwind info was not generated for this target" +# endif +#endif + +#if defined(PY_HAVE_PERF_TRAMPOLINE) || defined(PY_HAVE_JIT_GDB_UNWIND) + +#if defined(PY_HAVE_JIT_GDB_UNWIND) +# include +#endif +#include +#include + +// ============================================================================= +// DWARF CONSTANTS +// ============================================================================= + +/* + * DWARF (Debug With Arbitrary Record Formats) constants + * + * DWARF is a debugging data format used to provide stack unwinding information. + * These constants define the various encoding types and opcodes used in + * DWARF Call Frame Information (CFI) records. + */ + +/* DWARF Call Frame Information version */ +#define DWRF_CIE_VERSION 1 + +/* DWARF CFA (Call Frame Address) opcodes */ +enum { + DWRF_CFA_nop = 0x0, // No operation + DWRF_CFA_offset_extended = 0x5, // Extended offset instruction + DWRF_CFA_def_cfa = 0xc, // Define CFA rule + DWRF_CFA_def_cfa_register = 0xd, // Define CFA register + DWRF_CFA_def_cfa_offset = 0xe, // Define CFA offset + DWRF_CFA_offset_extended_sf = 0x11, // Extended signed offset + DWRF_CFA_advance_loc = 0x40, // Advance location counter + DWRF_CFA_offset = 0x80, // Simple offset instruction + DWRF_CFA_restore = 0xc0 // Restore register +}; + +/* + * Architecture-specific DWARF register numbers + * + * These constants define the register numbering scheme used by DWARF + * for each supported architecture. The numbers must match the ABI + * specification for proper stack unwinding. + */ +enum { +#ifdef __x86_64__ + /* x86_64 register numbering (note: order is defined by x86_64 ABI) */ + DWRF_REG_AX, // RAX + DWRF_REG_DX, // RDX + DWRF_REG_CX, // RCX + DWRF_REG_BX, // RBX + DWRF_REG_SI, // RSI + DWRF_REG_DI, // RDI + DWRF_REG_BP, // RBP + DWRF_REG_SP, // RSP + DWRF_REG_8, // R8 + DWRF_REG_9, // R9 + DWRF_REG_10, // R10 + DWRF_REG_11, // R11 + DWRF_REG_12, // R12 + DWRF_REG_13, // R13 + DWRF_REG_14, // R14 + DWRF_REG_15, // R15 + DWRF_REG_RA, // Return address (RIP) +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + /* AArch64 register numbering */ + DWRF_REG_FP = 29, // Frame Pointer + DWRF_REG_RA = 30, // Link register (return address) + DWRF_REG_SP = 31, // Stack pointer +#else +# error "Unsupported target architecture" +#endif +}; + +// ============================================================================= +// ELF OBJECT CONTEXT +// ============================================================================= + +/* + * Context for building ELF/DWARF structures + * + * This structure maintains state while constructing DWARF unwind information. + * It acts as a simple buffer manager with pointers to track current position + * and important landmarks within the buffer. + */ +typedef struct ELFObjectContext { + uint8_t* p; // Current write position in buffer + uint8_t* startp; // Start of buffer (for offset calculations) + uint8_t* fde_p; // Start of FDE data (for PC-relative calculations) + uintptr_t code_addr; // Address of the code section + size_t code_size; // Size of the code section +} ELFObjectContext; + +// ============================================================================= +// DWARF GENERATION UTILITIES +// ============================================================================= + +/* + * Append a null-terminated string to the ELF context buffer. + * + * Args: + * ctx: ELF object context + * str: String to append (must be null-terminated) + * + * Returns: Offset from start of buffer where string was written + */ +static uint32_t elfctx_append_string(ELFObjectContext* ctx, const char* str) { + uint8_t* p = ctx->p; + uint32_t ofs = (uint32_t)(p - ctx->startp); + + /* Copy string including null terminator */ + do { + *p++ = (uint8_t)*str; + } while (*str++); + + ctx->p = p; + return ofs; +} + +/* + * Append a SLEB128 (Signed Little Endian Base 128) value + * + * SLEB128 is a variable-length encoding used extensively in DWARF. + * It efficiently encodes small numbers in fewer bytes. + * + * Args: + * ctx: ELF object context + * v: Signed value to encode + */ +static void elfctx_append_sleb128(ELFObjectContext* ctx, int32_t v) { + uint8_t* p = ctx->p; + + /* Encode 7 bits at a time, with continuation bit in MSB */ + for (; (uint32_t)(v + 0x40) >= 0x80; v >>= 7) { + *p++ = (uint8_t)((v & 0x7f) | 0x80); // Set continuation bit + } + *p++ = (uint8_t)(v & 0x7f); // Final byte without continuation bit + + ctx->p = p; +} + +/* + * Append a ULEB128 (Unsigned Little Endian Base 128) value + * + * Similar to SLEB128 but for unsigned values. + * + * Args: + * ctx: ELF object context + * v: Unsigned value to encode + */ +static void elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v) { + uint8_t* p = ctx->p; + + /* Encode 7 bits at a time, with continuation bit in MSB */ + for (; v >= 0x80; v >>= 7) { + *p++ = (char)((v & 0x7f) | 0x80); // Set continuation bit + } + *p++ = (char)v; // Final byte without continuation bit + + ctx->p = p; +} + +/* + * Macros for generating DWARF structures + * + * These macros provide a convenient way to write various data types + * to the DWARF buffer while automatically advancing the pointer. + */ +#define DWRF_U8(x) (*p++ = (x)) // Write unsigned 8-bit +#define DWRF_I8(x) (*(int8_t*)p = (x), p++) // Write signed 8-bit +#define DWRF_U16(x) (*(uint16_t*)p = (x), p += 2) // Write unsigned 16-bit +#define DWRF_U32(x) (*(uint32_t*)p = (x), p += 4) // Write unsigned 32-bit +#define DWRF_ADDR(x) (*(uintptr_t*)p = (x), p += sizeof(uintptr_t)) // Write address +#define DWRF_UV(x) (ctx->p = p, elfctx_append_uleb128(ctx, (x)), p = ctx->p) // Write ULEB128 +#define DWRF_SV(x) (ctx->p = p, elfctx_append_sleb128(ctx, (x)), p = ctx->p) // Write SLEB128 +#define DWRF_STR(str) (ctx->p = p, elfctx_append_string(ctx, (str)), p = ctx->p) // Write string + +/* Align to specified boundary with NOP instructions */ +#define DWRF_ALIGNNOP(s) \ + while ((uintptr_t)p & ((s)-1)) { \ + *p++ = DWRF_CFA_nop; \ + } + +/* Write a DWARF section with automatic size calculation */ +#define DWRF_SECTION(name, stmt) \ + { \ + uint32_t* szp_##name = (uint32_t*)p; \ + p += 4; \ + stmt; \ + *szp_##name = (uint32_t)((p - (uint8_t*)szp_##name) - 4); \ + } + +// ============================================================================= +// DWARF EH FRAME GENERATION +// ============================================================================= + +static void elf_init_ehframe_perf(ELFObjectContext* ctx); +#if defined(PY_HAVE_JIT_GDB_UNWIND) +static void elf_init_ehframe_gdb(ELFObjectContext* ctx); +#endif + +static inline void elf_init_ehframe(ELFObjectContext* ctx, int absolute_addr) { + if (absolute_addr) { +#if defined(PY_HAVE_JIT_GDB_UNWIND) + elf_init_ehframe_gdb(ctx); +#else + Py_UNREACHABLE(); +#endif + } + else { + elf_init_ehframe_perf(ctx); + } +} + +size_t +_PyJitUnwind_EhFrameSize(int absolute_addr) +{ + /* The .eh_frame we emit is small and bounded; keep a generous buffer. */ + uint8_t scratch[512]; + _Static_assert(sizeof(scratch) >= 256, + "scratch buffer may be too small for elf_init_ehframe"); + ELFObjectContext ctx; + ctx.code_size = 1; + ctx.code_addr = 0; + ctx.startp = ctx.p = scratch; + ctx.fde_p = NULL; + /* Generate once into scratch to learn the required size. */ + elf_init_ehframe(&ctx, absolute_addr); + ptrdiff_t size = ctx.p - ctx.startp; + assert(size <= (ptrdiff_t)sizeof(scratch)); + return (size_t)size; +} + +size_t +_PyJitUnwind_BuildEhFrame(uint8_t *buffer, size_t buffer_size, + const void *code_addr, size_t code_size, + int absolute_addr) +{ + if (buffer == NULL || code_addr == NULL || code_size == 0) { + return 0; + } + /* Generate the frame twice: once to size-check, once to write. */ + size_t required = _PyJitUnwind_EhFrameSize(absolute_addr); + if (required == 0 || required > buffer_size) { + return 0; + } + ELFObjectContext ctx; + ctx.code_size = code_size; + ctx.code_addr = (uintptr_t)code_addr; + ctx.startp = ctx.p = buffer; + ctx.fde_p = NULL; + elf_init_ehframe(&ctx, absolute_addr); + size_t written = (size_t)(ctx.p - ctx.startp); + /* The frame size is independent of code_addr/code_size (fixed-width fields). */ + assert(written == required); + return written; +} + +/* + * Generate a minimal .eh_frame for a single JIT code region. + * + * The .eh_frame section contains Call Frame Information (CFI) that describes + * how to unwind the stack at any point in the code. This is essential for + * unwinding through JIT-generated code. + * + * The generated data contains: + * 1. A CIE (Common Information Entry) describing the calling convention. + * 2. An FDE (Frame Description Entry) describing how to unwind the JIT frame. + * + * Two flavors are emitted, dispatched on the absolute_addr flag: + * + * - absolute_addr == 0 (elf_init_ehframe_perf): PC-relative FDE address + * encoding for perf's synthesized DSO layout. The CIE describes the + * trampoline's entry state and the FDE walks through the prologue and + * epilogue with advance_loc instructions. This matches the pre-existing + * perf_jit_trampoline behavior byte-for-byte. + * + * - absolute_addr == 1 (elf_init_ehframe_gdb): absolute FDE address + * encoding for the GDB JIT in-memory ELF. The CIE describes the + * steady-state frame layout (CFA = %rbp+16 / x29+16, with saved fp and + * return-address column at fixed offsets) and the FDE emits no further + * CFI. The same rule applies at every PC in the registered region, + * which is correct for executor stencils (they pin the frame pointer + * across the region). This is the GDB-side fix; see elf_init_ehframe_gdb + * for details. + */ +static void elf_init_ehframe_perf(ELFObjectContext* ctx) { + int fde_ptr_enc = DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4; + uint8_t* p = ctx->p; + uint8_t* framep = p; // Remember start of frame data + + /* + * DWARF Unwind Table for Trampoline Function + * + * This section defines DWARF Call Frame Information (CFI) using encoded macros + * like `DWRF_U8`, `DWRF_UV`, and `DWRF_SECTION` to describe how the trampoline function + * preserves and restores registers. This is used by profiling tools (e.g., `perf`) + * and debuggers for stack unwinding in JIT-compiled code. + * + * ------------------------------------------------- + * TO REGENERATE THIS TABLE FROM GCC OBJECTS: + * ------------------------------------------------- + * + * 1. Create a trampoline source file (e.g., `trampoline.c`): + * + * #include + * typedef PyObject* (*py_evaluator)(void*, void*, int); + * PyObject* trampoline(void *ts, void *f, int throwflag, py_evaluator evaluator) { + * return evaluator(ts, f, throwflag); + * } + * + * 2. Compile to an object file with frame pointer preservation: + * + * gcc trampoline.c -I. -I./Include -O2 -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -c + * + * 3. Extract DWARF unwind info from the object file: + * + * readelf -w trampoline.o + * + * Example output from `.eh_frame`: + * + * 00000000 CIE + * Version: 1 + * Augmentation: "zR" + * Code alignment factor: 4 + * Data alignment factor: -8 + * Return address column: 30 + * DW_CFA_def_cfa: r31 (sp) ofs 0 + * + * 00000014 FDE cie=00000000 pc=0..14 + * DW_CFA_advance_loc: 4 + * DW_CFA_def_cfa_offset: 16 + * DW_CFA_offset: r29 at cfa-16 + * DW_CFA_offset: r30 at cfa-8 + * DW_CFA_advance_loc: 12 + * DW_CFA_restore: r30 + * DW_CFA_restore: r29 + * DW_CFA_def_cfa_offset: 0 + * + * -- These values can be verified by comparing with `readelf -w` or `llvm-dwarfdump --eh-frame`. + * + * ---------------------------------- + * HOW TO TRANSLATE TO DWRF_* MACROS: + * ---------------------------------- + * + * After compiling your trampoline with: + * + * gcc trampoline.c -I. -I./Include -O2 -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -c + * + * run: + * + * readelf -w trampoline.o + * + * to inspect the generated `.eh_frame` data. You will see two main components: + * + * 1. A CIE (Common Information Entry): shared configuration used by all FDEs. + * 2. An FDE (Frame Description Entry): function-specific unwind instructions. + * + * --------------------- + * Translating the CIE: + * --------------------- + * From `readelf -w`, you might see: + * + * 00000000 0000000000000010 00000000 CIE + * Version: 1 + * Augmentation: "zR" + * Code alignment factor: 4 + * Data alignment factor: -8 + * Return address column: 30 + * Augmentation data: 1b + * DW_CFA_def_cfa: r31 (sp) ofs 0 + * + * Map this to: + * + * DWRF_SECTION(CIE, + * DWRF_U32(0); // CIE ID (always 0 for CIEs) + * DWRF_U8(DWRF_CIE_VERSION); // Version: 1 + * DWRF_STR("zR"); // Augmentation string "zR" + * DWRF_UV(4); // Code alignment factor = 4 + * DWRF_SV(-8); // Data alignment factor = -8 + * DWRF_U8(DWRF_REG_RA); // Return address register (e.g., x30 = 30) + * DWRF_UV(1); // Augmentation data length = 1 + * DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); // Encoding for FDE pointers + * + * DWRF_U8(DWRF_CFA_def_cfa); // DW_CFA_def_cfa + * DWRF_UV(DWRF_REG_SP); // Register: SP (r31) + * DWRF_UV(0); // Offset = 0 + * + * DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer size boundary + * ) + * + * Notes: + * - Use `DWRF_UV` for unsigned LEB128, `DWRF_SV` for signed LEB128. + * - `DWRF_REG_RA` and `DWRF_REG_SP` are architecture-defined constants. + * + * --------------------- + * Translating the FDE: + * --------------------- + * From `readelf -w`: + * + * 00000014 0000000000000020 00000018 FDE cie=00000000 pc=0000000000000000..0000000000000014 + * DW_CFA_advance_loc: 4 + * DW_CFA_def_cfa_offset: 16 + * DW_CFA_offset: r29 at cfa-16 + * DW_CFA_offset: r30 at cfa-8 + * DW_CFA_advance_loc: 12 + * DW_CFA_restore: r30 + * DW_CFA_restore: r29 + * DW_CFA_def_cfa_offset: 0 + * + * Map the FDE header and instructions to: + * + * DWRF_SECTION(FDE, + * DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (relative from here) + * DWRF_U32(pc_relative_offset); // PC-relative location of the code (calculated dynamically) + * DWRF_U32(ctx->code_size); // Code range covered by this FDE + * DWRF_U8(0); // Augmentation data length (none) + * + * DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance location by 1 unit (1 * 4 = 4 bytes) + * DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 + * DWRF_UV(16); + * + * DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // Save x29 (frame pointer) + * DWRF_UV(2); // At offset 2 * 8 = 16 bytes + * + * DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // Save x30 (return address) + * DWRF_UV(1); // At offset 1 * 8 = 8 bytes + * + * DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance location by 3 units (3 * 4 = 12 bytes) + * + * DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // Restore x30 + * DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // Restore x29 + * + * DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + * DWRF_UV(0); + * ) + * + * To regenerate: + * 1. Get the `code alignment factor`, `data alignment factor`, and `RA column` from the CIE. + * 2. Note the range of the function from the FDE's `pc=...` line and map it to the JIT code as + * the code is in a different address space every time. + * 3. For each `DW_CFA_*` entry, use the corresponding `DWRF_*` macro: + * - `DW_CFA_def_cfa_offset` → DWRF_U8(DWRF_CFA_def_cfa_offset), DWRF_UV(value) + * - `DW_CFA_offset: rX` → DWRF_U8(DWRF_CFA_offset | reg), DWRF_UV(offset) + * - `DW_CFA_restore: rX` → DWRF_U8(DWRF_CFA_offset | reg) // restore is same as reusing offset + * - `DW_CFA_advance_loc: N` → DWRF_U8(DWRF_CFA_advance_loc | (N / code_alignment_factor)) + * 4. Use `DWRF_REG_FP`, `DWRF_REG_RA`, etc., for register numbers. + * 5. Use `sizeof(uintptr_t)` (typically 8) for pointer size calculations and alignment. + */ + + /* + * Emit DWARF EH CIE (Common Information Entry) + * + * The CIE describes the calling conventions and basic unwinding rules + * that apply to all functions in this compilation unit. + */ + DWRF_SECTION(CIE, + DWRF_U32(0); // CIE ID (0 indicates this is a CIE) + DWRF_U8(DWRF_CIE_VERSION); // CIE version (1) + DWRF_STR("zR"); // Augmentation string ("zR" = has LSDA) +#ifdef __x86_64__ + DWRF_UV(1); // Code alignment factor (x86_64: 1 byte) +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + DWRF_UV(4); // Code alignment factor (AArch64: 4 bytes per instruction) +#endif + DWRF_SV(-(int64_t)sizeof(uintptr_t)); // Data alignment factor (negative) + DWRF_U8(DWRF_REG_RA); // Return address register number + DWRF_UV(1); // Augmentation data length + DWRF_U8(fde_ptr_enc); // FDE pointer encoding + + /* Initial CFI instructions - describe default calling convention */ +#ifdef __x86_64__ + /* x86_64 initial CFI state */ + DWRF_U8(DWRF_CFA_def_cfa); // Define CFA (Call Frame Address) + DWRF_UV(DWRF_REG_SP); // CFA = SP register + DWRF_UV(sizeof(uintptr_t)); // CFA = SP + pointer_size + DWRF_U8(DWRF_CFA_offset|DWRF_REG_RA); // Return address is saved + DWRF_UV(1); // At offset 1 from CFA +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + /* AArch64 initial CFI state */ + DWRF_U8(DWRF_CFA_def_cfa); // Define CFA (Call Frame Address) + DWRF_UV(DWRF_REG_SP); // CFA = SP register + DWRF_UV(0); // CFA = SP + 0 (AArch64 starts with offset 0) + // No initial register saves in AArch64 CIE +#endif + DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer boundary + ) + + /* + * Emit DWARF EH FDE (Frame Description Entry) + * + * The FDE describes unwinding information specific to this function. + * It references the CIE and provides function-specific CFI instructions. + * + * The PC-relative offset is calculated after the entire EH frame is built + * to ensure accurate positioning relative to the synthesized DSO layout. + */ + DWRF_SECTION(FDE, + DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (backwards reference) + /* + * In perf jitdump mode the FDE PC field is encoded PC-relative and + * points back to code_start. Record where that field lives so we can + * patch in the final offset after the rest of the synthetic DSO + * layout is known. + */ + ctx->fde_p = p; // Remember where PC offset field is located for later calculation + DWRF_U32(0); // Placeholder for PC-relative offset (calculated below) + DWRF_U32(ctx->code_size); // Address range covered by this FDE (code length) + DWRF_U8(0); // Augmentation data length (none) + + /* + * Architecture-specific CFI instructions + * + * These instructions describe how registers are saved and restored + * during function calls. Each architecture has different calling + * conventions and register usage patterns. + */ +#ifdef __x86_64__ + /* x86_64 calling convention unwinding rules */ +# if defined(__CET__) && (__CET__ & 1) + DWRF_U8(DWRF_CFA_advance_loc | 4); // Advance past endbr64 (4 bytes) +# endif + DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance past push %rbp (1 byte) + DWRF_U8(DWRF_CFA_def_cfa_offset); // def_cfa_offset 16 + DWRF_UV(16); // New offset: SP + 16 + DWRF_U8(DWRF_CFA_offset | DWRF_REG_BP); // offset r6 at cfa-16 + DWRF_UV(2); // Offset factor: 2 * 8 = 16 bytes + DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance past mov %rsp,%rbp (3 bytes) + DWRF_U8(DWRF_CFA_def_cfa_register); // def_cfa_register r6 + DWRF_UV(DWRF_REG_BP); // Use base pointer register + DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance past call *%rcx (2 bytes) + pop %rbp (1 byte) = 3 + DWRF_U8(DWRF_CFA_def_cfa); // def_cfa r7 ofs 8 + DWRF_UV(DWRF_REG_SP); // Use stack pointer register + DWRF_UV(8); // New offset: SP + 8 +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + /* AArch64 calling convention unwinding rules */ + DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance by 1 instruction (4 bytes) + DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 + DWRF_UV(16); // Stack pointer moved by 16 bytes + DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // x29 (frame pointer) saved + DWRF_UV(2); // At CFA-16 (2 * 8 = 16 bytes from CFA) + DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // x30 (link register) saved + DWRF_UV(1); // At CFA-8 (1 * 8 = 8 bytes from CFA) + DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance by 3 instructions (12 bytes) + DWRF_U8(DWRF_CFA_def_cfa_register); // CFA = FP (x29) + 16 + DWRF_UV(DWRF_REG_FP); + DWRF_U8(DWRF_CFA_restore | DWRF_REG_RA); // Restore x30 - NO DWRF_UV() after this! + DWRF_U8(DWRF_CFA_restore | DWRF_REG_FP); // Restore x29 - NO DWRF_UV() after this! + DWRF_U8(DWRF_CFA_def_cfa); // CFA = SP + 0 (stack restored) + DWRF_UV(DWRF_REG_SP); + DWRF_UV(0); + +#else +# error "Unsupported target architecture" +#endif + + DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer boundary + ) + + ctx->p = p; // Update context pointer to end of generated data + + /* Calculate and update the PC-relative offset in the FDE + * + * When perf processes the jitdump, it creates a synthesized DSO with this layout: + * + * Synthesized DSO Memory Layout: + * ┌─────────────────────────────────────────────────────────────┐ < code_start + * │ Code Section │ + * │ (round_up(code_size, 8) bytes) │ + * ├─────────────────────────────────────────────────────────────┤ < start of EH frame data + * │ EH Frame Data │ + * │ ┌─────────────────────────────────────────────────────┐ │ + * │ │ CIE data │ │ + * │ └─────────────────────────────────────────────────────┘ │ + * │ ┌─────────────────────────────────────────────────────┐ │ + * │ │ FDE Header: │ │ + * │ │ - CIE offset (4 bytes) │ │ + * │ │ - PC offset (4 bytes) <─ fde_offset_in_frame ─────┼────┼─> points to code_start + * │ │ - address range (4 bytes) │ │ (this specific field) + * │ │ CFI Instructions... │ │ + * │ └─────────────────────────────────────────────────────┘ │ + * ├─────────────────────────────────────────────────────────────┤ < reference_point + * │ EhFrameHeader │ + * │ (navigation metadata) │ + * └─────────────────────────────────────────────────────────────┘ + * + * The PC offset field in the FDE must contain the distance from itself to code_start: + * + * distance = code_start - fde_pc_field + * + * Where: + * fde_pc_field_location = reference_point - eh_frame_size + fde_offset_in_frame + * code_start_location = reference_point - eh_frame_size - round_up(code_size, 8) + * + * Therefore: + * distance = code_start_location - fde_pc_field_location + * = (ref - eh_frame_size - rounded_code_size) - (ref - eh_frame_size + fde_offset_in_frame) + * = -rounded_code_size - fde_offset_in_frame + * = -(round_up(code_size, 8) + fde_offset_in_frame) + * + * Note: fde_offset_in_frame is the offset from EH frame start to the PC offset field. + * + */ + int32_t rounded_code_size = + (int32_t)_Py_SIZE_ROUND_UP(ctx->code_size, 8); + int32_t fde_offset_in_frame = (int32_t)(ctx->fde_p - framep); + *(int32_t *)ctx->fde_p = -(rounded_code_size + fde_offset_in_frame); +} + +/* + * Build .eh_frame data for the GDB JIT interface. + * + * The executor runs inside the frame established by _PyJIT_Entry, but the + * synthetic executor FDE collapses that state into a single logical JIT frame + * that unwinds directly into _PyEval_*. Executor stencils never touch the + * frame pointer - enforced by Tools/jit/_optimizers.py _validate() and + * -mframe-pointer=reserved - so the steady-state rule is valid at every PC + * and the FDE body is empty. Tools/jit/_targets.py derives the initial CFI + * rules from the row active at the executor call in the compiled shim object. + */ +#if defined(PY_HAVE_JIT_GDB_UNWIND) +static void elf_init_ehframe_gdb(ELFObjectContext* ctx) { + int fde_ptr_enc = DWRF_EH_PE_absptr; + uint8_t* p = ctx->p; + uint8_t* framep = p; + + DWRF_SECTION(CIE, + DWRF_U32(0); // CIE ID + DWRF_U8(DWRF_CIE_VERSION); + DWRF_STR("zR"); // aug data length + FDE ptr encoding follow + DWRF_UV(JIT_UNWIND_CODE_ALIGNMENT_FACTOR); + DWRF_SV(JIT_UNWIND_DATA_ALIGNMENT_FACTOR); + DWRF_U8(JIT_UNWIND_RA_REG); + DWRF_UV(1); // Augmentation data length + DWRF_U8(fde_ptr_enc); // FDE pointer encoding + + /* Executor steady-state rule (our invariant, not the compiler's). */ + DWRF_U8(DWRF_CFA_def_cfa); + DWRF_UV(JIT_UNWIND_CFA_REG); + DWRF_UV(JIT_UNWIND_CFA_OFFSET); + DWRF_U8(DWRF_CFA_offset | JIT_UNWIND_FP_REG); + DWRF_UV(JIT_UNWIND_FP_OFFSET); + DWRF_U8(DWRF_CFA_offset | JIT_UNWIND_RA_REG); + DWRF_UV(JIT_UNWIND_RA_OFFSET); + DWRF_ALIGNNOP(sizeof(uintptr_t)); + ) + + DWRF_SECTION(FDE, + DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (backwards reference) + DWRF_ADDR(ctx->code_addr); // Absolute code start + DWRF_ADDR((uintptr_t)ctx->code_size); // Code range covered + DWRF_U8(0); // Augmentation data length (none) + DWRF_ALIGNNOP(sizeof(uintptr_t)); + ) + + ctx->p = p; +} +#endif + +#if defined(PY_HAVE_JIT_GDB_UNWIND) +enum { + JIT_NOACTION = 0, + JIT_REGISTER_FN = 1, + JIT_UNREGISTER_FN = 2, +}; + +struct jit_code_entry { + struct jit_code_entry *next; + struct jit_code_entry *prev; + const char *symfile_addr; + uint64_t symfile_size; + const void *code_addr; +}; + +struct jit_descriptor { + uint32_t version; + uint32_t action_flag; + struct jit_code_entry *relevant_entry; + struct jit_code_entry *first_entry; +}; + +PyMutex _Py_jit_debug_mutex = {0}; + +Py_EXPORTED_SYMBOL volatile struct jit_descriptor __jit_debug_descriptor = { + 1, JIT_NOACTION, NULL, NULL +}; + +Py_EXPORTED_SYMBOL void __attribute__((noinline)) +__jit_debug_register_code(void) +{ + /* Keep this call visible to debuggers and not optimized away. */ + (void)__jit_debug_descriptor.action_flag; +#if defined(__GNUC__) || defined(__clang__) + __asm__ __volatile__("" ::: "memory"); +#endif +} + +static uint16_t +gdb_jit_machine_id(void) +{ + /* Map the current target to ELF e_machine; return 0 to skip registration. */ +#if defined(__x86_64__) || defined(_M_X64) + return EM_X86_64; +#elif defined(__aarch64__) && !defined(__ILP32__) + return EM_AARCH64; +#else + return 0; +#endif +} + +static struct jit_code_entry * +gdb_jit_register_code( + const void *code_addr, + size_t code_size, + const char *symname, + const uint8_t *eh_frame, + size_t eh_frame_size +) +{ + /* + * Build a minimal in-memory ELF for GDB's JIT interface and link it into + * __jit_debug_descriptor so debuggers can resolve JIT code. + */ + if (code_addr == NULL || code_size == 0 || symname == NULL) { + return NULL; + } + + const uint16_t machine = gdb_jit_machine_id(); + if (machine == 0) { + return NULL; + } + + enum { + SH_NULL = 0, + SH_TEXT, + SH_EH_FRAME, + SH_SHSTRTAB, + SH_STRTAB, + SH_SYMTAB, + SH_NUM, + }; + static const char shstrtab[] = + "\0.text\0.eh_frame\0.shstrtab\0.strtab\0.symtab"; + _Static_assert(sizeof(shstrtab) == + 1 + sizeof(".text") + sizeof(".eh_frame") + + sizeof(".shstrtab") + sizeof(".strtab") + sizeof(".symtab"), + "shstrtab size mismatch"); + const size_t shstrtab_size = sizeof(shstrtab); + const size_t sh_text = 1; + const size_t sh_eh_frame = sh_text + sizeof(".text"); + const size_t sh_shstrtab = sh_eh_frame + sizeof(".eh_frame"); + const size_t sh_strtab = sh_shstrtab + sizeof(".shstrtab"); + const size_t sh_symtab = sh_strtab + sizeof(".strtab"); + const size_t text_size = code_size; + const size_t text_padded = _Py_SIZE_ROUND_UP(text_size, 8); + const size_t strtab_size = 1 + strlen(symname) + 1; + const size_t symtab_size = 3 * sizeof(Elf64_Sym); + + size_t offset = sizeof(Elf64_Ehdr); + offset = _Py_SIZE_ROUND_UP(offset, 16); + const size_t text_off = offset; + const size_t eh_off = text_off + text_padded; + offset = eh_off + eh_frame_size; + const size_t shstr_off = offset; + offset += shstrtab_size; + const size_t str_off = offset; + offset += strtab_size; + /* Elf64_Sym requires 8-byte alignment for st_value/st_size. */ + offset = _Py_SIZE_ROUND_UP(offset, 8); + const size_t sym_off = offset; + offset += symtab_size; + offset = _Py_SIZE_ROUND_UP(offset, sizeof(Elf64_Shdr)); + const size_t sh_off = offset; + + const size_t shnum = SH_NUM; + const size_t total_size = sh_off + shnum * sizeof(Elf64_Shdr); + uint8_t *buf = (uint8_t *)PyMem_RawMalloc(total_size); + if (buf == NULL) { + return NULL; + } + memset(buf, 0, total_size); + + Elf64_Ehdr *ehdr = (Elf64_Ehdr *)buf; + memcpy(ehdr->e_ident, ELFMAG, SELFMAG); + ehdr->e_ident[EI_CLASS] = ELFCLASS64; + ehdr->e_ident[EI_DATA] = ELFDATA2LSB; + ehdr->e_ident[EI_VERSION] = EV_CURRENT; + ehdr->e_ident[EI_OSABI] = ELFOSABI_NONE; + ehdr->e_type = ET_DYN; + ehdr->e_machine = machine; + ehdr->e_version = EV_CURRENT; + ehdr->e_entry = 0; + ehdr->e_phoff = 0; + ehdr->e_shoff = sh_off; + ehdr->e_ehsize = sizeof(Elf64_Ehdr); + ehdr->e_shentsize = sizeof(Elf64_Shdr); + ehdr->e_shnum = shnum; + ehdr->e_shstrndx = SH_SHSTRTAB; + + memcpy(buf + text_off, code_addr, text_size); + memcpy(buf + eh_off, eh_frame, eh_frame_size); + + char *shstr = (char *)(buf + shstr_off); + memcpy(shstr, shstrtab, shstrtab_size); + + char *strtab = (char *)(buf + str_off); + strtab[0] = '\0'; + memcpy(strtab + 1, symname, strlen(symname)); + strtab[strtab_size - 1] = '\0'; + + Elf64_Sym *syms = (Elf64_Sym *)(buf + sym_off); + memset(syms, 0, symtab_size); + /* Section symbol for .text (local) */ + syms[1].st_info = ELF64_ST_INFO(STB_LOCAL, STT_SECTION); + syms[1].st_shndx = 1; + /* Function symbol */ + syms[2].st_name = 1; + syms[2].st_info = ELF64_ST_INFO(STB_GLOBAL, STT_FUNC); + syms[2].st_other = STV_DEFAULT; + syms[2].st_shndx = 1; + /* For ET_DYN/ET_EXEC, st_value is the absolute virtual address. */ + syms[2].st_value = (Elf64_Addr)(uintptr_t)code_addr; + syms[2].st_size = code_size; + + Elf64_Shdr *shdrs = (Elf64_Shdr *)(buf + sh_off); + memset(shdrs, 0, shnum * sizeof(Elf64_Shdr)); + + shdrs[SH_TEXT].sh_name = sh_text; + shdrs[SH_TEXT].sh_type = SHT_PROGBITS; + shdrs[SH_TEXT].sh_flags = SHF_ALLOC | SHF_EXECINSTR; + shdrs[SH_TEXT].sh_addr = (Elf64_Addr)(uintptr_t)code_addr; + shdrs[SH_TEXT].sh_offset = text_off; + shdrs[SH_TEXT].sh_size = text_size; + shdrs[SH_TEXT].sh_addralign = 16; + + shdrs[SH_EH_FRAME].sh_name = sh_eh_frame; + shdrs[SH_EH_FRAME].sh_type = SHT_PROGBITS; + shdrs[SH_EH_FRAME].sh_flags = SHF_ALLOC; + shdrs[SH_EH_FRAME].sh_addr = + (Elf64_Addr)((uintptr_t)code_addr + text_padded); + shdrs[SH_EH_FRAME].sh_offset = eh_off; + shdrs[SH_EH_FRAME].sh_size = eh_frame_size; + shdrs[SH_EH_FRAME].sh_addralign = 8; + + shdrs[SH_SHSTRTAB].sh_name = sh_shstrtab; + shdrs[SH_SHSTRTAB].sh_type = SHT_STRTAB; + shdrs[SH_SHSTRTAB].sh_offset = shstr_off; + shdrs[SH_SHSTRTAB].sh_size = shstrtab_size; + shdrs[SH_SHSTRTAB].sh_addralign = 1; + + shdrs[SH_STRTAB].sh_name = sh_strtab; + shdrs[SH_STRTAB].sh_type = SHT_STRTAB; + shdrs[SH_STRTAB].sh_offset = str_off; + shdrs[SH_STRTAB].sh_size = strtab_size; + shdrs[SH_STRTAB].sh_addralign = 1; + + shdrs[SH_SYMTAB].sh_name = sh_symtab; + shdrs[SH_SYMTAB].sh_type = SHT_SYMTAB; + shdrs[SH_SYMTAB].sh_offset = sym_off; + shdrs[SH_SYMTAB].sh_size = symtab_size; + shdrs[SH_SYMTAB].sh_link = SH_STRTAB; + shdrs[SH_SYMTAB].sh_info = 2; + shdrs[SH_SYMTAB].sh_addralign = 8; + shdrs[SH_SYMTAB].sh_entsize = sizeof(Elf64_Sym); + + struct jit_code_entry *entry = PyMem_RawMalloc(sizeof(*entry)); + if (entry == NULL) { + PyMem_RawFree(buf); + return NULL; + } + entry->symfile_addr = (const char *)buf; + entry->symfile_size = total_size; + entry->code_addr = code_addr; + + PyMutex_Lock(&_Py_jit_debug_mutex); + entry->prev = NULL; + entry->next = __jit_debug_descriptor.first_entry; + if (entry->next != NULL) { + entry->next->prev = entry; + } + __jit_debug_descriptor.first_entry = entry; + __jit_debug_descriptor.relevant_entry = entry; + __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; + __jit_debug_register_code(); + __jit_debug_descriptor.action_flag = JIT_NOACTION; + __jit_debug_descriptor.relevant_entry = NULL; + PyMutex_Unlock(&_Py_jit_debug_mutex); + return entry; +} +#endif // defined(PY_HAVE_JIT_GDB_UNWIND) + +void * +_PyJitUnwind_GdbRegisterCode(const void *code_addr, + size_t code_size, + const char *entry, + const char *filename) +{ +#if defined(PY_HAVE_JIT_GDB_UNWIND) + /* GDB expects a stable symbol name and absolute addresses in .eh_frame. */ + if (entry == NULL) { + entry = ""; + } + if (filename == NULL) { + filename = ""; + } + size_t name_size = snprintf(NULL, 0, "py::%s:%s", entry, filename) + 1; + char *name = (char *)PyMem_RawMalloc(name_size); + if (name == NULL) { + return NULL; + } + snprintf(name, name_size, "py::%s:%s", entry, filename); + + uint8_t buffer[1024]; + size_t eh_frame_size = _PyJitUnwind_BuildEhFrame( + buffer, sizeof(buffer), code_addr, code_size, 1); + if (eh_frame_size == 0) { + PyMem_RawFree(name); + return NULL; + } + + void *handle = gdb_jit_register_code(code_addr, code_size, name, + buffer, eh_frame_size); + PyMem_RawFree(name); + return handle; +#else + (void)code_addr; + (void)code_size; + (void)entry; + (void)filename; + return NULL; +#endif +} + +void +_PyJitUnwind_GdbUnregisterCode(void *handle) +{ +#if defined(PY_HAVE_JIT_GDB_UNWIND) + struct jit_code_entry *entry = (struct jit_code_entry *)handle; + if (entry == NULL) { + return; + } + + PyMutex_Lock(&_Py_jit_debug_mutex); + if (entry->prev != NULL) { + entry->prev->next = entry->next; + } + else { + __jit_debug_descriptor.first_entry = entry->next; + } + if (entry->next != NULL) { + entry->next->prev = entry->prev; + } + + __jit_debug_descriptor.relevant_entry = entry; + __jit_debug_descriptor.action_flag = JIT_UNREGISTER_FN; + __jit_debug_register_code(); + __jit_debug_descriptor.action_flag = JIT_NOACTION; + __jit_debug_descriptor.relevant_entry = NULL; + + PyMutex_Unlock(&_Py_jit_debug_mutex); + + PyMem_RawFree((void *)entry->symfile_addr); + PyMem_RawFree(entry); +#else + (void)handle; +#endif +} + +#endif // defined(PY_HAVE_PERF_TRAMPOLINE) || defined(PY_HAVE_JIT_GDB_UNWIND) diff --git a/Python/lock.c b/Python/lock.c index 752a5899e08..af136fefd29 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -248,7 +248,16 @@ _PyRawMutex_LockSlow(_PyRawMutex *m) // Wait for us to be woken up. Note that we still have to lock the // mutex ourselves: it is NOT handed off to us. - _PySemaphore_Wait(&waiter.sema, -1); + // + // Loop until we observe an actual wakeup. A return of Py_PARK_INTR + // could otherwise let us exit _PySemaphore_Wait and destroy + // `waiter.sema` while _PyRawMutex_UnlockSlow's matching + // _PySemaphore_Wakeup is still pending, since the unlocker has + // already CAS-removed us from the waiter list without any handshake. + int res; + do { + res = _PySemaphore_Wait(&waiter.sema, -1); + } while (res != Py_PARK_OK); } _PySemaphore_Destroy(&waiter.sema); diff --git a/Python/marshal.c b/Python/marshal.c index b60a36e128c..dace22da0d4 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -382,7 +382,6 @@ static int w_ref(PyObject *v, char *flag, WFILE *p) { _Py_hashtable_entry_t *entry; - int w; if (p->version < 3 || p->hashtable == NULL) return 0; /* not writing object references */ @@ -399,20 +398,28 @@ w_ref(PyObject *v, char *flag, WFILE *p) entry = _Py_hashtable_get_entry(p->hashtable, v); if (entry != NULL) { /* write the reference index to the stream */ - w = (int)(uintptr_t)entry->value; + uintptr_t w = (uintptr_t)entry->value; + if (w & 0x80000000LU) { + PyErr_Format(PyExc_ValueError, "cannot marshal recursion %T objects", v); + goto err; + } /* we don't store "long" indices in the dict */ - assert(0 <= w && w <= 0x7fffffff); + assert(w <= 0x7fffffff); w_byte(TYPE_REF, p); - w_long(w, p); + w_long((int)w, p); return 1; } else { - size_t s = p->hashtable->nentries; + size_t w = p->hashtable->nentries; /* we don't support long indices */ - if (s >= 0x7fffffff) { + if (w >= 0x7fffffff) { PyErr_SetString(PyExc_ValueError, "too many objects"); goto err; } - w = (int)s; + // Corresponding code should call w_complete() after + // writing the object. + if (PyCode_Check(v) || PySlice_Check(v) || PyFrozenDict_CheckExact(v)) { + w |= 0x80000000LU; + } if (_Py_hashtable_set(p->hashtable, Py_NewRef(v), (void *)(uintptr_t)w) < 0) { Py_DECREF(v); @@ -426,6 +433,27 @@ w_ref(PyObject *v, char *flag, WFILE *p) return 1; } +static void +w_complete(PyObject *v, WFILE *p) +{ + if (p->version < 3 || p->hashtable == NULL) { + return; + } + if (_PyObject_IsUniquelyReferenced(v)) { + return; + } + + _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(p->hashtable, v); + if (entry == NULL) { + return; + } + assert(entry != NULL); + uintptr_t w = (uintptr_t)entry->value; + assert(w & 0x80000000LU); + w &= ~0x80000000LU; + entry->value = (void *)(uintptr_t)w; +} + static void w_complex_object(PyObject *v, char flag, WFILE *p); @@ -599,6 +627,9 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(value, p); } w_object((PyObject *)NULL, p); + if (PyFrozenDict_CheckExact(v)) { + w_complete(v, p); + } } else if (PyAnySet_CheckExact(v)) { PyObject *value; @@ -684,6 +715,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(co->co_linetable, p); w_object(co->co_exceptiontable, p); Py_DECREF(co_code); + w_complete(v, p); } else if (PyObject_CheckBuffer(v)) { /* Write unknown bytes-like objects as a bytes object */ @@ -709,6 +741,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(slice->start, p); w_object(slice->stop, p); w_object(slice->step, p); + w_complete(v, p); } else { W_TYPE(TYPE_UNKNOWN, p); @@ -1433,9 +1466,19 @@ r_object(RFILE *p) case TYPE_DICT: case TYPE_FROZENDICT: v = PyDict_New(); - R_REF(v); - if (v == NULL) + if (v == NULL) { break; + } + if (type == TYPE_DICT) { + R_REF(v); + } + else { + idx = r_ref_reserve(flag, p); + if (idx < 0) { + Py_CLEAR(v); + break; + } + } for (;;) { PyObject *key, *val; key = r_object(p); @@ -1458,13 +1501,7 @@ r_object(RFILE *p) Py_CLEAR(v); } if (type == TYPE_FROZENDICT && v != NULL) { - PyObject *frozendict = PyFrozenDict_New(v); - if (frozendict != NULL) { - Py_SETREF(v, frozendict); - } - else { - Py_CLEAR(v); - } + Py_SETREF(v, PyFrozenDict_New(v)); } retval = v; break; diff --git a/Python/optimizer.c b/Python/optimizer.c index 60f3e541be2..11658fca0da 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -551,8 +551,6 @@ dynamic_exit_uop[MAX_UOP_ID + 1] = { }; -#define CONFIDENCE_RANGE 1000 -#define CONFIDENCE_CUTOFF 333 #ifdef Py_DEBUG #define DPRINTF(level, ...) \ @@ -564,12 +562,13 @@ dynamic_exit_uop[MAX_UOP_ID + 1] = { static inline void add_to_trace( - _PyJitUopBuffer *trace, + _PyJitTracerState *tracer, uint16_t opcode, uint16_t oparg, uint64_t operand, uint32_t target) { + _PyJitUopBuffer *trace = &tracer->code_buffer; _PyUOpInstruction *inst = trace->next; inst->opcode = opcode; inst->format = UOP_FORMAT_TARGET; @@ -578,6 +577,7 @@ add_to_trace( inst->operand0 = operand; #ifdef Py_STATS inst->execution_count = 0; + inst->fitness = tracer->translator_state.fitness; #endif trace->next++; } @@ -585,7 +585,7 @@ add_to_trace( #ifdef Py_DEBUG #define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \ - add_to_trace(trace, (OPCODE), (OPARG), (OPERAND), (TARGET)); \ + add_to_trace(tracer, (OPCODE), (OPARG), (OPERAND), (TARGET)); \ if (lltrace >= 2) { \ printf("%4d ADD_TO_TRACE: ", uop_buffer_length(trace)); \ _PyUOpPrint(uop_buffer_last(trace)); \ @@ -593,13 +593,61 @@ add_to_trace( } #else #define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \ - add_to_trace(trace, (OPCODE), (OPARG), (OPERAND), (TARGET)) + add_to_trace(tracer, (OPCODE), (OPARG), (OPERAND), (TARGET)) #endif #define INSTR_IP(INSTR, CODE) \ ((uint32_t)((INSTR) - ((_Py_CODEUNIT *)(CODE)->co_code_adaptive))) +/* Branch penalty: 0 for a fully biased branch and FITNESS_BRANCH_BALANCED for + * a balanced or fully off-trace branch. This keeps any single branch from + * consuming more than one balanced-branch cost. + */ +static inline int +compute_branch_penalty(uint16_t history) +{ + bool branch_taken = history & 1; + int taken_count = _Py_popcount32((uint32_t)history); + int on_trace_count = branch_taken ? taken_count : 16 - taken_count; + int off_trace = 16 - on_trace_count; + int penalty = off_trace * FITNESS_BRANCH_BALANCED / 8; + if (penalty > FITNESS_BRANCH_BALANCED) { + penalty = FITNESS_BRANCH_BALANCED; + } + return penalty; +} + +/* Compute exit quality for the current trace position. + * Higher values mean better places to stop the trace. */ +static inline int32_t +compute_exit_quality(_Py_CODEUNIT *target_instr, int opcode, + const _PyJitTracerState *tracer) +{ + if (target_instr == tracer->initial_state.close_loop_instr) { + return EXIT_QUALITY_CLOSE_LOOP; + } + else if (target_instr->op.code == ENTER_EXECUTOR) { + return EXIT_QUALITY_ENTER_EXECUTOR; + } + else if (opcode == JUMP_BACKWARD_JIT || + opcode == JUMP_BACKWARD || + opcode == JUMP_BACKWARD_NO_INTERRUPT) { + return EXIT_QUALITY_BACKWARD_EDGE; + } + else if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]] > 0) { + return EXIT_QUALITY_SPECIALIZABLE; + } + return EXIT_QUALITY_DEFAULT; +} + +/* Frame penalty: (MAX_ABSTRACT_FRAME_DEPTH-1) pushes exhaust fitness. */ +static inline int32_t +compute_frame_penalty(uint16_t fitness_initial) +{ + return (int32_t)fitness_initial / (MAX_ABSTRACT_FRAME_DEPTH - 1) + 1; +} + static int is_terminator(const _PyUOpInstruction *uop) { @@ -612,6 +660,44 @@ is_terminator(const _PyUOpInstruction *uop) ); } +static PyObject * +record_trace_transform_to_type(PyObject *value) +{ + PyObject *tp = Py_NewRef((PyObject *)Py_TYPE(value)); + Py_DECREF(value); + return tp; +} + +/* _RECORD_NOS_GEN_FUNC and _RECORD_3OS_GEN_FUNC record the raw receiver. + * If it is a generator, return its function object; otherwise return NULL. + */ +static PyObject * +record_trace_transform_gen_func(PyObject *value) +{ + PyObject *func = NULL; + if (PyGen_Check(value)) { + _PyStackRef f = ((PyGenObject *)value)->gi_iframe.f_funcobj; + if (!PyStackRef_IsNull(f)) { + func = Py_NewRef(PyStackRef_AsPyObjectBorrow(f)); + } + } + Py_DECREF(value); + return func; +} + +/* _RECORD_BOUND_METHOD records the raw callable. + * Keep it only for bound methods; otherwise return NULL. + */ +static PyObject * +record_trace_transform_bound_method(PyObject *value) +{ + if (Py_TYPE(value) == &PyMethod_Type) { + return value; + } + Py_DECREF(value); + return NULL; +} + /* Returns 1 on success (added to trace), 0 on trace end. */ // gh-142543: inlining this function causes stack overflows @@ -736,13 +822,11 @@ _PyJit_translate_single_bytecode_to_trace( DPRINTF(2, "Unsupported: oparg too large\n"); unsupported: { - // Rewind to previous instruction and replace with _EXIT_TRACE. _PyUOpInstruction *curr = uop_buffer_last(trace); while (curr->opcode != _SET_IP && uop_buffer_length(trace) > 2) { trace->next--; curr = uop_buffer_last(trace); } - assert(curr->opcode == _SET_IP || uop_buffer_length(trace) == 2); if (curr->opcode == _SET_IP) { int32_t old_target = (int32_t)uop_get_target(curr); curr->opcode = _DEOPT; @@ -765,10 +849,29 @@ _PyJit_translate_single_bytecode_to_trace( return 1; } + // Stop the trace if fitness has dropped below the exit quality threshold. + _PyJitTracerTranslatorState *ts = &tracer->translator_state; + int32_t eq = compute_exit_quality(target_instr, opcode, tracer); + DPRINTF(3, "Fitness check: %s(%d) fitness=%d, exit_quality=%d, depth=%d\n", + _PyOpcode_OpName[opcode], oparg, ts->fitness, eq, ts->frame_depth); + + if (ts->fitness < eq) { + // Heuristic exit: leave operand1=0 so the side exit increments chain_depth. + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + OPT_STAT_INC(fitness_terminated_traces); + DPRINTF(2, "Fitness terminated: %s(%d) fitness=%d < exit_quality=%d\n", + _PyOpcode_OpName[opcode], oparg, ts->fitness, eq); + goto done; + } + + // Snapshot remaining space so the later fitness charge reflects all buffer + // space this bytecode consumed, including reserved tail slots. + int32_t remaining_before = uop_buffer_remaining_space(trace); + // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT trace->end -= 2; - const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; + const _PyOpcodeRecordSlotMap *record_slot_map = &_PyOpcode_RecordSlotMaps[opcode]; assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); assert(!_PyErr_Occurred(tstate)); @@ -790,13 +893,11 @@ _PyJit_translate_single_bytecode_to_trace( // _GUARD_IP leads to an exit. trace->end -= needs_guard_ip; +#if Py_DEBUG + const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; int space_needed = expansion->nuops + needs_guard_ip + 2 + (!OPCODE_HAS_NO_SAVE_IP(opcode)); - if (uop_buffer_remaining_space(trace) < space_needed) { - DPRINTF(2, "No room for expansions and guards (need %d, got %d)\n", - space_needed, uop_buffer_remaining_space(trace)); - OPT_STAT_INC(trace_too_long); - goto done; - } + assert(uop_buffer_remaining_space(trace) > space_needed); +#endif ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); @@ -818,6 +919,12 @@ _PyJit_translate_single_bytecode_to_trace( assert(jump_happened ? (next_instr == computed_jump_instr) : (next_instr == computed_next_instr)); uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_happened]; ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(jump_happened ? computed_next_instr : computed_jump_instr, old_code)); + int bp = compute_branch_penalty(target_instr[1].cache); + tracer->translator_state.fitness -= bp; + DPRINTF(3, " branch penalty: -%d (history=0x%04x, taken=%d) -> fitness=%d\n", + bp, target_instr[1].cache, jump_happened, + tracer->translator_state.fitness); + break; } case JUMP_BACKWARD_JIT: @@ -825,29 +932,9 @@ _PyJit_translate_single_bytecode_to_trace( case JUMP_BACKWARD_NO_JIT: case JUMP_BACKWARD: ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); - _Py_FALLTHROUGH; - case JUMP_BACKWARD_NO_INTERRUPT: - { - if ((next_instr != tracer->initial_state.close_loop_instr) && - (next_instr != tracer->initial_state.start_instr) && - uop_buffer_length(&tracer->code_buffer) > CODE_SIZE_NO_PROGRESS && - // For side exits, we don't want to terminate them early. - tracer->initial_state.exit == NULL && - // These are coroutines, and we want to unroll those usually. - opcode != JUMP_BACKWARD_NO_INTERRUPT) { - // We encountered a JUMP_BACKWARD but not to the top of our own loop. - // We don't want to continue tracing as we might get stuck in the - // inner loop. Instead, end the trace where the executor of the - // inner loop might start and let the traces rejoin. - OPT_STAT_INC(inner_loop); - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); - uop_buffer_last(trace)->operand1 = true; // is_control_flow - DPRINTF(2, "JUMP_BACKWARD not to top ends trace %p %p %p\n", next_instr, - tracer->initial_state.close_loop_instr, tracer->initial_state.start_instr); - goto done; - } break; - } + case JUMP_BACKWARD_NO_INTERRUPT: + break; case RESUME: case RESUME_CHECK: @@ -948,9 +1035,49 @@ _PyJit_translate_single_bytecode_to_trace( assert(next->op.code == STORE_FAST); operand = next->op.arg; } + else if (uop == _PUSH_FRAME) { + _PyJitTracerTranslatorState *ts_depth = &tracer->translator_state; + ts_depth->frame_depth++; + assert(ts_depth->frame_depth < MAX_ABSTRACT_FRAME_DEPTH); + int32_t frame_penalty = compute_frame_penalty(tstate->interp->opt_config.fitness_initial); + ts_depth->fitness -= frame_penalty; + DPRINTF(3, " _PUSH_FRAME: depth=%d, penalty=-%d -> fitness=%d\n", + ts_depth->frame_depth, frame_penalty, + ts_depth->fitness); + } + else if (uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { + _PyJitTracerTranslatorState *ts_depth = &tracer->translator_state; + int32_t frame_penalty = compute_frame_penalty(tstate->interp->opt_config.fitness_initial); + if (ts_depth->frame_depth <= 0) { + // Returning past the traced root is normal for guarded + // caller continuation. Charge a small penalty so these + // paths still terminate. + int32_t underflow_penalty = frame_penalty / 4; + ts_depth->fitness -= underflow_penalty; + DPRINTF(3, " %s: underflow penalty=-%d -> fitness=%d\n", + _PyOpcode_uop_name[uop], underflow_penalty, + ts_depth->fitness); + } + else { + // Symmetric with push: net-zero frame impact. + ts_depth->fitness += frame_penalty; + ts_depth->frame_depth--; + DPRINTF(3, " %s: return reward=+%d, depth=%d -> fitness=%d\n", + _PyOpcode_uop_name[uop], frame_penalty, + ts_depth->frame_depth, + ts_depth->fitness); + } + } else if (_PyUop_Flags[uop] & HAS_RECORDS_VALUE_FLAG) { - PyObject *recorded_value = tracer->prev_state.recorded_values[record_idx]; - tracer->prev_state.recorded_values[record_idx] = NULL; + assert(record_idx < record_slot_map->count); + uint8_t record_slot = record_slot_map->slots[record_idx]; + assert(record_slot < tracer->prev_state.recorded_count); + PyObject *recorded_value = tracer->prev_state.recorded_values[record_slot]; + tracer->prev_state.recorded_values[record_slot] = NULL; + if ((record_slot_map->transform_mask & (1u << record_idx)) && + recorded_value != NULL) { + recorded_value = _PyOpcode_RecordTransformValue(uop, recorded_value); + } record_idx++; operand = (uintptr_t)recorded_value; } @@ -990,13 +1117,20 @@ _PyJit_translate_single_bytecode_to_trace( ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); goto done; } - DPRINTF(2, "Trace continuing\n"); + // Charge fitness by trace-buffer capacity consumed for this bytecode, + // including both emitted uops and tail reservations. + { + int32_t slots_used = remaining_before - uop_buffer_remaining_space(trace); + tracer->translator_state.fitness -= slots_used; + DPRINTF(3, " per-insn cost: -%d -> fitness=%d\n", slots_used, + tracer->translator_state.fitness); + } + DPRINTF(2, "Trace continuing (fitness=%d)\n", tracer->translator_state.fitness); return 1; done: DPRINTF(2, "Trace done\n"); if (!is_terminator(uop_buffer_last(trace))) { ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); - uop_buffer_last(trace)->operand1 = true; // is_control_flow } return 0; } @@ -1048,6 +1182,9 @@ _PyJit_TryInitializeTracing( /* Set up tracing buffer*/ _PyJitUopBuffer *trace = &tracer->code_buffer; uop_buffer_init(trace, &tracer->uop_array[0], UOP_MAX_TRACE_LENGTH); + _PyJitTracerTranslatorState *ts = &tracer->translator_state; + ts->fitness = tstate->interp->opt_config.fitness_initial; + ts->frame_depth = 0; ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)start_instr, INSTR_IP(start_instr, code)); ADD_TO_TRACE(_MAKE_WARM, 0, 0, 0); @@ -1077,6 +1214,9 @@ _PyJit_TryInitializeTracing( assert(curr_instr->op.code == JUMP_BACKWARD_JIT || curr_instr->op.code == RESUME_CHECK_JIT || (exit != NULL)); tracer->initial_state.jump_backward_instr = curr_instr; + DPRINTF(3, "Fitness init: chain_depth=%d, fitness=%d\n", + chain_depth, ts->fitness); + tracer->is_tracing = true; return 1; } @@ -1208,6 +1348,7 @@ static void make_exit(_PyUOpInstruction *inst, int opcode, int target, bool is_c inst->target = target; inst->operand1 = is_control_flow; #ifdef Py_STATS + inst->fitness = 0; inst->execution_count = 0; #endif } @@ -1307,6 +1448,7 @@ allocate_executor(int exit_count, int length) res->trace = (_PyUOpInstruction *)(res->exits + exit_count); res->code_size = length; res->exit_count = exit_count; + res->jit_gdb_handle = NULL; return res; } @@ -2008,8 +2150,8 @@ write_row_for_uop(_PyExecutorObject *executor, uint32_t i, FILE *out) #ifdef Py_STATS const char *bg_color = get_background_color(inst, executor->trace[0].execution_count); const char *color = get_foreground_color(inst, executor->trace[0].execution_count); - fprintf(out, " %s  --  %" PRIu64 "\n", - i, color, bg_color, color, opname, inst->execution_count); + fprintf(out, " %s [%d] --  %" PRIu64 "\n", + i, color, bg_color, color, opname, inst->fitness, inst->execution_count); #else const char *color = (_PyUop_Uncached[inst->opcode] == _DEOPT) ? RED : BLACK; fprintf(out, " %s op0=%" PRIu64 "\n", i, color, opname, inst->operand0); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 095bcfc639b..9f6ce206ef4 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -235,6 +235,9 @@ add_op(JitOptContext *ctx, _PyUOpInstruction *this_instr, out->target = this_instr->target; out->operand0 = (operand0); out->operand1 = this_instr->operand1; +#ifdef Py_STATS + out->fitness = this_instr->fitness; +#endif ctx->out_buffer.next++; } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 6e4882143fb..33b5257fd58 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -293,6 +293,7 @@ dummy_func(void) { || oparg == NB_INPLACE_TRUE_DIVIDE); bool is_remainder = (oparg == NB_REMAINDER || oparg == NB_INPLACE_REMAINDER); + int emit_op = _BINARY_OP; // Promote probable-float operands to known floats via speculative // guards. _RECORD_TOS_TYPE / _RECORD_NOS_TYPE in the BINARY_OP macro // record the observed operand type during tracing, which @@ -300,7 +301,7 @@ dummy_func(void) { // narrowing unlocks a meaningful downstream win: // - NB_TRUE_DIVIDE: enables the specialized float path below. // - NB_REMAINDER: lets the float result type propagate. - // NB_POWER is excluded — speculative guards there regressed + // NB_POWER is excluded: speculative guards there regressed // test_power_type_depends_on_input_values (GH-127844). if (is_truediv || is_remainder) { if (!sym_has_type(rhs) @@ -318,17 +319,17 @@ dummy_func(void) { } if (is_truediv && lhs_float && rhs_float) { if (PyJitRef_IsUnique(lhs)) { - ADD_OP(_BINARY_OP_TRUEDIV_FLOAT_INPLACE, 0, 0); + emit_op = _BINARY_OP_TRUEDIV_FLOAT_INPLACE; l = sym_new_null(ctx); r = rhs; } else if (PyJitRef_IsUnique(rhs)) { - ADD_OP(_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT, 0, 0); + emit_op = _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT; l = lhs; r = sym_new_null(ctx); } else { - ADD_OP(_BINARY_OP_TRUEDIV_FLOAT, 0, 0); + emit_op = _BINARY_OP_TRUEDIV_FLOAT; l = lhs; r = rhs; } @@ -382,6 +383,7 @@ dummy_func(void) { else { res = PyJitRef_MakeUnique(sym_new_type(ctx, &PyFloat_Type)); } + ADD_OP(emit_op, oparg, 0); } op(_BINARY_OP_ADD_INT, (left, right -- res, l, r)) { @@ -934,6 +936,13 @@ dummy_func(void) { assert(oparg >= 2); } + op(_RROT_3, (bottom, middle, top -- bottom, middle, top)) { + JitOptRef temp = top; + top = middle; + middle = bottom; + bottom = temp; + } + op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr, o)) { attr = sym_new_not_null(ctx); (void)offset; @@ -2330,7 +2339,10 @@ dummy_func(void) { goto error; } if (_Py_IsImmortal(temp)) { - ADD_OP(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + ADD_OP(_SWAP, 2, 0); + optimize_pop_top(ctx, this_instr, null); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + ADD_OP(_SWAP, 3, 0); } res = sym_new_const(ctx, temp); Py_DECREF(temp); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index c3c889e9de9..8f208beb864 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -638,10 +638,9 @@ if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { - // Replace with _LOAD_CONST_INLINE_BORROW + _SWAP + _SWAP since we have two inputs and an immortal result + // Replace with _LOAD_CONST_INLINE_BORROW + _RROT_3 since we have two inputs and an immortal result ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - ADD_OP(_SWAP, 3, 0); - ADD_OP(_SWAP, 2, 0); + ADD_OP(_RROT_3, 0, 0); } } CHECK_STACK_BOUNDS(1); @@ -710,10 +709,9 @@ if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { - // Replace with _LOAD_CONST_INLINE_BORROW + _SWAP + _SWAP since we have two inputs and an immortal result + // Replace with _LOAD_CONST_INLINE_BORROW + _RROT_3 since we have two inputs and an immortal result ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - ADD_OP(_SWAP, 3, 0); - ADD_OP(_SWAP, 2, 0); + ADD_OP(_RROT_3, 0, 0); } } CHECK_STACK_BOUNDS(1); @@ -782,10 +780,9 @@ if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { - // Replace with _LOAD_CONST_INLINE_BORROW + _SWAP + _SWAP since we have two inputs and an immortal result + // Replace with _LOAD_CONST_INLINE_BORROW + _RROT_3 since we have two inputs and an immortal result ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - ADD_OP(_SWAP, 3, 0); - ADD_OP(_SWAP, 2, 0); + ADD_OP(_RROT_3, 0, 0); } } CHECK_STACK_BOUNDS(1); @@ -1605,10 +1602,9 @@ if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { - // Replace with _LOAD_CONST_INLINE_BORROW + _SWAP + _SWAP since we have two inputs and an immortal result + // Replace with _LOAD_CONST_INLINE_BORROW + _RROT_3 since we have two inputs and an immortal result ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - ADD_OP(_SWAP, 3, 0); - ADD_OP(_SWAP, 2, 0); + ADD_OP(_RROT_3, 0, 0); } } CHECK_STACK_BOUNDS(1); @@ -2935,10 +2931,9 @@ if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { - // Replace with _LOAD_CONST_INLINE_BORROW + _SWAP + _SWAP since we have two inputs and an immortal result + // Replace with _LOAD_CONST_INLINE_BORROW + _RROT_3 since we have two inputs and an immortal result ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - ADD_OP(_SWAP, 3, 0); - ADD_OP(_SWAP, 2, 0); + ADD_OP(_RROT_3, 0, 0); } } CHECK_STACK_BOUNDS(1); @@ -3010,10 +3005,9 @@ if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { - // Replace with _LOAD_CONST_INLINE_BORROW + _SWAP + _SWAP since we have two inputs and an immortal result + // Replace with _LOAD_CONST_INLINE_BORROW + _RROT_3 since we have two inputs and an immortal result ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - ADD_OP(_SWAP, 3, 0); - ADD_OP(_SWAP, 2, 0); + ADD_OP(_RROT_3, 0, 0); } } CHECK_STACK_BOUNDS(1); @@ -3074,10 +3068,9 @@ if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { - // Replace with _LOAD_CONST_INLINE_BORROW + _SWAP + _SWAP since we have two inputs and an immortal result + // Replace with _LOAD_CONST_INLINE_BORROW + _RROT_3 since we have two inputs and an immortal result ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - ADD_OP(_SWAP, 3, 0); - ADD_OP(_SWAP, 2, 0); + ADD_OP(_RROT_3, 0, 0); } } CHECK_STACK_BOUNDS(1); @@ -3156,10 +3149,9 @@ if (sym_is_const(ctx, b)) { PyObject *result = sym_get_const(ctx, b); if (_Py_IsImmortal(result)) { - // Replace with _LOAD_CONST_INLINE_BORROW + _SWAP + _SWAP since we have two inputs and an immortal result + // Replace with _LOAD_CONST_INLINE_BORROW + _RROT_3 since we have two inputs and an immortal result ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - ADD_OP(_SWAP, 3, 0); - ADD_OP(_SWAP, 2, 0); + ADD_OP(_RROT_3, 0, 0); } } CHECK_STACK_BOUNDS(1); @@ -3265,10 +3257,9 @@ if (sym_is_const(ctx, b)) { PyObject *result = sym_get_const(ctx, b); if (_Py_IsImmortal(result)) { - // Replace with _LOAD_CONST_INLINE_BORROW + _SWAP + _SWAP since we have two inputs and an immortal result + // Replace with _LOAD_CONST_INLINE_BORROW + _RROT_3 since we have two inputs and an immortal result ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - ADD_OP(_SWAP, 3, 0); - ADD_OP(_SWAP, 2, 0); + ADD_OP(_RROT_3, 0, 0); } } CHECK_STACK_BOUNDS(1); @@ -3332,10 +3323,9 @@ if (sym_is_const(ctx, b)) { PyObject *result = sym_get_const(ctx, b); if (_Py_IsImmortal(result)) { - // Replace with _LOAD_CONST_INLINE_BORROW + _SWAP + _SWAP since we have two inputs and an immortal result + // Replace with _LOAD_CONST_INLINE_BORROW + _RROT_3 since we have two inputs and an immortal result ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - ADD_OP(_SWAP, 3, 0); - ADD_OP(_SWAP, 2, 0); + ADD_OP(_RROT_3, 0, 0); } } CHECK_STACK_BOUNDS(1); @@ -4514,11 +4504,13 @@ case _CALL_LEN: { JitOptRef arg; + JitOptRef null; JitOptRef callable; JitOptRef res; JitOptRef a; JitOptRef c; arg = stack_pointer[-1]; + null = stack_pointer[-2]; callable = stack_pointer[-3]; res = sym_new_type(ctx, &PyLong_Type); Py_ssize_t length = sym_tuple_length(arg); @@ -4544,7 +4536,10 @@ goto error; } if (_Py_IsImmortal(temp)) { - ADD_OP(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + ADD_OP(_SWAP, 2, 0); + optimize_pop_top(ctx, this_instr, null); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + ADD_OP(_SWAP, 3, 0); } res = sym_new_const(ctx, temp); CHECK_STACK_BOUNDS(-2); @@ -5215,10 +5210,9 @@ if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { - // Replace with _LOAD_CONST_INLINE_BORROW + _SWAP + _SWAP since we have two inputs and an immortal result + // Replace with _LOAD_CONST_INLINE_BORROW + _RROT_3 since we have two inputs and an immortal result ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - ADD_OP(_SWAP, 3, 0); - ADD_OP(_SWAP, 2, 0); + ADD_OP(_RROT_3, 0, 0); } } CHECK_STACK_BOUNDS(1); @@ -5237,6 +5231,7 @@ || oparg == NB_INPLACE_TRUE_DIVIDE); bool is_remainder = (oparg == NB_REMAINDER || oparg == NB_INPLACE_REMAINDER); + int emit_op = _BINARY_OP; if (is_truediv || is_remainder) { if (!sym_has_type(rhs) && sym_get_probable_type(rhs) == &PyFloat_Type) { @@ -5253,17 +5248,17 @@ } if (is_truediv && lhs_float && rhs_float) { if (PyJitRef_IsUnique(lhs)) { - ADD_OP(_BINARY_OP_TRUEDIV_FLOAT_INPLACE, 0, 0); + emit_op = _BINARY_OP_TRUEDIV_FLOAT_INPLACE; l = sym_new_null(ctx); r = rhs; } else if (PyJitRef_IsUnique(rhs)) { - ADD_OP(_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT, 0, 0); + emit_op = _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT; l = lhs; r = sym_new_null(ctx); } else { - ADD_OP(_BINARY_OP_TRUEDIV_FLOAT, 0, 0); + emit_op = _BINARY_OP_TRUEDIV_FLOAT; l = lhs; r = rhs; } @@ -5299,6 +5294,7 @@ else { res = PyJitRef_MakeUnique(sym_new_type(ctx, &PyFloat_Type)); } + ADD_OP(emit_op, oparg, 0); CHECK_STACK_BOUNDS(1); stack_pointer[-2] = res; stack_pointer[-1] = l; @@ -5497,16 +5493,20 @@ break; } - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW: { - JitOptRef res; - JitOptRef a; - JitOptRef c; - res = sym_new_not_null(ctx); - a = sym_new_not_null(ctx); - c = sym_new_not_null(ctx); - stack_pointer[-3] = res; - stack_pointer[-2] = a; - stack_pointer[-1] = c; + case _RROT_3: { + JitOptRef top; + JitOptRef middle; + JitOptRef bottom; + top = stack_pointer[-1]; + middle = stack_pointer[-2]; + bottom = stack_pointer[-3]; + JitOptRef temp = top; + top = middle; + middle = bottom; + bottom = temp; + stack_pointer[-3] = bottom; + stack_pointer[-2] = middle; + stack_pointer[-1] = top; break; } diff --git a/Python/parking_lot.c b/Python/parking_lot.c index 99c1ad848be..8823d77719c 100644 --- a/Python/parking_lot.c +++ b/Python/parking_lot.c @@ -61,7 +61,9 @@ _PySemaphore_Init(_PySemaphore *sema) NULL // unnamed ); if (!sema->platform_sem) { - Py_FatalError("parking_lot: CreateSemaphore failed"); + _Py_FatalErrorFormat(__func__, + "parking_lot: CreateSemaphore failed (error: %u)", + GetLastError()); } #elif defined(_Py_USE_SEMAPHORES) if (sem_init(&sema->platform_sem, /*pshared=*/0, /*value=*/0) < 0) { @@ -141,8 +143,8 @@ _PySemaphore_Wait(_PySemaphore *sema, PyTime_t timeout) } else { _Py_FatalErrorFormat(__func__, - "unexpected error from semaphore: %u (error: %u)", - wait, GetLastError()); + "unexpected error from semaphore: %u (error: %u, handle: %p)", + wait, GetLastError(), sema->platform_sem); } #elif defined(_Py_USE_SEMAPHORES) int err; @@ -230,7 +232,9 @@ _PySemaphore_Wakeup(_PySemaphore *sema) { #if defined(MS_WINDOWS) if (!ReleaseSemaphore(sema->platform_sem, 1, NULL)) { - Py_FatalError("parking_lot: ReleaseSemaphore failed"); + _Py_FatalErrorFormat(__func__, + "parking_lot: ReleaseSemaphore failed (error: %u, handle: %p)", + GetLastError(), sema->platform_sem); } #elif defined(_Py_USE_SEMAPHORES) int err = sem_post(&sema->platform_sem); diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index 0ba856ea610..0c460282fec 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -62,6 +62,7 @@ #include "pycore_frame.h" #include "pycore_interp.h" #include "pycore_mmap.h" // _PyAnnotateMemoryMap() +#include "pycore_jit_unwind.h" #include "pycore_runtime.h" // _PyRuntime #ifdef PY_HAVE_PERF_TRAMPOLINE @@ -73,6 +74,7 @@ #include // File control operations #include // Standard I/O operations #include // Standard library functions +#include // memcpy, strlen #include // Memory mapping functions (mmap) #include // System data types #include // System calls (sysconf, getpid) @@ -246,6 +248,25 @@ typedef struct { */ } CodeUnwindingInfoEvent; +/* + * EH Frame Header structure for DWARF unwinding + * + * This header provides metadata about the .eh_frame data that follows. + * It uses PC-relative and data-relative encodings to keep the synthesized + * DSO self-contained when perf injects it. + */ +typedef struct __attribute__((packed)) { + uint8_t version; + uint8_t eh_frame_ptr_enc; + uint8_t fde_count_enc; + uint8_t table_enc; + int32_t eh_frame_ptr; + uint32_t eh_fde_count; + int32_t from; + int32_t to; +} EhFrameHeader; +_Static_assert(sizeof(EhFrameHeader) == 20, "EhFrameHeader layout mismatch"); + // ============================================================================= // GLOBAL STATE MANAGEMENT // ============================================================================= @@ -259,10 +280,11 @@ typedef struct { */ typedef struct { FILE* perf_map; // File handle for the jitdump file - PyThread_type_lock map_lock; // Thread synchronization lock + PyMutex map_lock; // Thread synchronization lock void* mapped_buffer; // Memory-mapped region (signals perf we're active) size_t mapped_size; // Size of the mapped region - int code_id; // Counter for unique code region identifiers + uint32_t code_id; // Counter for unique code region identifiers + uint64_t build_id_salt; // Per-process salt for unique synthetic DSOs } PerfMapJitState; /* Global singleton instance */ @@ -316,40 +338,6 @@ static int64_t get_current_time_microseconds(void) { return ((int64_t)(tv.tv_sec) * 1000000) + tv.tv_usec; } -// ============================================================================= -// UTILITY FUNCTIONS -// ============================================================================= - -/* - * Round up a value to the next multiple of a given number - * - * This is essential for maintaining proper alignment requirements in the - * jitdump format. Many structures need to be aligned to specific boundaries - * (typically 8 or 16 bytes) for efficient processing by perf. - * - * Args: - * value: The value to round up - * multiple: The multiple to round up to - * - * Returns: The smallest value >= input that is a multiple of 'multiple' - */ -static size_t round_up(int64_t value, int64_t multiple) { - if (multiple == 0) { - return value; // Avoid division by zero - } - - int64_t remainder = value % multiple; - if (remainder == 0) { - return value; // Already aligned - } - - /* Calculate how much to add to reach the next multiple */ - int64_t difference = multiple - remainder; - int64_t rounded_up_value = value + difference; - - return rounded_up_value; -} - // ============================================================================= // FILE I/O UTILITIES // ============================================================================= @@ -406,623 +394,6 @@ static void perf_map_jit_write_header(int pid, FILE* out_file) { perf_map_jit_write_fully(&header, sizeof(header)); } -// ============================================================================= -// DWARF CONSTANTS AND UTILITIES -// ============================================================================= - -/* - * DWARF (Debug With Arbitrary Record Formats) constants - * - * DWARF is a debugging data format used to provide stack unwinding information. - * These constants define the various encoding types and opcodes used in - * DWARF Call Frame Information (CFI) records. - */ - -/* DWARF Call Frame Information version */ -#define DWRF_CIE_VERSION 1 - -/* DWARF CFA (Call Frame Address) opcodes */ -enum { - DWRF_CFA_nop = 0x0, // No operation - DWRF_CFA_offset_extended = 0x5, // Extended offset instruction - DWRF_CFA_def_cfa = 0xc, // Define CFA rule - DWRF_CFA_def_cfa_register = 0xd, // Define CFA register - DWRF_CFA_def_cfa_offset = 0xe, // Define CFA offset - DWRF_CFA_offset_extended_sf = 0x11, // Extended signed offset - DWRF_CFA_advance_loc = 0x40, // Advance location counter - DWRF_CFA_offset = 0x80, // Simple offset instruction - DWRF_CFA_restore = 0xc0 // Restore register -}; - -/* DWARF Exception Handling pointer encodings */ -enum { - DWRF_EH_PE_absptr = 0x00, // Absolute pointer - DWRF_EH_PE_omit = 0xff, // Omitted value - - /* Data type encodings */ - DWRF_EH_PE_uleb128 = 0x01, // Unsigned LEB128 - DWRF_EH_PE_udata2 = 0x02, // Unsigned 2-byte - DWRF_EH_PE_udata4 = 0x03, // Unsigned 4-byte - DWRF_EH_PE_udata8 = 0x04, // Unsigned 8-byte - DWRF_EH_PE_sleb128 = 0x09, // Signed LEB128 - DWRF_EH_PE_sdata2 = 0x0a, // Signed 2-byte - DWRF_EH_PE_sdata4 = 0x0b, // Signed 4-byte - DWRF_EH_PE_sdata8 = 0x0c, // Signed 8-byte - DWRF_EH_PE_signed = 0x08, // Signed flag - - /* Reference type encodings */ - DWRF_EH_PE_pcrel = 0x10, // PC-relative - DWRF_EH_PE_textrel = 0x20, // Text-relative - DWRF_EH_PE_datarel = 0x30, // Data-relative - DWRF_EH_PE_funcrel = 0x40, // Function-relative - DWRF_EH_PE_aligned = 0x50, // Aligned - DWRF_EH_PE_indirect = 0x80 // Indirect -}; - -/* Additional DWARF constants for debug information */ -enum { DWRF_TAG_compile_unit = 0x11 }; -enum { DWRF_children_no = 0, DWRF_children_yes = 1 }; -enum { - DWRF_AT_name = 0x03, // Name attribute - DWRF_AT_stmt_list = 0x10, // Statement list - DWRF_AT_low_pc = 0x11, // Low PC address - DWRF_AT_high_pc = 0x12 // High PC address -}; -enum { - DWRF_FORM_addr = 0x01, // Address form - DWRF_FORM_data4 = 0x06, // 4-byte data - DWRF_FORM_string = 0x08 // String form -}; - -/* Line number program opcodes */ -enum { - DWRF_LNS_extended_op = 0, // Extended opcode - DWRF_LNS_copy = 1, // Copy operation - DWRF_LNS_advance_pc = 2, // Advance program counter - DWRF_LNS_advance_line = 3 // Advance line number -}; - -/* Line number extended opcodes */ -enum { - DWRF_LNE_end_sequence = 1, // End of sequence - DWRF_LNE_set_address = 2 // Set address -}; - -/* - * Architecture-specific DWARF register numbers - * - * These constants define the register numbering scheme used by DWARF - * for each supported architecture. The numbers must match the ABI - * specification for proper stack unwinding. - */ -enum { -#ifdef __x86_64__ - /* x86_64 register numbering (note: order is defined by x86_64 ABI) */ - DWRF_REG_AX, // RAX - DWRF_REG_DX, // RDX - DWRF_REG_CX, // RCX - DWRF_REG_BX, // RBX - DWRF_REG_SI, // RSI - DWRF_REG_DI, // RDI - DWRF_REG_BP, // RBP - DWRF_REG_SP, // RSP - DWRF_REG_8, // R8 - DWRF_REG_9, // R9 - DWRF_REG_10, // R10 - DWRF_REG_11, // R11 - DWRF_REG_12, // R12 - DWRF_REG_13, // R13 - DWRF_REG_14, // R14 - DWRF_REG_15, // R15 - DWRF_REG_RA, // Return address (RIP) -#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) - /* AArch64 register numbering */ - DWRF_REG_FP = 29, // Frame Pointer - DWRF_REG_RA = 30, // Link register (return address) - DWRF_REG_SP = 31, // Stack pointer -#else -# error "Unsupported target architecture" -#endif -}; - -/* DWARF encoding constants used in EH frame headers */ -static const uint8_t DwarfUData4 = 0x03; // Unsigned 4-byte data -static const uint8_t DwarfSData4 = 0x0b; // Signed 4-byte data -static const uint8_t DwarfPcRel = 0x10; // PC-relative encoding -static const uint8_t DwarfDataRel = 0x30; // Data-relative encoding - -// ============================================================================= -// ELF OBJECT CONTEXT -// ============================================================================= - -/* - * Context for building ELF/DWARF structures - * - * This structure maintains state while constructing DWARF unwind information. - * It acts as a simple buffer manager with pointers to track current position - * and important landmarks within the buffer. - */ -typedef struct ELFObjectContext { - uint8_t* p; // Current write position in buffer - uint8_t* startp; // Start of buffer (for offset calculations) - uint8_t* eh_frame_p; // Start of EH frame data (for relative offsets) - uint8_t* fde_p; // Start of FDE data (for PC-relative calculations) - uint32_t code_size; // Size of the code being described -} ELFObjectContext; - -/* - * EH Frame Header structure for DWARF unwinding - * - * This structure provides metadata about the DWARF unwinding information - * that follows. It's required by the perf jitdump format to enable proper - * stack unwinding during profiling. - */ -typedef struct { - unsigned char version; // EH frame version (always 1) - unsigned char eh_frame_ptr_enc; // Encoding of EH frame pointer - unsigned char fde_count_enc; // Encoding of FDE count - unsigned char table_enc; // Encoding of table entries - int32_t eh_frame_ptr; // Pointer to EH frame data - int32_t eh_fde_count; // Number of FDEs (Frame Description Entries) - int32_t from; // Start address of code range - int32_t to; // End address of code range -} EhFrameHeader; - -// ============================================================================= -// DWARF GENERATION UTILITIES -// ============================================================================= - -/* - * Append a null-terminated string to the ELF context buffer - * - * Args: - * ctx: ELF object context - * str: String to append (must be null-terminated) - * - * Returns: Offset from start of buffer where string was written - */ -static uint32_t elfctx_append_string(ELFObjectContext* ctx, const char* str) { - uint8_t* p = ctx->p; - uint32_t ofs = (uint32_t)(p - ctx->startp); - - /* Copy string including null terminator */ - do { - *p++ = (uint8_t)*str; - } while (*str++); - - ctx->p = p; - return ofs; -} - -/* - * Append a SLEB128 (Signed Little Endian Base 128) value - * - * SLEB128 is a variable-length encoding used extensively in DWARF. - * It efficiently encodes small numbers in fewer bytes. - * - * Args: - * ctx: ELF object context - * v: Signed value to encode - */ -static void elfctx_append_sleb128(ELFObjectContext* ctx, int32_t v) { - uint8_t* p = ctx->p; - - /* Encode 7 bits at a time, with continuation bit in MSB */ - for (; (uint32_t)(v + 0x40) >= 0x80; v >>= 7) { - *p++ = (uint8_t)((v & 0x7f) | 0x80); // Set continuation bit - } - *p++ = (uint8_t)(v & 0x7f); // Final byte without continuation bit - - ctx->p = p; -} - -/* - * Append a ULEB128 (Unsigned Little Endian Base 128) value - * - * Similar to SLEB128 but for unsigned values. - * - * Args: - * ctx: ELF object context - * v: Unsigned value to encode - */ -static void elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v) { - uint8_t* p = ctx->p; - - /* Encode 7 bits at a time, with continuation bit in MSB */ - for (; v >= 0x80; v >>= 7) { - *p++ = (char)((v & 0x7f) | 0x80); // Set continuation bit - } - *p++ = (char)v; // Final byte without continuation bit - - ctx->p = p; -} - -/* - * Macros for generating DWARF structures - * - * These macros provide a convenient way to write various data types - * to the DWARF buffer while automatically advancing the pointer. - */ -#define DWRF_U8(x) (*p++ = (x)) // Write unsigned 8-bit -#define DWRF_I8(x) (*(int8_t*)p = (x), p++) // Write signed 8-bit -#define DWRF_U16(x) (*(uint16_t*)p = (x), p += 2) // Write unsigned 16-bit -#define DWRF_U32(x) (*(uint32_t*)p = (x), p += 4) // Write unsigned 32-bit -#define DWRF_ADDR(x) (*(uintptr_t*)p = (x), p += sizeof(uintptr_t)) // Write address -#define DWRF_UV(x) (ctx->p = p, elfctx_append_uleb128(ctx, (x)), p = ctx->p) // Write ULEB128 -#define DWRF_SV(x) (ctx->p = p, elfctx_append_sleb128(ctx, (x)), p = ctx->p) // Write SLEB128 -#define DWRF_STR(str) (ctx->p = p, elfctx_append_string(ctx, (str)), p = ctx->p) // Write string - -/* Align to specified boundary with NOP instructions */ -#define DWRF_ALIGNNOP(s) \ - while ((uintptr_t)p & ((s)-1)) { \ - *p++ = DWRF_CFA_nop; \ - } - -/* Write a DWARF section with automatic size calculation */ -#define DWRF_SECTION(name, stmt) \ - { \ - uint32_t* szp_##name = (uint32_t*)p; \ - p += 4; \ - stmt; \ - *szp_##name = (uint32_t)((p - (uint8_t*)szp_##name) - 4); \ - } - -// ============================================================================= -// DWARF EH FRAME GENERATION -// ============================================================================= - -static void elf_init_ehframe(ELFObjectContext* ctx); - -/* - * Initialize DWARF .eh_frame section for a code region - * - * The .eh_frame section contains Call Frame Information (CFI) that describes - * how to unwind the stack at any point in the code. This is essential for - * proper profiling as it allows perf to generate accurate call graphs. - * - * The function generates two main components: - * 1. CIE (Common Information Entry) - describes calling conventions - * 2. FDE (Frame Description Entry) - describes specific function unwinding - * - * Args: - * ctx: ELF object context containing code size and buffer pointers - */ -static size_t calculate_eh_frame_size(void) { - /* Calculate the EH frame size for the trampoline function */ - extern void *_Py_trampoline_func_start; - extern void *_Py_trampoline_func_end; - - size_t code_size = (char*)&_Py_trampoline_func_end - (char*)&_Py_trampoline_func_start; - - ELFObjectContext ctx; - char buffer[1024]; // Buffer for DWARF data (1KB should be sufficient) - ctx.code_size = code_size; - ctx.startp = ctx.p = (uint8_t*)buffer; - ctx.fde_p = NULL; - - elf_init_ehframe(&ctx); - return ctx.p - ctx.startp; -} - -static void elf_init_ehframe(ELFObjectContext* ctx) { - uint8_t* p = ctx->p; - uint8_t* framep = p; // Remember start of frame data - - /* - * DWARF Unwind Table for Trampoline Function - * - * This section defines DWARF Call Frame Information (CFI) using encoded macros - * like `DWRF_U8`, `DWRF_UV`, and `DWRF_SECTION` to describe how the trampoline function - * preserves and restores registers. This is used by profiling tools (e.g., `perf`) - * and debuggers for stack unwinding in JIT-compiled code. - * - * ------------------------------------------------- - * TO REGENERATE THIS TABLE FROM GCC OBJECTS: - * ------------------------------------------------- - * - * 1. Create a trampoline source file (e.g., `trampoline.c`): - * - * #include - * typedef PyObject* (*py_evaluator)(void*, void*, int); - * PyObject* trampoline(void *ts, void *f, int throwflag, py_evaluator evaluator) { - * return evaluator(ts, f, throwflag); - * } - * - * 2. Compile to an object file with frame pointer preservation: - * - * gcc trampoline.c -I. -I./Include -O2 -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -c - * - * 3. Extract DWARF unwind info from the object file: - * - * readelf -w trampoline.o - * - * Example output from `.eh_frame`: - * - * 00000000 CIE - * Version: 1 - * Augmentation: "zR" - * Code alignment factor: 4 - * Data alignment factor: -8 - * Return address column: 30 - * DW_CFA_def_cfa: r31 (sp) ofs 0 - * - * 00000014 FDE cie=00000000 pc=0..14 - * DW_CFA_advance_loc: 4 - * DW_CFA_def_cfa_offset: 16 - * DW_CFA_offset: r29 at cfa-16 - * DW_CFA_offset: r30 at cfa-8 - * DW_CFA_advance_loc: 12 - * DW_CFA_restore: r30 - * DW_CFA_restore: r29 - * DW_CFA_def_cfa_offset: 0 - * - * -- These values can be verified by comparing with `readelf -w` or `llvm-dwarfdump --eh-frame`. - * - * ---------------------------------- - * HOW TO TRANSLATE TO DWRF_* MACROS: - * ---------------------------------- - * - * After compiling your trampoline with: - * - * gcc trampoline.c -I. -I./Include -O2 -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -c - * - * run: - * - * readelf -w trampoline.o - * - * to inspect the generated `.eh_frame` data. You will see two main components: - * - * 1. A CIE (Common Information Entry): shared configuration used by all FDEs. - * 2. An FDE (Frame Description Entry): function-specific unwind instructions. - * - * --------------------- - * Translating the CIE: - * --------------------- - * From `readelf -w`, you might see: - * - * 00000000 0000000000000010 00000000 CIE - * Version: 1 - * Augmentation: "zR" - * Code alignment factor: 4 - * Data alignment factor: -8 - * Return address column: 30 - * Augmentation data: 1b - * DW_CFA_def_cfa: r31 (sp) ofs 0 - * - * Map this to: - * - * DWRF_SECTION(CIE, - * DWRF_U32(0); // CIE ID (always 0 for CIEs) - * DWRF_U8(DWRF_CIE_VERSION); // Version: 1 - * DWRF_STR("zR"); // Augmentation string "zR" - * DWRF_UV(4); // Code alignment factor = 4 - * DWRF_SV(-8); // Data alignment factor = -8 - * DWRF_U8(DWRF_REG_RA); // Return address register (e.g., x30 = 30) - * DWRF_UV(1); // Augmentation data length = 1 - * DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); // Encoding for FDE pointers - * - * DWRF_U8(DWRF_CFA_def_cfa); // DW_CFA_def_cfa - * DWRF_UV(DWRF_REG_SP); // Register: SP (r31) - * DWRF_UV(0); // Offset = 0 - * - * DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer size boundary - * ) - * - * Notes: - * - Use `DWRF_UV` for unsigned LEB128, `DWRF_SV` for signed LEB128. - * - `DWRF_REG_RA` and `DWRF_REG_SP` are architecture-defined constants. - * - * --------------------- - * Translating the FDE: - * --------------------- - * From `readelf -w`: - * - * 00000014 0000000000000020 00000018 FDE cie=00000000 pc=0000000000000000..0000000000000014 - * DW_CFA_advance_loc: 4 - * DW_CFA_def_cfa_offset: 16 - * DW_CFA_offset: r29 at cfa-16 - * DW_CFA_offset: r30 at cfa-8 - * DW_CFA_advance_loc: 12 - * DW_CFA_restore: r30 - * DW_CFA_restore: r29 - * DW_CFA_def_cfa_offset: 0 - * - * Map the FDE header and instructions to: - * - * DWRF_SECTION(FDE, - * DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (relative from here) - * DWRF_U32(pc_relative_offset); // PC-relative location of the code (calculated dynamically) - * DWRF_U32(ctx->code_size); // Code range covered by this FDE - * DWRF_U8(0); // Augmentation data length (none) - * - * DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance location by 1 unit (1 * 4 = 4 bytes) - * DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 - * DWRF_UV(16); - * - * DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // Save x29 (frame pointer) - * DWRF_UV(2); // At offset 2 * 8 = 16 bytes - * - * DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // Save x30 (return address) - * DWRF_UV(1); // At offset 1 * 8 = 8 bytes - * - * DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance location by 3 units (3 * 4 = 12 bytes) - * - * DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // Restore x30 - * DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // Restore x29 - * - * DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP - * DWRF_UV(0); - * ) - * - * To regenerate: - * 1. Get the `code alignment factor`, `data alignment factor`, and `RA column` from the CIE. - * 2. Note the range of the function from the FDE's `pc=...` line and map it to the JIT code as - * the code is in a different address space every time. - * 3. For each `DW_CFA_*` entry, use the corresponding `DWRF_*` macro: - * - `DW_CFA_def_cfa_offset` → DWRF_U8(DWRF_CFA_def_cfa_offset), DWRF_UV(value) - * - `DW_CFA_offset: rX` → DWRF_U8(DWRF_CFA_offset | reg), DWRF_UV(offset) - * - `DW_CFA_restore: rX` → DWRF_U8(DWRF_CFA_offset | reg) // restore is same as reusing offset - * - `DW_CFA_advance_loc: N` → DWRF_U8(DWRF_CFA_advance_loc | (N / code_alignment_factor)) - * 4. Use `DWRF_REG_FP`, `DWRF_REG_RA`, etc., for register numbers. - * 5. Use `sizeof(uintptr_t)` (typically 8) for pointer size calculations and alignment. - */ - - /* - * Emit DWARF EH CIE (Common Information Entry) - * - * The CIE describes the calling conventions and basic unwinding rules - * that apply to all functions in this compilation unit. - */ - DWRF_SECTION(CIE, - DWRF_U32(0); // CIE ID (0 indicates this is a CIE) - DWRF_U8(DWRF_CIE_VERSION); // CIE version (1) - DWRF_STR("zR"); // Augmentation string ("zR" = has LSDA) -#ifdef __x86_64__ - DWRF_UV(1); // Code alignment factor (x86_64: 1 byte) -#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) - DWRF_UV(4); // Code alignment factor (AArch64: 4 bytes per instruction) -#endif - DWRF_SV(-(int64_t)sizeof(uintptr_t)); // Data alignment factor (negative) - DWRF_U8(DWRF_REG_RA); // Return address register number - DWRF_UV(1); // Augmentation data length - DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); // FDE pointer encoding - - /* Initial CFI instructions - describe default calling convention */ -#ifdef __x86_64__ - /* x86_64 initial CFI state */ - DWRF_U8(DWRF_CFA_def_cfa); // Define CFA (Call Frame Address) - DWRF_UV(DWRF_REG_SP); // CFA = SP register - DWRF_UV(sizeof(uintptr_t)); // CFA = SP + pointer_size - DWRF_U8(DWRF_CFA_offset|DWRF_REG_RA); // Return address is saved - DWRF_UV(1); // At offset 1 from CFA -#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) - /* AArch64 initial CFI state */ - DWRF_U8(DWRF_CFA_def_cfa); // Define CFA (Call Frame Address) - DWRF_UV(DWRF_REG_SP); // CFA = SP register - DWRF_UV(0); // CFA = SP + 0 (AArch64 starts with offset 0) - // No initial register saves in AArch64 CIE -#endif - DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer boundary - ) - - ctx->eh_frame_p = p; // Remember start of FDE data - - /* - * Emit DWARF EH FDE (Frame Description Entry) - * - * The FDE describes unwinding information specific to this function. - * It references the CIE and provides function-specific CFI instructions. - * - * The PC-relative offset is calculated after the entire EH frame is built - * to ensure accurate positioning relative to the synthesized DSO layout. - */ - DWRF_SECTION(FDE, - DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (backwards reference) - ctx->fde_p = p; // Remember where PC offset field is located for later calculation - DWRF_U32(0); // Placeholder for PC-relative offset (calculated at end of elf_init_ehframe) - DWRF_U32(ctx->code_size); // Address range covered by this FDE (code length) - DWRF_U8(0); // Augmentation data length (none) - - /* - * Architecture-specific CFI instructions - * - * These instructions describe how registers are saved and restored - * during function calls. Each architecture has different calling - * conventions and register usage patterns. - */ -#ifdef __x86_64__ - /* x86_64 calling convention unwinding rules with frame pointer */ -# if defined(__CET__) && (__CET__ & 1) - DWRF_U8(DWRF_CFA_advance_loc | 4); // Advance past endbr64 (4 bytes) -# endif - DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance past push %rbp (1 byte) - DWRF_U8(DWRF_CFA_def_cfa_offset); // def_cfa_offset 16 - DWRF_UV(16); // New offset: SP + 16 - DWRF_U8(DWRF_CFA_offset | DWRF_REG_BP); // offset r6 at cfa-16 - DWRF_UV(2); // Offset factor: 2 * 8 = 16 bytes - DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance past mov %rsp,%rbp (3 bytes) - DWRF_U8(DWRF_CFA_def_cfa_register); // def_cfa_register r6 - DWRF_UV(DWRF_REG_BP); // Use base pointer register - DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance past call *%rcx (2 bytes) + pop %rbp (1 byte) = 3 - DWRF_U8(DWRF_CFA_def_cfa); // def_cfa r7 ofs 8 - DWRF_UV(DWRF_REG_SP); // Use stack pointer register - DWRF_UV(8); // New offset: SP + 8 -#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) - /* AArch64 calling convention unwinding rules */ - DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance by 1 instruction (4 bytes) - DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 - DWRF_UV(16); // Stack pointer moved by 16 bytes - DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // x29 (frame pointer) saved - DWRF_UV(2); // At CFA-16 (2 * 8 = 16 bytes from CFA) - DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // x30 (link register) saved - DWRF_UV(1); // At CFA-8 (1 * 8 = 8 bytes from CFA) - DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance by 3 instructions (12 bytes) - DWRF_U8(DWRF_CFA_restore | DWRF_REG_RA); // Restore x30 - NO DWRF_UV() after this! - DWRF_U8(DWRF_CFA_restore | DWRF_REG_FP); // Restore x29 - NO DWRF_UV() after this! - DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 0 (stack restored) - DWRF_UV(0); // Back to original stack position -#else -# error "Unsupported target architecture" -#endif - - DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer boundary - ) - - ctx->p = p; // Update context pointer to end of generated data - - /* Calculate and update the PC-relative offset in the FDE - * - * When perf processes the jitdump, it creates a synthesized DSO with this layout: - * - * Synthesized DSO Memory Layout: - * ┌─────────────────────────────────────────────────────────────┐ < code_start - * │ Code Section │ - * │ (round_up(code_size, 8) bytes) │ - * ├─────────────────────────────────────────────────────────────┤ < start of EH frame data - * │ EH Frame Data │ - * │ ┌─────────────────────────────────────────────────────┐ │ - * │ │ CIE data │ │ - * │ └─────────────────────────────────────────────────────┘ │ - * │ ┌─────────────────────────────────────────────────────┐ │ - * │ │ FDE Header: │ │ - * │ │ - CIE offset (4 bytes) │ │ - * │ │ - PC offset (4 bytes) <─ fde_offset_in_frame ─────┼────┼─> points to code_start - * │ │ - address range (4 bytes) │ │ (this specific field) - * │ │ CFI Instructions... │ │ - * │ └─────────────────────────────────────────────────────┘ │ - * ├─────────────────────────────────────────────────────────────┤ < reference_point - * │ EhFrameHeader │ - * │ (navigation metadata) │ - * └─────────────────────────────────────────────────────────────┘ - * - * The PC offset field in the FDE must contain the distance from itself to code_start: - * - * distance = code_start - fde_pc_field - * - * Where: - * fde_pc_field_location = reference_point - eh_frame_size + fde_offset_in_frame - * code_start_location = reference_point - eh_frame_size - round_up(code_size, 8) - * - * Therefore: - * distance = code_start_location - fde_pc_field_location - * = (ref - eh_frame_size - rounded_code_size) - (ref - eh_frame_size + fde_offset_in_frame) - * = -rounded_code_size - fde_offset_in_frame - * = -(round_up(code_size, 8) + fde_offset_in_frame) - * - * Note: fde_offset_in_frame is the offset from EH frame start to the PC offset field, - * - */ - if (ctx->fde_p != NULL) { - int32_t fde_offset_in_frame = (ctx->fde_p - ctx->startp); - int32_t rounded_code_size = round_up(ctx->code_size, 8); - int32_t pc_relative_offset = -(rounded_code_size + fde_offset_in_frame); - - - // Update the PC-relative offset in the FDE - *(int32_t*)ctx->fde_p = pc_relative_offset; - } -} - // ============================================================================= // JITDUMP INITIALIZATION // ============================================================================= @@ -1042,6 +413,12 @@ static void elf_init_ehframe(ELFObjectContext* ctx) { * Returns: Pointer to initialized state, or NULL on failure */ static void* perf_map_jit_init(void) { + PyMutex_Lock(&perf_jit_map_state.map_lock); + if (perf_jit_map_state.perf_map != NULL) { + PyMutex_Unlock(&perf_jit_map_state.map_lock); + return &perf_jit_map_state; + } + char filename[100]; int pid = getpid(); @@ -1051,6 +428,7 @@ static void* perf_map_jit_init(void) { /* Create/open the jitdump file with appropriate permissions */ const int fd = open(filename, O_CREAT | O_TRUNC | O_RDWR, 0666); if (fd == -1) { + PyMutex_Unlock(&perf_jit_map_state.map_lock); return NULL; // Failed to create file } @@ -1058,6 +436,7 @@ static void* perf_map_jit_init(void) { const long page_size = sysconf(_SC_PAGESIZE); if (page_size == -1) { close(fd); + PyMutex_Unlock(&perf_jit_map_state.map_lock); return NULL; // Failed to get page size } @@ -1086,6 +465,7 @@ static void* perf_map_jit_init(void) { if (perf_jit_map_state.mapped_buffer == MAP_FAILED) { perf_jit_map_state.mapped_buffer = NULL; close(fd); + PyMutex_Unlock(&perf_jit_map_state.map_lock); return NULL; // Memory mapping failed } (void)_PyAnnotateMemoryMap(perf_jit_map_state.mapped_buffer, page_size, @@ -1098,6 +478,7 @@ static void* perf_map_jit_init(void) { perf_jit_map_state.perf_map = fdopen(fd, "w+"); if (perf_jit_map_state.perf_map == NULL) { close(fd); + PyMutex_Unlock(&perf_jit_map_state.map_lock); return NULL; // Failed to create FILE* } @@ -1113,28 +494,18 @@ static void* perf_map_jit_init(void) { /* Write the jitdump file header */ perf_map_jit_write_header(pid, perf_jit_map_state.perf_map); - /* - * Initialize thread synchronization lock - * - * Multiple threads may attempt to write to the jitdump file - * simultaneously. This lock ensures thread-safe access to the - * global jitdump state. - */ - perf_jit_map_state.map_lock = PyThread_allocate_lock(); - if (perf_jit_map_state.map_lock == NULL) { - fclose(perf_jit_map_state.perf_map); - return NULL; // Failed to create lock - } - /* Initialize code ID counter */ perf_jit_map_state.code_id = 0; + perf_jit_map_state.build_id_salt = + ((uint64_t)pid << 32) ^ (uint64_t)get_current_monotonic_ticks(); /* Calculate padding size based on actual unwind info requirements */ - size_t eh_frame_size = calculate_eh_frame_size(); + size_t eh_frame_size = _PyJitUnwind_EhFrameSize(0); size_t unwind_data_size = sizeof(EhFrameHeader) + eh_frame_size; - trampoline_api.code_padding = round_up(unwind_data_size, 16); + trampoline_api.code_padding = _Py_SIZE_ROUND_UP(unwind_data_size, 16); trampoline_api.code_alignment = 32; + PyMutex_Unlock(&perf_jit_map_state.map_lock); return &perf_jit_map_state; } @@ -1143,54 +514,31 @@ static void* perf_map_jit_init(void) { // ============================================================================= /* - * Write a complete jitdump entry for a Python function + * Write a complete jitdump entry for a code region with a provided name. * - * This is the main function called by Python's trampoline system whenever - * a new piece of JIT-compiled code needs to be recorded. It writes both - * the unwinding information and the code load event to the jitdump file. - * - * The function performs these steps: - * 1. Initialize jitdump system if not already done - * 2. Extract function name and filename from Python code object - * 3. Generate DWARF unwinding information - * 4. Write unwinding info event to jitdump file - * 5. Write code load event to jitdump file - * - * Args: - * state: Jitdump state (currently unused, uses global state) - * code_addr: Address where the compiled code resides - * code_size: Size of the compiled code in bytes - * co: Python code object containing metadata - * - * IMPORTANT: This function signature is part of Python's internal API - * and must not be changed without coordinating with core Python development. + * This shares the same implementation as the trampoline callback, but + * allows callers that don't have a PyCodeObject to reuse the jitdump + * infrastructure. */ -static void perf_map_jit_write_entry(void *state, const void *code_addr, - unsigned int code_size, PyCodeObject *co) +static void perf_map_jit_write_entry_with_name( + void *state, + const void *code_addr, + size_t code_size, + const char *entry, + const char *filename +) { /* Initialize jitdump system on first use */ - if (perf_jit_map_state.perf_map == NULL) { - void* ret = perf_map_jit_init(); - if(ret == NULL){ - return; // Initialization failed, silently abort - } + void* ret = perf_map_jit_init(); + if (ret == NULL) { + return; // Initialization failed, silently abort } - /* - * Extract function information from Python code object - * - * We create a human-readable function name by combining the qualified - * name (includes class/module context) with the filename. This helps - * developers identify functions in perf reports. - */ - const char *entry = ""; - if (co->co_qualname != NULL) { - entry = PyUnicode_AsUTF8(co->co_qualname); + if (entry == NULL) { + entry = ""; } - - const char *filename = ""; - if (co->co_filename != NULL) { - filename = PyUnicode_AsUTF8(co->co_filename); + if (filename == NULL) { + filename = ""; } /* @@ -1218,15 +566,20 @@ static void perf_map_jit_write_entry(void *state, const void *code_addr, * Without it, perf cannot generate accurate call graphs, especially * in optimized code where frame pointers may be omitted. */ - ELFObjectContext ctx; - char buffer[1024]; // Buffer for DWARF data (1KB should be sufficient) - ctx.code_size = code_size; - ctx.startp = ctx.p = (uint8_t*)buffer; - ctx.fde_p = NULL; // Initialize to NULL, will be set when FDE is written + uint8_t buffer[1024]; // Buffer for DWARF data (1KB should be sufficient) + size_t eh_frame_size = _PyJitUnwind_BuildEhFrame( + buffer, sizeof(buffer), code_addr, code_size, 0); + if (eh_frame_size == 0) { + PyMem_RawFree(perf_map_entry); + return; + } - /* Generate EH frame (Exception Handling frame) data */ - elf_init_ehframe(&ctx); - int eh_frame_size = ctx.p - ctx.startp; + /* + * A logical jitdump entry is written as multiple records and also consumes + * a process-global code_id. Serialize the whole sequence so concurrent JIT + * compilation cannot interleave records or reuse an ID. + */ + PyMutex_Lock(&perf_jit_map_state.map_lock); /* * Write Code Unwinding Information Event @@ -1244,12 +597,12 @@ static void perf_map_jit_write_entry(void *state, const void *code_addr, assert(ev2.unwind_data_size <= (uint64_t)trampoline_api.code_padding); ev2.eh_frame_hdr_size = sizeof(EhFrameHeader); - ev2.mapped_size = round_up(ev2.unwind_data_size, 16); // 16-byte alignment + ev2.mapped_size = _Py_SIZE_ROUND_UP(ev2.unwind_data_size, 16); // 16-byte alignment /* Calculate total event size with padding */ - int content_size = sizeof(ev2) + sizeof(EhFrameHeader) + eh_frame_size; - int padding_size = round_up(content_size, 8) - content_size; // 8-byte align - ev2.base.size = content_size + padding_size; + int content_size = (int)(sizeof(ev2) + sizeof(EhFrameHeader) + eh_frame_size); + int padding_size = (int)_Py_SIZE_ROUND_UP((size_t)content_size, 8) - content_size; // 8-byte align + ev2.base.size = (uint32_t)(content_size + padding_size); /* Write the unwinding info event header */ perf_map_jit_write_fully(&ev2, sizeof(ev2)); @@ -1263,20 +616,21 @@ static void perf_map_jit_write_entry(void *state, const void *code_addr, */ EhFrameHeader f; f.version = 1; - f.eh_frame_ptr_enc = DwarfSData4 | DwarfPcRel; // PC-relative signed 4-byte - f.fde_count_enc = DwarfUData4; // Unsigned 4-byte count - f.table_enc = DwarfSData4 | DwarfDataRel; // Data-relative signed 4-byte + f.eh_frame_ptr_enc = DWRF_EH_PE_sdata4 | DWRF_EH_PE_pcrel; + f.fde_count_enc = DWRF_EH_PE_udata4; + f.table_enc = DWRF_EH_PE_sdata4 | DWRF_EH_PE_datarel; /* Calculate relative offsets for EH frame navigation */ - f.eh_frame_ptr = -(eh_frame_size + 4 * sizeof(unsigned char)); + f.eh_frame_ptr = -(int32_t)(eh_frame_size + 4 * sizeof(unsigned char)); f.eh_fde_count = 1; // We generate exactly one FDE per function - f.from = -(round_up(code_size, 8) + eh_frame_size); - - int cie_size = ctx.eh_frame_p - ctx.startp; - f.to = -(eh_frame_size - cie_size); + f.from = -(int32_t)(_Py_SIZE_ROUND_UP(code_size, 8) + eh_frame_size); + uint32_t cie_payload_size; + memcpy(&cie_payload_size, buffer, sizeof(cie_payload_size)); + int cie_size = (int)(sizeof(cie_payload_size) + cie_payload_size); + f.to = -(int32_t)(eh_frame_size - cie_size); /* Write EH frame data and header */ - perf_map_jit_write_fully(ctx.startp, eh_frame_size); + perf_map_jit_write_fully(buffer, eh_frame_size); perf_map_jit_write_fully(&f, sizeof(f)); /* Write padding to maintain alignment */ @@ -1313,12 +667,86 @@ static void perf_map_jit_write_entry(void *state, const void *code_addr, /* Write code load event and associated data */ perf_map_jit_write_fully(&ev, sizeof(ev)); perf_map_jit_write_fully(perf_map_entry, name_length+1); // Include null terminator - perf_map_jit_write_fully((void*)(base), size); // Copy actual machine code + /* + * Ensure each synthetic DSO has unique .text bytes. + * + * perf merges DSOs that share a build-id. Since trampolines can share + * identical code and unwind bytes, perf may resolve all JIT frames to + * the first symbol it saw (including entries from previous runs when + * build-id caching is enabled). Patch a small marker in the emitted + * bytes to make the build-id depend on a per-process salt and code id + * without modifying the live code. + */ + uint64_t marker = perf_jit_map_state.build_id_salt ^ + ((uint64_t)perf_jit_map_state.code_id << 32) ^ + (uint64_t)code_size; + if (size >= sizeof(marker)) { + size_t prefix = size - sizeof(marker); + perf_map_jit_write_fully((void *)(base), prefix); + perf_map_jit_write_fully(&marker, sizeof(marker)); + } + else if (size > 0) { + uint8_t tmp[sizeof(marker)]; + memcpy(tmp, (void *)(base), size); + for (size_t i = 0; i < size; i++) { + tmp[i] ^= (uint8_t)(marker >> (i * 8)); + } + perf_map_jit_write_fully(tmp, size); + } /* Clean up allocated memory */ + PyMutex_Unlock(&perf_jit_map_state.map_lock); PyMem_RawFree(perf_map_entry); } +/* + * Write a complete jitdump entry for a Python function + * + * This is the main function called by Python's trampoline system whenever + * a new piece of JIT-compiled code needs to be recorded. It writes both + * the unwinding information and the code load event to the jitdump file. + * + * The function performs these steps: + * 1. Initialize jitdump system if not already done + * 2. Extract function name and filename from Python code object + * 3. Generate DWARF unwinding information + * 4. Write unwinding info event to jitdump file + * 5. Write code load event to jitdump file + * + * Args: + * state: Jitdump state (currently unused, uses global state) + * code_addr: Address where the compiled code resides + * code_size: Size of the compiled code in bytes + * co: Python code object containing metadata + * + * IMPORTANT: This function signature is part of Python's internal API + * and must not be changed without coordinating with core Python development. + */ +static void perf_map_jit_write_entry(void *state, const void *code_addr, + size_t code_size, PyCodeObject *co) +{ + const char *entry = ""; + const char *filename = ""; + if (co != NULL) { + if (co->co_qualname != NULL) { + entry = PyUnicode_AsUTF8(co->co_qualname); + } + if (co->co_filename != NULL) { + filename = PyUnicode_AsUTF8(co->co_filename); + } + } + perf_map_jit_write_entry_with_name(state, code_addr, code_size, + entry, filename); +} + +void +_PyPerfJit_WriteNamedCode(const void *code_addr, size_t code_size, + const char *entry, const char *filename) +{ + perf_map_jit_write_entry_with_name( + NULL, code_addr, code_size, entry, filename); +} + // ============================================================================= // CLEANUP AND FINALIZATION // ============================================================================= @@ -1346,15 +774,12 @@ static int perf_map_jit_fini(void* state) { * writing to the file when we close it. This prevents corruption * and ensures all data is properly flushed. */ + PyMutex_Lock(&perf_jit_map_state.map_lock); if (perf_jit_map_state.perf_map != NULL) { - PyThread_acquire_lock(perf_jit_map_state.map_lock, 1); fclose(perf_jit_map_state.perf_map); // This also flushes buffers - PyThread_release_lock(perf_jit_map_state.map_lock); - - /* Clean up synchronization primitive */ - PyThread_free_lock(perf_jit_map_state.map_lock); perf_jit_map_state.perf_map = NULL; } + PyMutex_Unlock(&perf_jit_map_state.map_lock); /* * Unmap the memory region diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 0d835f3b7f5..58c61e64bfc 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -243,7 +243,7 @@ perf_trampoline_code_watcher(PyCodeEvent event, PyCodeObject *co) static void perf_map_write_entry(void *state, const void *code_addr, - unsigned int code_size, PyCodeObject *co) + size_t code_size, PyCodeObject *co) { const char *entry = ""; if (co->co_qualname != NULL) { diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index d9fc28475a4..57ce519c3c1 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -37,9 +37,6 @@ #include "pycore_uniqueid.h" // _PyObject_FinalizeUniqueIdPool() #include "pycore_warnings.h" // _PyWarnings_InitState() #include "pycore_weakref.h" // _PyWeakref_GET_REF() -#ifdef _Py_JIT -#include "pycore_jit.h" // _PyJIT_Fini() -#endif #if defined(PYMALLOC_USE_HUGEPAGES) && defined(MS_WINDOWS) #include @@ -1641,18 +1638,12 @@ Py_InitializeFromConfig(const PyConfig *config) void Py_InitializeEx(int install_sigs) { - PyStatus status; - - status = _PyRuntime_Initialize(); - if (_PyStatus_EXCEPTION(status)) { - Py_ExitStatusException(status); - } - if (Py_IsInitialized()) { /* bpo-33932: Calling Py_Initialize() twice does nothing. */ return; } + PyStatus status; PyConfig config; _PyConfig_InitCompatConfig(&config); @@ -2537,11 +2528,6 @@ _Py_Finalize(_PyRuntimeState *runtime) finalize_interp_clear(tstate); -#ifdef _Py_JIT - /* Free JIT shim memory */ - _PyJIT_Fini(); -#endif - #ifdef Py_TRACE_REFS /* Display addresses (& refcnts) of all objects still alive. * An address can be used to find the repr of the object, printed @@ -3356,7 +3342,7 @@ _Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp, /* display the current Python stack */ #ifndef Py_GIL_DISABLED - _Py_DumpTracebackThreads(fd, interp, tstate); + _Py_DumpTracebackThreads(fd, interp, tstate, 0); #else _Py_DumpTraceback(fd, tstate); #endif diff --git a/Python/pystate.c b/Python/pystate.c index d6a26f3339b..2df24597e65 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -489,11 +489,6 @@ free_interpreter(PyInterpreterState *interp) static inline int check_interpreter_whence(long); #endif -extern _Py_CODEUNIT * -_Py_LazyJitShim( - struct _PyExecutorObject *exec, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate -); - /* Get the interpreter state to a minimal consistent state. Further init happens in pylifecycle.c before it can be used. All fields not initialized here are expected to be zeroed out, @@ -635,6 +630,11 @@ init_interpreter(PyInterpreterState *interp, "PYTHON_JIT_SIDE_EXIT_INITIAL_BACKOFF", SIDE_EXIT_INITIAL_BACKOFF, 0, MAX_BACKOFF); + // Trace fitness configuration + init_policy(&interp->opt_config.fitness_initial, + "PYTHON_JIT_FITNESS_INITIAL", + FITNESS_INITIAL, EXIT_QUALITY_CLOSE_LOOP, UOP_MAX_TRACE_LENGTH - 1); + interp->opt_config.specialization_enabled = !is_env_enabled("PYTHON_SPECIALIZATION_OFF"); interp->opt_config.uops_optimize_enabled = !is_env_disabled("PYTHON_UOPS_OPTIMIZE"); if (interp != &runtime->_main_interpreter) { diff --git a/Python/pystats.c b/Python/pystats.c index a057ad88456..2fac2db1b73 100644 --- a/Python/pystats.c +++ b/Python/pystats.c @@ -274,6 +274,7 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) fprintf(out, "Optimization low confidence: %" PRIu64 "\n", stats->low_confidence); fprintf(out, "Optimization unknown callee: %" PRIu64 "\n", stats->unknown_callee); fprintf(out, "Executors invalidated: %" PRIu64 "\n", stats->executors_invalidated); + fprintf(out, "Optimization fitness terminated: %" PRIu64 "\n", stats->fitness_terminated_traces); print_histogram(out, "Trace length", stats->trace_length_hist); print_histogram(out, "Trace run length", stats->trace_run_length_hist); diff --git a/Python/record_functions.c.h b/Python/record_functions.c.h index dff13bfb45e..504f6e1d990 100644 --- a/Python/record_functions.c.h +++ b/Python/record_functions.c.h @@ -103,19 +103,45 @@ void _PyOpcode_RecordFunction_CODE(_PyInterpreterFrame *frame, _PyStackRef *stac #define _RECORD_3OS_GEN_FUNC_INDEX 3 #define _RECORD_NOS_GEN_FUNC_INDEX 4 #define _RECORD_CALLABLE_INDEX 5 -#define _RECORD_BOUND_METHOD_INDEX 6 -#define _RECORD_CALLABLE_KW_INDEX 7 -#define _RECORD_4OS_INDEX 8 -#define _RECORD_NOS_TYPE_INDEX 9 +#define _RECORD_CALLABLE_KW_INDEX 6 +#define _RECORD_4OS_INDEX 7 const _PyOpcodeRecordEntry _PyOpcode_RecordEntries[256] = { + [TO_BOOL_BOOL] = {1, {_RECORD_TOS_TYPE_INDEX}}, + [TO_BOOL_NONE] = {1, {_RECORD_TOS_TYPE_INDEX}}, + [LOAD_SUPER_ATTR_ATTR] = {1, {_RECORD_NOS_INDEX}}, + [TO_BOOL] = {1, {_RECORD_TOS_TYPE_INDEX}}, + [TO_BOOL_INT] = {1, {_RECORD_TOS_TYPE_INDEX}}, + [TO_BOOL_LIST] = {1, {_RECORD_TOS_TYPE_INDEX}}, + [TO_BOOL_STR] = {1, {_RECORD_TOS_TYPE_INDEX}}, [TO_BOOL_ALWAYS_TRUE] = {1, {_RECORD_TOS_TYPE_INDEX}}, - [BINARY_OP_SUBSCR_GETITEM] = {1, {_RECORD_NOS_INDEX}}, + [BINARY_OP_MULTIPLY_INT] = {2, {_RECORD_NOS_INDEX, _RECORD_TOS_TYPE_INDEX}}, + [BINARY_OP_ADD_INT] = {2, {_RECORD_NOS_INDEX, _RECORD_TOS_TYPE_INDEX}}, + [BINARY_OP_SUBTRACT_INT] = {2, {_RECORD_NOS_INDEX, _RECORD_TOS_TYPE_INDEX}}, + [BINARY_OP_MULTIPLY_FLOAT] = {2, {_RECORD_NOS_INDEX, _RECORD_TOS_TYPE_INDEX}}, + [BINARY_OP_ADD_FLOAT] = {2, {_RECORD_NOS_INDEX, _RECORD_TOS_TYPE_INDEX}}, + [BINARY_OP_SUBTRACT_FLOAT] = {2, {_RECORD_NOS_INDEX, _RECORD_TOS_TYPE_INDEX}}, + [BINARY_OP_ADD_UNICODE] = {2, {_RECORD_NOS_INDEX, _RECORD_TOS_TYPE_INDEX}}, + [BINARY_OP_EXTEND] = {2, {_RECORD_NOS_INDEX, _RECORD_TOS_TYPE_INDEX}}, + [BINARY_OP_INPLACE_ADD_UNICODE] = {2, {_RECORD_NOS_INDEX, _RECORD_TOS_TYPE_INDEX}}, + [BINARY_OP_SUBSCR_LIST_INT] = {2, {_RECORD_NOS_INDEX, _RECORD_TOS_TYPE_INDEX}}, + [BINARY_OP_SUBSCR_LIST_SLICE] = {2, {_RECORD_NOS_INDEX, _RECORD_TOS_TYPE_INDEX}}, + [BINARY_OP_SUBSCR_STR_INT] = {2, {_RECORD_NOS_INDEX, _RECORD_TOS_TYPE_INDEX}}, + [BINARY_OP_SUBSCR_USTR_INT] = {2, {_RECORD_NOS_INDEX, _RECORD_TOS_TYPE_INDEX}}, + [BINARY_OP_SUBSCR_TUPLE_INT] = {2, {_RECORD_NOS_INDEX, _RECORD_TOS_TYPE_INDEX}}, + [BINARY_OP_SUBSCR_DICT] = {2, {_RECORD_NOS_INDEX, _RECORD_TOS_TYPE_INDEX}}, + [BINARY_OP_SUBSCR_GETITEM] = {2, {_RECORD_NOS_INDEX, _RECORD_TOS_TYPE_INDEX}}, + [SEND] = {1, {_RECORD_3OS_GEN_FUNC_INDEX}}, [SEND_GEN] = {1, {_RECORD_3OS_GEN_FUNC_INDEX}}, + [STORE_ATTR] = {1, {_RECORD_TOS_TYPE_INDEX}}, + [LOAD_SUPER_ATTR] = {1, {_RECORD_NOS_INDEX}}, [LOAD_SUPER_ATTR_METHOD] = {1, {_RECORD_NOS_INDEX}}, + [LOAD_ATTR] = {1, {_RECORD_TOS_TYPE_INDEX}}, [LOAD_ATTR_INSTANCE_VALUE] = {1, {_RECORD_TOS_TYPE_INDEX}}, + [LOAD_ATTR_MODULE] = {1, {_RECORD_TOS_TYPE_INDEX}}, [LOAD_ATTR_WITH_HINT] = {1, {_RECORD_TOS_TYPE_INDEX}}, [LOAD_ATTR_SLOT] = {1, {_RECORD_TOS_TYPE_INDEX}}, + [LOAD_ATTR_CLASS] = {1, {_RECORD_TOS_TYPE_INDEX}}, [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = {1, {_RECORD_TOS_TYPE_INDEX}}, [LOAD_ATTR_PROPERTY] = {1, {_RECORD_TOS_TYPE_INDEX}}, [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = {1, {_RECORD_TOS_TYPE_INDEX}}, @@ -125,6 +151,11 @@ const _PyOpcodeRecordEntry _PyOpcode_RecordEntries[256] = { [GET_ITER] = {1, {_RECORD_TOS_TYPE_INDEX}}, [GET_ITER_SELF] = {1, {_RECORD_TOS_TYPE_INDEX}}, [GET_ITER_VIRTUAL] = {1, {_RECORD_TOS_TYPE_INDEX}}, + [FOR_ITER] = {1, {_RECORD_NOS_GEN_FUNC_INDEX}}, + [FOR_ITER_VIRTUAL] = {1, {_RECORD_NOS_GEN_FUNC_INDEX}}, + [FOR_ITER_LIST] = {1, {_RECORD_NOS_GEN_FUNC_INDEX}}, + [FOR_ITER_TUPLE] = {1, {_RECORD_NOS_GEN_FUNC_INDEX}}, + [FOR_ITER_RANGE] = {1, {_RECORD_NOS_GEN_FUNC_INDEX}}, [FOR_ITER_GEN] = {1, {_RECORD_NOS_GEN_FUNC_INDEX}}, [LOAD_SPECIAL] = {1, {_RECORD_TOS_TYPE_INDEX}}, [LOAD_ATTR_METHOD_WITH_VALUES] = {1, {_RECORD_TOS_TYPE_INDEX}}, @@ -132,34 +163,104 @@ const _PyOpcodeRecordEntry _PyOpcode_RecordEntries[256] = { [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = {1, {_RECORD_TOS_TYPE_INDEX}}, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = {1, {_RECORD_TOS_TYPE_INDEX}}, [LOAD_ATTR_METHOD_LAZY_DICT] = {1, {_RECORD_TOS_TYPE_INDEX}}, + [CALL] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_PY_GENERAL] = {1, {_RECORD_CALLABLE_INDEX}}, - [CALL_BOUND_METHOD_GENERAL] = {1, {_RECORD_BOUND_METHOD_INDEX}}, + [CALL_BOUND_METHOD_GENERAL] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_NON_PY_GENERAL] = {1, {_RECORD_CALLABLE_INDEX}}, - [CALL_BOUND_METHOD_EXACT_ARGS] = {1, {_RECORD_BOUND_METHOD_INDEX}}, + [CALL_BOUND_METHOD_EXACT_ARGS] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_PY_EXACT_ARGS] = {1, {_RECORD_CALLABLE_INDEX}}, + [CALL_TYPE_1] = {1, {_RECORD_CALLABLE_INDEX}}, + [CALL_STR_1] = {1, {_RECORD_CALLABLE_INDEX}}, + [CALL_TUPLE_1] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_ALLOC_AND_ENTER_INIT] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_BUILTIN_CLASS] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_BUILTIN_O] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_BUILTIN_FAST] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = {1, {_RECORD_CALLABLE_INDEX}}, + [CALL_LEN] = {1, {_RECORD_CALLABLE_INDEX}}, + [CALL_ISINSTANCE] = {1, {_RECORD_CALLABLE_INDEX}}, + [CALL_LIST_APPEND] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_METHOD_DESCRIPTOR_O] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_METHOD_DESCRIPTOR_NOARGS] = {1, {_RECORD_CALLABLE_INDEX}}, + [CALL_METHOD_DESCRIPTOR_FAST] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_KW_PY] = {1, {_RECORD_CALLABLE_KW_INDEX}}, [CALL_KW_BOUND_METHOD] = {1, {_RECORD_CALLABLE_KW_INDEX}}, + [CALL_KW] = {1, {_RECORD_CALLABLE_KW_INDEX}}, + [CALL_KW_NON_PY] = {1, {_RECORD_CALLABLE_KW_INDEX}}, + [CALL_FUNCTION_EX] = {1, {_RECORD_4OS_INDEX}}, [CALL_EX_PY] = {1, {_RECORD_4OS_INDEX}}, - [BINARY_OP] = {2, {_RECORD_TOS_TYPE_INDEX, _RECORD_NOS_TYPE_INDEX}}, + [CALL_EX_NON_PY_GENERAL] = {1, {_RECORD_4OS_INDEX}}, + [BINARY_OP] = {2, {_RECORD_NOS_INDEX, _RECORD_TOS_TYPE_INDEX}}, }; -const _Py_RecordFuncPtr _PyOpcode_RecordFunctions[10] = { +const _PyOpcodeRecordSlotMap _PyOpcode_RecordSlotMaps[256] = { + [TO_BOOL_ALWAYS_TRUE] = {1, 0, {0}}, + [BINARY_OP_SUBSCR_GETITEM] = {1, 0, {0}}, + [SEND_GEN] = {1, 0, {0}}, + [LOAD_SUPER_ATTR_METHOD] = {1, 0, {0}}, + [LOAD_ATTR_INSTANCE_VALUE] = {1, 0, {0}}, + [LOAD_ATTR_WITH_HINT] = {1, 0, {0}}, + [LOAD_ATTR_SLOT] = {1, 0, {0}}, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = {1, 0, {0}}, + [LOAD_ATTR_PROPERTY] = {1, 0, {0}}, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = {1, 0, {0}}, + [STORE_ATTR_INSTANCE_VALUE] = {1, 0, {0}}, + [STORE_ATTR_WITH_HINT] = {1, 0, {0}}, + [STORE_ATTR_SLOT] = {1, 0, {0}}, + [GET_ITER] = {1, 0, {0}}, + [GET_ITER_SELF] = {1, 0, {0}}, + [GET_ITER_VIRTUAL] = {1, 0, {0}}, + [FOR_ITER_GEN] = {1, 0, {0}}, + [LOAD_SPECIAL] = {1, 0, {0}}, + [LOAD_ATTR_METHOD_WITH_VALUES] = {1, 0, {0}}, + [LOAD_ATTR_METHOD_NO_DICT] = {1, 0, {0}}, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = {1, 0, {0}}, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = {1, 0, {0}}, + [LOAD_ATTR_METHOD_LAZY_DICT] = {1, 0, {0}}, + [CALL_PY_GENERAL] = {1, 0, {0}}, + [CALL_BOUND_METHOD_GENERAL] = {1, 1, {0}}, + [CALL_NON_PY_GENERAL] = {1, 0, {0}}, + [CALL_BOUND_METHOD_EXACT_ARGS] = {1, 1, {0}}, + [CALL_PY_EXACT_ARGS] = {1, 0, {0}}, + [CALL_ALLOC_AND_ENTER_INIT] = {1, 0, {0}}, + [CALL_BUILTIN_CLASS] = {1, 0, {0}}, + [CALL_BUILTIN_O] = {1, 0, {0}}, + [CALL_BUILTIN_FAST] = {1, 0, {0}}, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = {1, 0, {0}}, + [CALL_METHOD_DESCRIPTOR_O] = {1, 0, {0}}, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = {1, 0, {0}}, + [CALL_METHOD_DESCRIPTOR_NOARGS] = {1, 0, {0}}, + [CALL_KW_PY] = {1, 0, {0}}, + [CALL_KW_BOUND_METHOD] = {1, 0, {0}}, + [CALL_EX_PY] = {1, 0, {0}}, + [BINARY_OP] = {2, 2, {1, 0}}, +}; + +const _Py_RecordFuncPtr _PyOpcode_RecordFunctions[8] = { [0] = NULL, [_RECORD_TOS_TYPE_INDEX] = _PyOpcode_RecordFunction_TOS_TYPE, [_RECORD_NOS_INDEX] = _PyOpcode_RecordFunction_NOS, [_RECORD_3OS_GEN_FUNC_INDEX] = _PyOpcode_RecordFunction_3OS_GEN_FUNC, [_RECORD_NOS_GEN_FUNC_INDEX] = _PyOpcode_RecordFunction_NOS_GEN_FUNC, [_RECORD_CALLABLE_INDEX] = _PyOpcode_RecordFunction_CALLABLE, - [_RECORD_BOUND_METHOD_INDEX] = _PyOpcode_RecordFunction_BOUND_METHOD, [_RECORD_CALLABLE_KW_INDEX] = _PyOpcode_RecordFunction_CALLABLE_KW, [_RECORD_4OS_INDEX] = _PyOpcode_RecordFunction_4OS, - [_RECORD_NOS_TYPE_INDEX] = _PyOpcode_RecordFunction_NOS_TYPE, }; + +PyObject * +_PyOpcode_RecordTransformValue(int uop, PyObject *value) +{ + switch (uop) { + case _RECORD_TOS_TYPE: + case _RECORD_NOS_TYPE: + return record_trace_transform_to_type(value); + case _RECORD_NOS_GEN_FUNC: + case _RECORD_3OS_GEN_FUNC: + return record_trace_transform_gen_func(value); + case _RECORD_BOUND_METHOD: + return record_trace_transform_bound_method(value); + default: + return value; + } +} diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 1ee0b3bec68..c6447d03369 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2707,7 +2707,7 @@ PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void) { PyAPI_FUNC(int) PyUnstable_WritePerfMapEntry( const void *code_addr, - unsigned int code_size, + size_t code_size, const char *entry_name ) { #ifndef MS_WINDOWS @@ -2718,7 +2718,7 @@ PyAPI_FUNC(int) PyUnstable_WritePerfMapEntry( } } PyThread_acquire_lock(perf_map_state.map_lock, 1); - fprintf(perf_map_state.perf_map, "%" PRIxPTR " %x %s\n", (uintptr_t) code_addr, code_size, entry_name); + fprintf(perf_map_state.perf_map, "%" PRIxPTR " %zx %s\n", (uintptr_t) code_addr, code_size, entry_name); fflush(perf_map_state.perf_map); PyThread_release_lock(perf_map_state.map_lock); #endif diff --git a/Python/traceback.c b/Python/traceback.c index 1e8c9c879f9..f0e0df7101b 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -55,7 +55,7 @@ #define MAX_STRING_LENGTH 500 #define MAX_FRAME_DEPTH 100 -#define MAX_NTHREADS 100 +#define DEFAULT_MAX_NTHREADS 100 /* Function from Parser/tokenizer/file_tokenizer.c */ extern char* _PyTokenizer_FindEncodingFilename(int, PyObject *); @@ -1265,8 +1265,13 @@ write_thread_id(int fd, PyThreadState *tstate, int is_current) handlers if signals were received. */ const char* _Py_NO_SANITIZE_THREAD _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, - PyThreadState *current_tstate) + PyThreadState *current_tstate, + Py_ssize_t max_threads) { + if (max_threads == 0) { + max_threads = DEFAULT_MAX_NTHREADS; + } + if (current_tstate == NULL) { /* _Py_DumpTracebackThreads() is called from signal handlers by faulthandler. @@ -1310,13 +1315,13 @@ _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, return "unable to get the thread head state"; /* Dump the traceback of each thread */ - unsigned int nthreads = 0; + Py_ssize_t nthreads = 0; _Py_BEGIN_SUPPRESS_IPH do { if (nthreads != 0) PUTS(fd, "\n"); - if (nthreads >= MAX_NTHREADS) { + if (nthreads >= max_threads) { PUTS(fd, "...\n"); break; } diff --git a/Tools/build/generate_levenshtein_examples.py b/Tools/build/generate_levenshtein_examples.py index 30dcc7cf1a1..2396c8040ca 100644 --- a/Tools/build/generate_levenshtein_examples.py +++ b/Tools/build/generate_levenshtein_examples.py @@ -13,7 +13,7 @@ _CASE_COST = 1 -def _substitution_cost(ch_a, ch_b): +def _substitution_cost(ch_a: str, ch_b: str) -> int: if ch_a == ch_b: return 0 if ch_a.lower() == ch_b.lower(): @@ -22,7 +22,7 @@ def _substitution_cost(ch_a, ch_b): @lru_cache(None) -def levenshtein(a, b): +def levenshtein(a: str, b: str) -> int: if not a or not b: return (len(a) + len(b)) * _MOVE_COST option1 = levenshtein(a[:-1], b[:-1]) + _substitution_cost(a[-1], b[-1]) @@ -31,7 +31,7 @@ def levenshtein(a, b): return min(option1, option2, option3) -def main(): +def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('output_path', metavar='FILE', type=str) parser.add_argument('--overwrite', dest='overwrite', action='store_const', @@ -48,7 +48,7 @@ def main(): ) return - examples = set() + examples: set[tuple[str, str, int]] = set() # Create a lot of non-empty examples, which should end up with a Gauss-like # distribution for even costs (moves) and odd costs (case substitutions). while len(examples) < 9990: diff --git a/Tools/build/mypy.ini b/Tools/build/mypy.ini index 7d341afd1cd..5465e2d4b61 100644 --- a/Tools/build/mypy.ini +++ b/Tools/build/mypy.ini @@ -9,6 +9,7 @@ files = Tools/build/consts_getter.py, Tools/build/deepfreeze.py, Tools/build/generate-build-details.py, + Tools/build/generate_levenshtein_examples.py, Tools/build/generate_sbom.py, Tools/build/generate_stdlib_module_names.py, Tools/build/verify_ensurepip_wheels.py, diff --git a/Tools/build/smelly.py b/Tools/build/smelly.py index 7197d70bc8b..17547d4d916 100755 --- a/Tools/build/smelly.py +++ b/Tools/build/smelly.py @@ -25,6 +25,8 @@ # "Legacy": some old symbols are prefixed by "PY_". EXCEPTIONS = frozenset({ 'PY_TIMEOUT_MAX', + '__jit_debug_descriptor', + '__jit_debug_register_code', }) IGNORED_EXTENSION = "_ctypes_test" diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index a251a045b91..a16d5773d55 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -324,8 +324,10 @@ def format_tsv_lines(lines): _abs('Objects/stringlib/unicode_format.h'): (10_000, 400), _abs('Objects/typeobject.c'): (380_000, 13_000), _abs('Python/compile.c'): (20_000, 500), + _abs('Python/jit_unwind.c'): (20_000, 300), _abs('Python/optimizer.c'): (100_000, 5_000), _abs('Python/parking_lot.c'): (40_000, 1000), + _abs('Python/perf_jit_trampoline.c'): (40_000, 1000), _abs('Python/pylifecycle.c'): (750_000, 5000), _abs('Python/pystate.c'): (750_000, 5000), _abs('Python/initconfig.c'): (50_000, 500), diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 74ca5628240..db575d870be 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -83,6 +83,7 @@ Objects/picklebufobject.c - PyPickleBuffer_Type - Objects/rangeobject.c - PyLongRangeIter_Type - Objects/rangeobject.c - PyRangeIter_Type - Objects/rangeobject.c - PyRange_Type - +Objects/sentinelobject.c - PySentinel_Type - Objects/setobject.c - PyFrozenSet_Type - Objects/setobject.c - PySetIter_Type - Objects/setobject.c - PySet_Type - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index d2489387f46..aa89e312b62 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -386,6 +386,8 @@ Python/intrinsics.c - _PyIntrinsics_UnaryFunctions - Python/intrinsics.c - _PyIntrinsics_BinaryFunctions - Python/lock.c - TIME_TO_BE_FAIR_NS - Python/opcode_targets.h - opcode_targets - +Python/jit_unwind.c - __jit_debug_descriptor - +Python/jit_unwind.c - _Py_jit_debug_mutex - Python/perf_trampoline.c - _Py_perfmap_callbacks - Python/perf_jit_trampoline.c - _Py_perfmap_jit_callbacks - Python/perf_jit_trampoline.c - perf_jit_map_state - diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 784e1a8a2d1..aa914783f7c 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -258,8 +258,7 @@ def replace_opcode_if_evaluates_pure( # usually for binary ops with passthrough references 2: [("_LOAD_CONST_INLINE_BORROW", "0, (uintptr_t)result"), - ("_SWAP", "3, 0"), - ("_SWAP", "2, 0")], + ("_RROT_3", "0, 0")], }, } diff --git a/Tools/cases_generator/record_function_generator.py b/Tools/cases_generator/record_function_generator.py index d7ae0ebf79f..6f518ffdcf2 100644 --- a/Tools/cases_generator/record_function_generator.py +++ b/Tools/cases_generator/record_function_generator.py @@ -28,6 +28,21 @@ # Must match MAX_RECORDED_VALUES in Include/internal/pycore_optimizer.h. MAX_RECORDED_VALUES = 3 +# Map `_RECORD_*` uops to the helper that converts a raw family-recorded +# value to the form the specialized member consumes. +_RECORD_TRANSFORM_HELPERS: dict[str, str] = { + "_RECORD_TOS_TYPE": "record_trace_transform_to_type", + "_RECORD_NOS_TYPE": "record_trace_transform_to_type", + "_RECORD_NOS_GEN_FUNC": "record_trace_transform_gen_func", + "_RECORD_3OS_GEN_FUNC": "record_trace_transform_gen_func", + "_RECORD_BOUND_METHOD": "record_trace_transform_bound_method", +} + +# Recorder uops whose slot kind differs from the leading word of their name. +_RECORD_SLOT_KIND_OVERRIDES: dict[str, str] = { + "_RECORD_BOUND_METHOD": "CALLABLE", +} + class RecorderEmitter(Emitter): def __init__(self, out: CWriter): @@ -52,9 +67,83 @@ def record_value( return True +def get_record_slot_kind(record_name: str) -> str: + if record_name in _RECORD_SLOT_KIND_OVERRIDES: + return _RECORD_SLOT_KIND_OVERRIDES[record_name] + if not record_name.startswith("_RECORD_"): + return record_name + return record_name.removeprefix("_RECORD_").partition("_")[0] + + +def get_instruction_record_names(inst: Instruction) -> list[str]: + return [part.name for part in inst.parts if part.properties.records_value] + + +def get_family_record_names( + family_head: Instruction, + family_members: list[Instruction], + instruction_records: dict[str, list[str]], + record_slot_keys: dict[str, str], +) -> list[str]: + member_records = [instruction_records[m.name] for m in family_members] + all_member_names = {n for names in member_records for n in names} + records: list[str] = [] + slot_index: dict[str, int] = {} + + def add(name: str) -> None: + kind = record_slot_keys[name] + # Prefer the raw recorder if any member uses it; otherwise the given form. + raw = f"_RECORD_{kind}" + source = raw if raw in all_member_names else name + existing = slot_index.get(kind) + if existing is None: + slot_index[kind] = len(records) + records.append(source) + elif records[existing] != source: + raise ValueError( + f"Family {family_head.name} has incompatible recorders for " + f"slot {kind}: {records[existing]} and {source}" + ) + + for names in member_records: + for name in names: + add(name) + # Family head supplies any slots no member exercises. + for name in instruction_records[family_head.name]: + if record_slot_keys[name] not in slot_index: + slot_index[record_slot_keys[name]] = len(records) + records.append(name) + return records + + +def get_record_consumer_layout( + inst_name: str, + source_records: list[str], + own_records: list[str], + record_slot_keys: dict[str, str], +) -> tuple[list[int], int]: + used = [False] * len(source_records) + slot_map: list[int] = [] + transform_mask = 0 + for i, own in enumerate(own_records): + own_kind = record_slot_keys[own] + for j, src in enumerate(source_records): + if not used[j] and record_slot_keys[src] == own_kind: + used[j] = True + slot_map.append(j) + if src != own: + transform_mask |= 1 << i + break + else: + raise ValueError( + f"Instruction {inst_name} has no compatible family slot for " + f"{own} in {source_records}" + ) + return slot_map, transform_mask + def generate_recorder_functions(filenames: list[str], analysis: Analysis, out: CWriter) -> None: - write_header(__file__, filenames, outfile) - outfile.write( + write_header(__file__, filenames, out.out) + out.out.write( """ #ifdef TIER_ONE #error "This file is for Tier 2 only" @@ -63,13 +152,10 @@ def generate_recorder_functions(filenames: list[str], analysis: Analysis, out: C ) args = "_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int oparg, PyObject **recorded_value" emitter = RecorderEmitter(out) - func_count = 0 nop = analysis.instructions["NOP"] - function_table: dict[str, int] = dict() - for name, uop in analysis.uops.items(): + for uop in analysis.uops.values(): if not uop.properties.records_value: continue - func_count += 1 out.emit(f"void _PyOpcode_RecordFunction{uop.name[7:]}({args}) {{\n") seen = {"unused"} for var in uop.stack.inputs: @@ -83,42 +169,109 @@ def generate_recorder_functions(filenames: list[str], analysis: Analysis, out: C out.emit("\n\n") def generate_recorder_tables(analysis: Analysis, out: CWriter) -> None: - record_function_indexes: dict[str, int] = dict() + instruction_records = { + inst.name: get_instruction_record_names(inst) + for inst in analysis.instructions.values() + } + record_uop_names = [ + name for name, uop in analysis.uops.items() if uop.properties.records_value + ] + record_slot_keys = {name: get_record_slot_kind(name) for name in record_uop_names} + family_record_table = { + family.name: get_family_record_names( + analysis.instructions[family.name], + family.members, + instruction_records, + record_slot_keys, + ) + for family in analysis.families.values() + } + record_table: dict[str, list[str]] = {} - index = 1 + record_consumer_table: dict[str, tuple[list[int], int]] = {} + record_function_indexes: dict[str, int] = {} for inst in analysis.instructions.values(): - if not inst.properties.records_value: + own_records = instruction_records[inst.name] + # TRACE_RECORD runs before execution, but specialization may rewrite + # the opcode before translation. Record the raw family shape (union + # of head + members) so any opcode in the family can be translated + # from the same recorded layout. + family = inst.family or analysis.families.get(inst.name) + records = family_record_table[family.name] if family is not None else own_records + if not records: continue - records: list[str] = [] - for part in inst.parts: - if not part.properties.records_value: - continue - if part.name not in record_function_indexes: - record_function_indexes[part.name] = index - index += 1 - records.append(part.name) - if records: - if len(records) > MAX_RECORDED_VALUES: - raise ValueError( - f"Instruction {inst.name} has {len(records)} recording ops, " - f"exceeds MAX_RECORDED_VALUES ({MAX_RECORDED_VALUES})" - ) - record_table[inst.name] = records - func_count = len(record_function_indexes) + if len(records) > MAX_RECORDED_VALUES: + raise ValueError( + f"Instruction {inst.name} has {len(records)} recording ops, " + f"exceeds MAX_RECORDED_VALUES ({MAX_RECORDED_VALUES})" + ) + record_table[inst.name] = records + for name in records: + if name not in record_function_indexes: + record_function_indexes[name] = len(record_function_indexes) + 1 + if own_records: + record_consumer_table[inst.name] = get_record_consumer_layout( + inst.name, records, own_records, record_slot_keys + ) for name, index in record_function_indexes.items(): out.emit(f"#define {name}_INDEX {index}\n") out.emit("\n") + out.emit("const _PyOpcodeRecordEntry _PyOpcode_RecordEntries[256] = {\n") - for inst_name, record_names in record_table.items(): - indices = ", ".join(f"{name}_INDEX" for name in record_names) - out.emit(f" [{inst_name}] = {{{len(record_names)}, {{{indices}}}}},\n") + for inst_name, records in record_table.items(): + indices = ", ".join(f"{name}_INDEX" for name in records) + out.emit(f" [{inst_name}] = {{{len(records)}, {{{indices}}}}},\n") out.emit("};\n\n") - out.emit(f"const _Py_RecordFuncPtr _PyOpcode_RecordFunctions[{func_count+1}] = {{\n") + + out.emit("const _PyOpcodeRecordSlotMap _PyOpcode_RecordSlotMaps[256] = {\n") + for inst_name, (slots, mask) in record_consumer_table.items(): + slot_list = ", ".join(str(s) for s in slots) + out.emit( + f" [{inst_name}] = {{{len(slots)}, {mask}, {{{slot_list}}}}},\n" + ) + out.emit("};\n\n") + + out.emit( + f"const _Py_RecordFuncPtr _PyOpcode_RecordFunctions" + f"[{len(record_function_indexes) + 1}] = {{\n" + ) out.emit(" [0] = NULL,\n") for name in record_function_indexes: out.emit(f" [{name}_INDEX] = _PyOpcode_RecordFunction{name[7:]},\n") out.emit("};\n") + generate_record_transform_dispatcher(record_uop_names, out) + + +def generate_record_transform_dispatcher( + record_uop_names: list[str], out: CWriter +) -> None: + """Emit a switch that converts a family-recorded value for a recorder uop. + + Only `_RECORD_*` uops that need conversion get a case; the default + returns the input value unchanged. Helpers live in Python/optimizer.c. + """ + cases: dict[str, list[str]] = {} + for record_name in record_uop_names: + helper = _RECORD_TRANSFORM_HELPERS.get(record_name) + if helper is None: + continue + cases.setdefault(helper, []).append(record_name) + out.emit("\n") + out.emit( + "PyObject *\n" + "_PyOpcode_RecordTransformValue(int uop, PyObject *value)\n" + "{\n" + ) + out.emit(" switch (uop) {\n") + for helper, names in cases.items(): + for name in names: + out.emit(f" case {name}:\n") + out.emit(f" return {helper}(value);\n") + out.emit(" default:\n") + out.emit(" return value;\n") + out.emit(" }\n") + out.emit("}\n") arg_parser = argparse.ArgumentParser( diff --git a/Tools/jit/README.md b/Tools/jit/README.md index 8eadb3349ba..2a687bb9e89 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -9,7 +9,12 @@ ## Installing LLVM The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). -LLVM version 21 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. +LLVM version 21 is the officially supported version. The tools `clang`, `llvm-readobj`, `llvm-objdump`, and `llvm-dwarfdump` need to be installed and discoverable (version suffixes, like `clang-21`, are okay). + +You can customize the LLVM configuration using environment variables before running configure: + +- LLVM_VERSION: Specify a different LLVM version (default: 21) +- LLVM_TOOLS_INSTALL_DIR: Point to a specific LLVM installation prefix when multiple installations exist (the tools are expected in `

/bin`) It's easy to install all of the required tools: @@ -62,7 +67,7 @@ ### Windows ### Dev Containers -If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no +If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no need to install LLVM as the Fedora 43 base image includes LLVM 21 out of the box. ## Building @@ -80,4 +85,9 @@ ## Miscellaneous [^pep-744]: [PEP 744](https://peps.python.org/pep-0744/) -[^why-llvm]: Clang is specifically needed because it's the only C compiler with support for guaranteed tail calls (`musttail`), which are required by CPython's continuation-passing-style approach to JIT compilation. Since LLVM also includes other functionalities we need (namely, object file parsing and disassembly), it's convenient to only support one toolchain at this time. +[^why-llvm]: Clang is specifically needed because it's the only C compiler with support for guaranteed tail calls (`musttail`), which are required by CPython's continuation-passing-style approach to JIT compilation. Since LLVM also includes other functionalities we need (namely, object file parsing, disassembly, and DWARF inspection), it's convenient to only support one toolchain at this time. + +### Understanding JIT behavior + +The [example_trace_dump.py](./example_trace_dump.py) script will (when configured as described in the script) dump out the +executors for a range of tiny programs to show the behavior of the JIT front-end. \ No newline at end of file diff --git a/Tools/jit/_dwarf.py b/Tools/jit/_dwarf.py new file mode 100644 index 00000000000..5b6b148562e --- /dev/null +++ b/Tools/jit/_dwarf.py @@ -0,0 +1,236 @@ +"""Utilities for deriving JIT unwind information from DWARF CFI.""" + +import dataclasses +import pathlib +import re +import typing + +_LLVMRun = typing.Callable[..., typing.Awaitable[str]] + + +@dataclasses.dataclass(frozen=True) +class UnwindInfo: + code_alignment_factor: int + data_alignment_factor: int + return_address_register: int + cfa_register: int + cfa_offset: int + frame_pointer_register: int + frame_pointer_offset: int + return_address_offset: int + + +@dataclasses.dataclass(frozen=True) +class ELFUnwindConfig: + frame_pointer: str + return_address: str + register_numbers: typing.Mapping[str, int] + call_instruction_prefixes: tuple[str, ...] + + def is_call_instruction(self, instruction: str) -> bool: + return instruction.startswith(self.call_instruction_prefixes) + + +@dataclasses.dataclass(frozen=True) +class _UnwindRow: + pc: int + cfa_register: str + cfa_offset: int + saved_registers: dict[str, int] + + +class ELFUnwindInfo: + def __init__( + self, + target_name: str, + *, + config: ELFUnwindConfig, + verbose: bool = False, + llvm_version: str, + llvm_tools_install_dir: str | None = None, + llvm_run: _LLVMRun, + ) -> None: + self.target_name = target_name + self.config = config + self.verbose = verbose + self.llvm_version = llvm_version + self.llvm_tools_install_dir = llvm_tools_install_dir + self.llvm_run = llvm_run + + @staticmethod + def _parse_dwarfdump_int( + dump: str, field: str, *, required: bool = True + ) -> int | None: + match = re.search(rf"^\s*{field}:\s+(-?\d+)$", dump, re.MULTILINE) + if match is None: + if required: + raise ValueError(f"missing {field} in llvm-dwarfdump output") + return None + return int(match.group(1)) + + @staticmethod + def _parse_dwarfdump_rows(dump: str) -> list[_UnwindRow]: + row_pattern = re.compile( + r"^\s*0x(?P[0-9a-f]+):\s+" + r"CFA=(?P[A-Z][A-Z0-9]*)" + r"(?P[+-]\d+)?" + r"(?::\s*(?P.*))?$" + ) + saved_pattern = re.compile( + r"(?P[A-Z][A-Z0-9]*)=\[CFA(?P[+-]\d+)?\]" + ) + rows = [] + for line in dump.splitlines(): + row_match = row_pattern.match(line) + if row_match is None: + continue + saved_registers = {} + saved = row_match["saved"] + if saved: + for saved_match in saved_pattern.finditer(saved): + offset = saved_match["offset"] + saved_registers[saved_match["register"]] = ( + int(offset) if offset is not None else 0 + ) + cfa_offset = row_match["cfa_offset"] + rows.append( + _UnwindRow( + pc=int(row_match["pc"], 16), + cfa_register=row_match["cfa_register"], + cfa_offset=int(cfa_offset) if cfa_offset is not None else 0, + saved_registers=saved_registers, + ) + ) + if not rows: + raise ValueError("missing interpreted CFI rows in llvm-dwarfdump output") + return rows + + @staticmethod + def _parse_objdump_instructions(dump: str) -> list[tuple[int, str]]: + instructions = [] + for line in dump.splitlines(): + match = re.match( + r"^\s*(?P[0-9a-f]+):\s+" + r"(?:(?:[0-9a-f]{2}|[0-9a-f]{8})\s+)+" + r"(?P.+)$", + line, + ) + if match: + instructions.append( + ( + int(match["pc"], 16), + re.sub(r"\s+", " ", match["instruction"].strip()), + ) + ) + if not instructions: + raise ValueError("missing instructions in llvm-objdump output") + return instructions + + def _reg_number(self, register: str) -> int: + try: + return self.config.register_numbers[register] + except KeyError as exc: + raise ValueError( + f"unsupported register {register!r} in llvm-dwarfdump output" + ) from exc + + @staticmethod + def _encoded_cfa_offset(byte_offset: int, data_alignment_factor: int) -> int: + if data_alignment_factor == 0: + raise ValueError("DWARF data alignment factor must not be zero") + if byte_offset % data_alignment_factor: + raise ValueError( + f"offset {byte_offset} is not a multiple of " + f"data alignment factor {data_alignment_factor}" + ) + return byte_offset // data_alignment_factor + + async def _read_objdump(self, output: pathlib.Path) -> str: + return await self.llvm_run( + "llvm-objdump", + ["-d", f"{output}"], + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, + ) + + async def _read_eh_frame(self, output: pathlib.Path) -> str: + return await self.llvm_run( + "llvm-dwarfdump", + ["--eh-frame", f"{output}"], + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, + ) + + def _executor_call_pc(self, disassembly: str) -> int: + calls = [ + pc + for pc, instruction in self._parse_objdump_instructions(disassembly) + if self.config.is_call_instruction(instruction) + ] + if len(calls) != 1: + raise ValueError( + f"{self.target_name} JIT shim should contain exactly one executor call" + ) + call_pc = calls[0] + return call_pc + + def _active_row(self, eh_frame: str, call_pc: int) -> _UnwindRow: + rows = self._parse_dwarfdump_rows(eh_frame) + active_rows = [row for row in rows if row.pc <= call_pc] + if not active_rows: + raise ValueError( + f"{self.target_name} JIT shim has no CFI row for executor call " + f"at 0x{call_pc:x}" + ) + return max(active_rows, key=lambda row: row.pc) + + def _check_saved_registers(self, row: _UnwindRow) -> None: + if ( + self.config.frame_pointer not in row.saved_registers + or self.config.return_address not in row.saved_registers + ): + raise ValueError( + f"{self.target_name} JIT shim CFI row at 0x{row.pc:x} " + f"does not save {self.config.frame_pointer} and " + f"{self.config.return_address}" + ) + + def _build_unwind_info(self, eh_frame: str, active_row: _UnwindRow) -> UnwindInfo: + code_alignment_factor = self._parse_dwarfdump_int( + eh_frame, "Code alignment factor" + ) + data_alignment_factor = self._parse_dwarfdump_int( + eh_frame, "Data alignment factor" + ) + return_address_register = self._parse_dwarfdump_int( + eh_frame, "Return address column" + ) + assert code_alignment_factor is not None + assert data_alignment_factor is not None + assert return_address_register is not None + return UnwindInfo( + code_alignment_factor=code_alignment_factor, + data_alignment_factor=data_alignment_factor, + return_address_register=return_address_register, + cfa_register=self._reg_number(active_row.cfa_register), + cfa_offset=active_row.cfa_offset, + frame_pointer_register=self._reg_number(self.config.frame_pointer), + frame_pointer_offset=self._encoded_cfa_offset( + active_row.saved_registers[self.config.frame_pointer], + data_alignment_factor, + ), + return_address_offset=self._encoded_cfa_offset( + active_row.saved_registers[self.config.return_address], + data_alignment_factor, + ), + ) + + async def extract(self, output: pathlib.Path) -> UnwindInfo: + disassembly = await self._read_objdump(output) + call_pc = self._executor_call_pc(disassembly) + eh_frame = await self._read_eh_frame(output) + active_row = self._active_row(eh_frame, call_pc) + self._check_saved_registers(active_row) + return self._build_unwind_info(eh_frame, active_row) diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index a4aaacdf412..96cf5fc4714 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -69,7 +69,9 @@ async def _check_tool_version( name: str, llvm_version: str, *, echo: bool = False ) -> bool: output = await _run(name, ["--version"], echo=echo) - _llvm_version_pattern = re.compile(rf"version\s+{llvm_version}\.\d+\.\d+\S*\s+") + _llvm_version_pattern = re.compile( + rf"(? str @_async_cache -async def _find_tool(tool: str, llvm_version: str, *, echo: bool = False) -> str | None: +async def _find_tool( + tool: str, + llvm_version: str, + llvm_tools_install_dir: str | None, + *, + echo: bool = False, +) -> str | None: + # Explicitly defined LLVM installation location + if llvm_tools_install_dir: + path = os.path.join(llvm_tools_install_dir, "bin", tool) + if await _check_tool_version(path, llvm_version, echo=echo): + return path # Unversioned executables: path = tool if await _check_tool_version(path, llvm_version, echo=echo): @@ -114,10 +127,11 @@ async def maybe_run( args: typing.Iterable[str], echo: bool = False, llvm_version: str = _LLVM_VERSION, + llvm_tools_install_dir: str | None = None, ) -> str | None: """Run an LLVM tool if it can be found. Otherwise, return None.""" - path = await _find_tool(tool, llvm_version, echo=echo) + path = await _find_tool(tool, llvm_version, llvm_tools_install_dir, echo=echo) return path and await _run(path, args, echo=echo) @@ -126,10 +140,17 @@ async def run( args: typing.Iterable[str], echo: bool = False, llvm_version: str = _LLVM_VERSION, + llvm_tools_install_dir: str | None = None, ) -> str: """Run an LLVM tool if it can be found. Otherwise, raise RuntimeError.""" - output = await maybe_run(tool, args, echo=echo, llvm_version=llvm_version) + output = await maybe_run( + tool, + args, + echo=echo, + llvm_version=llvm_version, + llvm_tools_install_dir=llvm_tools_install_dir, + ) if output is None: raise RuntimeError(f"Can't find {tool}-{llvm_version}!") return output diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index fd5c143b8a8..ceee383ea68 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -12,6 +12,7 @@ import typing import shlex +import _dwarf import _llvm import _optimizers import _schema @@ -37,6 +38,27 @@ ) +_ELF_UNWIND_AARCH64 = _dwarf.ELFUnwindConfig( + frame_pointer="W29", + return_address="W30", + register_numbers={ + "W29": 29, + "W30": 30, + }, + call_instruction_prefixes=("blr ",), +) + +_ELF_UNWIND_X86_64 = _dwarf.ELFUnwindConfig( + frame_pointer="RBP", + return_address="RIP", + register_numbers={ + "RBP": 6, + "RIP": 16, + }, + call_instruction_prefixes=("callq ", "call "), +) + + @dataclasses.dataclass class _Target(typing.Generic[_S, _R]): triple: str @@ -52,10 +74,18 @@ class _Target(typing.Generic[_S, _R]): verbose: bool = False cflags: str = "" frame_pointers: bool = False + unwind: _dwarf.ELFUnwindConfig | None = None llvm_version: str = _llvm._LLVM_VERSION + llvm_tools_install_dir: str | None = None known_symbols: dict[str, int] = dataclasses.field(default_factory=dict) pyconfig_dir: pathlib.Path = pathlib.Path.cwd().resolve() + def _compile_args(self) -> list[str]: + return list(self.args) + + def _shim_compile_args(self) -> list[str]: + return [] + def _get_nop(self) -> bytes: if re.fullmatch(r"aarch64-.*", self.triple): nop = b"\x1f\x20\x03\xd5" @@ -81,11 +111,41 @@ def _compute_digest(self) -> str: hasher.update(pathlib.Path(dirpath, filename).read_bytes()) return hasher.hexdigest() + def _write_generated_header( + self, + output: pathlib.Path, + *, + digest: str, + comment: str, + lines: typing.Iterable[str], + ) -> None: + output_new = output.with_name(f"{output.name}.new") + try: + with output_new.open("w") as file: + file.write(digest) + if comment: + file.write(f"// {comment}\n") + file.write("\n") + for line in lines: + file.write(f"{line}\n") + try: + output_new.replace(output) + except FileNotFoundError: + # another process probably already moved the file + if not output.is_file(): + raise + finally: + output_new.unlink(missing_ok=True) + async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: group = _stencils.StencilGroup() args = ["--disassemble", "--reloc", f"{path}"] output = await _llvm.maybe_run( - "llvm-objdump", args, echo=self.verbose, llvm_version=self.llvm_version + "llvm-objdump", + args, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) if output is not None: # Make sure that full paths don't leak out (for reproducibility): @@ -105,7 +165,11 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: f"{path}", ] output = await _llvm.run( - "llvm-readobj", args, echo=self.verbose, llvm_version=self.llvm_version + "llvm-readobj", + args, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) # --elf-output-style=JSON is only *slightly* broken on Mach-O... output = output.replace("PrivateExtern\n", "\n") @@ -130,12 +194,8 @@ def _handle_relocation( ) -> _stencils.Hole: raise NotImplementedError(type(self)) - async def _compile( - self, opname: str, c: pathlib.Path, tempdir: pathlib.Path - ) -> _stencils.StencilGroup: - s = tempdir / f"{opname}.s" - o = tempdir / f"{opname}.o" - args_s = [ + def _base_clang_args(self, opname: str, tempdir: pathlib.Path) -> list[str]: + return [ f"--target={self.triple}", "-DPy_BUILD_CORE_MODULE", "-D_DEBUG" if self.debug else "-DNDEBUG", @@ -158,48 +218,93 @@ async def _compile( # generates better code than -O2 (and -O2 usually generates better # code than -O3). As a nice benefit, it uses less memory too: "-Os", - "-S", # Shorten full absolute file paths in the generated code (like the # __FILE__ macro and assert failure messages) for reproducibility: f"-ffile-prefix-map={CPYTHON}=.", f"-ffile-prefix-map={tempdir}=.", - # This debug info isn't necessary, and bloats out the JIT'ed code. - # We *may* be able to re-enable this, process it, and JIT it for a - # nicer debugging experience... but that needs a lot more research: - "-fno-asynchronous-unwind-tables", # Don't call built-in functions that we can't find or patch: "-fno-builtin", # Don't call stack-smashing canaries that we can't find or patch: "-fno-stack-protector", "-std=c11", + ] + + async def _build_stencil_group( + self, opname: str, c: pathlib.Path, tempdir: pathlib.Path + ) -> _stencils.StencilGroup: + s = tempdir / f"{opname}.s" + o = tempdir / f"{opname}.o" + args_s = self._base_clang_args(opname, tempdir) + args_s += [ + "-S", + # Stencils do not need unwind info, and the optimizer does not + # preserve .cfi_* directives correctly. On Darwin, + # -fno-asynchronous-unwind-tables alone still leaves synchronous + # unwind directives in the assembly, so disable both forms here. + "-fno-unwind-tables", + "-fno-asynchronous-unwind-tables", "-o", f"{s}", f"{c}", ] - is_shim = opname == "shim" if self.frame_pointers: - frame_pointer = "all" if is_shim else "reserved" - args_s += ["-Xclang", f"-mframe-pointer={frame_pointer}"] - args_s += self.args + args_s += ["-Xclang", "-mframe-pointer=reserved"] + args_s += self._compile_args() # Allow user-provided CFLAGS to override any defaults args_s += shlex.split(self.cflags) await _llvm.run( - "clang", args_s, echo=self.verbose, llvm_version=self.llvm_version + "clang", + args_s, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) - if not is_shim: - self.optimizer( - s, - label_prefix=self.label_prefix, - symbol_prefix=self.symbol_prefix, - re_global=self.re_global, - frame_pointers=self.frame_pointers, - ).run() + self.optimizer( + s, + label_prefix=self.label_prefix, + symbol_prefix=self.symbol_prefix, + re_global=self.re_global, + frame_pointers=self.frame_pointers, + ).run() args_o = [f"--target={self.triple}", "-c", "-o", f"{o}", f"{s}"] await _llvm.run( - "clang", args_o, echo=self.verbose, llvm_version=self.llvm_version + "clang", + args_o, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) return await self._parse(o) + async def _build_shim_object(self, output: pathlib.Path) -> None: + with tempfile.TemporaryDirectory() as tempdir: + work = pathlib.Path(tempdir).resolve() + args_o = self._base_clang_args("shim", work) + args_o += self._shim_compile_args() + args_o += [ + "-c", + # The shim is a real function in the final binary, so + # keep unwind info for debuggers and stack walkers. + "-fasynchronous-unwind-tables", + ] + if self.frame_pointers: + args_o += ["-Xclang", "-mframe-pointer=all"] + args_o += self._compile_args() + args_o += shlex.split(self.cflags) + args_o += ["-o", f"{output}", f"{TOOLS_JIT / 'shim.c'}"] + await _llvm.run( + "clang", + args_o, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, + ) + + async def _get_shim_unwind_info( + self, output: pathlib.Path + ) -> _dwarf.UnwindInfo | None: + return None + async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: generated_cases = PYTHON_EXECUTOR_CASES_C_H.read_text() cases_and_opnames = sorted( @@ -214,8 +319,6 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: with tempfile.TemporaryDirectory() as tempdir: work = pathlib.Path(tempdir).resolve() async with asyncio.TaskGroup() as group: - coro = self._compile("shim", TOOLS_JIT / "shim.c", work) - tasks.append(group.create_task(coro, name="shim")) template = TOOLS_JIT_TEMPLATE_C.read_text() for case, opname in cases_and_opnames: # Write out a copy of the template with *only* this case @@ -225,7 +328,7 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: # all of the other cases): c = work / f"{opname}.c" c.write_text(template.replace("CASE", case)) - coro = self._compile(opname, c, work) + coro = self._build_stencil_group(opname, c, work) tasks.append(group.create_task(coro, name=opname)) stencil_groups = {task.get_name(): task.result() for task in tasks} for stencil_group in stencil_groups.values(): @@ -239,8 +342,10 @@ def build( comment: str = "", force: bool = False, jit_stencils: pathlib.Path, + jit_shim_object: pathlib.Path, + jit_unwind_info: pathlib.Path, ) -> None: - """Build jit_stencils.h in the given directory.""" + """Build jit_stencils.h and the shim object in the given directory.""" jit_stencils.parent.mkdir(parents=True, exist_ok=True) if not self.stable: warning = f"JIT support for {self.triple} is still experimental!" @@ -250,35 +355,47 @@ def build( outline = "=" * len(warning) print("\n".join(["", outline, warning, request, outline, ""])) digest = f"// {self._compute_digest()}\n" + # The generated headers include the input digest as their first line. + # If every generated artifact is current, skip the expensive rebuild. if ( not force and jit_stencils.exists() and jit_stencils.read_text().startswith(digest) + and jit_shim_object.exists() + and jit_unwind_info.exists() + and jit_unwind_info.read_text().startswith(digest) ): return + # Build the shim first so its compiled DWARF CFI can be used to derive + # the unwind rules emitted into jit_unwind_info-.h. + ASYNCIO_RUNNER.run(self._build_shim_object(jit_shim_object)) + unwind_info = ASYNCIO_RUNNER.run(self._get_shim_unwind_info(jit_shim_object)) + self._write_generated_header( + jit_unwind_info, + digest=digest, + comment=comment, + lines=_writer.dump_unwind_info(unwind_info), + ) + # Build the uop stencils after the shim metadata has been emitted. stencil_groups = ASYNCIO_RUNNER.run(self._build_stencils()) - jit_stencils_new = jit_stencils.parent / "jit_stencils.h.new" - try: - with jit_stencils_new.open("w") as file: - file.write(digest) - if comment: - file.write(f"// {comment}\n") - file.write("\n") - for line in _writer.dump(stencil_groups, self.known_symbols): - file.write(f"{line}\n") - try: - jit_stencils_new.replace(jit_stencils) - except FileNotFoundError: - # another process probably already moved the file - if not jit_stencils.is_file(): - raise - finally: - jit_stencils_new.unlink(missing_ok=True) + self._write_generated_header( + jit_stencils, + digest=digest, + comment=comment, + lines=_writer.dump(stencil_groups, self.known_symbols), + ) class _COFF( _Target[_schema.COFFSection, _schema.COFFRelocation] ): # pylint: disable = too-few-public-methods + def _shim_compile_args(self) -> list[str]: + # The shim is part of pythoncore, not a shared extension. + # On Windows, Py_BUILD_CORE_MODULE makes public APIs import from + # pythonXY.lib, which creates a self-dependency when linking + # pythoncore.dll. Build the shim with builtin/core semantics. + return ["-UPy_BUILD_CORE_MODULE", "-DPy_BUILD_CORE_BUILTIN"] + def _handle_section( self, section: _schema.COFFSection, group: _stencils.StencilGroup ) -> None: @@ -379,6 +496,10 @@ class _COFF64(_COFF): symbol_prefix = "" re_global = re.compile(r'\s*\.def\s+(?P